From d8c4d95937d1cfd5a7dff2e28d695cf2bcc65350 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 4 Jun 2025 16:52:35 +0800 Subject: [PATCH 001/128] GitHub: enable the blank issues support --- .github/ISSUE_TEMPLATE/config.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 05a81306669..74483115591 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,23 +1,3 @@ -# -------------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed -# with this work for additional information regarding copyright -# ownership. The ASF licenses this file to You under the Apache -# License, Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. -# -# -------------------------------------------------------------------- - blank_issues_enabled: true contact_links: - name: 🙏🏻 Q&A From 894867ba1aaf557c1a13c81365f362b58e09bb78 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Mon, 20 Mar 2023 07:51:33 -0700 Subject: [PATCH 002/128] Generate gp_ view for desired pg_ system views For a selected list of PG system views (started with 'pg_'prefix ), we will create a corresponding 'gp_' view for each one in the list. Each 'gp_' view is basically a UNION ALL of the results of running the corresponding 'pg_' view on all segments (including the coordinator). Note that, these views do not aggregate the results. The aggregate version of the views will be named with a '_summary' appendix (such as 'gp_stat_all_tables_summary'). To add a new 'pg_' view to this list, simply put the name in file 'src/backend/catalog/system_views_gp.in'. This commit adds an initial list of views that we think make sense to have 'gp_' views. With this change, we also remove the existing definition of gp_stat_archiver view and let it be generated automatically. We also had gp_stat_replication but it carries additional column than pg_stat_replication so it cannot use the automatic way. --- src/backend/catalog/.gitignore | 1 + src/backend/catalog/Makefile | 8 +++-- src/backend/catalog/system_views.sql | 6 +--- src/backend/catalog/system_views_gp.in | 48 ++++++++++++++++++++++++++ src/bin/initdb/initdb.c | 4 +++ src/include/catalog/catversion.h | 2 +- 6 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 src/backend/catalog/system_views_gp.in diff --git a/src/backend/catalog/.gitignore b/src/backend/catalog/.gitignore index 6c4c6d228db..3912b022a03 100644 --- a/src/backend/catalog/.gitignore +++ b/src/backend/catalog/.gitignore @@ -8,3 +8,4 @@ /pg_*_d.h /gp_*_d.h /bki-stamp +/system_views_gp.sql diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index b95f92f2e6e..3c59fd3accf 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -56,6 +56,9 @@ OBJS += pg_extprotocol.o \ gp_matview_aux.o \ pg_directory_table.o storage_directory_table.o +GP_SYSVIEW_IN = system_views_gp.in +GP_SYSVIEW_SQL = system_views_gp.sql + CATALOG_JSON:= $(addprefix $(top_srcdir)/gpMgmt/bin/gppylib/data/, $(addsuffix .json,$(GP_MAJORVERSION))) include $(top_srcdir)/src/backend/common.mk @@ -133,7 +136,7 @@ POSTGRES_BKI_DATA += $(addprefix $(top_srcdir)/src/include/catalog/,\ $(top_builddir)/src/include/catalog/gp_version_at_initdb.dat -all: distprep generated-header-symlinks +all: distprep generated-header-symlinks $(GP_SYSVIEW_SQL) distprep: bki-stamp @@ -197,6 +200,7 @@ ifeq ($(USE_INTERNAL_FTS_FOUND), false) endif $(INSTALL_DATA) $(srcdir)/system_functions.sql '$(DESTDIR)$(datadir)/system_functions.sql' $(INSTALL_DATA) $(srcdir)/system_views.sql '$(DESTDIR)$(datadir)/system_views.sql' + $(INSTALL_DATA) $(srcdir)/$(GP_SYSVIEW_SQL) '$(DESTDIR)$(datadir)/$(GP_SYSVIEW_SQL)' $(INSTALL_DATA) $(srcdir)/information_schema.sql '$(DESTDIR)$(datadir)/information_schema.sql' $(INSTALL_DATA) $(call vpathsearch,cdb_schema.sql) '$(DESTDIR)$(datadir)/cdb_init.d/cdb_schema.sql' $(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt' @@ -217,4 +221,4 @@ endif clean: maintainer-clean: clean - rm -f bki-stamp postgres.bki system_constraints.sql $(GENERATED_HEADERS) + rm -f bki-stamp postgres.bki system_constraints.sql $(GENERATED_HEADERS) $(GP_SYSVIEW_SQL) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 86e938a3b87..5b5b9e5c644 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1095,6 +1095,7 @@ $$ $$ LANGUAGE SQL EXECUTE ON ALL SEGMENTS; +-- This view has an additional column than pg_stat_replication so cannot be generated using system_views_gp.in CREATE VIEW gp_stat_replication AS SELECT *, pg_catalog.gp_replication_error() AS sync_error FROM pg_catalog.gp_stat_get_master_replication() AS R @@ -1802,11 +1803,6 @@ UNION ALL SELECT gp_segment_id, gp_get_suboverflowed_backends() FROM gp_dist_random('gp_id') order by 1; -CREATE OR REPLACE VIEW gp_stat_archiver AS - SELECT -1 AS gp_segment_id, * FROM pg_stat_archiver - UNION - SELECT gp_execution_segment() AS gp_segment_id, * FROM gp_dist_random('pg_stat_archiver'); - CREATE FUNCTION gp_get_session_endpoints (OUT gp_segment_id int, OUT auth_token text, OUT cursorname text, OUT sessionid int, OUT hostname varchar(64), OUT port int, OUT username text, OUT state text, diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in new file mode 100644 index 00000000000..5c0a8c2dc7e --- /dev/null +++ b/src/backend/catalog/system_views_gp.in @@ -0,0 +1,48 @@ +# This file lists all the PG system views 'pg_%' that we would like to create an +# MPP-aware view 'gp_%' out of. The generated 'gp_%' view definitions will be placed +# in system_views_gp.sql, and initialized at the same time as system_views.sql. +pg_backend_memory_contexts +pg_config +pg_cursors +pg_file_settings +pg_replication_origin_status +pg_replication_slots +pg_settings +pg_stat_activity +pg_stat_archiver +pg_stat_bgwriter +pg_stat_database +pg_stat_database_conflicts +pg_stat_gssapi +pg_stat_operations +pg_stat_progress_analyze +pg_stat_progress_basebackup +pg_stat_progress_cluster +pg_stat_progress_copy +pg_stat_progress_create_index +pg_stat_progress_vacuum +pg_stat_slru +pg_stat_ssl +pg_stat_subscription +pg_stat_sys_indexes +pg_stat_sys_tables +pg_stat_user_functions +pg_stat_user_indexes +pg_stat_user_tables +pg_stat_wal +pg_stat_wal_receiver +pg_stat_xact_all_tables +pg_stat_xact_sys_tables +pg_stat_xact_user_functions +pg_stat_xact_user_tables +pg_statio_all_indexes +pg_statio_all_sequences +pg_statio_all_tables +pg_statio_sys_indexes +pg_statio_sys_sequences +pg_statio_sys_tables +pg_statio_user_indexes +pg_statio_user_sequences +pg_statio_user_tables +pg_stats +pg_stats_ext diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index f7c34a3e208..e4732816499 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -175,6 +175,7 @@ static char *external_fts_files; #endif static char *system_functions_file; static char *system_views_file; +static char *system_views_gp_file; static bool success = false; static bool made_new_pgdata = false; static bool found_existing_pgdata = false; @@ -2833,6 +2834,7 @@ setup_data_file_paths(void) set_input(&system_constraints_file, "system_constraints.sql"); set_input(&system_functions_file, "system_functions.sql"); set_input(&system_views_file, "system_views.sql"); + set_input(&system_views_gp_file, "system_views_gp.sql"); set_input(&cdb_init_d_dir, "cdb_init.d"); @@ -2866,6 +2868,7 @@ setup_data_file_paths(void) #endif check_input(system_functions_file); check_input(system_views_file); + check_input(system_views_gp_file); } @@ -3233,6 +3236,7 @@ initialize_data_directory(void) */ setup_run_file(cmdfd, system_views_file); + setup_run_file(cmdfd, system_views_gp_file); setup_description(cmdfd); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 86910a0dada..026192b3674 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -56,6 +56,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302502091 +#define CATALOG_VERSION_NO 302506101 #endif From c2a606f3f078d074539d5008465e91ba67164524 Mon Sep 17 00:00:00 2001 From: wangxiaoran Date: Tue, 10 Jun 2025 17:03:18 +0800 Subject: [PATCH 003/128] Fix system_views_gp.in and fix test query_conflict Some pg_ views have been modified by cbdb: the gp_segment_id colmun has been added to them. So they are failed to be transformed from the pg_ views to gp_ views (see commit 5028222620d410fe3d4c60f732a599e269006968) So just remove them from system_vies_gp.in. Maybe better to fix them later. --- pom.xml | 1 + src/backend/catalog/system_views_gp.in | 20 +++++++++---------- .../input/hot_standby/query_conflict.source | 4 ++-- .../output/hot_standby/query_conflict.source | 10 +++++----- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 0e000093399..f89b9d0bb8a 100644 --- a/pom.xml +++ b/pom.xml @@ -1050,6 +1050,7 @@ code or new licensing patterns. src/backend/postmaster/test/checkpointer_test.c src/backend/postmaster/README.auto-ANALYZE src/backend/mock.mk + src/backend/catalog/system_views_gp.in src/backend/catalog/storage_tablespace.c src/backend/catalog/test/storage_tablespace_test.c src/backend/catalog/sql_features.txt diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in index 5c0a8c2dc7e..d46dde3191e 100644 --- a/src/backend/catalog/system_views_gp.in +++ b/src/backend/catalog/system_views_gp.in @@ -1,7 +1,7 @@ # This file lists all the PG system views 'pg_%' that we would like to create an # MPP-aware view 'gp_%' out of. The generated 'gp_%' view definitions will be placed # in system_views_gp.sql, and initialized at the same time as system_views.sql. -pg_backend_memory_contexts +#pg_backend_memory_contexts pg_config pg_cursors pg_file_settings @@ -11,16 +11,16 @@ pg_settings pg_stat_activity pg_stat_archiver pg_stat_bgwriter -pg_stat_database +#pg_stat_database pg_stat_database_conflicts pg_stat_gssapi pg_stat_operations -pg_stat_progress_analyze -pg_stat_progress_basebackup -pg_stat_progress_cluster -pg_stat_progress_copy -pg_stat_progress_create_index -pg_stat_progress_vacuum +#pg_stat_progress_analyze +#pg_stat_progress_basebackup +#pg_stat_progress_cluster +#pg_stat_progress_copy +#pg_stat_progress_create_index +#pg_stat_progress_vacuum pg_stat_slru pg_stat_ssl pg_stat_subscription @@ -29,7 +29,7 @@ pg_stat_sys_tables pg_stat_user_functions pg_stat_user_indexes pg_stat_user_tables -pg_stat_wal +#pg_stat_wal pg_stat_wal_receiver pg_stat_xact_all_tables pg_stat_xact_sys_tables @@ -44,5 +44,5 @@ pg_statio_sys_tables pg_statio_user_indexes pg_statio_user_sequences pg_statio_user_tables -pg_stats +#pg_stats ERROR: column "most_common_vals" has pseudo-type anyarray pg_stats_ext diff --git a/src/test/isolation2/input/hot_standby/query_conflict.source b/src/test/isolation2/input/hot_standby/query_conflict.source index 0e2706bfa5a..5f2aee3be53 100644 --- a/src/test/isolation2/input/hot_standby/query_conflict.source +++ b/src/test/isolation2/input/hot_standby/query_conflict.source @@ -126,8 +126,8 @@ select gp_inject_fault('after_open_temp_file', 'reset',dbid) from gp_segment_con -1S<: -1Sq: --- conflict has been recorded --1S: select max(confl_tablespace) from gp_stat_database_conflicts where datname = 'isolation2-hot-standby'; +-- conflict has been recorded. The query has multiple slices +-1S: select max(confl_tablespace) >= 1 from gp_stat_database_conflicts where datname = 'isolation2-hot-standby'; -- cleanup !\retcode rm -rf @testtablespace@/hs_tablespace_directory; diff --git a/src/test/isolation2/output/hot_standby/query_conflict.source b/src/test/isolation2/output/hot_standby/query_conflict.source index 397e3977d12..909d2532df3 100644 --- a/src/test/isolation2/output/hot_standby/query_conflict.source +++ b/src/test/isolation2/output/hot_standby/query_conflict.source @@ -269,11 +269,11 @@ ERROR: canceling statement due to conflict with recovery (seg1 slice3 127.0.1. DETAIL: User was or might have been using tablespace that must be dropped. -1Sq: ... --- conflict has been recorded --1S: select max(confl_tablespace) from gp_stat_database_conflicts where datname = 'isolation2-hot-standby'; - max ------ - 1 +-- conflict has been recorded. The query has multiple slices +-1S: select max(confl_tablespace) >= 1 from gp_stat_database_conflicts where datname = 'isolation2-hot-standby'; + ?column? +---------- + t (1 row) -- cleanup From 4699ddb7b56cc950827c1fecce1d6e48c19a9a9f Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Mon, 20 Mar 2023 14:29:50 -0700 Subject: [PATCH 004/128] Fix names of pg_stat_all_tables|indexes We used to not have a very clear naming guideline for the existing 'pg_%' system views and the MPP versions of them. As an example, we renamed PG's pg_stat_all_tables and pg_stat_all_indexes to have an '_internal' appendix, and used their original names to collect aggregated results from all segments (commit e6f93033289e). However, with the previous commit, we now let all existing PG system views to have their original names, while add corresponding 'gp_%' views for the non-aggregated results from all segments, and 'gp_%_summary' views for aggregated results from all segments. Therefore, we now revert pg_stat_all_tables and pg_stat_all_indexes back to their original definitions, which just collect stats from a single segment. Then, we add them to sytem_views_gp.in to produce gp_stat_all_tables and gp_stat_all_indexes which collect non-aggregated results from all segments. Finally, we rename the aggregate version of those views to be gp_stat_all_tables_summary and gp_stat_all_indexes_summary. Because views pg_stat_user_tables and pg_stat_user_indexes use the above sumary views, we have to add _summary views for these two views as well. We will add _summary for other system views later. Modify regress test accordingly. --- src/backend/catalog/system_views.sql | 34 ++++++--- src/backend/catalog/system_views_gp.in | 2 + src/test/regress/expected/pg_stat.out | 16 ++-- .../regress/input/pgstat_qd_tabstat.source | 76 +++++++++---------- .../regress/output/pgstat_qd_tabstat.source | 76 +++++++++---------- src/test/regress/sql/pg_stat.sql | 16 ++-- 6 files changed, 116 insertions(+), 104 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 5b5b9e5c644..55bd43d57b7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -657,7 +657,7 @@ REVOKE EXECUTE ON FUNCTION pg_get_backend_memory_contexts() FROM PUBLIC; -- Statistics views -CREATE VIEW pg_stat_all_tables_internal AS +CREATE VIEW pg_stat_all_tables AS SELECT C.oid AS relid, N.nspname AS schemaname, @@ -691,7 +691,7 @@ CREATE VIEW pg_stat_all_tables_internal AS -- Gather data from segments on user tables, and use data on coordinator on system tables. -CREATE VIEW pg_stat_all_tables AS +CREATE VIEW gp_stat_all_tables_summary AS SELECT s.relid, s.schemaname, @@ -742,7 +742,7 @@ FROM max(analyze_count) as analyze_count, max(autoanalyze_count) as autoanalyze_count FROM - gp_dist_random('pg_stat_all_tables_internal') allt + gp_dist_random('pg_stat_all_tables') allt inner join pg_class c on allt.relid = c.oid left outer join gp_distribution_policy d @@ -760,9 +760,9 @@ FROM SELECT * FROM - pg_stat_all_tables_internal + pg_stat_all_tables WHERE - relid < 16384) m, pg_stat_all_tables_internal s + relid < 16384) m, pg_stat_all_tables s WHERE m.relid = s.relid; CREATE VIEW pg_stat_xact_all_tables AS @@ -812,7 +812,12 @@ CREATE VIEW pg_stat_user_tables AS -- since we don't have segments. -- We create a new view for single node mode. CREATE VIEW pg_stat_user_tables_single_node AS - SELECT * FROM pg_stat_all_tables_internal + SELECT * FROM pg_stat_all_tables + WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_stat_user_tables_summary AS + SELECT * FROM gp_stat_all_tables_summary WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND schemaname !~ '^pg_toast'; @@ -856,7 +861,7 @@ CREATE VIEW pg_statio_user_tables AS WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND schemaname !~ '^pg_toast'; -CREATE VIEW pg_stat_all_indexes_internal AS +CREATE VIEW pg_stat_all_indexes AS SELECT C.oid AS relid, I.oid AS indexrelid, @@ -874,7 +879,7 @@ CREATE VIEW pg_stat_all_indexes_internal AS -- Gather data from segments on user tables, and use data on coordinator on system tables. -CREATE VIEW pg_stat_all_indexes AS +CREATE VIEW gp_stat_all_indexes_summary AS SELECT s.relid, s.indexrelid, @@ -895,7 +900,7 @@ FROM sum(idx_tup_read) as idx_tup_read, sum(idx_tup_fetch) as idx_tup_fetch FROM - gp_dist_random('pg_stat_all_indexes_internal') + gp_dist_random('pg_stat_all_indexes') WHERE relid >= 16384 GROUP BY relid, indexrelid, schemaname, relname, indexrelname @@ -905,10 +910,10 @@ FROM SELECT * FROM - pg_stat_all_indexes_internal + pg_stat_all_indexes WHERE - relid < 16384) m, pg_stat_all_indexes_internal s -WHERE m.indexrelid = s.indexrelid; + relid < 16384) m, pg_stat_all_indexes s +WHERE m.relid = s.relid; CREATE VIEW pg_stat_sys_indexes AS SELECT * FROM pg_stat_all_indexes @@ -920,6 +925,11 @@ CREATE VIEW pg_stat_user_indexes AS WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND schemaname !~ '^pg_toast'; +CREATE VIEW gp_stat_user_indexes_summary AS + SELECT * FROM gp_stat_all_indexes_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND + schemaname !~ '^pg_toast'; + CREATE VIEW pg_statio_all_indexes AS SELECT C.oid AS relid, diff --git a/src/backend/catalog/system_views_gp.in b/src/backend/catalog/system_views_gp.in index d46dde3191e..cd865cea662 100644 --- a/src/backend/catalog/system_views_gp.in +++ b/src/backend/catalog/system_views_gp.in @@ -9,6 +9,8 @@ pg_replication_origin_status pg_replication_slots pg_settings pg_stat_activity +pg_stat_all_indexes +pg_stat_all_tables pg_stat_archiver pg_stat_bgwriter #pg_stat_database diff --git a/src/test/regress/expected/pg_stat.out b/src/test/regress/expected/pg_stat.out index 80e4e4ff132..8c2b2e81d32 100644 --- a/src/test/regress/expected/pg_stat.out +++ b/src/test/regress/expected/pg_stat.out @@ -5,7 +5,7 @@ create table pg_stat_test(a int); select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup -from pg_stat_all_tables where relname = 'pg_stat_test'; +from gp_stat_all_tables_summary where relname = 'pg_stat_test'; schemaname | relname | seq_scan | seq_tup_read | idx_scan | idx_tup_fetch | n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup ------------+--------------+----------+--------------+----------+---------------+-----------+-----------+-----------+---------------+------------+------------ public | pg_stat_test | 0 | 0 | | | 0 | 0 | 0 | 0 | 0 | 0 @@ -14,7 +14,7 @@ from pg_stat_all_tables where relname = 'pg_stat_test'; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup -from pg_stat_user_tables where relname = 'pg_stat_test'; +from gp_stat_user_tables_summary where relname = 'pg_stat_test'; schemaname | relname | seq_scan | seq_tup_read | idx_scan | idx_tup_fetch | n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup ------------+--------------+----------+--------------+----------+---------------+-----------+-----------+-----------+---------------+------------+------------ public | pg_stat_test | 0 | 0 | | | 0 | 0 | 0 | 0 | 0 | 0 @@ -22,14 +22,14 @@ from pg_stat_user_tables where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_all_indexes where relname = 'pg_stat_test'; +from gp_stat_all_indexes_summary where relname = 'pg_stat_test'; schemaname | relname | indexrelname | idx_scan | idx_tup_read | idx_tup_fetch ------------+---------+--------------+----------+--------------+--------------- (0 rows) select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_user_indexes where relname = 'pg_stat_test'; +from gp_stat_user_indexes_summary where relname = 'pg_stat_test'; schemaname | relname | indexrelname | idx_scan | idx_tup_read | idx_tup_fetch ------------+---------+--------------+----------+--------------+--------------- (0 rows) @@ -63,7 +63,7 @@ reset enable_seqscan; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze -from pg_stat_all_tables where relname = 'pg_stat_test'; +from gp_stat_all_tables_summary where relname = 'pg_stat_test'; schemaname | relname | seq_scan | seq_tup_read | idx_scan | idx_tup_fetch | n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze ------------+--------------+----------+--------------+----------+---------------+-----------+-----------+-----------+---------------+------------+------------+--------------------- public | pg_stat_test | 12 | 391 | 1 | 0 | 110 | 0 | 19 | 0 | 91 | 19 | 129 @@ -72,7 +72,7 @@ from pg_stat_all_tables where relname = 'pg_stat_test'; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze -from pg_stat_user_tables where relname = 'pg_stat_test'; +from gp_stat_user_tables_summary where relname = 'pg_stat_test'; schemaname | relname | seq_scan | seq_tup_read | idx_scan | idx_tup_fetch | n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze ------------+--------------+----------+--------------+----------+---------------+-----------+-----------+-----------+---------------+------------+------------+--------------------- public | pg_stat_test | 12 | 391 | 1 | 0 | 110 | 0 | 19 | 0 | 91 | 19 | 129 @@ -80,7 +80,7 @@ from pg_stat_user_tables where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_all_indexes where relname = 'pg_stat_test'; +from gp_stat_all_indexes_summary where relname = 'pg_stat_test'; schemaname | relname | indexrelname | idx_scan | idx_tup_read | idx_tup_fetch ------------+--------------+--------------------------+----------+--------------+--------------- public | pg_stat_test | pg_stat_user_table_index | 1 | 1 | 0 @@ -88,7 +88,7 @@ from pg_stat_all_indexes where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_user_indexes where relname = 'pg_stat_test'; +from gp_stat_user_indexes_summary where relname = 'pg_stat_test'; schemaname | relname | indexrelname | idx_scan | idx_tup_read | idx_tup_fetch ------------+--------------+--------------------------+----------+--------------+--------------- public | pg_stat_test | pg_stat_user_table_index | 1 | 1 | 0 diff --git a/src/test/regress/input/pgstat_qd_tabstat.source b/src/test/regress/input/pgstat_qd_tabstat.source index 0fb8c0ccb71..e9f76ccf0d6 100644 --- a/src/test/regress/input/pgstat_qd_tabstat.source +++ b/src/test/regress/input/pgstat_qd_tabstat.source @@ -10,7 +10,7 @@ copy table_for_docopy (i, j) from stdin; 3 hello3 \. select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_docopy'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_docopy'::regclass; CREATE TABLE data_tbl (a int,b char) distributed by (a); INSERT INTO data_tbl values(1,'1'); @@ -21,7 +21,7 @@ COPY data_tbl TO '/tmp/data_tbl.csv' on segment; create table copy_on_segment (a int,b char); COPY copy_on_segment from '/tmp/data_tbl.csv' on segment log errors segment reject limit 3 rows; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'copy_on_segment'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'copy_on_segment'::regclass; -- Test pgstat table stat in initplan on QD @@ -34,26 +34,26 @@ copy table_for_initplan (i, j, k) from stdin; explain (costs off) with updated AS (update table_for_initplan set k = 33 where i = 3 returning k) select table_for_initplan.*, (select sum(k) from updated) from table_for_initplan; with updated AS (update table_for_initplan set k = 33 where i = 3 returning k) select table_for_initplan.*, (select sum(k) from updated) from table_for_initplan; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_initplan'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_initplan'::regclass; -- Test pgstat table stat in CTAS on QD create table table_for_ctas with (autovacuum_enabled=false) as select i, 'hello' || i from generate_series(1, 100) f(i); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_ctas'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_ctas'::regclass; select i, 'hello' || i into table_for_insert_into from generate_series(1, 100) f(i); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_insert_into'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_insert_into'::regclass; -- Test pgstat table stat in ALTER TABLE SET DISTRIBUTED BY on QD create table table_for_set_distributed_by(i int, j varchar) distributed by (i); insert into table_for_set_distributed_by select i, 'hello' || i from generate_series(1, 333) f(i); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_set_distributed_by'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_set_distributed_by'::regclass; alter table table_for_set_distributed_by set distributed by (j); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_set_distributed_by'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_set_distributed_by'::regclass; -- Test pgstat table stat in execution of funciton on QD @@ -68,7 +68,7 @@ $$ language plpgsql volatile; select update_table_for_function(); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_function'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_function'::regclass; -- Test pgstat table stat in ALTER TABLE EXPAND TABLE on QD; @@ -78,11 +78,11 @@ create table table_for_expand(i int, j varchar) distributed by (i); insert into table_for_expand select i, 'hello' || i from generate_series(1, 333) f(i); select count(distinct gp_segment_id) from table_for_expand; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_expand'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_expand'::regclass; alter table table_for_expand expand table; select count(distinct gp_segment_id) from table_for_expand; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_expand'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_expand'::regclass; select gp_debug_reset_create_table_default_numsegments(); @@ -103,7 +103,7 @@ update table_for_iud set j = 'heroes never die' where i >= 300; release savepoint level3; commit; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_iud'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_iud'::regclass; begin; savepoint level1; @@ -120,14 +120,14 @@ rollback to savepoint level3; delete from table_for_iud where i <= 200; commit; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_iud'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_iud'::regclass; -- Test pgstat table stat in TRUNCATE on QD create table table_for_truncate(i int, j varchar) distributed by (i); insert into table_for_truncate select i, 'hello' || i from generate_series(1, 777) f(i); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_truncate'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_truncate'::regclass; begin; savepoint level1; savepoint level2; @@ -141,12 +141,12 @@ delete from table_for_truncate where i >= 700; update table_for_truncate set j = 'D' where i <= 200; commit; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate'::regclass; create table table_for_truncate_abort(i int, j varchar) distributed by (i); insert into table_for_truncate_abort select i, 'hello' || i from generate_series(1, 777) f(i); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate_abort'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate_abort'::regclass; begin; savepoint level1; savepoint level2; @@ -160,7 +160,7 @@ delete from table_for_truncate_abort where i < 700; update table_for_truncate_abort set j = 'D' where i >= 200; rollback; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate_abort'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate_abort'::regclass; -- Test pgstat table stat for partition table on QD @@ -171,17 +171,17 @@ PARTITION BY RANGE (rank) DEFAULT PARTITION extra ); insert into rankpart select i, i % 10, i from generate_series(1, 1000)i; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; begin; delete from rankpart where id <= 100; rollback; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; copy rankpart (id, rank, product) from stdin; 1001 1 1001 @@ -196,8 +196,8 @@ copy rankpart (id, rank, product) from stdin; 1010 6 1010 \. select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; begin; update rankpart set rank = 1 where id > 1005; @@ -209,9 +209,9 @@ release savepoint level2; rollback to savepoint level1; commit; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; begin; savepoint level1_1; @@ -219,9 +219,9 @@ insert into rankpart select i, i % 10, i from generate_series(2001, 3000)i; insert into rankpart select i, i % 10, i from generate_series(3001, 4000)i; commit; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; -- Test pgstat matview stat with distributed policy. @@ -229,11 +229,11 @@ create table base_table(i int, j int, z int ) distributed by (i); insert into base_table select i,i,i from generate_series(1, 100) i; create materialized view mt as select * from base_table where z>=50; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; insert into base_table select i,i,i from generate_series(1, 100) i; refresh materialized view mt; select pg_sleep(0.77) from gp_dist_random('gp_id'); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; -- pg_stat_all_tables collects gpstats across segments select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; @@ -245,11 +245,11 @@ create table base_table(i int, j int, z int ) distributed replicated; insert into base_table select i,i,i from generate_series(1, 100) i; create materialized view mt as select * from base_table where z>=50 distributed replicated; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; insert into base_table select i,i,i from generate_series(1, 100) i; refresh materialized view mt; select pg_sleep(0.77) from gp_dist_random('gp_id'); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; -- pg_stat_all_tables collects gpstats across segments select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; @@ -263,15 +263,15 @@ insert into tabstat_ao select 1,1; delete from tabstat_ao; select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select count(*) from pg_stat_all_tables +select count(*) from gp_stat_all_tables_summary where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass) OR relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass) OR relid = (select visimaprelid from pg_appendonly where relid = 'tabstat_ao'::regclass); select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. -select n_tup_ins from pg_stat_all_tables where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); -select n_tup_ins from pg_stat_all_tables where relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); -select n_tup_ins from pg_stat_all_tables where relid = (select visimaprelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select visimaprelid from pg_appendonly where relid = 'tabstat_ao'::regclass); drop table tabstat_ao; diff --git a/src/test/regress/output/pgstat_qd_tabstat.source b/src/test/regress/output/pgstat_qd_tabstat.source index ca612e18b9c..591f262f7c5 100644 --- a/src/test/regress/output/pgstat_qd_tabstat.source +++ b/src/test/regress/output/pgstat_qd_tabstat.source @@ -10,7 +10,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_docopy'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_docopy'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 3 | 0 | 0 | 0 | 3 | 0 | 3 @@ -32,7 +32,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'copy_on_segment'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'copy_on_segment'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 4 | 0 | 0 | 0 | 4 | 0 | 4 @@ -70,7 +70,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_initplan'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_initplan'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 3 | 1 | 0 | 0 | 3 | 1 | 4 @@ -86,7 +86,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_ctas'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_ctas'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 100 | 0 | 0 | 0 | 100 | 0 | 100 @@ -101,7 +101,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_insert_into'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_insert_into'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 100 | 0 | 0 | 0 | 100 | 0 | 100 @@ -116,7 +116,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_set_distributed_by'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_set_distributed_by'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 333 | 0 | 0 | 0 | 333 | 0 | 333 @@ -129,7 +129,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_set_distributed_by'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_set_distributed_by'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 333 | 0 | 0 | 0 | 333 | 0 | 333 @@ -157,7 +157,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_function'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_function'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 333 | 0 | 200 | 0 | 133 | 200 | 533 @@ -185,7 +185,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_expand'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_expand'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 333 | 0 | 0 | 0 | 333 | 0 | 333 @@ -204,7 +204,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_expand'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_expand'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 333 | 0 | 0 | 0 | 333 | 0 | 333 @@ -238,7 +238,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_iud'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_iud'::regclass; n_live_tup | n_dead_tup ------------+------------ 333 | 34 @@ -264,7 +264,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_iud'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_iud'::regclass; n_live_tup | n_dead_tup ------------+------------ 133 | 713 @@ -279,7 +279,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'table_for_truncate'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'table_for_truncate'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 777 | 0 | 0 | 0 | 777 | 0 | 777 @@ -303,7 +303,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate'::regclass; n_live_tup | n_dead_tup ------------+------------ 699 | 301 @@ -317,7 +317,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate_abort'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate_abort'::regclass; n_live_tup | n_dead_tup ------------+------------ 777 | 0 @@ -341,7 +341,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_live_tup, n_dead_tup from pg_stat_all_tables_internal where relid = 'table_for_truncate_abort'::regclass; +select n_live_tup, n_dead_tup from pg_stat_all_tables where relid = 'table_for_truncate_abort'::regclass; n_live_tup | n_dead_tup ------------+------------ 777 | 223 @@ -360,19 +360,19 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 500 | 0 | 0 | 0 | 500 | 0 | 500 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 400 | 0 | 0 | 0 | 400 | 0 | 400 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 100 | 0 | 0 | 0 | 100 | 0 | 100 @@ -387,19 +387,19 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 500 | 0 | 50 | 0 | 500 | 0 | 500 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 400 | 0 | 40 | 0 | 400 | 0 | 400 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 100 | 0 | 10 | 0 | 100 | 0 | 100 @@ -412,13 +412,13 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 505 | 0 | 50 | 0 | 505 | 0 | 505 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 405 | 0 | 40 | 0 | 405 | 0 | 405 @@ -439,19 +439,19 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 510 | 0 | 55 | 0 | 510 | 0 | 510 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 410 | 0 | 50 | 0 | 400 | 10 | 410 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 100 | 0 | 10 | 0 | 100 | 0 | 100 @@ -468,19 +468,19 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_2'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_2'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 1510 | 0 | 55 | 0 | 1510 | 0 | 1510 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_3'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_3'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 1210 | 0 | 50 | 0 | 1200 | 10 | 1210 (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'rankpart_1_prt_extra'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'rankpart_1_prt_extra'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 300 | 0 | 10 | 0 | 300 | 0 | 300 @@ -498,7 +498,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 51 | 0 | 0 | 0 | 51 | 0 | 51 @@ -514,7 +514,7 @@ select pg_sleep(0.77) from gp_dist_random('gp_id'); -- Force pgstat_report_stat( (3 rows) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 153 | 0 | 0 | 0 | 102 | 0 | 153 @@ -539,7 +539,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 51 | 0 | 0 | 0 | 51 | 0 | 51 @@ -555,7 +555,7 @@ select pg_sleep(0.77) from gp_dist_random('gp_id'); -- Force pgstat_report_stat( (3 rows) -select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables_internal where relid = 'mt'::regclass; +select n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze from pg_stat_all_tables where relid = 'mt'::regclass; n_tup_ins | n_tup_upd | n_tup_del | n_tup_hot_upd | n_live_tup | n_dead_tup | n_mod_since_analyze -----------+-----------+-----------+---------------+------------+------------+--------------------- 153 | 0 | 0 | 0 | 102 | 0 | 153 @@ -580,7 +580,7 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select count(*) from pg_stat_all_tables +select count(*) from gp_stat_all_tables_summary where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass) OR relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass) @@ -596,19 +596,19 @@ select pg_sleep(0.77); -- Force pgstat_report_stat() to send tabstat. (1 row) -select n_tup_ins from pg_stat_all_tables where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select segrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); n_tup_ins ----------- 1 (1 row) -select n_tup_ins from pg_stat_all_tables where relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select blkdirrelid from pg_appendonly where relid = 'tabstat_ao'::regclass); n_tup_ins ----------- 1 (1 row) -select n_tup_ins from pg_stat_all_tables where relid = (select visimaprelid from pg_appendonly where relid = 'tabstat_ao'::regclass); +select n_tup_ins from gp_stat_all_tables_summary where relid = (select visimaprelid from pg_appendonly where relid = 'tabstat_ao'::regclass); n_tup_ins ----------- 1 diff --git a/src/test/regress/sql/pg_stat.sql b/src/test/regress/sql/pg_stat.sql index d9fc37850b0..383a9149186 100644 --- a/src/test/regress/sql/pg_stat.sql +++ b/src/test/regress/sql/pg_stat.sql @@ -6,17 +6,17 @@ create table pg_stat_test(a int); select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup -from pg_stat_all_tables where relname = 'pg_stat_test'; +from gp_stat_all_tables_summary where relname = 'pg_stat_test'; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup -from pg_stat_user_tables where relname = 'pg_stat_test'; +from gp_stat_user_tables_summary where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_all_indexes where relname = 'pg_stat_test'; +from gp_stat_all_indexes_summary where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_user_indexes where relname = 'pg_stat_test'; +from gp_stat_user_indexes_summary where relname = 'pg_stat_test'; begin; -- make analyze same transcation with insert to avoid double the pgstat causes by unorder message read. insert into pg_stat_test select * from generate_series(1, 100); @@ -42,17 +42,17 @@ reset enable_seqscan; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze -from pg_stat_all_tables where relname = 'pg_stat_test'; +from gp_stat_all_tables_summary where relname = 'pg_stat_test'; select schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze -from pg_stat_user_tables where relname = 'pg_stat_test'; +from gp_stat_user_tables_summary where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_all_indexes where relname = 'pg_stat_test'; +from gp_stat_all_indexes_summary where relname = 'pg_stat_test'; select schemaname, relname, indexrelname, idx_scan, idx_tup_read, idx_tup_fetch -from pg_stat_user_indexes where relname = 'pg_stat_test'; +from gp_stat_user_indexes_summary where relname = 'pg_stat_test'; reset optimizer; reset max_parallel_workers_per_gather; From 3ce93173469001e916384f282b9579388ee89fc0 Mon Sep 17 00:00:00 2001 From: Alexandra Wang Date: Thu, 27 Apr 2023 16:53:32 -0700 Subject: [PATCH 005/128] Add gp_stat_progress_%_summary system views Added the following views: gp_stat_progress_vacuum_summary gp_stat_progress_analyze_summary gp_stat_progress_cluster_summary gp_stat_progress_create_index_summary Also replaced pg_stat_progress_* views with gp_stat_progress_* views for existing tests. --- src/backend/catalog/Makefile | 4 +- .../catalog/system_views_gp_summary.sql | 109 ++++++++++ src/backend/commands/analyze.c | 1 + src/bin/initdb/initdb.c | 6 +- src/include/catalog/catversion.h | 2 +- .../isolation2/expected/analyze_progress.out | 95 +++++++++ .../expected/ao_index_build_progress.out | 88 ++++++-- .../expected/vacuum_progress_row.out | 189 +++++++++++------- src/test/isolation2/isolation2_schedule | 1 + src/test/isolation2/sql/analyze_progress.sql | 42 ++++ .../sql/ao_index_build_progress.sql | 44 ++-- src/test/isolation2/sql/setup.sql | 21 ++ .../isolation2/sql/vacuum_progress_row.sql | 168 +++++----------- 13 files changed, 539 insertions(+), 231 deletions(-) create mode 100644 src/backend/catalog/system_views_gp_summary.sql create mode 100644 src/test/isolation2/expected/analyze_progress.out create mode 100644 src/test/isolation2/sql/analyze_progress.sql diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 3c59fd3accf..e2ad48a8699 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -201,6 +201,7 @@ endif $(INSTALL_DATA) $(srcdir)/system_functions.sql '$(DESTDIR)$(datadir)/system_functions.sql' $(INSTALL_DATA) $(srcdir)/system_views.sql '$(DESTDIR)$(datadir)/system_views.sql' $(INSTALL_DATA) $(srcdir)/$(GP_SYSVIEW_SQL) '$(DESTDIR)$(datadir)/$(GP_SYSVIEW_SQL)' + $(INSTALL_DATA) $(srcdir)/system_views_gp_summary.sql '$(DESTDIR)$(datadir)/system_views_gp_summary.sql' $(INSTALL_DATA) $(srcdir)/information_schema.sql '$(DESTDIR)$(datadir)/information_schema.sql' $(INSTALL_DATA) $(call vpathsearch,cdb_schema.sql) '$(DESTDIR)$(datadir)/cdb_init.d/cdb_schema.sql' $(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt' @@ -212,7 +213,8 @@ installdirs: .PHONY: uninstall-data uninstall-data: - rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt fix-CVE-2024-4317.sql) + rm -f $(addprefix '$(DESTDIR)$(datadir)'/, postgres.bki system_constraints.sql system_functions.sql system_views.sql system_views_gp_summary.sql information_schema.sql cdb_init.d/cdb_schema.sql cdb_init.d/gp_toolkit.sql sql_features.txt fix-CVE-2024-4317.sql) + ifeq ($(USE_INTERNAL_FTS_FOUND), false) rm -f $(addprefix '$(DESTDIR)$(datadir)'/, external_fts.sql) endif diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql new file mode 100644 index 00000000000..58f4768c124 --- /dev/null +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -0,0 +1,109 @@ +/* + * Greenplum System Summary Views + * + * Portions Copyright (c) 2006-2010, Greenplum inc. + * Portions Copyright (c) 2012-Present VMware, Inc. or its affiliates. + * Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + * src/backend/catalog/system_views_gp_summary.sql + * + + * This file contains summary views for various Greenplum system catalog + * views. These summary views are designed to provide aggregated or averaged + * information for partitioned and replicated tables, considering multiple + * segments in a Greenplum database. + * + * Note: this file is read in single-user -j mode, which means that the + * command terminator is semicolon-newline-newline; whenever the backend + * sees that, it stops and executes what it's got. If you write a lot of + * statements without empty lines between, they'll all get quoted to you + * in any error message about one of them, so don't do that. Also, you + * cannot write a semicolon immediately followed by an empty line in a + * string literal (including a function body!) or a multiline comment. + */ + +CREATE VIEW gp_stat_progress_vacuum_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.phase, + case when d.policytype = 'r' then (sum(a.heap_blks_total)/d.numsegments)::bigint else sum(a.heap_blks_total) end heap_blks_total, + case when d.policytype = 'r' then (sum(a.heap_blks_scanned)/d.numsegments)::bigint else sum(a.heap_blks_scanned) end heap_blks_scanned, + case when d.policytype = 'r' then (sum(a.heap_blks_vacuumed)/d.numsegments)::bigint else sum(a.heap_blks_vacuumed) end heap_blks_vacuumed, + case when d.policytype = 'r' then (sum(a.index_vacuum_count)/d.numsegments)::bigint else sum(a.index_vacuum_count) end index_vacuum_count, + case when d.policytype = 'r' then (sum(a.max_dead_tuples)/d.numsegments)::bigint else sum(a.max_dead_tuples) end max_dead_tuples, + case when d.policytype = 'r' then (sum(a.num_dead_tuples)/d.numsegments)::bigint else sum(a.num_dead_tuples) end num_dead_tuples +FROM gp_stat_progress_vacuum a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_vacuum a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.phase, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_analyze_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.phase, + case when d.policytype = 'r' then (sum(a.sample_blks_total)/d.numsegments)::bigint else sum(a.sample_blks_total) end sample_blks_total, + case when d.policytype = 'r' then (sum(a.sample_blks_scanned)/d.numsegments)::bigint else sum(a.sample_blks_scanned) end sample_blks_scanned, + case when d.policytype = 'r' then (sum(a.ext_stats_total)/d.numsegments)::bigint else sum(a.ext_stats_total) end ext_stats_total, + case when d.policytype = 'r' then (sum(a.ext_stats_computed)/d.numsegments)::bigint else sum(a.ext_stats_computed) end ext_stats_computed, + case when d.policytype = 'r' then (sum(a.child_tables_total)/d.numsegments)::bigint else sum(a.child_tables_total) end child_tables_total, + case when d.policytype = 'r' then (sum(a.child_tables_done)/d.numsegments)::bigint else sum(a.child_tables_done) end child_tables_done +FROM gp_stat_progress_analyze a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_analyze a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.phase, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_cluster_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.command, + a.phase, + a.cluster_index_relid, + case when d.policytype = 'r' then (sum(a.heap_tuples_scanned)/d.numsegments)::bigint else sum(a.heap_tuples_scanned) end heap_tuples_scanned, + case when d.policytype = 'r' then (sum(a.heap_tuples_written)/d.numsegments)::bigint else sum(a.heap_tuples_written) end heap_tuples_written, + case when d.policytype = 'r' then (sum(a.heap_blks_total)/d.numsegments)::bigint else sum(a.heap_blks_total) end heap_blks_total, + case when d.policytype = 'r' then (sum(a.heap_blks_scanned)/d.numsegments)::bigint else sum(a.heap_blks_scanned) end heap_blks_scanned, + case when d.policytype = 'r' then (sum(a.index_rebuild_count)/d.numsegments)::bigint else sum(a.index_rebuild_count) end index_rebuild_count +FROM gp_stat_progress_cluster a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_cluster a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.command, a.phase, a.cluster_index_relid, d.policytype, d.numsegments; + +CREATE OR REPLACE VIEW gp_stat_progress_create_index_summary AS +SELECT + max(coalesce(a1.pid, 0)) as pid, + a.datid, + a.datname, + a.relid, + a.index_relid, + a.command, + a.phase, + case when d.policytype = 'r' then (sum(a.lockers_total)/d.numsegments)::bigint else sum(a.lockers_total) end lockers_total, + case when d.policytype = 'r' then (sum(a.lockers_done)/d.numsegments)::bigint else sum(a.lockers_done) end lockers_done, + max(a.current_locker_pid) as current_locker_pid, + case when d.policytype = 'r' then (sum(a.blocks_total)/d.numsegments)::bigint else sum(a.blocks_total) end blocks_total, + case when d.policytype = 'r' then (sum(a.blocks_done)/d.numsegments)::bigint else sum(a.blocks_done) end blocks_done, + case when d.policytype = 'r' then (sum(a.tuples_total)/d.numsegments)::bigint else sum(a.tuples_total) end tuples_total, + case when d.policytype = 'r' then (sum(a.tuples_done)/d.numsegments)::bigint else sum(a.tuples_done) end tuples_done, + case when d.policytype = 'r' then (sum(a.partitions_total)/d.numsegments)::bigint else sum(a.partitions_total) end partitions_total, + case when d.policytype = 'r' then (sum(a.partitions_done)/d.numsegments)::bigint else sum(a.partitions_done) end partitions_done +FROM gp_stat_progress_create_index a + JOIN pg_class c ON a.relid = c.oid + LEFT JOIN gp_distribution_policy d ON c.oid = d.localoid + LEFT JOIN gp_stat_progress_create_index a1 ON a.pid = a1.pid AND a1.gp_segment_id = -1 +WHERE a.gp_segment_id > -1 +GROUP BY a.datid, a.datname, a.relid, a.index_relid, a.command, a.phase, d.policytype, d.numsegments; diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 8d329d44503..c15dcdc8213 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -1896,6 +1896,7 @@ acquire_sample_rows(Relation onerel, int elevel, pgstat_progress_update_param(PROGRESS_ANALYZE_BLOCKS_DONE, ++blksdone); + SIMPLE_FAULT_INJECTOR("analyze_block"); } ExecDropSingleTupleTableSlot(slot); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index e4732816499..708cf77ffdf 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -176,6 +176,7 @@ static char *external_fts_files; static char *system_functions_file; static char *system_views_file; static char *system_views_gp_file; +static char *system_views_gp_summary_file; static bool success = false; static bool made_new_pgdata = false; static bool found_existing_pgdata = false; @@ -1706,8 +1707,6 @@ setup_run_file(FILE *cmdfd, const char *filename) } PG_CMD_PUTS("\n\n"); - - free(lines); } /* @@ -2835,6 +2834,7 @@ setup_data_file_paths(void) set_input(&system_functions_file, "system_functions.sql"); set_input(&system_views_file, "system_views.sql"); set_input(&system_views_gp_file, "system_views_gp.sql"); + set_input(&system_views_gp_summary_file, "system_views_gp_summary.sql"); set_input(&cdb_init_d_dir, "cdb_init.d"); @@ -2869,6 +2869,7 @@ setup_data_file_paths(void) check_input(system_functions_file); check_input(system_views_file); check_input(system_views_gp_file); + check_input(system_views_gp_summary_file); } @@ -3237,6 +3238,7 @@ initialize_data_directory(void) setup_run_file(cmdfd, system_views_file); setup_run_file(cmdfd, system_views_gp_file); + setup_run_file(cmdfd, system_views_gp_summary_file); setup_description(cmdfd); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 026192b3674..ac649996d98 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -56,6 +56,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302506101 +#define CATALOG_VERSION_NO 302509031 #endif diff --git a/src/test/isolation2/expected/analyze_progress.out b/src/test/isolation2/expected/analyze_progress.out new file mode 100644 index 00000000000..97c02fbe617 --- /dev/null +++ b/src/test/isolation2/expected/analyze_progress.out @@ -0,0 +1,95 @@ +-- Test gp_stat_progress_analyze_summary +-- setup hash distributed table +CREATE TABLE t_analyze_part (a INT, b INT) DISTRIBUTED BY (a); +CREATE +INSERT INTO t_analyze_part SELECT i, i FROM generate_series(1, 100000) i; +INSERT 100000 + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- session 1: analyze the table +1&: ANALYZE t_analyze_part; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 60 blocks have been scanned as aggregated progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + has_pid | datname | relid | phase | sample_blks_total | sample_blks_scanned +---------+----------------+----------------+-----------------------+-------------------+--------------------- + t | isolation2test | t_analyze_part | acquiring sample rows | 111 | 60 +(1 row) + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> +ANALYZE + +-- teardown +DROP TABLE t_analyze_part; +DROP + +-- setup replicated table +CREATE TABLE t_analyze_repl (a INT, b INT) DISTRIBUTED REPLICATED; +CREATE +INSERT INTO t_analyze_repl SELECT i, i FROM generate_series(1, 100000) i; +INSERT 100000 + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- session 1: analyze the table +1&: ANALYZE t_analyze_repl; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 20 blocks have been scanned as average progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + has_pid | datname | relid | phase | sample_blks_total | sample_blks_scanned +---------+----------------+----------------+-----------------------+-------------------+--------------------- + t | isolation2test | t_analyze_repl | acquiring sample rows | 111 | 20 +(1 row) + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> +ANALYZE + +-- teardown +DROP TABLE t_analyze_repl; +DROP + diff --git a/src/test/isolation2/expected/ao_index_build_progress.out b/src/test/isolation2/expected/ao_index_build_progress.out index 1048076ce9f..2cc97d09031 100644 --- a/src/test/isolation2/expected/ao_index_build_progress.out +++ b/src/test/isolation2/expected/ao_index_build_progress.out @@ -8,36 +8,52 @@ CREATE -- Insert all tuples to seg1. INSERT INTO ao_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; INSERT 100000 +INSERT INTO ao_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT 100000 +INSERT INTO ao_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; +INSERT 100000 -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON ao_index_build_progress(i); -- Wait until some AO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -1U: SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'ao_index_build_progress'::regclass; +SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'ao_index_build_progress'::regclass; command | phase | blocks_total_actual | blocks_total_reported | blocks_done_reported --------------+--------------------------------+---------------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 10 | 10 | 2 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'ao_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 30 | 6 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE @@ -49,38 +65,54 @@ CREATE -- Insert all tuples to seg1. INSERT INTO aoco_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; INSERT 100000 +INSERT INTO aoco_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT 100000 +INSERT INTO aoco_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; +INSERT 100000 -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON aoco_index_build_progress(i); -- Wait until some AOCO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -- Note: all blocks for the relation have to be scanned as we are building an -- index for the first time and a block directory has to be created. -1U: SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'aoco_index_build_progress'::regclass; +SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; command | phase | blocks_total_actual | blocks_total_reported | blocks_done_reported --------------+--------------------------------+---------------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 20 | 20 | 4 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 60 | 12 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE @@ -88,36 +120,48 @@ CREATE -- Repeat the test for another index build -- Suspend execution when some blocks have been read. -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1&: CREATE INDEX ON aoco_index_build_progress(j); -- Wait until some AOCO varblocks have been read. -SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: -(1 row) + Success: + Success: +(3 rows) -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the size of col j's segfile. -- Note: since we already had a block directory prior to the index build on --- column 'j', only column 'j' will be scanned. CBDB_CHERRY_PICK_MERGE_FIXME: fix when ao blkdir will be supported -1U: SELECT command, phase, ((pg_stat_file(pg_relation_filepath('aoco_index_build_progress') || '.' || 129)).size + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM pg_stat_progress_create_index WHERE relid = 'aoco_index_build_progress'::regclass; +-- column 'j', only column 'j' will be scanned. +1U: SELECT command, phase, ((pg_stat_file(pg_relation_filepath('aoco_index_build_progress') || '.' || 129)).size + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported FROM gp_stat_progress_create_index WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; command | phase | col_j_blocks | blocks_total_reported | blocks_done_reported --------------+--------------------------------+--------------+-----------------------+---------------------- CREATE INDEX | building index: scanning table | 8 | 20 | 4 (1 row) +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; + command | phase | blocks_total | blocks_done +--------------+--------------------------------+--------------+------------- + CREATE INDEX | building index: scanning table | 24 | 9 +(1 row) -SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> CREATE diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out index 0f1b3e65ef5..619b3e41b80 100644 --- a/src/test/isolation2/expected/vacuum_progress_row.out +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -16,22 +16,21 @@ CREATE CREATE INDEX on vacuum_progress_ao_row(j); CREATE --- Insert all tuples to seg1 from two current sessions so that data are stored --- in two segment files. +-- Insert from two current sessions so that data are stored in two segment files. 1: BEGIN; BEGIN 2: BEGIN; BEGIN -1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -- Commit so that the logical EOF of segno 2 is non-zero. 2: COMMIT; COMMIT 2: BEGIN; BEGIN -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; INSERT 100000 -- Abort so that segno 2 has dead tuples after its logical EOF 2: ABORT; @@ -53,6 +52,11 @@ SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_ (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | last_vacuum | vacuum_count +------------+------------+-------------+-------------- + 33327 | 66654 | | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | last_vacuum | vacuum_count ------------+------------+-------------+-------------- 100000 | 200000 | | 0 (1 row) @@ -60,11 +64,13 @@ SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables -- Perform VACUUM and observe the progress -- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1: set Debug_appendonly_print_compaction to on; SET @@ -75,85 +81,120 @@ SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2 Success: (1 row) -- We are in pre_cleanup phase and some blocks should've been vacuumed by now -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized pre-cleanup | 165 | 0 | 110 | 0 | 100000 | 0 + vacuum_progress_ao_row | append-optimized pre-cleanup | 55 | 0 | 37 | 0 | 33327 | 0 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 166 | 0 | 111 | 0 | 100000 | 0 (1 row) -- Resume execution and suspend again in the middle of compact phase -SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized compact | 165 | 0 | 110 | 0 | 100000 | 199 + vacuum_progress_ao_row | append-optimized compact | 55 | 0 | 37 | 0 | 33327 | 227 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 166 | 0 | 111 | 0 | 100000 | 594 (1 row) -- Resume execution and suspend again after compacting all segfiles -SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- After compacting all segfiles we expect 50000 dead tuples -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized compact | 165 | 55 | 110 | 0 | 100000 | 50000 + vacuum_progress_ao_row | append-optimized compact | 55 | 19 | 37 | 0 | 33327 | 16622 +(1 row) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 166 | 57 | 111 | 0 | 100000 | 50000 (1 row) -- Resume execution and entering post_cleaup phase, suspend at the end of it. -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: + Success: +(3 rows) +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) -- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized post-cleanup | 165 | 55 | 110 | 0 | 100000 | 50000 + vacuum_progress_ao_row | append-optimized post-cleanup | 55 | 19 | 37 | 0 | 33327 | 16622 (1 row) -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized post-cleanup | 166 | 57 | 111 | 0 | 100000 | 50000 +(1 row) + +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) + Success: + Success: +(3 rows) 1<: <... completed> VACUUM @@ -166,63 +207,68 @@ VACUUM SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; relpages | reltuples | relallvisible ----------+-----------+--------------- - 83 | 50000 | 0 + 84 | 50000 | 0 (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count ------------+------------+-----------------+-------------- - 50000 | 0 | t | 1 + 16705 | 0 | t | 1 (1 row) -- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_inject_fault ----------------- Success: (1 row) -1&: VACUUM vacuum_progress_ao_row; --- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: +(2 rows) +1&: VACUUM vacuum_progress_ao_row; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes for segment 0. +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_wait_until_triggered_fault ------------------------------- Success: (1 row) --- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP for segment 1 and 2, there should be only 1. +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: +(2 rows) +-- Segment 0 is in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 0; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ------------------------+-------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | vacuuming indexes | 83 | 0 | 0 | 1 | 50000 | 0 + vacuum_progress_ao_row | vacuuming indexes | 28 | 0 | 0 | 1 | 16737 | 0 (1 row) +-- Segment 1 and 2 are in truncate segments phase (part of ao post_cleanup phase), heap_blks_vacuumed should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > 0; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 28 | 0 | 19 | 2 | 16558 | 0 + vacuum_progress_ao_row | append-optimized pre-cleanup | 28 | 0 | 19 | 2 | 16705 | 0 +(2 rows) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 56 | 0 | 38 | 4 | 33263 | 0 + vacuum_progress_ao_row | vacuuming indexes | 28 | 0 | 0 | 1 | 16737 | 0 +(2 rows) --- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; gp_inject_fault ----------------- Success: -(1 row) -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; - gp_inject_fault ------------------ Success: -(1 row) -SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; - gp_wait_until_triggered_fault -------------------------------- - Success: -(1 row) --- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples -------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_row | append-optimized pre-cleanup | 83 | 0 | 55 | 2 | 50000 | 0 -(1 row) - -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + Success: +(3 rows) +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; gp_inject_fault ----------------- Success: @@ -230,25 +276,34 @@ SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) 1<: <... completed> VACUUM --- Vacuum has finished, nothing should show up in the progress view. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Vacuum has finished, nothing should show up in the view. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- +(0 rows) +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- (0 rows) -- pg_class and collected stats view should be updated after the 2nd VACUUM -1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); - wait_until_dead_tup_change_to -------------------------------- - OK +1U: SELECT wait_until_vacuum_count_change_to('vacuum_progress_ao_row'::regclass::oid, 2); + wait_until_vacuum_count_change_to +----------------------------------- + OK (1 row) SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; relpages | reltuples | relallvisible ----------+-----------+--------------- - 28 | 50000 | 0 + 30 | 50000 | 0 (1 row) SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 16705 | 0 | t | 2 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count ------------+------------+-----------------+-------------- 50000 | 0 | t | 2 (1 row) diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index c4318923c1f..4a0f9dc6925 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -237,6 +237,7 @@ test: idle_gang_cleaner # test idle_in_transaction_session_timeout test: ao_index_build_progress +test: analyze_progress # Tests for FTS test: fts_errors diff --git a/src/test/isolation2/sql/analyze_progress.sql b/src/test/isolation2/sql/analyze_progress.sql new file mode 100644 index 00000000000..311ab161eb2 --- /dev/null +++ b/src/test/isolation2/sql/analyze_progress.sql @@ -0,0 +1,42 @@ +-- Test gp_stat_progress_analyze_summary +-- setup hash distributed table +CREATE TABLE t_analyze_part (a INT, b INT) DISTRIBUTED BY (a); +INSERT INTO t_analyze_part SELECT i, i FROM generate_series(1, 100000) i; + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 1: analyze the table +1&: ANALYZE t_analyze_part; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 60 blocks have been scanned as aggregated progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +1<: + +-- teardown +DROP TABLE t_analyze_part; + +-- setup replicated table +CREATE TABLE t_analyze_repl (a INT, b INT) DISTRIBUTED REPLICATED; +INSERT INTO t_analyze_repl SELECT i, i FROM generate_series(1, 100000) i; + +-- Suspend analyze after scanning 20 blocks on each segment +SELECT gp_inject_fault('analyze_block', 'suspend', '', '', '', 20, 20, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 1: analyze the table +1&: ANALYZE t_analyze_repl; +SELECT gp_wait_until_triggered_fault('analyze_block', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; + +-- session 2: query pg_stat_progress_analyze while the analyze is running, the view should indicate 20 blocks have been scanned as average progress of 3 segments +2: SELECT pid IS NOT NULL as has_pid, datname, relid::regclass, phase, sample_blks_total, sample_blks_scanned FROM gp_stat_progress_analyze_summary; + +-- Reset fault injector +SELECT gp_inject_fault('analyze_block', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +1<: + +-- teardown +DROP TABLE t_analyze_repl; diff --git a/src/test/isolation2/sql/ao_index_build_progress.sql b/src/test/isolation2/sql/ao_index_build_progress.sql index 19aade99e29..c5e34d1eab7 100644 --- a/src/test/isolation2/sql/ao_index_build_progress.sql +++ b/src/test/isolation2/sql/ao_index_build_progress.sql @@ -7,30 +7,34 @@ CREATE TABLE ao_index_build_progress(i int, j bigint) USING ao_row -- Insert all tuples to seg1. INSERT INTO ao_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; +INSERT INTO ao_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT INTO ao_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 10, 10, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON ao_index_build_progress(i); -- Wait until some AO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 10, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -1U: SELECT command, phase, +SELECT command, phase, (pg_relation_size('ao_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'ao_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'ao_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'ao_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: @@ -40,31 +44,35 @@ CREATE TABLE aoco_index_build_progress(i int, j int ENCODING (compresstype=zstd, -- Insert all tuples to seg1. INSERT INTO aoco_index_build_progress SELECT 0, i FROM generate_series(1, 100000) i; +INSERT INTO aoco_index_build_progress SELECT 2, i FROM generate_series(1, 100000) i; +INSERT INTO aoco_index_build_progress SELECT 5, i FROM generate_series(1, 100000) i; -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON aoco_index_build_progress(i); -- Wait until some AOCO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the relation's on-disk size. -- Note: all blocks for the relation have to be scanned as we are building an -- index for the first time and a block directory has to be created. -1U: SELECT command, phase, +SELECT command, phase, (pg_relation_size('aoco_index_build_progress') + (current_setting('block_size')::int - 1)) / current_setting('block_size')::int AS blocks_total_actual, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'aoco_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: @@ -72,13 +80,13 @@ SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', d -- Suspend execution when some blocks have been read. SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'suspend', '', '', '', 5, 5, 0, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1&: CREATE INDEX ON aoco_index_build_progress(j); -- Wait until some AOCO varblocks have been read. SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_success', 5, dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- By now, we should have reported some blocks (of size 'block_size') as "done", -- as well as a total number of blocks that matches the size of col j's segfile. @@ -90,10 +98,12 @@ SELECT gp_wait_until_triggered_fault('AppendOnlyStorageRead_ReadNextBlock_succes AS col_j_blocks, blocks_total AS blocks_total_reported, blocks_done AS blocks_done_reported - FROM pg_stat_progress_create_index - WHERE relid = 'aoco_index_build_progress'::regclass; + FROM gp_stat_progress_create_index + WHERE gp_segment_id = 1 AND relid = 'aoco_index_build_progress'::regclass; +-- The same should be true for the summary view, and the total number of blocks should be tripled. +SELECT command, phase, blocks_total, blocks_done FROM gp_stat_progress_create_index_summary WHERE relid = 'aoco_index_build_progress'::regclass; SELECT gp_inject_fault('AppendOnlyStorageRead_ReadNextBlock_success', 'reset', dbid) - FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: diff --git a/src/test/isolation2/sql/setup.sql b/src/test/isolation2/sql/setup.sql index 949f22f0002..6241413ba3b 100644 --- a/src/test/isolation2/sql/setup.sql +++ b/src/test/isolation2/sql/setup.sql @@ -461,6 +461,27 @@ begin end; /* in func */ $$ language plpgsql; +-- Helper function that ensures stats collector receives stat from the latest operation. +create or replace function wait_until_vacuum_count_change_to(relid oid, stat_val_expected bigint) + returns text as $$ +declare + stat_val int; /* in func */ + i int; /* in func */ +begin + i := 0; /* in func */ + while i < 1200 loop + select pg_stat_get_vacuum_count(relid) into stat_val; /* in func */ + if stat_val = stat_val_expected then /* in func */ + return 'OK'; /* in func */ + end if; /* in func */ + perform pg_sleep(0.1); /* in func */ + perform pg_stat_clear_snapshot(); /* in func */ + i := i + 1; /* in func */ + end loop; /* in func */ + return 'Fail'; /* in func */ +end; /* in func */ +$$ language plpgsql; + -- Helper function to get the number of blocks in a relation. CREATE OR REPLACE FUNCTION nblocks(rel regclass) RETURNS int AS $$ /* in func */ BEGIN /* in func */ diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql index 1ccf6ca090f..93da2a47bbc 100644 --- a/src/test/isolation2/sql/vacuum_progress_row.sql +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -11,16 +11,15 @@ CREATE TABLE vacuum_progress_ao_row(i int, j int); CREATE INDEX on vacuum_progress_ao_row(i); CREATE INDEX on vacuum_progress_ao_row(j); --- Insert all tuples to seg1 from two current sessions so that data are stored --- in two segment files. +-- Insert from two current sessions so that data are stored in two segment files. 1: BEGIN; 2: BEGIN; -1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; -- Commit so that the logical EOF of segno 2 is non-zero. 2: COMMIT; 2: BEGIN; -2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; -- Abort so that segno 2 has dead tuples after its logical EOF 2: ABORT; 2q: @@ -32,40 +31,46 @@ DELETE FROM vacuum_progress_ao_row where j % 2 = 0; -- Lookup pg_class and collected stats view before VACUUM SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; -SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables WHERE relname = 'vacuum_progress_ao_row' and gp_segment_id = 1; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; -- Perform VACUUM and observe the progress -- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1: set Debug_appendonly_print_compaction to on; 1&: VACUUM vacuum_progress_ao_row; SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We are in pre_cleanup phase and some blocks should've been vacuumed by now -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and suspend again in the middle of compact phase -SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and suspend again after compacting all segfiles -SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- After compacting all segfiles we expect 50000 dead tuples -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- Resume execution and entering post_cleaup phase, suspend at the end of it. -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; -SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; 1<: -- pg_class and collected stats view should be updated after the 1st VACUUM @@ -74,113 +79,34 @@ SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_ SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; -- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; 1&: VACUUM vacuum_progress_ao_row; --- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. -SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; --- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - --- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; -SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; --- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; - -SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes for segment 0. +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP for segment 1 and 2, there should be only 1. +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content > 0 AND role = 'p'; +-- Segment 0 is in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 0; +-- Segment 1 and 2 are in truncate segments phase (part of ao post_cleanup phase), heap_blks_vacuumed should increase to 1. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > 0; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 0 AND role = 'p'; 1<: --- Vacuum has finished, nothing should show up in the progress view. -1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +-- Vacuum has finished, nothing should show up in the view. +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id = 1; +select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; -- pg_class and collected stats view should be updated after the 2nd VACUUM -1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); +1U: SELECT wait_until_vacuum_count_change_to('vacuum_progress_ao_row'::regclass::oid, 2); SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; -SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables WHERE relname = 'vacuum_progress_ao_row' and gp_segment_id = 1; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM gp_stat_all_tables_summary WHERE relname = 'vacuum_progress_ao_row'; --- open if system views gp_stat_progress_vacuum* are enabled ---1q: ----- Test vacuum worker process is changed at post-cleanup phase due to mirror down. ----- Current behavior is it will clear previous compact phase num_dead_tuples in post-cleanup ----- phase (at injecting point vacuum_ao_post_cleanup_end), which is different from above case ----- in which vacuum worker isn't changed. ---ALTER SYSTEM SET gp_fts_mark_mirror_down_grace_period to 10; ---ALTER SYSTEM SET gp_fts_probe_interval to 10; ---SELECT gp_segment_id, pg_reload_conf() FROM gp_id UNION SELECT gp_segment_id, pg_reload_conf() FROM gp_dist_random('gp_id'); --- ---DROP TABLE IF EXISTS vacuum_progress_ao_row; ---CREATE TABLE vacuum_progress_ao_row(i int, j int); ---CREATE INDEX on vacuum_progress_ao_row(i); ---CREATE INDEX on vacuum_progress_ao_row(j); ---1: BEGIN; ---2: BEGIN; ---1: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: COMMIT; ---2: BEGIN; ---2: INSERT INTO vacuum_progress_ao_row SELECT i, i FROM generate_series(1, 100000) i; ---2: ABORT; ---2q: ---1: ABORT; ---DELETE FROM vacuum_progress_ao_row where j % 2 = 0; --- ----- Suspend execution at the end of compact phase. ---2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ---1: set debug_appendonly_print_compaction to on; ---1&: vacuum vacuum_progress_ao_row; --- ---2: SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 3, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ----- Non-zero progressing data num_dead_tuples is showed up. ---select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; ---select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- ----- Resume execution of compact phase and block at syncrep. ---2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; ---2: SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- stop the mirror should turn off syncrep ---2: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE content=1 AND role = 'm'; --- ----- Resume walsender to detect mirror down and suspend at the beginning ----- of post-cleanup taken over by a new vacuum worker. ---2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- resume walsender and let it exit so that mirror stop can be detected ---2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; ----- Ensure we enter into the target logic which stops cumulative data but ----- initializes a new vacrelstats at the beginning of post-cleanup phase. ----- Also all segments should reach to the same "vacuum_worker_changed" point ----- due to FTS version being changed. ---2: SELECT gp_wait_until_triggered_fault('vacuum_worker_changed', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ----- now seg1's mirror is marked as down ---2: SELECT content, role, preferred_role, mode, status FROM gp_segment_configuration WHERE content > -1; --- ----- Resume execution and entering post_cleaup phase, suspend at the end of it. ---2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ---2: SELECT gp_inject_fault('vacuum_worker_changed', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; ---2: SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ----- The previous collected num_dead_tuples in compact phase is zero. ---select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; ---select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; --- ---2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; --- ---1<: --- ----- restore environment ---1: reset debug_appendonly_print_compaction; --- ---2: SELECT pg_ctl_start(datadir, port) FROM gp_segment_configuration WHERE role = 'm' AND content = 1; ---2: SELECT wait_until_all_segments_synchronized(); --- ----- Cleanup ---SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; ---reset Debug_appendonly_print_compaction; ---reset default_table_access_method; ---ALTER SYSTEM RESET gp_fts_mark_mirror_down_grace_period; ---ALTER SYSTEM RESET gp_fts_probe_interval; ---SELECT gp_segment_id, pg_reload_conf() FROM gp_id UNION SELECT gp_segment_id, pg_reload_conf() FROM gp_dist_random('gp_id'); --- \ No newline at end of file +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; +reset Debug_appendonly_print_compaction; +reset default_table_access_method; From e6ea65aa9e7783e92abcc34af6d970005723a6bb Mon Sep 17 00:00:00 2001 From: Andrew Repp Date: Fri, 7 Apr 2023 16:34:44 -0500 Subject: [PATCH 006/128] Add gp summary system views These summary views offer basic aggregation of the gp_stat_* views across Greenplum coordinator and segments. Aggregation logic applied as follows: * Time related (last_%): use max() * Transaction related, not innately summable (number of commits/rollbacks) : use max() * Table specific: sum()/numsegments for replicated tables, sum() for distributed tables * Innately summable stats, if no particular table is involved: use sum() * pid: use coordinator's pid (not used here, but this is the convention in other gp_%_summary views) --- src/backend/catalog/system_views.sql | 122 ------ .../catalog/system_views_gp_summary.sql | 381 ++++++++++++++++++ 2 files changed, 381 insertions(+), 122 deletions(-) diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 55bd43d57b7..e68262830d4 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -689,81 +689,6 @@ CREATE VIEW pg_stat_all_tables AS WHERE C.relkind IN ('r', 't', 'm', 'o', 'b', 'M', 'p') GROUP BY C.oid, N.nspname, C.relname; --- Gather data from segments on user tables, and use data on coordinator on system tables. - -CREATE VIEW gp_stat_all_tables_summary AS -SELECT - s.relid, - s.schemaname, - s.relname, - m.seq_scan, - m.seq_tup_read, - m.idx_scan, - m.idx_tup_fetch, - m.n_tup_ins, - m.n_tup_upd, - m.n_tup_del, - m.n_tup_hot_upd, - m.n_live_tup, - m.n_dead_tup, - m.n_mod_since_analyze, - m.n_ins_since_vacuum, - s.last_vacuum, - s.last_autovacuum, - s.last_analyze, - s.last_autoanalyze, - s.vacuum_count, - s.autovacuum_count, - s.analyze_count, - s.autoanalyze_count -FROM - (SELECT - allt.relid, - allt.schemaname, - allt.relname, - case when d.policytype = 'r' then (sum(seq_scan)/d.numsegments)::bigint else sum(seq_scan) end seq_scan, - case when d.policytype = 'r' then (sum(seq_tup_read)/d.numsegments)::bigint else sum(seq_tup_read) end seq_tup_read, - case when d.policytype = 'r' then (sum(idx_scan)/d.numsegments)::bigint else sum(idx_scan) end idx_scan, - case when d.policytype = 'r' then (sum(idx_tup_fetch)/d.numsegments)::bigint else sum(idx_tup_fetch) end idx_tup_fetch, - case when d.policytype = 'r' then (sum(n_tup_ins)/d.numsegments)::bigint else sum(n_tup_ins) end n_tup_ins, - case when d.policytype = 'r' then (sum(n_tup_upd)/d.numsegments)::bigint else sum(n_tup_upd) end n_tup_upd, - case when d.policytype = 'r' then (sum(n_tup_del)/d.numsegments)::bigint else sum(n_tup_del) end n_tup_del, - case when d.policytype = 'r' then (sum(n_tup_hot_upd)/d.numsegments)::bigint else sum(n_tup_hot_upd) end n_tup_hot_upd, - case when d.policytype = 'r' then (sum(n_live_tup)/d.numsegments)::bigint else sum(n_live_tup) end n_live_tup, - case when d.policytype = 'r' then (sum(n_dead_tup)/d.numsegments)::bigint else sum(n_dead_tup) end n_dead_tup, - case when d.policytype = 'r' then (sum(n_mod_since_analyze)/d.numsegments)::bigint else sum(n_mod_since_analyze) end n_mod_since_analyze, - case when d.policytype = 'r' then (sum(n_ins_since_vacuum)/d.numsegments)::bigint else sum(n_ins_since_vacuum) end n_ins_since_vacuum, - max(last_vacuum) as last_vacuum, - max(last_autovacuum) as last_autovacuum, - max(last_analyze) as last_analyze, - max(last_autoanalyze) as last_autoanalyze, - max(vacuum_count) as vacuum_count, - max(autovacuum_count) as autovacuum_count, - max(analyze_count) as analyze_count, - max(autoanalyze_count) as autoanalyze_count - FROM - gp_dist_random('pg_stat_all_tables') allt - inner join pg_class c - on allt.relid = c.oid - left outer join gp_distribution_policy d - on allt.relid = d.localoid - WHERE - relid >= 16384 - and ( - d.localoid is not null - or c.relkind in ('o', 'b', 'M') - ) - GROUP BY allt.relid, allt.schemaname, allt.relname, d.policytype, d.numsegments - - UNION ALL - - SELECT - * - FROM - pg_stat_all_tables - WHERE - relid < 16384) m, pg_stat_all_tables s -WHERE m.relid = s.relid; CREATE VIEW pg_stat_xact_all_tables AS SELECT @@ -816,10 +741,6 @@ CREATE VIEW pg_stat_user_tables_single_node AS WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND schemaname !~ '^pg_toast'; -CREATE VIEW gp_stat_user_tables_summary AS - SELECT * FROM gp_stat_all_tables_summary - WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND - schemaname !~ '^pg_toast'; CREATE VIEW pg_stat_xact_user_tables AS SELECT * FROM pg_stat_xact_all_tables @@ -877,44 +798,6 @@ CREATE VIEW pg_stat_all_indexes AS LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE C.relkind IN ('r', 't', 'm', 'o', 'b', 'M'); --- Gather data from segments on user tables, and use data on coordinator on system tables. - -CREATE VIEW gp_stat_all_indexes_summary AS -SELECT - s.relid, - s.indexrelid, - s.schemaname, - s.relname, - s.indexrelname, - m.idx_scan, - m.idx_tup_read, - m.idx_tup_fetch -FROM - (SELECT - relid, - indexrelid, - schemaname, - relname, - indexrelname, - sum(idx_scan) as idx_scan, - sum(idx_tup_read) as idx_tup_read, - sum(idx_tup_fetch) as idx_tup_fetch - FROM - gp_dist_random('pg_stat_all_indexes') - WHERE - relid >= 16384 - GROUP BY relid, indexrelid, schemaname, relname, indexrelname - - UNION ALL - - SELECT - * - FROM - pg_stat_all_indexes - WHERE - relid < 16384) m, pg_stat_all_indexes s -WHERE m.relid = s.relid; - CREATE VIEW pg_stat_sys_indexes AS SELECT * FROM pg_stat_all_indexes WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR @@ -925,11 +808,6 @@ CREATE VIEW pg_stat_user_indexes AS WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND schemaname !~ '^pg_toast'; -CREATE VIEW gp_stat_user_indexes_summary AS - SELECT * FROM gp_stat_all_indexes_summary - WHERE schemaname NOT IN ('pg_catalog', 'information_schema') AND - schemaname !~ '^pg_toast'; - CREATE VIEW pg_statio_all_indexes AS SELECT C.oid AS relid, diff --git a/src/backend/catalog/system_views_gp_summary.sql b/src/backend/catalog/system_views_gp_summary.sql index 58f4768c124..f83329203ab 100644 --- a/src/backend/catalog/system_views_gp_summary.sql +++ b/src/backend/catalog/system_views_gp_summary.sql @@ -22,6 +22,387 @@ * string literal (including a function body!) or a multiline comment. */ +CREATE VIEW gp_stat_archiver_summary AS +SELECT + sum(gsa.archived_count) as archived_count, + max(gsa.last_archived_wal) as last_archived_wal, + max(gsa.last_archived_time) as last_archived_time, + sum(gsa.failed_count) as failed_count, + max(gsa.last_failed_wal) as last_failed_wal, + max(gsa.last_failed_time) as last_failed_time, + max(gsa.stats_reset) as stats_reset +FROM + gp_stat_archiver gsa; + +CREATE VIEW gp_stat_bgwriter_summary AS +SELECT + sum(gsb.checkpoints_timed) as checkpoints_timed, + sum(gsb.checkpoints_req) as checkpoints_req, + sum(gsb.checkpoint_write_time) as checkpoint_write_time, + sum(gsb.checkpoint_sync_time) as checkpoint_sync_time, + sum(gsb.buffers_checkpoint) as buffers_checkpoint, + sum(gsb.buffers_clean) as buffers_clean, + sum(gsb.maxwritten_clean) as maxwritten_clean, + sum(gsb.buffers_backend) as buffers_backend, + sum(gsb.buffers_backend_fsync) as buffers_backend_fsync, + sum(gsb.buffers_alloc) as buffers_alloc, + max(gsb.stats_reset) as stats_reset +FROM + gp_stat_bgwriter gsb; + +CREATE VIEW gp_stat_wal_summary AS +SELECT + sum(gsw.wal_records) as wal_records, + sum(gsw.wal_fpw) as wal_fpw, + sum(gsw.wal_bytes) as wal_bytes, + sum(gsw.wal_buffers_full) as wal_buffers_full, + sum(gsw.wal_write) as wal_write, + sum(gsw.wal_sync) as wal_sync, + sum(gsw.wal_write_time) as wal_write_time, + sum(gsw.wal_sync_time) as wal_sync_time, + max(gsw.stats_reset) as stats_reset +from + gp_stat_wal gsw; + +CREATE VIEW gp_stat_database_summary AS +SELECT + sdb.datid, + sdb.datname, + sum(sdb.numbackends) as numbackends, + max(sdb.xact_commit) as xact_commit, + max(sdb.xact_rollback) as xact_rollback, + sum(sdb.blks_read) as blks_read, + sum(sdb.blks_hit) as blks_hit, + sum(sdb.tup_returned) as tup_returned, + sum(sdb.tup_fetched) as tup_fetched, + sum(sdb.tup_inserted) as tup_inserted, + sum(sdb.tup_updated) as tup_updated, + sum(sdb.tup_deleted) as tup_deleted, + max(sdb.conflicts) as conflicts, + sum(sdb.temp_files) as temp_files, + sum(sdb.temp_bytes) as temp_bytes, + sum(sdb.deadlocks) as deadlocks, + sum(sdb.checksum_failures) as checksum_failures, + max(sdb.checksum_last_failure) as checksum_last_failure, + sum(sdb.blk_read_time) as blk_read_time, + sum(sdb.blk_write_time) as blk_write_time, + max(sdb.stats_reset) as stats_reset +FROM + gp_stat_database sdb +GROUP BY + sdb.datid, + sdb.datname; + + +-- Gather data from segments on user tables, and use data on coordinator on system tables. +CREATE VIEW gp_stat_all_tables_summary AS +SELECT + s.relid, + s.schemaname, + s.relname, + m.seq_scan, + m.seq_tup_read, + m.idx_scan, + m.idx_tup_fetch, + m.n_tup_ins, + m.n_tup_upd, + m.n_tup_del, + m.n_tup_hot_upd, + m.n_live_tup, + m.n_dead_tup, + m.n_mod_since_analyze, + s.last_vacuum, + s.last_autovacuum, + s.last_analyze, + s.last_autoanalyze, + s.vacuum_count, + s.autovacuum_count, + s.analyze_count, + s.autoanalyze_count +FROM + (SELECT + allt.relid, + allt.schemaname, + allt.relname, + case when d.policytype = 'r' then (sum(seq_scan)/d.numsegments)::bigint else sum(seq_scan) end seq_scan, + case when d.policytype = 'r' then (sum(seq_tup_read)/d.numsegments)::bigint else sum(seq_tup_read) end seq_tup_read, + case when d.policytype = 'r' then (sum(idx_scan)/d.numsegments)::bigint else sum(idx_scan) end idx_scan, + case when d.policytype = 'r' then (sum(idx_tup_fetch)/d.numsegments)::bigint else sum(idx_tup_fetch) end idx_tup_fetch, + case when d.policytype = 'r' then (sum(n_tup_ins)/d.numsegments)::bigint else sum(n_tup_ins) end n_tup_ins, + case when d.policytype = 'r' then (sum(n_tup_upd)/d.numsegments)::bigint else sum(n_tup_upd) end n_tup_upd, + case when d.policytype = 'r' then (sum(n_tup_del)/d.numsegments)::bigint else sum(n_tup_del) end n_tup_del, + case when d.policytype = 'r' then (sum(n_tup_hot_upd)/d.numsegments)::bigint else sum(n_tup_hot_upd) end n_tup_hot_upd, + case when d.policytype = 'r' then (sum(n_live_tup)/d.numsegments)::bigint else sum(n_live_tup) end n_live_tup, + case when d.policytype = 'r' then (sum(n_dead_tup)/d.numsegments)::bigint else sum(n_dead_tup) end n_dead_tup, + case when d.policytype = 'r' then (sum(n_mod_since_analyze)/d.numsegments)::bigint else sum(n_mod_since_analyze) end n_mod_since_analyze, + max(last_vacuum) as last_vacuum, + max(last_autovacuum) as last_autovacuum, + max(last_analyze) as last_analyze, + max(last_autoanalyze) as last_autoanalyze, + max(vacuum_count) as vacuum_count, + max(autovacuum_count) as autovacuum_count, + max(analyze_count) as analyze_count, + max(autoanalyze_count) as autoanalyze_count + FROM + gp_dist_random('pg_stat_all_tables') allt + inner join pg_class c + on allt.relid = c.oid + left outer join gp_distribution_policy d + on allt.relid = d.localoid + WHERE + relid >= 16384 + and ( + d.localoid is not null + or c.relkind in ('o', 'b', 'M') + ) + GROUP BY allt.relid, allt.schemaname, allt.relname, d.policytype, d.numsegments + + UNION ALL + + SELECT + * + FROM + pg_stat_all_tables + WHERE + relid < 16384) m, pg_stat_all_tables s +WHERE m.relid = s.relid; + +CREATE VIEW gp_stat_user_tables_summary AS + SELECT * FROM gp_stat_all_tables_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_stat_sys_tables_summary AS + SELECT * FROM gp_stat_all_tables_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_stat_xact_all_tables_summary AS +SELECT + sxa.relid, + sxa.schemaname, + sxa.relname, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.seq_scan)/dst.numsegments)::bigint ELSE sum(sxa.seq_scan) END AS seq_scan, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.seq_tup_read)/dst.numsegments)::bigint ELSE sum(sxa.seq_tup_read) END AS seq_tup_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.idx_scan)/dst.numsegments)::bigint ELSE sum(sxa.idx_scan) END AS idx_scan, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.idx_tup_fetch)/dst.numsegments)::bigint ELSE sum(sxa.idx_tup_fetch) END AS idx_tup_fetch, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.n_tup_ins)/dst.numsegments)::bigint ELSE sum(sxa.n_tup_ins) END AS n_tup_ins, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.n_tup_upd)/dst.numsegments)::bigint ELSE sum(sxa.n_tup_upd) END AS n_tup_upd, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.n_tup_del)/dst.numsegments)::bigint ELSE sum(sxa.n_tup_del) END AS n_tup_del, + CASE WHEN dst.policytype = 'r' THEN (sum(sxa.n_tup_hot_upd)/dst.numsegments)::bigint ELSE sum(sxa.n_tup_hot_upd) END AS n_tup_hot_upd +FROM + gp_stat_xact_all_tables sxa + LEFT OUTER JOIN gp_distribution_policy dst + ON sxa.relid = dst.localoid +GROUP BY + sxa.relid, + sxa.schemaname, + sxa.relname, + dst.policytype, + dst.numsegments; + +CREATE VIEW gp_stat_xact_sys_tables_summary as + SELECT * FROM gp_stat_xact_all_tables_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_stat_xact_user_tables_summary AS + SELECT * FROM gp_stat_xact_all_tables_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +-- Gather data from segments on user tables, and use data on coordinator on system tables. +CREATE VIEW gp_stat_all_indexes_summary AS +SELECT + s.relid, + s.indexrelid, + s.schemaname, + s.relname, + s.indexrelname, + m.idx_scan, + m.idx_tup_read, + m.idx_tup_fetch +FROM + (SELECT + alli.relid, + alli.indexrelid, + alli.schemaname, + alli.relname, + alli.indexrelname, + case when d.policytype = 'r' then (sum(alli.idx_scan)/d.numsegments)::bigint else sum(alli.idx_scan) end idx_scan, + case when d.policytype = 'r' then (sum(alli.idx_tup_read)/d.numsegments)::bigint else sum(alli.idx_tup_read) end idx_tup_read, + case when d.policytype = 'r' then (sum(alli.idx_tup_fetch)/d.numsegments)::bigint else sum(alli.idx_tup_fetch) end idx_tup_fetch + FROM + gp_dist_random('pg_stat_all_indexes') alli + inner join pg_class c + on alli.relid = c.oid + left outer join gp_distribution_policy d + on alli.relid = d.localoid + WHERE + relid >= 16384 + GROUP BY alli.relid, alli.indexrelid, alli.schemaname, alli.relname, alli.indexrelname, d.policytype, d.numsegments + + UNION ALL + + SELECT + * + FROM + pg_stat_all_indexes + WHERE + relid < 16384) m, pg_stat_all_indexes s +WHERE m.relid = s.relid; + +CREATE VIEW gp_stat_sys_indexes_summary AS + SELECT * FROM gp_stat_all_indexes_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_stat_user_indexes_summary AS + SELECT * FROM gp_stat_all_indexes_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_statio_all_tables_summary as +SELECT + sat.relid, + sat.schemaname, + sat.relname, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.heap_blks_read)/dst.numsegments)::bigint ELSE sum(sat.heap_blks_read) END AS heap_blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.heap_blks_hit)/dst.numsegments)::bigint ELSE sum(sat.heap_blks_hit) END AS heap_blks_hit, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.idx_blks_read)/dst.numsegments)::bigint ELSE sum(sat.idx_blks_read) END AS idx_blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.idx_blks_hit)/dst.numsegments)::bigint ELSE sum(sat.idx_blks_hit) END AS idx_blks_hit, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.toast_blks_read)/dst.numsegments)::bigint ELSE sum(sat.toast_blks_read) END AS toast_blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.toast_blks_hit)/dst.numsegments)::bigint ELSE sum(sat.toast_blks_hit) END AS toast_blks_hit, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.tidx_blks_read)/dst.numsegments)::bigint ELSE sum(sat.tidx_blks_read) END AS tidx_blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sat.tidx_blks_hit)/dst.numsegments)::bigint ELSE sum(sat.tidx_blks_hit) END AS tidx_blks_hit +FROM + gp_statio_all_tables sat + LEFT OUTER JOIN gp_distribution_policy dst + ON sat.relid = dst.localoid +GROUP BY + sat.relid, + sat.schemaname, + sat.relname, + dst.policytype, + dst.numsegments; + +CREATE VIEW gp_statio_sys_tables_summary AS + SELECT * FROM gp_statio_all_tables_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_statio_user_tables_summary AS + SELECT * FROM gp_stat_all_tables_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_statio_all_sequences_summary as +SELECT + sas.relid, + sas.schemaname, + sas.relname, + CASE WHEN dst.policytype = 'r' THEN (sum(sas.blks_read)/dst.numsegments)::bigint ELSE sum(sas.blks_read) END AS blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sas.blks_hit)/dst.numsegments)::bigint ELSE sum(sas.blks_hit) END AS blks_hit +FROM + gp_statio_all_sequences sas + LEFT OUTER JOIN gp_distribution_policy dst + ON sas.relid = dst.localoid +GROUP BY + sas.relid, + sas.schemaname, + sas.relname, + dst.policytype, + dst.numsegments; + +CREATE VIEW gp_statio_sys_sequences_summary AS + SELECT * FROM gp_statio_all_sequences_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_statio_user_sequences_summary AS + SELECT * FROM gp_statio_all_sequences_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_statio_all_indexes_summary AS +SELECT + sai.relid, + sai.indexrelid, + sai.schemaname, + sai.relname, + sai.indexrelname, + CASE WHEN dst.policytype = 'r' THEN (sum(sai.idx_blks_read)/dst.numsegments)::bigint ELSE sum(sai.idx_blks_read) END AS idx_blks_read, + CASE WHEN dst.policytype = 'r' THEN (sum(sai.idx_blks_hit)/dst.numsegments)::bigint ELSE sum(sai.idx_blks_hit) END AS idx_blks_hit +FROM + gp_statio_all_indexes sai + LEFT OUTER JOIN gp_distribution_policy dst + ON sai.relid = dst.localoid +GROUP BY + sai.relid, + sai.indexrelid, + sai.schemaname, + sai.relname, + sai.indexrelname, + dst.policytype, + dst.numsegments; + +CREATE VIEW gp_statio_sys_indexes_summary AS + SELECT * FROM gp_statio_all_indexes_summary + WHERE schemaname IN ('pg_catalog', 'information_schema', 'pg_aoseg') OR + schemaname ~ '^pg_toast'; + +CREATE VIEW gp_statio_user_indexes_summary AS + SELECT * FROM gp_statio_all_indexes_summary + WHERE schemaname NOT IN ('pg_catalog', 'information_schema', 'pg_aoseg') AND + schemaname !~ '^pg_toast'; + +CREATE VIEW gp_stat_user_functions_summary AS +SELECT + guf.funcid, + guf.schemaname, + guf.funcname, + sum(guf.calls) AS calls, + sum(guf.total_time) AS total_time, + sum(guf.self_time) AS self_time +FROM + gp_stat_user_functions guf +GROUP BY + guf.funcid, + guf.schemaname, + guf.funcname; + +CREATE VIEW gp_stat_xact_user_functions_summary AS +SELECT + xuf.funcid, + xuf.schemaname, + xuf.funcname, + sum(xuf.calls) AS calls, + sum(xuf.total_time) AS total_time, + sum(xuf.self_time) AS self_time +FROM + gp_stat_xact_user_functions xuf +GROUP BY + xuf.funcid, + xuf.schemaname, + xuf.funcname; + +CREATE VIEW gp_stat_slru_summary AS +SELECT + gss.name, + sum(gss.blks_zeroed) AS blks_zeroed, + sum(gss.blks_hit) AS blks_hit, + sum(gss.blks_read) AS blks_read, + sum(gss.blks_written) AS blks_written, + sum(gss.blks_exists) AS blks_exists, + sum(gss.flushes) AS flushes, + sum(gss.truncates) AS truncates, + max(gss.stats_reset) AS stats_reset +FROM + gp_stat_slru gss +GROUP BY + gss.name; + + CREATE VIEW gp_stat_progress_vacuum_summary AS SELECT max(coalesce(a1.pid, 0)) as pid, From 9fc966fa098af4963f4818455c0206cd088fb27f Mon Sep 17 00:00:00 2001 From: Andrey Sokolov Date: Tue, 30 Dec 2025 10:51:13 +0300 Subject: [PATCH 007/128] Run gp_internal_tools tests --- .github/workflows/build-cloudberry.yml | 3 ++- gpcontrib/gp_internal_tools/Makefile | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index ca75f7b42e7..adb57fb85ec 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -312,7 +312,8 @@ jobs: "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck", - "gpcontrib/gp_exttable_fdw:installcheck"] + "gpcontrib/gp_exttable_fdw:installcheck", + "gpcontrib/gp_internal_tools:installcheck"] }, {"test":"ic-diskquota", "make_configs":["gpcontrib/diskquota:installcheck"], diff --git a/gpcontrib/gp_internal_tools/Makefile b/gpcontrib/gp_internal_tools/Makefile index 643a13f0118..829645e1268 100755 --- a/gpcontrib/gp_internal_tools/Makefile +++ b/gpcontrib/gp_internal_tools/Makefile @@ -4,6 +4,8 @@ DATA = gp_internal_tools--1.0.0.sql PG_CPPFLAGS = -I$(libpq_srcdir) +REGRESS = gp_session_state_memory + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) From d604ff1a1bec6733b3913ec1d85fda0914f30ee7 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Sat, 28 Feb 2026 18:36:03 +0800 Subject: [PATCH 008/128] initdb: fix stale errno handling in setup_cdb_schema() setup_cdb_schema() checked errno after a readdir() loop without resetting it beforehand. In some environments (e.g., Ubuntu 24.04), a stale errno value from operations inside the loop (such as pg_realloc or pg_strdup) could persist, causing readdir's normal termination to be misinterpreted as a failure (e.g., "Function not implemented"). This commit fixes the issue by adopting the standard PostgreSQL idiom: - Use "while (errno = 0, (file = readdir(dir)) != NULL)" to ensure errno is cleared strictly before each readdir() call. - Move closedir() after the errno check to prevent it from overwriting the error code from readdir(). - Add defensive error checking for the closedir() call itself. This ensures robust directory scanning and reliable error reporting during cluster initialization. --- src/bin/initdb/initdb.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 708cf77ffdf..8fdae656bc8 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -2024,7 +2024,7 @@ setup_cdb_schema(FILE *cmdfd) /* Collect all files with .sql suffix in array. */ nscripts = 0; - while ((file = readdir(dir)) != NULL) + while (errno = 0, (file = readdir(dir)) != NULL) { int namelen = strlen(file->d_name); @@ -2054,12 +2054,16 @@ setup_cdb_schema(FILE *cmdfd) errno = 0; #endif - closedir(dir); - if (errno != 0) { - /* some kind of I/O error? */ pg_log_error("error while reading cdb_init.d directory: %m"); + closedir(dir); + exit(1); + } + + if (closedir(dir)) + { + pg_log_error("error while closing cdb_init.d directory: %m"); exit(1); } From d6192cc6a71da4c8401c1418ac1b78fdbfa313be Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 2 Mar 2026 12:09:27 +0800 Subject: [PATCH 009/128] Fix invalid escape sequences in Python files Fix SyntaxWarning caused by invalid escape sequences in mainUtils.py and logfilter.py. These warnings appear on Python 3.12+ (e.g., Ubuntu 24.04) and will become SyntaxError in Python 3.14. Changes: - mainUtils.py: Use raw strings for shell commands containing '\$' - logfilter.py: Use raw docstrings for functions containing regex examples See: https://github.com/apache/cloudberry/issues/1587 --- gpMgmt/bin/gppylib/logfilter.py | 4 ++-- gpMgmt/bin/gppylib/mainUtils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gpMgmt/bin/gppylib/logfilter.py b/gpMgmt/bin/gppylib/logfilter.py index c427ac1a6cb..46c0b5e174a 100644 --- a/gpMgmt/bin/gppylib/logfilter.py +++ b/gpMgmt/bin/gppylib/logfilter.py @@ -67,7 +67,7 @@ def FilterLogEntries(iterable, filters=[], ibegin=0, jend=None): - """ + r""" Generator to consume the lines of a GPDB log file from iterable, yield the lines which satisfy the given criteria, and skip the rest. @@ -668,7 +668,7 @@ def MatchInFirstLine(iterable, regex): def NoMatchInFirstLine(iterable, regex): - """ + r""" Generator to filter a stream of groups. Skips those groups whose first line contains a match for the given regex; yields all other groups. diff --git a/gpMgmt/bin/gppylib/mainUtils.py b/gpMgmt/bin/gppylib/mainUtils.py index 553ca9d57c9..e947639591d 100644 --- a/gpMgmt/bin/gppylib/mainUtils.py +++ b/gpMgmt/bin/gppylib/mainUtils.py @@ -488,7 +488,7 @@ def parseStatusLine(line, isStart = False, isStop = False): def check_fts(fts): - fts_check_cmd= "ps -ef | awk '{print \$2, \$8}' | grep gpfts | grep -v grep" + fts_check_cmd= r"ps -ef | awk '{print \$2, \$8}' | grep gpfts | grep -v grep" process_cmd = "gpssh -h %s -e \"%s\" | wc -l" % (fts, fts_check_cmd) fts_process_res=int(subprocess.check_output(process_cmd, shell=True).decode().strip()) return fts_process_res == 2 @@ -500,7 +500,7 @@ def check_etcd(etcd): if etcd_process_res == 2: return True # for demo cluster - etcd_check_cmd = "ps -ef | awk '{print \$2, \$8}' | grep etcd | grep -v grep" + etcd_check_cmd = r"ps -ef | awk '{print \$2, \$8}' | grep etcd | grep -v grep" process_cmd = "gpssh -h %s -e \"%s\"| wc -l" % (etcd, etcd_check_cmd) etcd_process_res = int(subprocess.check_output(process_cmd, shell=True).decode().strip()) return etcd_process_res == 2 From 687d4fd1774da1aef38a64ae99721a4e4091c893 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 16 Mar 2026 12:00:20 +0800 Subject: [PATCH 010/128] Fix LICENSE path for ISC license file Correct a typo in LICENSE that referenced the ISC license file with a duplicated directory name. No functional change. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 28796e982e1..0ccd7072122 100644 --- a/LICENSE +++ b/LICENSE @@ -246,7 +246,7 @@ The PostgreSQL software includes: src/backend/utils/adt/inet_cidr_ntop.c src/backend/utils/adt/inet_net_pton.c - see licenses/licenses/LICENSE-isc.txt + see licenses/LICENSE-isc.txt ---------------------------- Perl Artistic License 2.0 (exception) From f309b7c8e0b58579b6c556ef3b1f43ff34e9b810 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Wed, 18 Mar 2026 13:55:57 +0100 Subject: [PATCH 011/128] [MINOR] Remove unused build script that used non-HTTPS to download code * remove unused build_xerces.py --- pom.xml | 1 - .../gporca/concourse/xerces-c/build_xerces.py | 93 ------------------- 2 files changed, 94 deletions(-) delete mode 100644 src/backend/gporca/concourse/xerces-c/build_xerces.py diff --git a/pom.xml b/pom.xml index f89b9d0bb8a..1b4daf82da5 100644 --- a/pom.xml +++ b/pom.xml @@ -573,7 +573,6 @@ code or new licensing patterns. src/backend/gporca/cmake/FindXerces.cmake src/backend/gporca/concourse/build_and_test.py - src/backend/gporca/concourse/xerces-c/build_xerces.py src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 src/backend/gporca/server/fixdxl.sh src/backend/gporca/server/include/unittest/gpopt/operators/CScalarIsDistinctFromTest.h diff --git a/src/backend/gporca/concourse/xerces-c/build_xerces.py b/src/backend/gporca/concourse/xerces-c/build_xerces.py deleted file mode 100644 index 8b49e13ac2a..00000000000 --- a/src/backend/gporca/concourse/xerces-c/build_xerces.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 - -import optparse -import os -import os.path -import subprocess -import sys -import tarfile -import urllib.request, urllib.error, urllib.parse -import hashlib - -XERCES_SOURCE_URL = "http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.2.tar.gz" -XERCES_SOURCE_DIR = "xerces-c-3.1.2" - -def num_cpus(): - # Use multiprocessing module, available in Python 2.6+ - try: - import multiprocessing - return multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - pass - - # Get POSIX system config value for number of processors. - posix_num_cpus = os.sysconf("SC_NPROCESSORS_ONLN") - if posix_num_cpus != -1: - return posix_num_cpus - - # Guess - return 2 - -def get_xerces_source(): - remote_src = urllib.request.urlopen(XERCES_SOURCE_URL) - local_src = open("xerces_src.tar.gz", "wb") - local_src.write(remote_src.read()) - local_src.close() - file_hash = hashlib.sha256(open('xerces_src.tar.gz','rb').read()).hexdigest() - actual_hash = "" - with open('xerces_patch/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256', 'r') as f: - actual_hash = f.read().strip() - if file_hash != actual_hash: - return 1 - tarball = tarfile.open("xerces_src.tar.gz", "r:gz") - for item in tarball: - tarball.extract(item, ".") - tarball.close() - return 0 - -def configure(cxx_compiler, cxxflags, cflags, output_dir): - os.mkdir("build") - environment = os.environ.copy() - if cxx_compiler: - environment["CXX"] = cxx_compiler - if cxxflags: - environment["CXXFLAGS"] = cxxflags - if cflags: - environment["CFLAGS"] = cflags - return subprocess.call( - [os.path.abspath(XERCES_SOURCE_DIR + "/configure"), "--prefix=" + os.path.abspath(output_dir)], - env = environment, - cwd = "build") - -def make(): - return subprocess.call(["make", "-j" + str(num_cpus())], cwd="build") - -def install(): - return subprocess.call(["make", "install"], cwd="build") - -def main(): - parser = optparse.OptionParser() - parser.add_option("--compiler", dest="compiler") - parser.add_option("--cxxflags", dest="cxxflags") - parser.add_option("--cflags", dest="cflags") - parser.add_option("--output_dir", dest="output_dir", default="install") - (options, args) = parser.parse_args() - if len(args) > 0: - print("Unknown arguments") - return 1 - status = get_xerces_source() - if status: - return status - status = configure(options.compiler, options.cxxflags, options.cflags, options.output_dir) - if status: - return status - status = make() - if status: - return status - status = install() - if status: - return status - return 0 - -if __name__ == "__main__": - sys.exit(main()) From 26be110854680e46a8a6cf1fead89cd4602aaad4 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 18 Mar 2026 03:14:39 +0800 Subject: [PATCH 012/128] Fix SIGSEGV on segments when creating in-place tablespace When CREATE TABLESPACE ... LOCATION '' is dispatched from QD to QE, the serialization converts the empty string to NULL. On the segment, pstrdup(stmt->location) crashes with SIGSEGV because stmt->location is NULL. Add a NULL guard to treat NULL the same as empty string, preserving in-place tablespace semantics. Fixes https://github.com/apache/cloudberry/issues/1627 --- src/backend/commands/tablespace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 2416522d016..00fa8bcfb2a 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -328,7 +328,7 @@ CreateTableSpace(CreateTableSpaceStmt *stmt) } if (!location) - location = pstrdup(stmt->location); + location = pstrdup(stmt->location ? stmt->location : ""); if (stmt->filehandler) fileHandler = pstrdup(stmt->filehandler); From d7f23315f4ae7be19a96e7687a1f61cd5264f2af Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 18 Mar 2026 03:15:50 +0800 Subject: [PATCH 013/128] Fix session lock leak in ALTER DATABASE SET TABLESPACE for utility mode movedb() acquires a session-level AccessExclusiveLock on the database via MoveDbSessionLockAcquire(). The release in CommitTransaction() only checked for GP_ROLE_DISPATCH and IS_SINGLENODE(), missing GP_ROLE_UTILITY. This caused the lock to leak in standalone backends (e.g. TAP tests), triggering a proc.c assertion failure at exit: FailedAssertion("SHMQueueEmpty(&(MyProc->myProcLocks[i]))") Add GP_ROLE_UTILITY to the release condition. Also fix a spurious "could not read symbolic link" log message when dropping in-place tablespaces: readlink() on a directory returns EINVAL, which is expected and can be safely skipped. Fixes https://github.com/apache/cloudberry/issues/1626 --- src/backend/access/transam/xact.c | 16 +++++++++------- src/backend/commands/tablespace.c | 15 ++++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index d6b44fc8412..62ddae7a649 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -3036,13 +3036,15 @@ CommitTransaction(void) DoPendingDbDeletes(true); /* - * Only QD holds the session level lock this long for a movedb operation. - * This is to prevent another transaction from moving database objects into - * the source database oid directory while it is being deleted. We don't - * worry about aborts as we release session level locks automatically during - * an abort as opposed to a commit. - */ - if(Gp_role == GP_ROLE_DISPATCH || IS_SINGLENODE()) + * Release the session level lock held for a movedb operation. This is to + * prevent another transaction from moving database objects into the source + * database oid directory while it is being deleted. We don't worry about + * aborts as we release session level locks automatically during an abort + * as opposed to a commit. We must also release in utility mode (e.g. + * standalone backends used in TAP tests). + */ + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_UTILITY || + IS_SINGLENODE()) MoveDbSessionLockRelease(); /* diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 00fa8bcfb2a..e6822521694 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -1230,14 +1230,19 @@ destroy_tablespace_directories(Oid tablespaceoid, bool redo) linkloc = pstrdup(linkloc_with_version_dir); get_parent_directory(linkloc); - /* Remove the symlink target directory if it exists or is valid. */ + /* + * Remove the symlink target directory if it exists or is valid. + * If linkloc is a directory (e.g. in-place tablespace), readlink() + * will fail with EINVAL, which we can safely skip. + */ rllen = readlink(linkloc, link_target_dir, sizeof(link_target_dir)); if(rllen < 0) { - ereport(redo ? LOG : ERROR, - (errcode_for_file_access(), - errmsg("could not read symbolic link \"%s\": %m", - linkloc))); + if (errno != EINVAL) + ereport(redo ? LOG : ERROR, + (errcode_for_file_access(), + errmsg("could not read symbolic link \"%s\": %m", + linkloc))); } else if(rllen >= sizeof(link_target_dir)) { From 53a0407183d56c5c31798c58cc9149da73089d61 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 18 Mar 2026 07:45:52 +0800 Subject: [PATCH 014/128] Fix PostgresNode.pm for TAP tests on CBDB Two fixes: - Use TestLib::slurp_file instead of PostgreSQL::Test::Utils::slurp_file in adjust_conf(). PostgresNode.pm only imports TestLib, not PostgreSQL::Test::Utils, so the latter is undefined and causes t/101_restore_point_and_startup_pause to fail. - Replace cp with install -m 644 in enable_archiving() archive_command. coreutils 8.32 (Rocky 8, Ubuntu 22.04) uses copy_file_range() in cp which crashes on Docker overlayfs. install does not use copy_file_range(), avoiding the crash. Also add ic-recovery test suite to rocky8 and deb CI pipelines. --- .github/workflows/build-cloudberry-rocky8.yml | 4 +++ .../cloudberry/scripts/parse-results.pl | 31 ++++++++++++++++--- src/test/perl/PostgresNode.pm | 4 +-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 2abf88060e3..e0936c725c8 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -339,6 +339,10 @@ jobs: }, {"test":"ic-cbdb-parallel", "make_configs":["src/test/regress:installcheck-cbdb-parallel"] + }, + {"test":"ic-recovery", + "make_configs":["src/test/recovery:installcheck"], + "enable_core_check":false } ] }' diff --git a/devops/build/automation/cloudberry/scripts/parse-results.pl b/devops/build/automation/cloudberry/scripts/parse-results.pl index d09085d5fb9..2c754bcae9d 100755 --- a/devops/build/automation/cloudberry/scripts/parse-results.pl +++ b/devops/build/automation/cloudberry/scripts/parse-results.pl @@ -110,7 +110,7 @@ my @ignored_test_list = (); while (<$fh>) { - # Match the summary lines + # Match the summary lines (pg_regress format) if (/All (\d+) tests passed\./) { $status = 'passed'; $total_tests = $1; @@ -132,8 +132,22 @@ $status = 'failed'; $failed_tests = $1 - $3; $ignored_tests = $3; - $total_tests = $2; - $passed_tests = $2 - $1; + + # TAP/prove summary format: "Files=N, Tests=N, ..." + } elsif (/^Files=\d+, Tests=(\d+),/) { + $total_tests = $1; + + # TAP/prove result: "Result: PASS" or "Result: FAIL" + } elsif (/^Result: PASS/) { + $status = 'passed'; + $passed_tests = $total_tests; + $failed_tests = 0; + } elsif (/^Result: FAIL/) { + $status = 'failed'; + + # TAP individual test failure: " t/xxx.pl (Wstat: ...)" + } elsif (/^\s+(t\/\S+\.pl)\s+\(Wstat:/) { + push @failed_test_list, $1; } # Capture failed tests @@ -150,8 +164,15 @@ # Close the log file close $fh; -# Validate failed test count matches found test names -if ($status eq 'failed' && scalar(@failed_test_list) != $failed_tests) { +# For TAP format, derive failed/passed counts from collected test names +if ($status eq 'failed' && $failed_tests == 0 && scalar(@failed_test_list) > 0) { + $failed_tests = scalar(@failed_test_list); + $passed_tests = $total_tests - $failed_tests if $total_tests > 0; +} + +# Validate failed test count matches found test names (pg_regress format only) +if ($status eq 'failed' && $failed_tests > 0 && scalar(@failed_test_list) > 0 + && scalar(@failed_test_list) != $failed_tests) { print "Error: Found $failed_tests failed tests in summary but found " . scalar(@failed_test_list) . " failed test names\n"; print "Failed test names found:\n"; foreach my $test (@failed_test_list) { diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index b9896d5076a..50ba9d041bc 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -708,7 +708,7 @@ sub adjust_conf my $conffile = $self->data_dir . '/' . $filename; - my $contents = PostgreSQL::Test::Utils::slurp_file($conffile); + my $contents = TestLib::slurp_file($conffile); my @lines = split(/\n/, $contents); my @result; my $eq = $skip_equals ? '' : '= '; @@ -1296,7 +1296,7 @@ sub enable_archiving my $copy_command = $TestLib::windows_os ? qq{copy "%p" "$path\\\\%f"} - : qq{cp "%p" "$path/%f"}; + : qq{install -m 644 "%p" "$path/%f"}; # Enable archive_mode and archive_command on node $self->append_conf( From 54162d6e505065449cda8bfd0ab8e522685f86cf Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Sat, 14 Mar 2026 03:50:51 +0800 Subject: [PATCH 015/128] ORCA: Fix use-after-free in flatten_join_alias_var_optimizer Guard pfree/list_free calls with pointer-equality checks to avoid freeing live nodes when flatten_join_alias_vars returns the same pointer unchanged (e.g., outer-reference Vars with varlevelsup > 0). The unconditional pfree(havingQual) freed the Var node, whose memory was later reused by palloc for a T_List. copyObjectImpl then copied the wrong node type into havingQual, causing ORCA to encounter an unexpected RangeTblEntry and fall back to the Postgres planner. Applies the same guard pattern to all six fields: targetList, returningList, havingQual, scatterClause, limitOffset, limitCount. Reported-in: https://github.com/apache/cloudberry/issues/1618 --- src/backend/optimizer/util/clauses.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 2669f2c3017..9085f3876a4 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -5522,35 +5522,40 @@ flatten_join_alias_var_optimizer(Query *query, int queryLevel) if (NIL != targetList) { queryNew->targetList = (List *) flatten_join_alias_vars(queryNew, (Node *) targetList); - list_free(targetList); + if (targetList != queryNew->targetList) + list_free(targetList); } - List * returningList = queryNew->returningList; + List *returningList = queryNew->returningList; if (NIL != returningList) { queryNew->returningList = (List *) flatten_join_alias_vars(queryNew, (Node *) returningList); - list_free(returningList); + if (returningList != queryNew->returningList) + list_free(returningList); } Node *havingQual = queryNew->havingQual; if (NULL != havingQual) { queryNew->havingQual = flatten_join_alias_vars(queryNew, havingQual); - pfree(havingQual); + if (havingQual != queryNew->havingQual) + pfree(havingQual); } List *scatterClause = queryNew->scatterClause; if (NIL != scatterClause) { queryNew->scatterClause = (List *) flatten_join_alias_vars(queryNew, (Node *) scatterClause); - list_free(scatterClause); + if (scatterClause != queryNew->scatterClause) + list_free(scatterClause); } Node *limitOffset = queryNew->limitOffset; if (NULL != limitOffset) { queryNew->limitOffset = flatten_join_alias_vars(queryNew, limitOffset); - pfree(limitOffset); + if (limitOffset != queryNew->limitOffset) + pfree(limitOffset); } List *windowClause = queryNew->windowClause; @@ -5577,7 +5582,8 @@ flatten_join_alias_var_optimizer(Query *query, int queryLevel) if (NULL != limitCount) { queryNew->limitCount = flatten_join_alias_vars(queryNew, limitCount); - pfree(limitCount); + if (limitCount != queryNew->limitCount) + pfree(limitCount); } return queryNew; From 54f44b1284a7031b66629765552c6e3c696e0be1 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Sat, 14 Mar 2026 03:51:08 +0800 Subject: [PATCH 016/128] ORCA: Fix incorrect decorrelation of GROUP BY () HAVING Force correlated execution (SubPlan) for scalar subqueries with GROUP BY () and a correlated HAVING clause. Previously ORCA decorrelated such subqueries into Left Outer Join + COALESCE(count,0), which incorrectly returned 0 instead of NULL when the HAVING condition was false. Add FHasCorrelatedSelectAboveGbAgg() to detect the pattern where NormalizeHaving() has converted the HAVING clause into a CLogicalSelect with outer refs above a CLogicalGbAgg with empty grouping columns. When detected, set m_fCorrelatedExecution = true in Psd() to bypass the COALESCE decorrelation path. Update groupingsets_optimizer.out expected output to reflect the new ORCA SubPlan format instead of Postgres planner fallback. Reported-in: https://github.com/apache/cloudberry/issues/1618 --- .../expected/groupingsets_optimizer.out | 28 ++--- .../libgpopt/src/xforms/CSubqueryHandler.cpp | 115 +++++++++++++++++- src/backend/optimizer/util/clauses.c | 7 +- .../expected/groupingsets_optimizer.out | 28 ++--- 4 files changed, 148 insertions(+), 30 deletions(-) diff --git a/contrib/pax_storage/src/test/regress/expected/groupingsets_optimizer.out b/contrib/pax_storage/src/test/regress/expected/groupingsets_optimizer.out index b3da68b1f9d..382fb46fdfd 100644 --- a/contrib/pax_storage/src/test/regress/expected/groupingsets_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/groupingsets_optimizer.out @@ -949,21 +949,21 @@ select v.c, (select count(*) from gstest2 group by () having v.c) explain (costs off) select v.c, (select count(*) from gstest2 group by () having v.c) from (values (false),(true)) v(c) order by v.c; - QUERY PLAN --------------------------------------------------------------------------- - Sort - Sort Key: "*VALUES*".column1 - -> Values Scan on "*VALUES*" - SubPlan 1 - -> Aggregate - Group Key: () - Filter: "*VALUES*".column1 - -> Result - One-Time Filter: "*VALUES*".column1 - -> Materialize - -> Gather Motion 3:1 (slice1; segments: 3) + QUERY PLAN +-------------------------------------------------------------------- + Result + -> Sort + Sort Key: "Values".column1 + -> Values Scan on "Values" + SubPlan 1 + -> Result + One-Time Filter: "Values".column1 + -> Finalize Aggregate + -> Materialize + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate -> Seq Scan on gstest2 - Optimizer: Postgres query optimizer + Optimizer: GPORCA (13 rows) -- HAVING with GROUPING queries diff --git a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp index 791ee8345a9..a3c333c5ab1 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp @@ -308,6 +308,93 @@ CSubqueryHandler::FProjectCountSubquery(CExpression *pexprSubquery, } +//--------------------------------------------------------------------------- +// @function: +// FContainsEmptyGbAgg +// +// @doc: +// Return true if pexpr contains a GbAgg with empty grouping columns +// (i.e., GROUP BY ()) +// +//--------------------------------------------------------------------------- +static BOOL +FContainsEmptyGbAgg(CExpression *pexpr) +{ + if (COperator::EopLogicalGbAgg == pexpr->Pop()->Eopid()) + { + return 0 == CLogicalGbAgg::PopConvert(pexpr->Pop())->Pdrgpcr()->Size(); + } + const ULONG arity = pexpr->Arity(); + for (ULONG ul = 0; ul < arity; ul++) + { + CExpression *pexprChild = (*pexpr)[ul]; + if (pexprChild->Pop()->FLogical() && FContainsEmptyGbAgg(pexprChild)) + { + return true; + } + } + return false; +} + + +//--------------------------------------------------------------------------- +// @function: +// FHasCorrelatedSelectAboveGbAgg +// +// @doc: +// Return true if pexpr has a CLogicalSelect with outer references in its +// filter predicate that sits above a GROUP BY () aggregate. This pattern +// arises when a correlated scalar subquery has a correlated HAVING clause, +// e.g. "SELECT count(*) FROM t GROUP BY () HAVING outer_col". +// +// When such a pattern exists the scalar subquery must NOT be decorrelated +// with COALESCE(count,0) semantics: if the HAVING condition is false the +// subquery should return NULL (no rows), not 0. +// +//--------------------------------------------------------------------------- +static BOOL +FHasCorrelatedSelectAboveGbAgg(CExpression *pexpr) +{ + // Stop recursion at a GbAgg boundary: we are looking for a Select + // that sits *above* a GbAgg, so once we reach the GbAgg there is + // nothing more to check in this branch. + if (COperator::EopLogicalGbAgg == pexpr->Pop()->Eopid()) + { + return false; + } + + if (COperator::EopLogicalSelect == pexpr->Pop()->Eopid() && + pexpr->HasOuterRefs()) + { + // The Select has outer references somewhere in its subtree. + // Check whether they originate from the filter (child 1) rather + // than from the logical child (child 0). If the logical child has + // no outer refs but the Select as a whole does, the outer refs must + // come from the filter predicate — exactly the correlated-HAVING + // pattern we want to detect. + CExpression *pexprLogicalChild = (*pexpr)[0]; + if (!pexprLogicalChild->HasOuterRefs() && + FContainsEmptyGbAgg(pexprLogicalChild)) + { + return true; + } + } + + // Recurse into logical children only. + const ULONG arity = pexpr->Arity(); + for (ULONG ul = 0; ul < arity; ul++) + { + CExpression *pexprChild = (*pexpr)[ul]; + if (pexprChild->Pop()->FLogical() && + FHasCorrelatedSelectAboveGbAgg(pexprChild)) + { + return true; + } + } + return false; +} + + //--------------------------------------------------------------------------- // @function: // CSubqueryHandler::SSubqueryDesc::SetCorrelatedExecution @@ -382,6 +469,21 @@ CSubqueryHandler::Psd(CMemoryPool *mp, CExpression *pexprSubquery, // set flag of correlated execution psd->SetCorrelatedExecution(); + // A correlated scalar subquery of the form + // SELECT count(*) FROM t GROUP BY () HAVING + // must execute as a correlated SubPlan. After NormalizeHaving() the HAVING + // clause becomes a CLogicalSelect with outer refs sitting above the GbAgg. + // If we decorrelate such a subquery the join filter replaces the HAVING + // condition, but a LEFT JOIN returns 0 (not NULL) for count(*) when no + // rows match — which is semantically wrong. Forcing correlated execution + // preserves the correct NULL-when-no-rows semantics. + if (!psd->m_fCorrelatedExecution && psd->m_fHasCountAgg && + psd->m_fHasOuterRefs && + FHasCorrelatedSelectAboveGbAgg(pexprInner)) + { + psd->m_fCorrelatedExecution = true; + } + return psd; } @@ -753,8 +855,19 @@ CSubqueryHandler::FCreateOuterApplyForScalarSubquery( *ppexprNewOuter = pexprPrj; BOOL fGeneratedByQuantified = popSubquery->FGeneratedByQuantified(); + + // When GROUP BY () has a correlated HAVING clause (now represented as a + // CLogicalSelect with outer refs sitting above the GbAgg), the subquery + // must return NULL — not 0 — when the HAVING condition is false. + // Applying COALESCE(count,0) would incorrectly convert that NULL to 0, + // so we skip the special count(*) semantics in that case. + BOOL fCorrelatedHavingAboveEmptyGby = + (fHasCountAggMatchingColumn && 0 == pgbAgg->Pdrgpcr()->Size() && + FHasCorrelatedSelectAboveGbAgg((*pexprSubquery)[0])); + if (fGeneratedByQuantified || - (fHasCountAggMatchingColumn && 0 == pgbAgg->Pdrgpcr()->Size())) + (fHasCountAggMatchingColumn && 0 == pgbAgg->Pdrgpcr()->Size() && + !fCorrelatedHavingAboveEmptyGby)) { CMDAccessor *md_accessor = COptCtxt::PoctxtFromTLS()->Pmda(); const IMDTypeInt8 *pmdtypeint8 = md_accessor->PtMDType(); diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 9085f3876a4..a039fdd87d2 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -5539,7 +5539,12 @@ flatten_join_alias_var_optimizer(Query *query, int queryLevel) { queryNew->havingQual = flatten_join_alias_vars(queryNew, havingQual); if (havingQual != queryNew->havingQual) - pfree(havingQual); + { + if (IsA(havingQual, List)) + list_free((List *) havingQual); + else + pfree(havingQual); + } } List *scatterClause = queryNew->scatterClause; diff --git a/src/test/regress/expected/groupingsets_optimizer.out b/src/test/regress/expected/groupingsets_optimizer.out index 08ef4c1a68c..3718069c36c 100644 --- a/src/test/regress/expected/groupingsets_optimizer.out +++ b/src/test/regress/expected/groupingsets_optimizer.out @@ -958,21 +958,21 @@ select v.c, (select count(*) from gstest2 group by () having v.c) explain (costs off) select v.c, (select count(*) from gstest2 group by () having v.c) from (values (false),(true)) v(c) order by v.c; - QUERY PLAN --------------------------------------------------------------------------- - Sort - Sort Key: "*VALUES*".column1 - -> Values Scan on "*VALUES*" - SubPlan 1 - -> Aggregate - Group Key: () - Filter: "*VALUES*".column1 - -> Result - One-Time Filter: "*VALUES*".column1 - -> Materialize - -> Gather Motion 3:1 (slice1; segments: 3) + QUERY PLAN +-------------------------------------------------------------------- + Result + -> Sort + Sort Key: "Values".column1 + -> Values Scan on "Values" + SubPlan 1 + -> Result + One-Time Filter: "Values".column1 + -> Finalize Aggregate + -> Materialize + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate -> Seq Scan on gstest2 - Optimizer: Postgres query optimizer + Optimizer: GPORCA (13 rows) -- HAVING with GROUPING queries From c6e7a5ca19cf1c08b821b38be279f482bac2c070 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Thu, 5 Mar 2026 15:45:39 +0800 Subject: [PATCH 017/128] Support AQUMV exact-match for multi-table JOIN queries Add a new AQUMV code path that rewrites multi-table JOIN queries to scan materialized views when the query exactly matches the MV definition. This compares the saved raw parse tree against stored viewQuery from gp_matview_aux, bypassing the single-table AQUMV logic entirely. This enables significant query acceleration for common analytical patterns: instead of repeatedly computing expensive multi-table joins at query time, the planner can directly read pre-computed results from the materialized view, turning O(N*M) join operations into a simple sequential scan. For example, given: CREATE MATERIALIZED VIEW mv AS SELECT t1.a, t2.b FROM t1 JOIN t2 ON t1.a = t2.a; -- Before (GUC off): original join plan Gather Motion 3:1 -> Hash Join Hash Cond: (t1.a = t2.a) -> Seq Scan on t1 -> Hash -> Seq Scan on t2 -- After (GUC on): rewritten to MV scan Gather Motion 3:1 -> Seq Scan on mv --- src/backend/optimizer/plan/aqumv.c | 345 +++++ src/backend/optimizer/plan/planner.c | 18 + src/include/nodes/pathnodes.h | 2 + src/include/optimizer/aqumv.h | 1 + src/test/regress/expected/matview_data.out | 1341 +++++++++++++++++++- src/test/regress/sql/matview_data.sql | 535 ++++++++ 6 files changed, 2206 insertions(+), 36 deletions(-) diff --git a/src/backend/optimizer/plan/aqumv.c b/src/backend/optimizer/plan/aqumv.c index 4a576061780..2084bcbc181 100644 --- a/src/backend/optimizer/plan/aqumv.c +++ b/src/backend/optimizer/plan/aqumv.c @@ -996,3 +996,348 @@ groupby_query_rewrite(PlannerInfo *subroot, subroot->append_rel_list = NIL; return true; } + +/* + * aqumv_query_is_exact_match + * + * Compare two Query trees for semantic identity. Both should be at the + * same preprocessing stage (raw parser output). Returns true only if + * they are structurally identical in all query-semantics fields. + */ +static bool +aqumv_query_is_exact_match(Query *raw_parse, Query *viewQuery) +{ + /* Both must be CMD_SELECT */ + if (raw_parse->commandType != CMD_SELECT || + viewQuery->commandType != CMD_SELECT) + return false; + + /* Same number of range table entries */ + if (list_length(raw_parse->rtable) != list_length(viewQuery->rtable)) + return false; + + /* Compare range tables (table OIDs, join types, aliases structure) */ + if (!equal(raw_parse->rtable, viewQuery->rtable)) + return false; + + /* Compare join tree (FROM clause + WHERE quals) */ + if (!equal(raw_parse->jointree, viewQuery->jointree)) + return false; + + /* Compare target list entries: expressions and sort/group refs */ + if (list_length(raw_parse->targetList) != list_length(viewQuery->targetList)) + return false; + { + ListCell *lc1, *lc2; + forboth(lc1, raw_parse->targetList, lc2, viewQuery->targetList) + { + TargetEntry *tle1 = lfirst_node(TargetEntry, lc1); + TargetEntry *tle2 = lfirst_node(TargetEntry, lc2); + if (!equal(tle1->expr, tle2->expr)) + return false; + if (tle1->resjunk != tle2->resjunk) + return false; + if (tle1->ressortgroupref != tle2->ressortgroupref) + return false; + } + } + + /* Compare GROUP BY, HAVING, ORDER BY, DISTINCT, LIMIT */ + if (!equal(raw_parse->groupClause, viewQuery->groupClause)) + return false; + if (!equal(raw_parse->havingQual, viewQuery->havingQual)) + return false; + if (!equal(raw_parse->sortClause, viewQuery->sortClause)) + return false; + if (!equal(raw_parse->distinctClause, viewQuery->distinctClause)) + return false; + if (!equal(raw_parse->limitCount, viewQuery->limitCount)) + return false; + if (!equal(raw_parse->limitOffset, viewQuery->limitOffset)) + return false; + + /* Compare boolean flags */ + if (raw_parse->hasAggs != viewQuery->hasAggs) + return false; + if (raw_parse->hasWindowFuncs != viewQuery->hasWindowFuncs) + return false; + if (raw_parse->hasDistinctOn != viewQuery->hasDistinctOn) + return false; + + return true; +} + +/* + * answer_query_using_materialized_views_for_join + * + * Handle multi-table JOIN queries via exact-match comparison. + * This is completely independent from the single-table AQUMV code path. + * + * We compare the saved raw parse tree (before any planner preprocessing) + * against the stored viewQuery from gp_matview_aux. On exact match, + * rewrite the query to a simple SELECT FROM mv. + */ +RelOptInfo * +answer_query_using_materialized_views_for_join(PlannerInfo *root, AqumvContext aqumv_context) +{ + RelOptInfo *current_rel = aqumv_context->current_rel; + query_pathkeys_callback qp_callback = aqumv_context->qp_callback; + Query *parse = root->parse; + Query *raw_parse = root->aqumv_raw_parse; + RelOptInfo *mv_final_rel = current_rel; + Relation matviewRel; + Relation mvauxDesc; + TupleDesc mvaux_tupdesc; + SysScanDesc mvscan; + HeapTuple tup; + Form_gp_matview_aux mvaux_tup; + bool need_close = false; + + /* Must have the saved raw parse tree. */ + if (raw_parse == NULL) + return mv_final_rel; + + /* Must be a join query (more than one table in FROM). */ + if (list_length(raw_parse->rtable) <= 1) + return mv_final_rel; + + /* Basic eligibility checks (same as single-table AQUMV). */ + if (parse->commandType != CMD_SELECT || + parse->rowMarks != NIL || + parse->scatterClause != NIL || + parse->cteList != NIL || + parse->setOperations != NULL || + parse->hasModifyingCTE || + parse->parentStmtType == PARENTSTMTTYPE_REFRESH_MATVIEW || + parse->parentStmtType == PARENTSTMTTYPE_CTAS || + contain_mutable_functions((Node *) raw_parse) || + parse->hasSubLinks) + return mv_final_rel; + + mvauxDesc = table_open(GpMatviewAuxId, AccessShareLock); + mvaux_tupdesc = RelationGetDescr(mvauxDesc); + + mvscan = systable_beginscan(mvauxDesc, InvalidOid, false, + NULL, 0, NULL); + + while (HeapTupleIsValid(tup = systable_getnext(mvscan))) + { + Datum view_query_datum; + char *view_query_str; + bool is_null; + Query *viewQuery; + RangeTblEntry *mvrte; + PlannerInfo *subroot; + TupleDesc mv_tupdesc; + + CHECK_FOR_INTERRUPTS(); + if (need_close) + table_close(matviewRel, AccessShareLock); + + mvaux_tup = (Form_gp_matview_aux) GETSTRUCT(tup); + matviewRel = table_open(mvaux_tup->mvoid, AccessShareLock); + need_close = true; + + if (!RelationIsPopulated(matviewRel)) + continue; + + /* MV must be up-to-date (IVM is always current). */ + if (!RelationIsIVM(matviewRel) && + !MatviewIsGeneralyUpToDate(RelationGetRelid(matviewRel))) + continue; + + /* Get a copy of view query. */ + view_query_datum = heap_getattr(tup, + Anum_gp_matview_aux_view_query, + mvaux_tupdesc, + &is_null); + + view_query_str = TextDatumGetCString(view_query_datum); + viewQuery = copyObject(stringToNode(view_query_str)); + pfree(view_query_str); + Assert(IsA(viewQuery, Query)); + + /* Skip single-table viewQueries (handled by existing AQUMV). */ + if (list_length(viewQuery->rtable) <= 1) + continue; + + /* Exact match comparison between raw parse and view query. */ + if (!aqumv_query_is_exact_match(raw_parse, viewQuery)) + continue; + + /* + * We have an exact match. Rewrite viewQuery to: + * SELECT mv.col1, mv.col2, ... FROM mv + */ + mv_tupdesc = RelationGetDescr(matviewRel); + + /* Build new target list referencing MV columns. */ + { + List *new_tlist = NIL; + ListCell *lc; + int attnum = 0; + + foreach(lc, viewQuery->targetList) + { + TargetEntry *old_tle = lfirst_node(TargetEntry, lc); + TargetEntry *new_tle; + Var *newVar; + Form_pg_attribute attr; + + if (old_tle->resjunk) + continue; + + attnum++; + attr = TupleDescAttr(mv_tupdesc, attnum - 1); + + newVar = makeVar(1, + attr->attnum, + attr->atttypid, + attr->atttypmod, + attr->attcollation, + 0); + newVar->location = -1; + + new_tle = makeTargetEntry((Expr *) newVar, + (AttrNumber) attnum, + old_tle->resname, + false); + new_tlist = lappend(new_tlist, new_tle); + } + + viewQuery->targetList = new_tlist; + } + + /* Create new RTE for the MV. */ + mvrte = makeNode(RangeTblEntry); + mvrte->rtekind = RTE_RELATION; + mvrte->relid = RelationGetRelid(matviewRel); + mvrte->relkind = RELKIND_MATVIEW; + mvrte->rellockmode = AccessShareLock; + mvrte->inh = false; + mvrte->inFromCl = true; + + /* Build eref with column names from the MV's TupleDesc. */ + { + Alias *eref = makeAlias(RelationGetRelationName(matviewRel), NIL); + int i; + for (i = 0; i < mv_tupdesc->natts; i++) + { + Form_pg_attribute attr = TupleDescAttr(mv_tupdesc, i); + if (!attr->attisdropped) + eref->colnames = lappend(eref->colnames, + makeString(pstrdup(NameStr(attr->attname)))); + else + eref->colnames = lappend(eref->colnames, + makeString(pstrdup(""))); + } + mvrte->eref = eref; + mvrte->alias = makeAlias(RelationGetRelationName(matviewRel), NIL); + } + + viewQuery->rtable = list_make1(mvrte); + viewQuery->jointree = makeFromExpr(list_make1(makeNode(RangeTblRef)), NULL); + ((RangeTblRef *) linitial(viewQuery->jointree->fromlist))->rtindex = 1; + + /* Clear aggregation/grouping/sorting state — all materialized. */ + viewQuery->hasAggs = false; + viewQuery->groupClause = NIL; + viewQuery->havingQual = NULL; + viewQuery->sortClause = NIL; + viewQuery->distinctClause = NIL; + viewQuery->hasDistinctOn = false; + viewQuery->hasWindowFuncs = false; + viewQuery->hasTargetSRFs = false; + viewQuery->limitCount = parse->limitCount; + viewQuery->limitOffset = parse->limitOffset; + viewQuery->limitOption = parse->limitOption; + + /* Create subroot for planning the MV scan. */ + subroot = (PlannerInfo *) palloc(sizeof(PlannerInfo)); + memcpy(subroot, root, sizeof(PlannerInfo)); + subroot->parent_root = root; + subroot->eq_classes = NIL; + subroot->plan_params = NIL; + subroot->outer_params = NULL; + subroot->init_plans = NIL; + subroot->agginfos = NIL; + subroot->aggtransinfos = NIL; + subroot->parse = viewQuery; + subroot->tuple_fraction = root->tuple_fraction; + subroot->limit_tuples = root->limit_tuples; + subroot->append_rel_list = NIL; + subroot->hasHavingQual = false; + subroot->hasNonPartialAggs = false; + subroot->hasNonSerialAggs = false; + subroot->numOrderedAggs = 0; + subroot->hasNonCombine = false; + subroot->numPureOrderedAggs = 0; + + subroot->processed_tlist = NIL; + preprocess_targetlist(subroot); + + /* Compute final locus for the MV scan. */ + { + PathTarget *newtarget = make_pathtarget_from_tlist(subroot->processed_tlist); + subroot->final_locus = cdbllize_get_final_locus(subroot, newtarget); + } + + /* + * Plan the MV scan. + * + * We need a clean qp_extra with no groupClause or activeWindows, + * because the rewritten viewQuery is a simple SELECT from the MV + * with no GROUP BY, windowing, etc. The standard_qp_callback uses + * qp_extra->groupClause to compute group_pathkeys, which would fail + * if it still contained the original query's GROUP BY expressions. + * + * standard_qp_extra is { List *activeWindows; List *groupClause; }, + * so a zeroed struct of that size works correctly (both fields NIL). + */ + { + char clean_qp_extra[2 * sizeof(List *)]; + memset(clean_qp_extra, 0, sizeof(clean_qp_extra)); + mv_final_rel = query_planner(subroot, qp_callback, clean_qp_extra); + } + + /* Cost-based decision: use MV only if cheaper. */ + if (mv_final_rel->cheapest_total_path->total_cost < current_rel->cheapest_total_path->total_cost) + { + root->parse = viewQuery; + root->processed_tlist = subroot->processed_tlist; + root->agginfos = subroot->agginfos; + root->aggtransinfos = subroot->aggtransinfos; + root->simple_rte_array = subroot->simple_rte_array; + root->simple_rel_array = subroot->simple_rel_array; + root->simple_rel_array_size = subroot->simple_rel_array_size; + root->hasNonPartialAggs = subroot->hasNonPartialAggs; + root->hasNonSerialAggs = subroot->hasNonSerialAggs; + root->numOrderedAggs = subroot->numOrderedAggs; + root->hasNonCombine = subroot->hasNonCombine; + root->numPureOrderedAggs = subroot->numPureOrderedAggs; + root->hasHavingQual = subroot->hasHavingQual; + root->group_pathkeys = subroot->group_pathkeys; + root->sort_pathkeys = subroot->sort_pathkeys; + root->query_pathkeys = subroot->query_pathkeys; + root->distinct_pathkeys = subroot->distinct_pathkeys; + root->eq_classes = subroot->eq_classes; + root->append_rel_list = subroot->append_rel_list; + current_rel = mv_final_rel; + table_close(matviewRel, NoLock); + need_close = false; + break; + } + else + { + /* MV is not cheaper, reset and try next. */ + mv_final_rel = current_rel; + } + } + + if (need_close) + table_close(matviewRel, AccessShareLock); + systable_endscan(mvscan); + table_close(mvauxDesc, AccessShareLock); + + return current_rel; +} diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index afbf249fc33..1b83b83e762 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -953,6 +953,17 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->partColsUpdated = false; root->is_correlated_subplan = false; + /* + * Save a copy of the raw parse tree for AQUMV join exact-match. + * This must be done before any preprocessing modifies the parse tree. + */ + if (Gp_role == GP_ROLE_DISPATCH && + enable_answer_query_using_materialized_views && + parent_root == NULL) + root->aqumv_raw_parse = copyObject(parse); + else + root->aqumv_raw_parse = NULL; + /* * If there is a WITH list, process each WITH query and either convert it * to RTE_SUBQUERY RTE(s) or build an initplan SubPlan structure for it. @@ -1965,6 +1976,13 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) /* Do the real work. */ current_rel = answer_query_using_materialized_views(root, aqumv_context); + + /* Try join AQUMV if single-table didn't rewrite. */ + if (current_rel == aqumv_context->current_rel) + { + current_rel = answer_query_using_materialized_views_for_join(root, aqumv_context); + } + /* parse tree may be rewriten. */ parse = root->parse; } diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 41220887050..05cac0c9043 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -505,6 +505,8 @@ struct PlannerInfo int numPureOrderedAggs; /* CDB: number that use ORDER BY/WITHIN GROUP, not counting DISTINCT */ bool hasNonCombine; /* CDB: any agg func w/o a combine func? */ bool is_from_orca; /* true if this PlannerInfo was created from Orca*/ + + Query *aqumv_raw_parse; /* Raw parse tree for AQUMV join exact-match */ }; /* diff --git a/src/include/optimizer/aqumv.h b/src/include/optimizer/aqumv.h index 6e51d4dbc92..2bb4122cf11 100644 --- a/src/include/optimizer/aqumv.h +++ b/src/include/optimizer/aqumv.h @@ -44,5 +44,6 @@ typedef struct AqumvContextData { typedef AqumvContextData *AqumvContext; extern RelOptInfo* answer_query_using_materialized_views(PlannerInfo *root, AqumvContextData *aqumv_context); +extern RelOptInfo* answer_query_using_materialized_views_for_join(PlannerInfo *root, AqumvContextData *aqumv_context); #endif /* AQUMV_H */ diff --git a/src/test/regress/expected/matview_data.out b/src/test/regress/expected/matview_data.out index 85697e90072..e415ceaf363 100644 --- a/src/test/regress/expected/matview_data.out +++ b/src/test/regress/expected/matview_data.out @@ -791,6 +791,1274 @@ drop materialized view mv_join2; drop table jt3; drop table jt2; drop table jt1; +-- +-- Test AQUMV (Answer Query Using Materialized Views) with join queries. +-- Each matching test shows EXPLAIN + SELECT with GUC off (original plan), +-- then EXPLAIN + SELECT with GUC on (MV rewrite). Results must match. +-- +create table aqj_t1(a int, b int) distributed by (a); +create table aqj_t2(a int, b int) distributed by (a); +create table aqj_t3(a int, b int) distributed by (a); +insert into aqj_t1 select i, i*10 from generate_series(1, 100) i; +insert into aqj_t2 select i, i*100 from generate_series(1, 100) i; +insert into aqj_t3 select i, i*1000 from generate_series(1, 100) i; +analyze aqj_t1; +analyze aqj_t2; +analyze aqj_t3; +-- 1. Two-table INNER JOIN exact match +create materialized view mv_aqj_join2 as + select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_join2; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a order by 1 limit 5; + a | b +---+----- + 1 | 100 + 2 | 200 + 3 | 300 + 4 | 400 + 5 | 500 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_join2 + Optimizer: Postgres query optimizer +(3 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a order by 1 limit 5; + a | b +---+----- + 1 | 100 + 2 | 200 + 3 | 300 + 4 | 400 + 5 | 500 +(5 rows) + +-- 2. Join with WHERE clause +create materialized view mv_aqj_where as + select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_where; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + Filter: (a > 5) + -> Hash + -> Seq Scan on aqj_t2 + Filter: (a > 5) + Optimizer: Postgres query optimizer +(9 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5 order by 1 limit 5; + a | b +----+------ + 6 | 600 + 7 | 700 + 8 | 800 + 9 | 900 + 10 | 1000 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_where + Optimizer: Postgres query optimizer +(3 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5 order by 1 limit 5; + a | b +----+------ + 6 | 600 + 7 | 700 + 8 | 800 + 9 | 900 + 10 | 1000 +(5 rows) + +-- 3. Join with GROUP BY + aggregate +create materialized view mv_aqj_agg as + select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_agg; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; + QUERY PLAN +------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> HashAggregate + Group Key: aqj_t1.a + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(9 rows) + +select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a order by 1 limit 5; + a | cnt +---+----- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_agg + Optimizer: Postgres query optimizer +(3 rows) + +select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a order by 1 limit 5; + a | cnt +---+----- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 +(5 rows) + +-- 4. Non-match: different WHERE clause (should show Hash Join, not MV) +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 10; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + Filter: (a > 10) + -> Hash + -> Seq Scan on aqj_t2 + Filter: (a > 10) + Optimizer: Postgres query optimizer +(9 rows) + +-- 5. Non-match: different target list +explain(costs off) select aqj_t1.b, aqj_t2.a from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +-- 6. Non-match: different join type (INNER vs LEFT) +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 left join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Left Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +-- 7. Three-table join +create materialized view mv_aqj_join3 as + select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_join3; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; + QUERY PLAN +------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t3.a) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + -> Hash + -> Seq Scan on aqj_t3 + Optimizer: Postgres query optimizer +(11 rows) + +select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a + order by 1 limit 5; + a | b | c +---+-----+------ + 1 | 100 | 1000 + 2 | 200 | 2000 + 3 | 300 | 3000 + 4 | 400 | 4000 + 5 | 500 | 5000 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_join3 + Optimizer: Postgres query optimizer +(3 rows) + +select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a + order by 1 limit 5; + a | b | c +---+-----+------ + 1 | 100 | 1000 + 2 | 200 | 2000 + 3 | 300 | 3000 + 4 | 400 | 4000 + 5 | 500 | 5000 +(5 rows) + +-- 8. Implicit join (FROM t1, t2 WHERE ...) +create materialized view mv_aqj_implicit as + select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_implicit; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a order by 1 limit 5; + a | b +---+----- + 1 | 100 + 2 | 200 + 3 | 300 + 4 | 400 + 5 | 500 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_implicit + Optimizer: Postgres query optimizer +(3 rows) + +select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a order by 1 limit 5; + a | b +---+----- + 1 | 100 + 2 | 200 + 3 | 300 + 4 | 400 + 5 | 500 +(5 rows) + +-- 9. MV not up-to-date: after INSERT on base table +insert into aqj_t1 values(999, 9990); +set enable_answer_query_using_materialized_views = on; +-- Should NOT use mv_aqj_join2 (status is 'i') +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +-- 10. After REFRESH: should use MV again +refresh materialized view mv_aqj_join2; +analyze mv_aqj_join2; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_join2 + Optimizer: Postgres query optimizer +(3 rows) + +-- 11. GUC off: should NOT use MV +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (aqj_t1.a = aqj_t2.a) + -> Seq Scan on aqj_t1 + -> Hash + -> Seq Scan on aqj_t2 + Optimizer: Postgres query optimizer +(7 rows) + +-- +-- More complex join AQUMV test cases with richer schemas +-- +create table aqj_orders( + order_id int, + customer_id int, + amount numeric(10,2), + status text, + order_date date +) distributed by (order_id); +create table aqj_customers( + customer_id int, + name text, + region text, + credit_limit numeric(10,2) +) distributed by (customer_id); +create table aqj_products( + product_id int, + name text, + category text, + price numeric(10,2) +) distributed by (product_id); +create table aqj_order_items( + item_id int, + order_id int, + product_id int, + quantity int +) distributed by (item_id); +insert into aqj_customers select i, 'cust_' || i, case when i % 3 = 0 then 'east' when i % 3 = 1 then 'west' else 'north' end, (i * 100)::numeric(10,2) from generate_series(1, 50) i; +insert into aqj_orders select i, (i % 50) + 1, (i * 10.5)::numeric(10,2), case when i % 4 = 0 then 'shipped' when i % 4 = 1 then 'pending' when i % 4 = 2 then 'delivered' else 'cancelled' end, '2024-01-01'::date + (i % 365) from generate_series(1, 200) i; +insert into aqj_products select i, 'prod_' || i, case when i % 5 = 0 then 'electronics' when i % 5 = 1 then 'books' when i % 5 = 2 then 'clothing' when i % 5 = 3 then 'food' else 'toys' end, (i * 5.99)::numeric(10,2) from generate_series(1, 30) i; +insert into aqj_order_items select i, (i % 200) + 1, (i % 30) + 1, (i % 10) + 1 from generate_series(1, 500) i; +analyze aqj_customers; +analyze aqj_orders; +analyze aqj_products; +analyze aqj_order_items; +-- 12. Join with multiple columns + WHERE on text column +create materialized view mv_aqj_orders_cust as + select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_orders_cust; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + Filter: (status = 'shipped'::text) + -> Hash + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(10 rows) + +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + order_id | amount | name | region +----------+--------+---------+-------- + 4 | 42.00 | cust_5 | north + 8 | 84.00 | cust_9 | east + 12 | 126.00 | cust_13 | west + 16 | 168.00 | cust_17 | north + 20 | 210.00 | cust_21 | east +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_orders_cust + Optimizer: Postgres query optimizer +(3 rows) + +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + order_id | amount | name | region +----------+--------+---------+-------- + 4 | 42.00 | cust_5 | north + 8 | 84.00 | cust_9 | east + 12 | 126.00 | cust_13 | west + 16 | 168.00 | cust_17 | north + 20 | 210.00 | cust_21 | east +(5 rows) + +-- 13. Four-table join +create materialized view mv_aqj_order_details as + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_order_details; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; + QUERY PLAN +------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (oi.product_id = p.product_id) + -> Hash Join + Hash Cond: (o.order_id = oi.order_id) + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + -> Hash + -> Seq Scan on aqj_customers c + -> Hash + -> Seq Scan on aqj_order_items oi + -> Hash + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on aqj_products p + Optimizer: Postgres query optimizer +(19 rows) + +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + order_id | customer_name | product_name | quantity | price +----------+---------------+--------------+----------+-------- + 1 | cust_2 | prod_11 | 1 | 65.89 + 1 | cust_2 | prod_21 | 1 | 125.79 + 2 | cust_3 | prod_12 | 2 | 71.88 + 2 | cust_3 | prod_2 | 2 | 11.98 + 2 | cust_3 | prod_22 | 2 | 131.78 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_order_details + Optimizer: Postgres query optimizer +(3 rows) + +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + order_id | customer_name | product_name | quantity | price +----------+---------------+--------------+----------+-------- + 1 | cust_2 | prod_11 | 1 | 65.89 + 1 | cust_2 | prod_21 | 1 | 125.79 + 2 | cust_3 | prod_12 | 2 | 71.88 + 2 | cust_3 | prod_2 | 2 | 11.98 + 2 | cust_3 | prod_22 | 2 | 131.78 +(5 rows) + +-- 14. GROUP BY on join with multiple aggregates: sum, count, avg +create materialized view mv_aqj_cust_summary as + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'region' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_cust_summary; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Finalize HashAggregate + Group Key: c.region + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: c.region + -> Streaming Partial HashAggregate + Group Key: c.region + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(14 rows) + +select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region + order by c.region; + region | order_count | total_amount | avg_amount +--------+-------------+--------------+----------------------- + east | 64 | 66864.00 | 1044.7500000000000000 + north | 68 | 71400.00 | 1050.0000000000000000 + west | 68 | 72786.00 | 1070.3823529411764706 +(3 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_cust_summary + Optimizer: Postgres query optimizer +(3 rows) + +select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region + order by c.region; + region | order_count | total_amount | avg_amount +--------+-------------+--------------+----------------------- + east | 64 | 66864.00 | 1044.7500000000000000 + north | 68 | 71400.00 | 1050.0000000000000000 + west | 68 | 72786.00 | 1070.3823529411764706 +(3 rows) + +-- 15. Join with expression in target list (arithmetic + function) +create materialized view mv_aqj_expr as + select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_expr; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; + QUERY PLAN +--------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(8 rows) + +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + order_id | amount_with_tax | name | region_upper +----------+-----------------+--------+-------------- + 1 | 11.550 | cust_2 | NORTH + 2 | 23.100 | cust_3 | EAST + 3 | 34.650 | cust_4 | WEST + 4 | 46.200 | cust_5 | NORTH + 5 | 57.750 | cust_6 | EAST +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_expr + Optimizer: Postgres query optimizer +(3 rows) + +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + order_id | amount_with_tax | name | region_upper +----------+-----------------+--------+-------------- + 1 | 11.550 | cust_2 | NORTH + 2 | 23.100 | cust_3 | EAST + 3 | 34.650 | cust_4 | WEST + 4 | 46.200 | cust_5 | NORTH + 5 | 57.750 | cust_6 | EAST +(5 rows) + +-- 16. Non-match: same tables + expressions, but extra WHERE (should NOT match mv_aqj_expr) +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where c.region = 'east'; + QUERY PLAN +--------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on aqj_customers c + Filter: (region = 'east'::text) + Optimizer: Postgres query optimizer +(9 rows) + +-- 17. Non-match: same tables but different aggregate target list +explain(costs off) + select c.region, sum(o.amount) as total_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Finalize HashAggregate + Group Key: c.region + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: c.region + -> Streaming Partial HashAggregate + Group Key: c.region + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(14 rows) + +-- 18. Non-match: different join order (o JOIN c vs c JOIN o) +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_customers c join aqj_orders o on o.customer_id = c.customer_id + where o.status = 'shipped'; + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + Filter: (status = 'shipped'::text) + -> Hash + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(10 rows) + +-- 19. Join with compound WHERE (multiple AND conditions) +create materialized view mv_aqj_compound_where as + select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_compound_where; +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + Filter: ((amount > '50'::numeric) AND (status = 'pending'::text)) + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on aqj_customers c + Filter: (region = 'west'::text) + Optimizer: Postgres query optimizer +(10 rows) + +select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50 + order by o.order_id limit 5; + order_id | amount | name +----------+--------+--------- + 9 | 94.50 | cust_10 + 21 | 220.50 | cust_22 + 33 | 346.50 | cust_34 + 45 | 472.50 | cust_46 + 53 | 556.50 | cust_4 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_compound_where + Optimizer: Postgres query optimizer +(3 rows) + +select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50 + order by o.order_id limit 5; + order_id | amount | name +----------+--------+--------- + 9 | 94.50 | cust_10 + 21 | 220.50 | cust_22 + 33 | 346.50 | cust_34 + 45 | 472.50 | cust_46 + 53 | 556.50 | cust_4 +(5 rows) + +-- 20. Self-join +create materialized view mv_aqj_selfjoin as + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_selfjoin; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; + QUERY PLAN +------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o1.customer_id = o2.customer_id) + Join Filter: (o1.order_id < o2.order_id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: o1.customer_id + -> Seq Scan on aqj_orders o1 + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: o2.customer_id + -> Seq Scan on aqj_orders o2 + Optimizer: Postgres query optimizer +(12 rows) + +select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id + order by o1.order_id, o2.order_id limit 5; + id1 | id2 | amt1 | amt2 +-----+-----+-------+--------- + 1 | 51 | 10.50 | 535.50 + 1 | 101 | 10.50 | 1060.50 + 1 | 151 | 10.50 | 1585.50 + 2 | 52 | 21.00 | 546.00 + 2 | 102 | 21.00 | 1071.00 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_selfjoin + Optimizer: Postgres query optimizer +(3 rows) + +select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id + order by o1.order_id, o2.order_id limit 5; + id1 | id2 | amt1 | amt2 +-----+-----+-------+--------- + 1 | 51 | 10.50 | 535.50 + 1 | 101 | 10.50 | 1060.50 + 1 | 151 | 10.50 | 1585.50 + 2 | 52 | 21.00 | 546.00 + 2 | 102 | 21.00 | 1071.00 +(5 rows) + +-- 21. GROUP BY with multi-column key on join +create materialized view mv_aqj_grp_multi as + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'region, status' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_grp_multi; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Finalize HashAggregate + Group Key: c.region, o.status + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: c.region, o.status + -> Streaming Partial HashAggregate + Group Key: c.region, o.status + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(14 rows) + +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + region | status | cnt | total +--------+-----------+-----+---------- + east | cancelled | 16 | 16968.00 + east | delivered | 16 | 16464.00 + east | pending | 16 | 16968.00 + east | shipped | 16 | 16464.00 + north | cancelled | 18 | 19425.00 + north | delivered | 16 | 16800.00 +(6 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_grp_multi + Optimizer: Postgres query optimizer +(3 rows) + +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + region | status | cnt | total +--------+-----------+-----+---------- + east | cancelled | 16 | 16968.00 + east | delivered | 16 | 16464.00 + east | pending | 16 | 16968.00 + east | shipped | 16 | 16464.00 + north | cancelled | 18 | 19425.00 + north | delivered | 16 | 16800.00 +(6 rows) + +-- 22. Four-table join with WHERE and aggregate +create materialized view mv_aqj_3way_agg as + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'region, category' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_3way_agg; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Finalize HashAggregate + Group Key: c.region, p.category + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: c.region, p.category + -> Streaming Partial HashAggregate + Group Key: c.region, p.category + -> Hash Join + Hash Cond: (oi.product_id = p.product_id) + -> Hash Join + Hash Cond: (oi.order_id = o.order_id) + -> Seq Scan on aqj_order_items oi + -> Hash + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice4; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + Filter: (status = 'delivered'::text) + -> Hash + -> Seq Scan on aqj_customers c + -> Hash + -> Broadcast Motion 3:3 (slice5; segments: 3) + -> Seq Scan on aqj_products p + Optimizer: Postgres query optimizer +(26 rows) + +select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category + order by c.region, p.category limit 6; + region | category | total_qty | line_count +--------+-------------+-----------+------------ + east | books | 30 | 5 + east | clothing | 20 | 10 + east | electronics | 50 | 5 + east | food | 80 | 10 + east | toys | 40 | 10 + north | books | 60 | 10 +(6 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_3way_agg + Optimizer: Postgres query optimizer +(3 rows) + +select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category + order by c.region, p.category limit 6; + region | category | total_qty | line_count +--------+-------------+-----------+------------ + east | books | 30 | 5 + east | clothing | 20 | 10 + east | electronics | 50 | 5 + east | food | 80 | 10 + east | toys | 40 | 10 + north | books | 60 | 10 +(6 rows) + +-- 23. Implicit four-table join (comma style) +create materialized view mv_aqj_implicit3 as + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_implicit3; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; + QUERY PLAN +------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (oi.product_id = p.product_id) + -> Hash Join + Hash Cond: (oi.order_id = o.order_id) + -> Seq Scan on aqj_order_items oi + -> Hash + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + Filter: (status = 'pending'::text) + -> Hash + -> Seq Scan on aqj_customers c + -> Hash + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on aqj_products p + Optimizer: Postgres query optimizer +(20 rows) + +select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending' + order by o.order_id, p.name limit 5; + order_id | name | product_name +----------+--------+-------------- + 1 | cust_2 | prod_11 + 1 | cust_2 | prod_21 + 5 | cust_6 | prod_15 + 5 | cust_6 | prod_25 + 5 | cust_6 | prod_5 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_implicit3 + Optimizer: Postgres query optimizer +(3 rows) + +select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending' + order by o.order_id, p.name limit 5; + order_id | name | product_name +----------+--------+-------------- + 1 | cust_2 | prod_11 + 1 | cust_2 | prod_21 + 5 | cust_6 | prod_15 + 5 | cust_6 | prod_25 + 5 | cust_6 | prod_5 +(5 rows) + +-- 24. Result correctness across DML + REFRESH cycle +insert into aqj_orders values(201, 1, 9999.99, 'shipped', '2025-12-31'); +set enable_answer_query_using_materialized_views = on; +-- Stale: should NOT use MV +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: o.customer_id + -> Seq Scan on aqj_orders o + Filter: (status = 'shipped'::text) + -> Hash + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(10 rows) + +-- Refresh and verify MV is used again +refresh materialized view mv_aqj_orders_cust; +analyze mv_aqj_orders_cust; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; + QUERY PLAN +------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on mv_aqj_orders_cust + Optimizer: Postgres query optimizer +(3 rows) + +-- The new row should appear in results via MV scan +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' and o.order_id = 201; + order_id | amount | name | region +----------+---------+--------+-------- + 201 | 9999.99 | cust_1 | west +(1 row) + +-- 25. Post-DML comprehensive: refresh all, then verify GUC off vs on results +refresh materialized view mv_aqj_order_details; +refresh materialized view mv_aqj_expr; +refresh materialized view mv_aqj_selfjoin; +refresh materialized view mv_aqj_grp_multi; +refresh materialized view mv_aqj_3way_agg; +refresh materialized view mv_aqj_implicit3; +analyze mv_aqj_order_details; +analyze mv_aqj_expr; +analyze mv_aqj_selfjoin; +analyze mv_aqj_grp_multi; +analyze mv_aqj_3way_agg; +analyze mv_aqj_implicit3; +-- Verify four-table join results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + order_id | customer_name | product_name | quantity | price +----------+---------------+--------------+----------+-------- + 1 | cust_2 | prod_11 | 1 | 65.89 + 1 | cust_2 | prod_21 | 1 | 125.79 + 2 | cust_3 | prod_12 | 2 | 71.88 + 2 | cust_3 | prod_2 | 2 | 11.98 + 2 | cust_3 | prod_22 | 2 | 131.78 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + order_id | customer_name | product_name | quantity | price +----------+---------------+--------------+----------+-------- + 1 | cust_2 | prod_11 | 1 | 65.89 + 1 | cust_2 | prod_21 | 1 | 125.79 + 2 | cust_3 | prod_12 | 2 | 71.88 + 2 | cust_3 | prod_2 | 2 | 11.98 + 2 | cust_3 | prod_22 | 2 | 131.78 +(5 rows) + +-- Verify expression MV results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + order_id | amount_with_tax | name | region_upper +----------+-----------------+--------+-------------- + 1 | 11.550 | cust_2 | NORTH + 2 | 23.100 | cust_3 | EAST + 3 | 34.650 | cust_4 | WEST + 4 | 46.200 | cust_5 | NORTH + 5 | 57.750 | cust_6 | EAST +(5 rows) + +set enable_answer_query_using_materialized_views = on; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + order_id | amount_with_tax | name | region_upper +----------+-----------------+--------+-------------- + 1 | 11.550 | cust_2 | NORTH + 2 | 23.100 | cust_3 | EAST + 3 | 34.650 | cust_4 | WEST + 4 | 46.200 | cust_5 | NORTH + 5 | 57.750 | cust_6 | EAST +(5 rows) + +-- Verify multi-key GROUP BY results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + region | status | cnt | total +--------+-----------+-----+---------- + east | cancelled | 16 | 16968.00 + east | delivered | 16 | 16464.00 + east | pending | 16 | 16968.00 + east | shipped | 16 | 16464.00 + north | cancelled | 18 | 19425.00 + north | delivered | 16 | 16800.00 +(6 rows) + +set enable_answer_query_using_materialized_views = on; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + region | status | cnt | total +--------+-----------+-----+---------- + east | cancelled | 16 | 16968.00 + east | delivered | 16 | 16464.00 + east | pending | 16 | 16968.00 + east | shipped | 16 | 16464.00 + north | cancelled | 18 | 19425.00 + north | delivered | 16 | 16800.00 +(6 rows) + +-- Clean up AQUMV join test objects +drop materialized view mv_aqj_implicit3; +drop materialized view mv_aqj_3way_agg; +drop materialized view mv_aqj_grp_multi; +drop materialized view mv_aqj_selfjoin; +drop materialized view mv_aqj_compound_where; +drop materialized view mv_aqj_expr; +drop materialized view mv_aqj_cust_summary; +drop materialized view mv_aqj_order_details; +drop materialized view mv_aqj_orders_cust; +drop materialized view mv_aqj_implicit; +drop materialized view mv_aqj_join3; +drop materialized view mv_aqj_agg; +drop materialized view mv_aqj_where; +drop materialized view mv_aqj_join2; +drop table aqj_order_items; +drop table aqj_products; +drop table aqj_customers; +drop table aqj_orders; +drop table aqj_t3; +drop table aqj_t2; +drop table aqj_t1; -- test drop table select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); mvname | datastatus @@ -925,12 +2193,12 @@ HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sur select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) insert into par_1_prt_1 values (1, 1, 1); @@ -938,9 +2206,9 @@ insert into par_1_prt_1 values (1, 1, 1); select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ + mv_par1_2 | u mv_par2 | u mv_par2_1 | u - mv_par1_2 | u mv_par1_1 | i mv_par1 | i mv_par | i @@ -951,12 +2219,12 @@ insert into par values (1, 2, 2); select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | i - mv_par2_1 | u mv_par1_2 | u + mv_par2_1 | u mv_par1_1 | i mv_par1 | i mv_par | i + mv_par2 | i (6 rows) refresh materialized view mv_par; @@ -971,11 +2239,11 @@ insert into par_1_prt_2_2_prt_1 values (1, 2, 1); select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | i - mv_par2_1 | i mv_par1_2 | u mv_par1 | u mv_par1_1 | u + mv_par2_1 | i + mv_par2 | i mv_par | i (6 rows) @@ -986,11 +2254,11 @@ truncate par_1_prt_2; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | e - mv_par2_1 | e mv_par1_2 | u mv_par1 | u mv_par1_1 | u + mv_par2_1 | e + mv_par2 | e mv_par | e (6 rows) @@ -1000,11 +2268,11 @@ truncate par_1_prt_2; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | e - mv_par2_1 | e mv_par1_2 | u mv_par1 | u mv_par1_1 | u + mv_par2_1 | e + mv_par2 | e mv_par | e (6 rows) @@ -1018,9 +2286,9 @@ vacuum full par_1_prt_1_2_prt_1; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ + mv_par1_2 | u mv_par2 | u mv_par2_1 | u - mv_par1_2 | u mv_par1_1 | r mv_par1 | r mv_par | r @@ -1038,8 +2306,8 @@ select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ mv_par2 | r - mv_par2_1 | r mv_par | r + mv_par2_1 | r mv_par1_2 | r mv_par1 | r mv_par1_1 | r @@ -1058,10 +2326,10 @@ NOTICE: table has parent, setting distribution columns to match parent table select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u mv_par1 | e mv_par | e (6 rows) @@ -1089,10 +2357,10 @@ alter table par_1_prt_1 detach partition par_1_prt_1_2_prt_1; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u mv_par1 | e mv_par | e (6 rows) @@ -1107,10 +2375,10 @@ alter table par_1_prt_1 attach partition new_par for values from (4) to (5); select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u mv_par1 | e mv_par | e (6 rows) @@ -1130,12 +2398,12 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) insert into par values(1, 1, 1), (1, 1, 2); @@ -1155,23 +2423,23 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) insert into par_1_prt_2_2_prt_1 values(2, 2, 1); select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | i - mv_par2_1 | i mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2_1 | i + mv_par2 | i mv_par | i (6 rows) @@ -1180,23 +2448,23 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) delete from par where b = 2 and c = 1; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | e - mv_par2_1 | e mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2_1 | e + mv_par2 | e mv_par | e (6 rows) @@ -1205,21 +2473,21 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) delete from par_1_prt_1_2_prt_2; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ + mv_par1_1 | u mv_par2 | u mv_par2_1 | u - mv_par1_1 | u mv_par1_2 | e mv_par1 | e mv_par | e @@ -1231,12 +2499,12 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) update par set c = 2 where b = 1 and c = 1; @@ -1257,12 +2525,12 @@ begin; select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; mvname | datastatus -----------+------------ - mv_par2 | u - mv_par2_1 | u mv_par | u mv_par1 | u mv_par1_1 | u mv_par1_2 | u + mv_par2 | u + mv_par2_1 | u (6 rows) update par set c = 2, a = 2 where b = 1 and c = 1; @@ -1281,6 +2549,7 @@ abort; -- Test report warning if extend protocol data is not consumed. --start_ignore drop extension gp_inject_fault; +ERROR: extension "gp_inject_fault" does not exist create extension gp_inject_fault; --end_ignore select gp_inject_fault_infinite('consume_extend_protocol_data', 'skip', dbid) diff --git a/src/test/regress/sql/matview_data.sql b/src/test/regress/sql/matview_data.sql index 059a5a97bf4..8b5a56986e0 100644 --- a/src/test/regress/sql/matview_data.sql +++ b/src/test/regress/sql/matview_data.sql @@ -343,6 +343,541 @@ drop table jt3; drop table jt2; drop table jt1; +-- +-- Test AQUMV (Answer Query Using Materialized Views) with join queries. +-- Each matching test shows EXPLAIN + SELECT with GUC off (original plan), +-- then EXPLAIN + SELECT with GUC on (MV rewrite). Results must match. +-- +create table aqj_t1(a int, b int) distributed by (a); +create table aqj_t2(a int, b int) distributed by (a); +create table aqj_t3(a int, b int) distributed by (a); +insert into aqj_t1 select i, i*10 from generate_series(1, 100) i; +insert into aqj_t2 select i, i*100 from generate_series(1, 100) i; +insert into aqj_t3 select i, i*1000 from generate_series(1, 100) i; +analyze aqj_t1; +analyze aqj_t2; +analyze aqj_t3; + +-- 1. Two-table INNER JOIN exact match +create materialized view mv_aqj_join2 as + select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; +analyze mv_aqj_join2; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a order by 1 limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a order by 1 limit 5; + +-- 2. Join with WHERE clause +create materialized view mv_aqj_where as + select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; +analyze mv_aqj_where; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5 order by 1 limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5; +select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 5 order by 1 limit 5; + +-- 3. Join with GROUP BY + aggregate +create materialized view mv_aqj_agg as + select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; +analyze mv_aqj_agg; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; +select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a order by 1 limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a; +select aqj_t1.a, count(*) as cnt from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a group by aqj_t1.a order by 1 limit 5; + +-- 4. Non-match: different WHERE clause (should show Hash Join, not MV) +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a where aqj_t1.a > 10; + +-- 5. Non-match: different target list +explain(costs off) select aqj_t1.b, aqj_t2.a from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + +-- 6. Non-match: different join type (INNER vs LEFT) +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 left join aqj_t2 on aqj_t1.a = aqj_t2.a; + +-- 7. Three-table join +create materialized view mv_aqj_join3 as + select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; +analyze mv_aqj_join3; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; +select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a + order by 1 limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a; +select aqj_t1.a, aqj_t2.b, aqj_t3.b as c + from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a join aqj_t3 on aqj_t2.a = aqj_t3.a + order by 1 limit 5; + +-- 8. Implicit join (FROM t1, t2 WHERE ...) +create materialized view mv_aqj_implicit as + select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; +analyze mv_aqj_implicit; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; +select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a order by 1 limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a; +select aqj_t1.a, aqj_t2.b from aqj_t1, aqj_t2 where aqj_t1.a = aqj_t2.a order by 1 limit 5; + +-- 9. MV not up-to-date: after INSERT on base table +insert into aqj_t1 values(999, 9990); +set enable_answer_query_using_materialized_views = on; +-- Should NOT use mv_aqj_join2 (status is 'i') +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + +-- 10. After REFRESH: should use MV again +refresh materialized view mv_aqj_join2; +analyze mv_aqj_join2; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + +-- 11. GUC off: should NOT use MV +set enable_answer_query_using_materialized_views = off; +explain(costs off) select aqj_t1.a, aqj_t2.b from aqj_t1 join aqj_t2 on aqj_t1.a = aqj_t2.a; + +-- +-- More complex join AQUMV test cases with richer schemas +-- + +create table aqj_orders( + order_id int, + customer_id int, + amount numeric(10,2), + status text, + order_date date +) distributed by (order_id); + +create table aqj_customers( + customer_id int, + name text, + region text, + credit_limit numeric(10,2) +) distributed by (customer_id); + +create table aqj_products( + product_id int, + name text, + category text, + price numeric(10,2) +) distributed by (product_id); + +create table aqj_order_items( + item_id int, + order_id int, + product_id int, + quantity int +) distributed by (item_id); + +insert into aqj_customers select i, 'cust_' || i, case when i % 3 = 0 then 'east' when i % 3 = 1 then 'west' else 'north' end, (i * 100)::numeric(10,2) from generate_series(1, 50) i; +insert into aqj_orders select i, (i % 50) + 1, (i * 10.5)::numeric(10,2), case when i % 4 = 0 then 'shipped' when i % 4 = 1 then 'pending' when i % 4 = 2 then 'delivered' else 'cancelled' end, '2024-01-01'::date + (i % 365) from generate_series(1, 200) i; +insert into aqj_products select i, 'prod_' || i, case when i % 5 = 0 then 'electronics' when i % 5 = 1 then 'books' when i % 5 = 2 then 'clothing' when i % 5 = 3 then 'food' else 'toys' end, (i * 5.99)::numeric(10,2) from generate_series(1, 30) i; +insert into aqj_order_items select i, (i % 200) + 1, (i % 30) + 1, (i % 10) + 1 from generate_series(1, 500) i; + +analyze aqj_customers; +analyze aqj_orders; +analyze aqj_products; +analyze aqj_order_items; + +-- 12. Join with multiple columns + WHERE on text column +create materialized view mv_aqj_orders_cust as + select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +analyze mv_aqj_orders_cust; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + +-- 13. Four-table join +create materialized view mv_aqj_order_details as + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; +analyze mv_aqj_order_details; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + +-- 14. GROUP BY on join with multiple aggregates: sum, count, avg +create materialized view mv_aqj_cust_summary as + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; +analyze mv_aqj_cust_summary; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; +select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region + order by c.region; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; +select c.region, count(*) as order_count, sum(o.amount) as total_amount, avg(o.amount) as avg_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region + order by c.region; + +-- 15. Join with expression in target list (arithmetic + function) +create materialized view mv_aqj_expr as + select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; +analyze mv_aqj_expr; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + +-- 16. Non-match: same tables + expressions, but extra WHERE (should NOT match mv_aqj_expr) +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where c.region = 'east'; + +-- 17. Non-match: same tables but different aggregate target list +explain(costs off) + select c.region, sum(o.amount) as total_amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region; + +-- 18. Non-match: different join order (o JOIN c vs c JOIN o) +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_customers c join aqj_orders o on o.customer_id = c.customer_id + where o.status = 'shipped'; + +-- 19. Join with compound WHERE (multiple AND conditions) +create materialized view mv_aqj_compound_where as + select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; +analyze mv_aqj_compound_where; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; +select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50 + order by o.order_id limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50; +select o.order_id, o.amount, c.name + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' and c.region = 'west' and o.amount > 50 + order by o.order_id limit 5; + +-- 20. Self-join +create materialized view mv_aqj_selfjoin as + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; +analyze mv_aqj_selfjoin; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; +select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id + order by o1.order_id, o2.order_id limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id; +select o1.order_id as id1, o2.order_id as id2, o1.amount as amt1, o2.amount as amt2 + from aqj_orders o1 join aqj_orders o2 on o1.customer_id = o2.customer_id + where o1.order_id < o2.order_id + order by o1.order_id, o2.order_id limit 5; + +-- 21. GROUP BY with multi-column key on join +create materialized view mv_aqj_grp_multi as + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; +analyze mv_aqj_grp_multi; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + +-- 22. Four-table join with WHERE and aggregate +create materialized view mv_aqj_3way_agg as + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; +analyze mv_aqj_3way_agg; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; +select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category + order by c.region, p.category limit 6; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category; +select c.region, p.category, sum(oi.quantity) as total_qty, count(*) as line_count + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + where o.status = 'delivered' + group by c.region, p.category + order by c.region, p.category limit 6; + +-- 23. Implicit four-table join (comma style) +create materialized view mv_aqj_implicit3 as + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; +analyze mv_aqj_implicit3; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; +select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending' + order by o.order_id, p.name limit 5; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending'; +select o.order_id, c.name, p.name as product_name + from aqj_orders o, aqj_customers c, aqj_order_items oi, aqj_products p + where o.customer_id = c.customer_id and o.order_id = oi.order_id and oi.product_id = p.product_id + and o.status = 'pending' + order by o.order_id, p.name limit 5; + +-- 24. Result correctness across DML + REFRESH cycle +insert into aqj_orders values(201, 1, 9999.99, 'shipped', '2025-12-31'); +set enable_answer_query_using_materialized_views = on; +-- Stale: should NOT use MV +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +-- Refresh and verify MV is used again +refresh materialized view mv_aqj_orders_cust; +analyze mv_aqj_orders_cust; +explain(costs off) select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped'; +-- The new row should appear in results via MV scan +select o.order_id, o.amount, c.name, c.region + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' and o.order_id = 201; + +-- 25. Post-DML comprehensive: refresh all, then verify GUC off vs on results +refresh materialized view mv_aqj_order_details; +refresh materialized view mv_aqj_expr; +refresh materialized view mv_aqj_selfjoin; +refresh materialized view mv_aqj_grp_multi; +refresh materialized view mv_aqj_3way_agg; +refresh materialized view mv_aqj_implicit3; +analyze mv_aqj_order_details; +analyze mv_aqj_expr; +analyze mv_aqj_selfjoin; +analyze mv_aqj_grp_multi; +analyze mv_aqj_3way_agg; +analyze mv_aqj_implicit3; + +-- Verify four-table join results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + +set enable_answer_query_using_materialized_views = on; +select o.order_id, c.name as customer_name, p.name as product_name, oi.quantity, p.price + from aqj_orders o + join aqj_customers c on o.customer_id = c.customer_id + join aqj_order_items oi on o.order_id = oi.order_id + join aqj_products p on oi.product_id = p.product_id + order by o.order_id, p.name limit 5; + +-- Verify expression MV results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + +set enable_answer_query_using_materialized_views = on; +select o.order_id, o.amount * 1.1 as amount_with_tax, c.name, upper(c.region) as region_upper + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + order by o.order_id limit 5; + +-- Verify multi-key GROUP BY results after DML+refresh +set enable_answer_query_using_materialized_views = off; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + +set enable_answer_query_using_materialized_views = on; +select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by c.region, o.status + order by c.region, o.status limit 6; + +-- Clean up AQUMV join test objects +drop materialized view mv_aqj_implicit3; +drop materialized view mv_aqj_3way_agg; +drop materialized view mv_aqj_grp_multi; +drop materialized view mv_aqj_selfjoin; +drop materialized view mv_aqj_compound_where; +drop materialized view mv_aqj_expr; +drop materialized view mv_aqj_cust_summary; +drop materialized view mv_aqj_order_details; +drop materialized view mv_aqj_orders_cust; +drop materialized view mv_aqj_implicit; +drop materialized view mv_aqj_join3; +drop materialized view mv_aqj_agg; +drop materialized view mv_aqj_where; +drop materialized view mv_aqj_join2; +drop table aqj_order_items; +drop table aqj_products; +drop table aqj_customers; +drop table aqj_orders; +drop table aqj_t3; +drop table aqj_t2; +drop table aqj_t1; + -- test drop table select mvname, datastatus from gp_matview_aux where mvname in ('mv0','mv1', 'mv2', 'mv3'); drop materialized view mv2; From 07b821fbf8d8141f47f963bdc8ea027a01bc1d54 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 10 Mar 2026 16:34:01 +0800 Subject: [PATCH 018/128] Address review feedback for AQUMV join exact-match Fix three issues in aqumv_query_is_exact_match(): - Add groupDistinct comparison (GROUP BY vs GROUP BY DISTINCT) - Add limitOption comparison (LIMIT vs FETCH FIRST WITH TIES) - Clear qp_extra in-place via aqumv_context->qp_extra instead of allocating a local char array; move standard_qp_extra typedef to planner.h so aqumv.c can reference the proper struct type Add test cases 26-28 to verify the new comparisons: - LIMIT vs FETCH FIRST WITH TIES non-match and exact match - GROUP BY DISTINCT vs GROUP BY non-match --- src/backend/optimizer/plan/aqumv.c | 23 +-- src/backend/optimizer/plan/planner.c | 7 +- src/include/optimizer/planner.h | 7 + src/test/regress/expected/matview_data.out | 155 ++++++++++++++++++++- src/test/regress/sql/matview_data.sql | 63 +++++++++ 5 files changed, 237 insertions(+), 18 deletions(-) diff --git a/src/backend/optimizer/plan/aqumv.c b/src/backend/optimizer/plan/aqumv.c index 2084bcbc181..15203a201a2 100644 --- a/src/backend/optimizer/plan/aqumv.c +++ b/src/backend/optimizer/plan/aqumv.c @@ -1045,6 +1045,8 @@ aqumv_query_is_exact_match(Query *raw_parse, Query *viewQuery) /* Compare GROUP BY, HAVING, ORDER BY, DISTINCT, LIMIT */ if (!equal(raw_parse->groupClause, viewQuery->groupClause)) return false; + if (raw_parse->groupDistinct != viewQuery->groupDistinct) + return false; if (!equal(raw_parse->havingQual, viewQuery->havingQual)) return false; if (!equal(raw_parse->sortClause, viewQuery->sortClause)) @@ -1055,6 +1057,8 @@ aqumv_query_is_exact_match(Query *raw_parse, Query *viewQuery) return false; if (!equal(raw_parse->limitOffset, viewQuery->limitOffset)) return false; + if (raw_parse->limitOption != viewQuery->limitOption) + return false; /* Compare boolean flags */ if (raw_parse->hasAggs != viewQuery->hasAggs) @@ -1285,20 +1289,19 @@ answer_query_using_materialized_views_for_join(PlannerInfo *root, AqumvContext a /* * Plan the MV scan. * - * We need a clean qp_extra with no groupClause or activeWindows, - * because the rewritten viewQuery is a simple SELECT from the MV - * with no GROUP BY, windowing, etc. The standard_qp_callback uses - * qp_extra->groupClause to compute group_pathkeys, which would fail - * if it still contained the original query's GROUP BY expressions. + * Clear qp_extra's groupClause and activeWindows because the + * rewritten viewQuery is a simple SELECT from the MV with no + * GROUP BY or windowing. standard_qp_callback would otherwise + * try to compute group_pathkeys from stale expressions. * - * standard_qp_extra is { List *activeWindows; List *groupClause; }, - * so a zeroed struct of that size works correctly (both fields NIL). + * Safe: grouping_planner() no longer reads qp_extra after AQUMV. */ { - char clean_qp_extra[2 * sizeof(List *)]; - memset(clean_qp_extra, 0, sizeof(clean_qp_extra)); - mv_final_rel = query_planner(subroot, qp_callback, clean_qp_extra); + standard_qp_extra *qp = (standard_qp_extra *) aqumv_context->qp_extra; + qp->activeWindows = NIL; + qp->groupClause = NIL; } + mv_final_rel = query_planner(subroot, qp_callback, aqumv_context->qp_extra); /* Cost-based decision: use MV only if cheaper. */ if (mv_final_rel->cheapest_total_path->total_cost < current_rel->cheapest_total_path->total_cost) diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 1b83b83e762..a173373dee2 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -121,12 +121,7 @@ create_upper_paths_hook_type create_upper_paths_hook = NULL; #define EXPRKIND_TABLEFUNC_LATERAL 12 #define EXPRKIND_WINDOW_BOUND 13 -/* Passthrough data for standard_qp_callback */ -typedef struct -{ - List *activeWindows; /* active windows, if any */ - List *groupClause; /* overrides parse->groupClause */ -} standard_qp_extra; +/* standard_qp_extra is defined in optimizer/planner.h */ /* * Data specific to grouping sets diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h index 610034b2c62..9715d9fb31a 100644 --- a/src/include/optimizer/planner.h +++ b/src/include/optimizer/planner.h @@ -66,4 +66,11 @@ extern bool optimizer_init; extern void preprocess_qual_conditions(PlannerInfo *root, Node *jtnode); +/* Passthrough data for standard_qp_callback */ +typedef struct +{ + List *activeWindows; /* active windows, if any */ + List *groupClause; /* overrides parse->groupClause */ +} standard_qp_extra; + #endif /* PLANNER_H */ diff --git a/src/test/regress/expected/matview_data.out b/src/test/regress/expected/matview_data.out index e415ceaf363..a624ab31f3e 100644 --- a/src/test/regress/expected/matview_data.out +++ b/src/test/regress/expected/matview_data.out @@ -2037,7 +2037,158 @@ select c.region, o.status, count(*) as cnt, sum(o.amount) as total north | delivered | 16 | 16800.00 (6 rows) +-- 26. Non-match: LIMIT vs FETCH FIRST WITH TIES (limitOption differs) +create materialized view mv_aqj_limit_test as + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_limit_test; +set enable_answer_query_using_materialized_views = on; +-- Same tables/WHERE/ORDER BY but FETCH FIRST WITH TIES: should NOT match +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id fetch first 5 rows with ties; + QUERY PLAN +--------------------------------------------------------------------------- + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: o.order_id + -> Limit + -> Sort + Sort Key: o.order_id + -> Hash Join + Hash Cond: (c.customer_id = o.customer_id) + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on aqj_customers c + -> Hash + -> Seq Scan on aqj_orders o + Filter: (status = 'shipped'::text) + Optimizer: Postgres query optimizer +(14 rows) + +-- Identical LIMIT query: should match +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + QUERY PLAN +------------------------------------------------- + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + -> Limit + -> Seq Scan on mv_aqj_limit_test + Optimizer: Postgres query optimizer +(5 rows) + +-- 27. Match: FETCH FIRST WITH TIES exact match +create materialized view mv_aqj_with_ties as + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'order_id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_aqj_with_ties; +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + QUERY PLAN +--------------------------------------------------------------------------- + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: o.order_id + -> Limit + -> Sort + Sort Key: o.order_id + -> Hash Join + Hash Cond: (c.customer_id = o.customer_id) + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on aqj_customers c + -> Hash + -> Seq Scan on aqj_orders o + Filter: (status = 'pending'::text) + Optimizer: Postgres query optimizer +(14 rows) + +select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + order_id | amount +----------+-------- + 1 | 10.50 + 5 | 52.50 + 9 | 94.50 + 13 | 136.50 + 17 | 178.50 +(5 rows) + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + QUERY PLAN +------------------------------------------------ + Limit + -> Gather Motion 3:1 (slice1; segments: 3) + -> Limit + -> Seq Scan on mv_aqj_with_ties + Optimizer: Postgres query optimizer +(5 rows) + +select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + order_id | amount +----------+-------- + 1 | 10.50 + 5 | 52.50 + 9 | 94.50 + 13 | 136.50 + 17 | 178.50 +(5 rows) + +-- 28. Non-match: GROUP BY vs GROUP BY DISTINCT (groupDistinct differs) +-- MV mv_aqj_grp_multi uses GROUP BY (groupDistinct=false, registered in catalog) +-- Query uses GROUP BY DISTINCT — should NOT match +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by distinct c.region, o.status; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Finalize HashAggregate + Group Key: c.region, o.status + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: c.region, o.status + -> Streaming Partial HashAggregate + Group Key: c.region, o.status + -> Hash Join + Hash Cond: (o.customer_id = c.customer_id) + -> Seq Scan on aqj_orders o + -> Hash + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on aqj_customers c + Optimizer: Postgres query optimizer +(14 rows) + -- Clean up AQUMV join test objects +drop materialized view mv_aqj_with_ties; +drop materialized view mv_aqj_limit_test; drop materialized view mv_aqj_implicit3; drop materialized view mv_aqj_3way_agg; drop materialized view mv_aqj_grp_multi; @@ -2412,10 +2563,10 @@ select mvname, datastatus from gp_matview_aux where mvname like 'mv_par%'; -----------+------------ mv_par2 | u mv_par2_1 | u - mv_par1_1 | i + mv_par1_2 | i mv_par1 | i mv_par | i - mv_par1_2 | i + mv_par1_1 | i (6 rows) abort; diff --git a/src/test/regress/sql/matview_data.sql b/src/test/regress/sql/matview_data.sql index 8b5a56986e0..65de9dd5c9b 100644 --- a/src/test/regress/sql/matview_data.sql +++ b/src/test/regress/sql/matview_data.sql @@ -855,7 +855,70 @@ select c.region, o.status, count(*) as cnt, sum(o.amount) as total group by c.region, o.status order by c.region, o.status limit 6; +-- 26. Non-match: LIMIT vs FETCH FIRST WITH TIES (limitOption differs) +create materialized view mv_aqj_limit_test as + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; +analyze mv_aqj_limit_test; + +set enable_answer_query_using_materialized_views = on; +-- Same tables/WHERE/ORDER BY but FETCH FIRST WITH TIES: should NOT match +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id fetch first 5 rows with ties; +-- Identical LIMIT query: should match +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'shipped' + order by o.order_id limit 5; + +-- 27. Match: FETCH FIRST WITH TIES exact match +create materialized view mv_aqj_with_ties as + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; +analyze mv_aqj_with_ties; + +set enable_answer_query_using_materialized_views = off; +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; +select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; +select o.order_id, o.amount + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + where o.status = 'pending' + order by o.order_id fetch first 5 rows with ties; + +-- 28. Non-match: GROUP BY vs GROUP BY DISTINCT (groupDistinct differs) +-- MV mv_aqj_grp_multi uses GROUP BY (groupDistinct=false, registered in catalog) +-- Query uses GROUP BY DISTINCT — should NOT match +set enable_answer_query_using_materialized_views = on; +explain(costs off) + select c.region, o.status, count(*) as cnt, sum(o.amount) as total + from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id + group by distinct c.region, o.status; + -- Clean up AQUMV join test objects +drop materialized view mv_aqj_with_ties; +drop materialized view mv_aqj_limit_test; drop materialized view mv_aqj_implicit3; drop materialized view mv_aqj_3way_agg; drop materialized view mv_aqj_grp_multi; From 1b13f98098677f03f7c8bd6022f7daccf3d936a4 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 10 Mar 2026 17:21:39 +0800 Subject: [PATCH 019/128] Fix ORDER BY lost after AQUMV join rewrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rewrite cleared sortClause, so grouping_planner() skipped adding a Sort node — queries with ORDER BY returned unsorted results from the MV scan. Fix: preserve sortClause and copy ressortgroupref to rewritten target entries so the upper planner generates Sort correctly. Before: Limit -> Gather -> Limit -> Seq Scan on mv After: Limit -> Gather -> Limit -> Sort -> Seq Scan on mv --- src/backend/optimizer/plan/aqumv.c | 5 +++-- src/test/regress/expected/matview_data.out | 22 ++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/backend/optimizer/plan/aqumv.c b/src/backend/optimizer/plan/aqumv.c index 15203a201a2..04c173f41de 100644 --- a/src/backend/optimizer/plan/aqumv.c +++ b/src/backend/optimizer/plan/aqumv.c @@ -1206,6 +1206,7 @@ answer_query_using_materialized_views_for_join(PlannerInfo *root, AqumvContext a (AttrNumber) attnum, old_tle->resname, false); + new_tle->ressortgroupref = old_tle->ressortgroupref; new_tlist = lappend(new_tlist, new_tle); } @@ -1243,11 +1244,11 @@ answer_query_using_materialized_views_for_join(PlannerInfo *root, AqumvContext a viewQuery->jointree = makeFromExpr(list_make1(makeNode(RangeTblRef)), NULL); ((RangeTblRef *) linitial(viewQuery->jointree->fromlist))->rtindex = 1; - /* Clear aggregation/grouping/sorting state — all materialized. */ + /* Clear aggregation/grouping state — already materialized in MV. */ viewQuery->hasAggs = false; viewQuery->groupClause = NIL; viewQuery->havingQual = NULL; - viewQuery->sortClause = NIL; + /* Keep sortClause: upper planner needs it to add Sort node. */ viewQuery->distinctClause = NIL; viewQuery->hasDistinctOn = false; viewQuery->hasWindowFuncs = false; diff --git a/src/test/regress/expected/matview_data.out b/src/test/regress/expected/matview_data.out index a624ab31f3e..9a9074cd2d5 100644 --- a/src/test/regress/expected/matview_data.out +++ b/src/test/regress/expected/matview_data.out @@ -2077,14 +2077,17 @@ explain(costs off) from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id where o.status = 'shipped' order by o.order_id limit 5; - QUERY PLAN -------------------------------------------------- + QUERY PLAN +------------------------------------------------------- Limit -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: order_id -> Limit - -> Seq Scan on mv_aqj_limit_test + -> Sort + Sort Key: order_id + -> Seq Scan on mv_aqj_limit_test Optimizer: Postgres query optimizer -(5 rows) +(8 rows) -- 27. Match: FETCH FIRST WITH TIES exact match create materialized view mv_aqj_with_ties as @@ -2138,14 +2141,17 @@ explain(costs off) from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id where o.status = 'pending' order by o.order_id fetch first 5 rows with ties; - QUERY PLAN ------------------------------------------------- + QUERY PLAN +------------------------------------------------------ Limit -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: order_id -> Limit - -> Seq Scan on mv_aqj_with_ties + -> Sort + Sort Key: order_id + -> Seq Scan on mv_aqj_with_ties Optimizer: Postgres query optimizer -(5 rows) +(8 rows) select o.order_id, o.amount from aqj_orders o join aqj_customers c on o.customer_id = c.customer_id From 743f68e0c96f8bcc7a3193ae6f4fdc11d98403ef Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Fri, 2 Dec 2022 17:50:26 -0800 Subject: [PATCH 020/128] Prevent pgstats from getting confused when relkind of a relation changes When the relkind of a relache entry changes, because a table is converted into a view, pgstats can get confused in 15+, leading to crashes or assertion failures. For HEAD, Tom fixed this in b23cd185fd5, by removing support for converting a table to a view, removing the source of the inconsistency. This commit just adds an assertion that a relcache entry's relkind does not change, just in case we end up with another case of that in the future. As there's no cases of changing relkind anymore, we can't add a test that that's handled correctly. For 15, fix the problem by not maintaining the association with the old pgstat entry when the relkind changes during a relcache invalidation processing. In that case the pgstat entry needs to be unlinked first, to avoid PgStat_TableStatus->relation getting out of sync. Also add a test reproducing the issues. No known problem exists in 11-14, so just add the test there. Reported-by: vignesh C Author: Andres Freund Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CALDaNm2yXz+zOtv7y5zBd5WKT8O0Ld3YxikuU3dcyCvxF7gypA@mail.gmail.com Discussion: https://postgr.es/m/CALDaNm3oZA-8Wbps2Jd1g5_Gjrr-x3YWrJPek-mF5Asrrvz2Dg@mail.gmail.com Backpatch: 15- --- src/test/regress/expected/create_view.out | 26 ++++++++++++++++++- .../expected/create_view_optimizer.out | 26 ++++++++++++++++++- src/test/regress/sql/create_view.sql | 22 ++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/create_view.out b/src/test/regress/expected/create_view.out index ac88c92f398..1eab17e3706 100644 --- a/src/test/regress/expected/create_view.out +++ b/src/test/regress/expected/create_view.out @@ -1999,6 +1999,29 @@ select pg_get_viewdef('tt26v', true); FROM ( VALUES (1,2,3)) v(x, y, z); (1 row) +-- Test that changing the relkind of a relcache entry doesn't cause +-- trouble. Prior instances of where it did: +-- CALDaNm2yXz+zOtv7y5zBd5WKT8O0Ld3YxikuU3dcyCvxF7gypA@mail.gmail.com +-- CALDaNm3oZA-8Wbps2Jd1g5_Gjrr-x3YWrJPek-mF5Asrrvz2Dg@mail.gmail.com +CREATE TABLE tt26(c int); +BEGIN; +CREATE TABLE tt27(c int); +SAVEPOINT q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +SELECT * FROM tt27; + c +--- +(0 rows) + +ROLLBACK TO q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +ROLLBACK; +BEGIN; +CREATE TABLE tt28(c int); +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +ERROR: "tt28" is already a view +ROLLBACK; -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. @@ -2046,7 +2069,7 @@ drop cascades to view aliased_view_2 drop cascades to view aliased_view_3 drop cascades to view aliased_view_4 DROP SCHEMA testviewschm2 CASCADE; -NOTICE: drop cascades to 76 other objects +NOTICE: drop cascades to 77 other objects DETAIL: drop cascades to table t1 drop cascades to view temporal1 drop cascades to view temporal2 @@ -2121,5 +2144,6 @@ drop cascades to view tt23v drop cascades to view tt24v drop cascades to view tt25v drop cascades to view tt26v +drop cascades to table tt26 drop cascades to table tdis drop cascades to view tdis_v1 diff --git a/src/test/regress/expected/create_view_optimizer.out b/src/test/regress/expected/create_view_optimizer.out index ece2034d92d..8eb7f64038d 100755 --- a/src/test/regress/expected/create_view_optimizer.out +++ b/src/test/regress/expected/create_view_optimizer.out @@ -1997,6 +1997,29 @@ select pg_get_viewdef('tt26v', true); FROM ( VALUES (1,2,3)) v(x, y, z); (1 row) +-- Test that changing the relkind of a relcache entry doesn't cause +-- trouble. Prior instances of where it did: +-- CALDaNm2yXz+zOtv7y5zBd5WKT8O0Ld3YxikuU3dcyCvxF7gypA@mail.gmail.com +-- CALDaNm3oZA-8Wbps2Jd1g5_Gjrr-x3YWrJPek-mF5Asrrvz2Dg@mail.gmail.com +CREATE TABLE tt26(c int); +BEGIN; +CREATE TABLE tt27(c int); +SAVEPOINT q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +SELECT * FROM tt27; + c +--- +(0 rows) + +ROLLBACK TO q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +ROLLBACK; +BEGIN; +CREATE TABLE tt28(c int); +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +ERROR: "tt28" is already a view +ROLLBACK; -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. @@ -2044,7 +2067,7 @@ drop cascades to view aliased_view_2 drop cascades to view aliased_view_3 drop cascades to view aliased_view_4 DROP SCHEMA testviewschm2 CASCADE; -NOTICE: drop cascades to 76 other objects +NOTICE: drop cascades to 77 other objects DETAIL: drop cascades to table t1 drop cascades to view temporal1 drop cascades to view temporal2 @@ -2119,5 +2142,6 @@ drop cascades to view tt23v drop cascades to view tt24v drop cascades to view tt25v drop cascades to view tt26v +drop cascades to table tt26 drop cascades to table tdis drop cascades to view tdis_v1 diff --git a/src/test/regress/sql/create_view.sql b/src/test/regress/sql/create_view.sql index f35364b8a23..427d6242747 100644 --- a/src/test/regress/sql/create_view.sql +++ b/src/test/regress/sql/create_view.sql @@ -693,6 +693,28 @@ select x + y + z as c1, from (values(1,2,3)) v(x,y,z); select pg_get_viewdef('tt26v', true); + +-- Test that changing the relkind of a relcache entry doesn't cause +-- trouble. Prior instances of where it did: +-- CALDaNm2yXz+zOtv7y5zBd5WKT8O0Ld3YxikuU3dcyCvxF7gypA@mail.gmail.com +-- CALDaNm3oZA-8Wbps2Jd1g5_Gjrr-x3YWrJPek-mF5Asrrvz2Dg@mail.gmail.com +CREATE TABLE tt26(c int); + +BEGIN; +CREATE TABLE tt27(c int); +SAVEPOINT q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +SELECT * FROM tt27; +ROLLBACK TO q; +CREATE RULE "_RETURN" AS ON SELECT TO tt27 DO INSTEAD SELECT * FROM tt26; +ROLLBACK; + +BEGIN; +CREATE TABLE tt28(c int); +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; +ROLLBACK; + -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); create view tdis_v1 as select a,b,c, -1::int from tdis group by 1,2,3,4; From 4084cf1558629df6a5db9d3a8d438690c6346e5f Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Mon, 5 Aug 2024 06:05:23 -0700 Subject: [PATCH 021/128] Restrict accesses to non-system views and foreign tables during pg_dump. When pg_dump retrieves the list of database objects and performs the data dump, there was possibility that objects are replaced with others of the same name, such as views, and access them. This vulnerability could result in code execution with superuser privileges during the pg_dump process. This issue can arise when dumping data of sequences, foreign tables (only 13 or later), or tables registered with a WHERE clause in the extension configuration table. To address this, pg_dump now utilizes the newly introduced restrict_nonsystem_relation_kind GUC parameter to restrict the accesses to non-system views and foreign tables during the dump process. This new GUC parameter is added to back branches too, but these changes do not require cluster recreation. Back-patch to all supported branches. Reviewed-by: Noah Misch Security: CVE-2024-7348 Backpatch-through: 12 --- .../postgres_fdw/expected/postgres_fdw.out | 11 ++++ contrib/postgres_fdw/sql/postgres_fdw.sql | 8 +++ doc/src/sgml/config.sgml | 17 +++++ doc/src/sgml/ref/pg_dump.sgml | 8 +++ src/backend/foreign/foreign.c | 10 +++ src/backend/optimizer/plan/createplan.c | 13 ++++ src/backend/optimizer/util/plancat.c | 12 ++++ src/backend/rewrite/rewriteHandler.c | 17 +++++ src/backend/tcop/postgres.c | 63 +++++++++++++++++++ src/backend/utils/misc/guc.c | 15 ++++- src/bin/pg_dump/pg_dump.c | 48 ++++++++++++++ src/include/tcop/tcopprot.h | 9 +++ src/include/utils/unsync_guc_name.h | 1 + src/test/regress/expected/create_view.out | 19 +++++- .../expected/create_view_optimizer.out | 19 +++++- src/test/regress/sql/create_view.sql | 10 +++ 16 files changed, 277 insertions(+), 3 deletions(-) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 10700d6fd4a..9b70906a4a9 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -707,6 +707,17 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft_empty ORDER BY c1; Remote SQL: SELECT c1, c2 FROM public.loct_empty ORDER BY c1 ASC NULLS LAST (3 rows) +-- test restriction on non-system foreign tables. +SET restrict_nonsystem_relation_kind TO 'foreign-table'; +SELECT * from ft1 where c1 < 1; -- ERROR +ERROR: access to non-system foreign table is restricted +INSERT INTO ft1 (c1) VALUES (1); -- ERROR +ERROR: access to non-system foreign table is restricted +DELETE FROM ft1 WHERE c1 = 1; -- ERROR +ERROR: access to non-system foreign table is restricted +TRUNCATE ft1; -- ERROR +ERROR: access to non-system foreign table is restricted +RESET restrict_nonsystem_relation_kind; -- =================================================================== -- WHERE with remotely-executable conditions -- =================================================================== diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 793dd64811d..ac290d3ba30 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -321,6 +321,14 @@ DELETE FROM loct_empty; ANALYZE ft_empty; EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft_empty ORDER BY c1; +-- test restriction on non-system foreign tables. +SET restrict_nonsystem_relation_kind TO 'foreign-table'; +SELECT * from ft1 where c1 < 1; -- ERROR +INSERT INTO ft1 (c1) VALUES (1); -- ERROR +DELETE FROM ft1 WHERE c1 = 1; -- ERROR +TRUNCATE ft1; -- ERROR +RESET restrict_nonsystem_relation_kind; + -- =================================================================== -- WHERE with remotely-executable conditions -- =================================================================== diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 23f60cad528..9b03793f74a 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -9027,6 +9027,23 @@ SET XML OPTION { DOCUMENT | CONTENT }; + + restrict_nonsystem_relation_kind (string) + + restrict_nonsystem_relation_kind + configuration parameter + + + + + This variable specifies relation kind to which access is restricted. + It contains a comma-separated list of relation kind. Currently, the + supported relation kinds are view and + foreign-table. + + + + diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index d3113d76a07..c405fef866b 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -831,6 +831,14 @@ PostgreSQL documentation The only exception is that an empty pattern is disallowed. + + + Using wildcards in may result + in access to unexpected foreign servers. Also, to use this option securely, + make sure that the named server must have a trusted owner. + + + When is specified, diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index 2d60eff9459..7a00c5f06db 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -33,6 +33,7 @@ #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" +#include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/rel.h" @@ -590,6 +591,15 @@ GetFdwRoutine(Oid fdwhandler) Datum datum; FdwRoutine *routine; + /* Check if the access to foreign tables is restricted */ + if (unlikely((restrict_nonsystem_relation_kind & RESTRICT_RELKIND_FOREIGN_TABLE) != 0)) + { + /* there must not be built-in FDW handler */ + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access to non-system foreign table is restricted"))); + } + datum = OidFunctionCall0(fdwhandler); routine = (FdwRoutine *) DatumGetPointer(datum); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 53c1bf7d338..20c1587b062 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -47,6 +47,7 @@ #include "parser/parsetree.h" #include "partitioning/partdesc.h" #include "partitioning/partprune.h" +#include "tcop/tcopprot.h" #include "utils/lsyscache.h" #include "utils/uri.h" @@ -8482,7 +8483,19 @@ make_modifytable(PlannerInfo *root, Plan *subplan, Assert(rte->rtekind == RTE_RELATION); if (rte->relkind == RELKIND_FOREIGN_TABLE) + { + /* Check if the access to foreign tables is restricted */ + if (unlikely((restrict_nonsystem_relation_kind & RESTRICT_RELKIND_FOREIGN_TABLE) != 0)) + { + /* there must not be built-in foreign tables */ + Assert(rte->relid >= FirstNormalObjectId); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access to non-system foreign table is restricted"))); + } + fdwroutine = GetFdwRoutineByRelId(rte->relid); + } else fdwroutine = NULL; } diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 46d85f3c324..8971c42fe77 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -49,6 +49,7 @@ #include "rewrite/rewriteManip.h" #include "statistics/statistics.h" #include "storage/bufmgr.h" +#include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/partcache.h" @@ -484,6 +485,17 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, /* Grab foreign-table info using the relcache, while we have it */ if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { + /* Check if the access to foreign tables is restricted */ + if (unlikely((restrict_nonsystem_relation_kind & RESTRICT_RELKIND_FOREIGN_TABLE) != 0)) + { + /* there must not be built-in foreign tables */ + Assert(RelationGetRelid(relation) >= FirstNormalObjectId); + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access to non-system foreign table is restricted"))); + } + rel->serverid = GetForeignServerIdByRelId(RelationGetRelid(relation)); rel->segSeverids = GetForeignServerSegsByRelId(RelationGetRelid(relation)); rel->fdwroutine = GetFdwRoutineForRelation(relation, true); diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c index 23c528b60f9..9da36260c77 100644 --- a/src/backend/rewrite/rewriteHandler.c +++ b/src/backend/rewrite/rewriteHandler.c @@ -43,6 +43,7 @@ #include "rewrite/rewriteManip.h" #include "rewrite/rewriteSearchCycle.h" #include "rewrite/rowsecurity.h" +#include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/rel.h" @@ -1815,6 +1816,14 @@ ApplyRetrieveRule(Query *parsetree, if (rule->qual != NULL) elog(ERROR, "cannot handle qualified ON SELECT rule"); + /* Check if the expansion of non-system views are restricted */ + if (unlikely((restrict_nonsystem_relation_kind & RESTRICT_RELKIND_VIEW) != 0 && + RelationGetRelid(relation) >= FirstNormalObjectId)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access to non-system view \"%s\" is restricted", + RelationGetRelationName(relation)))); + if (rt_index == parsetree->resultRelation) { /* @@ -3261,6 +3270,14 @@ rewriteTargetView(Query *parsetree, Relation view) } } + /* Check if the expansion of non-system views are restricted */ + if (unlikely((restrict_nonsystem_relation_kind & RESTRICT_RELKIND_VIEW) != 0 && + RelationGetRelid(view) >= FirstNormalObjectId)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access to non-system view \"%s\" is restricted", + RelationGetRelationName(view)))); + /* * For INSERT/UPDATE the modified columns must all be updatable. Note that * we get the modified columns from the query's targetlist, not from the diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 37cdcfba46a..6ae0202b396 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -89,6 +89,7 @@ #include "utils/snapmgr.h" #include "utils/timeout.h" #include "utils/timestamp.h" +#include "utils/varlena.h" #include "cdb/cdbutil.h" #include "cdb/cdbvars.h" @@ -151,6 +152,8 @@ cancel_pending_hook_type cancel_pending_hook = NULL; * Hook for query execution. */ exec_simple_query_hook_type exec_simple_query_hook = NULL; +/* flags for non-system relation kinds to restrict use */ +int restrict_nonsystem_relation_kind; /* ---------------- * private typedefs etc @@ -4558,6 +4561,66 @@ assign_max_stack_depth(int newval, void *extra) max_stack_depth_bytes = newval_bytes; } +/* + * GUC check_hook for restrict_nonsystem_relation_kind + */ +bool +check_restrict_nonsystem_relation_kind(char **newval, void **extra, GucSource source) +{ + char *rawstring; + List *elemlist; + ListCell *l; + int flags = 0; + + /* Need a modifiable copy of string */ + rawstring = pstrdup(*newval); + + if (!SplitIdentifierString(rawstring, ',', &elemlist)) + { + /* syntax error in list */ + GUC_check_errdetail("List syntax is invalid."); + pfree(rawstring); + list_free(elemlist); + return false; + } + + foreach(l, elemlist) + { + char *tok = (char *) lfirst(l); + + if (pg_strcasecmp(tok, "view") == 0) + flags |= RESTRICT_RELKIND_VIEW; + else if (pg_strcasecmp(tok, "foreign-table") == 0) + flags |= RESTRICT_RELKIND_FOREIGN_TABLE; + else + { + GUC_check_errdetail("Unrecognized key word: \"%s\".", tok); + pfree(rawstring); + list_free(elemlist); + return false; + } + } + + pfree(rawstring); + list_free(elemlist); + + /* Save the flags in *extra, for use by the assign function */ + *extra = malloc(sizeof(int)); + *((int *) *extra) = flags; + + return true; +} + +/* + * GUC assign_hook for restrict_nonsystem_relation_kind + */ +void +assign_restrict_nonsystem_relation_kind(const char *newval, void *extra) +{ + int *flags = (int *) extra; + + restrict_nonsystem_relation_kind = *flags; +} /* * set_debug_options --- apply "-d N" command line option diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 168f6113ce2..bc3e76d0314 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -686,6 +686,8 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; static char *file_encryption_method_str; +static char *restrict_nonsystem_relation_kind_string; + /* should be static, but commands/variable.c needs to get at this */ char *role_string; @@ -4761,7 +4763,18 @@ static struct config_string ConfigureNamesString[] = "", NULL, NULL, NULL }, - + + { + {"restrict_nonsystem_relation_kind", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Sets relation kinds of non-system relation to restrict use"), + NULL, + GUC_LIST_INPUT | GUC_NOT_IN_SAMPLE + }, + &restrict_nonsystem_relation_kind_string, + "", + check_restrict_nonsystem_relation_kind, assign_restrict_nonsystem_relation_kind, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, NULL, NULL, NULL, NULL diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a484f693b43..e85e39cf9f5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -362,6 +362,7 @@ static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, const char *prefix, Archive *fout); static char *get_synchronized_snapshot(Archive *fout); static void setupDumpWorker(Archive *AHX); +static void set_restrict_relation_kind(Archive *AH, const char *value); static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); @@ -1468,6 +1469,13 @@ setup_connection(Archive *AH, const char *dumpencoding, ExecuteSqlStatement(AH, "SET row_security = off"); } + /* + * For security reasons, we restrict the expansion of non-system views and + * access to foreign tables during the pg_dump process. This restriction + * is adjusted when dumping foreign table data. + */ + set_restrict_relation_kind(AH, "view, foreign-table"); + /* * Start transaction-snapshot mode transaction to dump consistent data. */ @@ -2362,6 +2370,11 @@ dumpTableData_copy(Archive *fout, const void *dcontext) */ if (tdinfo->filtercond || tbinfo->relkind == RELKIND_FOREIGN_TABLE) { + /* Temporary allows to access to foreign tables to dump data */ + if (tbinfo->relkind == RELKIND_FOREIGN_TABLE) + set_restrict_relation_kind(fout, "view"); + + /* Note: this syntax is only supported in 8.2 and up */ appendPQExpBufferStr(q, "COPY (SELECT "); /* klugery to get rid of parens in column list */ if (strlen(column_list) > 2) @@ -2473,6 +2486,11 @@ dumpTableData_copy(Archive *fout, const void *dcontext) classname); destroyPQExpBuffer(q); + + /* Revert back the setting */ + if (tbinfo->relkind == RELKIND_FOREIGN_TABLE) + set_restrict_relation_kind(fout, "view, foreign-table"); + return 1; } @@ -2499,6 +2517,10 @@ dumpTableData_insert(Archive *fout, const void *dcontext) int rows_per_statement = dopt->dump_inserts; int rows_this_statement = 0; + /* Temporary allows to access to foreign tables to dump data */ + if (tbinfo->relkind == RELKIND_FOREIGN_TABLE) + set_restrict_relation_kind(fout, "view"); + /* * If we're going to emit INSERTs with column names, the most efficient * way to deal with generated columns is to exclude them entirely. For @@ -2738,6 +2760,10 @@ dumpTableData_insert(Archive *fout, const void *dcontext) destroyPQExpBuffer(insertStmt); free(attgenerated); + /* Revert back the setting */ + if (tbinfo->relkind == RELKIND_FOREIGN_TABLE) + set_restrict_relation_kind(fout, "view, foreign-table"); + return 1; } @@ -4819,6 +4845,28 @@ is_superuser(Archive *fout) return false; } +/* + * Set the given value to restrict_nonsystem_relation_kind value. Since + * restrict_nonsystem_relation_kind is introduced in minor version releases, + * the setting query is effective only where available. + */ +static void +set_restrict_relation_kind(Archive *AH, const char *value) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + appendPQExpBuffer(query, + "SELECT set_config(name, '%s', false) " + "FROM pg_settings " + "WHERE name = 'restrict_nonsystem_relation_kind'", + value); + res = ExecuteSqlQuery(AH, query->data, PGRES_TUPLES_OK); + + PQclear(res); + destroyPQExpBuffer(query); +} + /* * getSubscriptions * get information about subscriptions diff --git a/src/include/tcop/tcopprot.h b/src/include/tcop/tcopprot.h index 33c929e9082..c7c534417a2 100644 --- a/src/include/tcop/tcopprot.h +++ b/src/include/tcop/tcopprot.h @@ -47,6 +47,12 @@ typedef enum extern PGDLLIMPORT int log_statement; +/* Flags for restrict_nonsystem_relation_kind value */ +#define RESTRICT_RELKIND_VIEW 0x01 +#define RESTRICT_RELKIND_FOREIGN_TABLE 0x02 + +extern PGDLLIMPORT int restrict_nonsystem_relation_kind; + extern List *pg_parse_query(const char *query_string); extern List *pg_rewrite_query(Query *query); extern List *pg_analyze_and_rewrite(RawStmt *parsetree, @@ -67,6 +73,9 @@ extern List *pg_plan_queries(List *querytrees, const char *query_string, extern bool check_max_stack_depth(int *newval, void **extra, GucSource source); extern void assign_max_stack_depth(int newval, void *extra); +extern bool check_restrict_nonsystem_relation_kind(char **newval, void **extra, + GucSource source); +extern void assign_restrict_nonsystem_relation_kind(const char *newval, void *extra); extern void die(SIGNAL_ARGS); extern void quickdie(SIGNAL_ARGS) pg_attribute_noreturn(); diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index cba11770a81..55a81df5bae 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -537,6 +537,7 @@ "resource_select_only", "restart_after_crash", "restore_command", + "restrict_nonsystem_relation_kind", "role", "runaway_detector_activation_percent", "segment_size", diff --git a/src/test/regress/expected/create_view.out b/src/test/regress/expected/create_view.out index 1eab17e3706..2a1fdca3562 100644 --- a/src/test/regress/expected/create_view.out +++ b/src/test/regress/expected/create_view.out @@ -2022,6 +2022,21 @@ CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; ERROR: "tt28" is already a view ROLLBACK; +-- test restriction on non-system view expansion. +create table tt27v_tbl (a int); +create view tt27v as select a from tt27v_tbl; +set restrict_nonsystem_relation_kind to 'view'; +select a from tt27v where a > 0; -- Error +ERROR: access to non-system view "tt27v" is restricted +insert into tt27v values (1); -- Error +ERROR: access to non-system view "tt27v" is restricted +select viewname from pg_views where viewname = 'tt27v'; -- Ok to access a system view. + viewname +---------- + tt27v +(1 row) + +reset restrict_nonsystem_relation_kind; -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. @@ -2069,7 +2084,7 @@ drop cascades to view aliased_view_2 drop cascades to view aliased_view_3 drop cascades to view aliased_view_4 DROP SCHEMA testviewschm2 CASCADE; -NOTICE: drop cascades to 77 other objects +NOTICE: drop cascades to 79 other objects DETAIL: drop cascades to table t1 drop cascades to view temporal1 drop cascades to view temporal2 @@ -2145,5 +2160,7 @@ drop cascades to view tt24v drop cascades to view tt25v drop cascades to view tt26v drop cascades to table tt26 +drop cascades to table tt27v_tbl +drop cascades to view tt27v drop cascades to table tdis drop cascades to view tdis_v1 diff --git a/src/test/regress/expected/create_view_optimizer.out b/src/test/regress/expected/create_view_optimizer.out index 8eb7f64038d..c17ae313cac 100755 --- a/src/test/regress/expected/create_view_optimizer.out +++ b/src/test/regress/expected/create_view_optimizer.out @@ -2020,6 +2020,21 @@ CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; ERROR: "tt28" is already a view ROLLBACK; +-- test restriction on non-system view expansion. +create table tt27v_tbl (a int); +create view tt27v as select a from tt27v_tbl; +set restrict_nonsystem_relation_kind to 'view'; +select a from tt27v where a > 0; -- Error +ERROR: access to non-system view "tt27v" is restricted +insert into tt27v values (1); -- Error +ERROR: access to non-system view "tt27v" is restricted +select viewname from pg_views where viewname = 'tt27v'; -- Ok to access a system view. + viewname +---------- + tt27v +(1 row) + +reset restrict_nonsystem_relation_kind; -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. @@ -2067,7 +2082,7 @@ drop cascades to view aliased_view_2 drop cascades to view aliased_view_3 drop cascades to view aliased_view_4 DROP SCHEMA testviewschm2 CASCADE; -NOTICE: drop cascades to 77 other objects +NOTICE: drop cascades to 79 other objects DETAIL: drop cascades to table t1 drop cascades to view temporal1 drop cascades to view temporal2 @@ -2143,5 +2158,7 @@ drop cascades to view tt24v drop cascades to view tt25v drop cascades to view tt26v drop cascades to table tt26 +drop cascades to table tt27v_tbl +drop cascades to view tt27v drop cascades to table tdis drop cascades to view tdis_v1 diff --git a/src/test/regress/sql/create_view.sql b/src/test/regress/sql/create_view.sql index 427d6242747..4605fdb71b1 100644 --- a/src/test/regress/sql/create_view.sql +++ b/src/test/regress/sql/create_view.sql @@ -715,6 +715,16 @@ CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; CREATE RULE "_RETURN" AS ON SELECT TO tt28 DO INSTEAD SELECT * FROM tt26; ROLLBACK; + +-- test restriction on non-system view expansion. +create table tt27v_tbl (a int); +create view tt27v as select a from tt27v_tbl; +set restrict_nonsystem_relation_kind to 'view'; +select a from tt27v where a > 0; -- Error +insert into tt27v values (1); -- Error +select viewname from pg_views where viewname = 'tt27v'; -- Ok to access a system view. +reset restrict_nonsystem_relation_kind; + -- test display negative operator of const-folder expression create table tdis(a int, b int, c int); create view tdis_v1 as select a,b,c, -1::int from tdis group by 1,2,3,4; From c4748066ed1e3ffe9514ff297eb32d6217df6d88 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Fri, 20 Mar 2026 11:33:36 +0800 Subject: [PATCH 022/128] Remove unused concourse/* from source This is a further update based on the PR: - https://github.com/apache/cloudberry/pull/1625 --- pom.xml | 2 - .../gporca/concourse/build_and_test.py | 110 ------------------ .../xerces-c/xerces-c-3.1.2.tar.gz.sha256 | 1 - 3 files changed, 113 deletions(-) delete mode 100755 src/backend/gporca/concourse/build_and_test.py delete mode 100644 src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 diff --git a/pom.xml b/pom.xml index 1b4daf82da5..5d27326e4b6 100644 --- a/pom.xml +++ b/pom.xml @@ -572,8 +572,6 @@ code or new licensing patterns. src/backend/gporca/**/Makefile src/backend/gporca/cmake/FindXerces.cmake - src/backend/gporca/concourse/build_and_test.py - src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 src/backend/gporca/server/fixdxl.sh src/backend/gporca/server/include/unittest/gpopt/operators/CScalarIsDistinctFromTest.h src/backend/gporca/server/dxl.xsd diff --git a/src/backend/gporca/concourse/build_and_test.py b/src/backend/gporca/concourse/build_and_test.py deleted file mode 100755 index e885843dc4c..00000000000 --- a/src/backend/gporca/concourse/build_and_test.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/python3 - -import optparse -import os -import shutil -import subprocess -import sys - -def num_cpus(): - # Use multiprocessing module, available in Python 2.6+ - try: - import multiprocessing - return multiprocessing.cpu_count() - except (ImportError, NotImplementedError): - pass - - # Get POSIX system config value for number of processors. - posix_num_cpus = os.sysconf("SC_NPROCESSORS_ONLN") - if posix_num_cpus != -1: - return posix_num_cpus - - # Guess - return 2 - -def install_dependencies(dependencies, output_dir): - for dependency in dependencies: - status = install_dependency(dependency, output_dir) - if status: - return status - -def install_dependency(dependency_name, output_dir): - return subprocess.call( - ["tar -xzf " + dependency_name + "/*.tar.gz -C " + output_dir], shell=True) - -def cmake_configure(src_dir, build_type, output_dir, cxx_compiler = None, cxxflags = None): - if os.path.exists("build"): - shutil.rmtree("build") - os.mkdir("build") - cmake_args = ["cmake", - "-D", "CMAKE_INSTALL_PREFIX=" + output_dir, - "-D", "CMAKE_BUILD_TYPE=" + build_type] - if cxx_compiler: - cmake_args.append("-D") - cmake_args.append("CMAKE_CXX_COMPILER=" + cxx_compiler) - if cxxflags: - cmake_args.append("-D") - cmake_args.append("CMAKE_CXX_FLAGS=" + cxxflags) - - cmake_args.append("../" + src_dir) - cmake_command = " ".join(cmake_args) - if os.path.exists('/opt/gcc_env.sh'): - cmake_command = "source /opt/gcc_env.sh && " + cmake_command - print(cmake_command) - return subprocess.call(cmake_command, cwd="build", shell=True) - -def make(): - return subprocess.call(["make", - "-j" + str(num_cpus()), - "-l" + str(2 * num_cpus()), - ], - cwd="build", - env=ccache_env() - ) - - -def ccache_env(): - env = os.environ.copy() - env['CCACHE_DIR'] = os.getcwd() + '/.ccache' - return env - - -def run_tests(): - return subprocess.call(["ctest", - "--output-on-failure", - "-j" + str(num_cpus()), - "--test-load", str(4 * num_cpus()), - ], - cwd="build") - -def main(): - parser = optparse.OptionParser() - parser.add_option("--build_type", dest="build_type", default="RELEASE") - parser.add_option("--compiler", dest="compiler") - parser.add_option("--cxxflags", dest="cxxflags") - parser.add_option("--output_dir", dest="output_dir", default="install") - parser.add_option("--skiptests", dest="skiptests", action="store_true", default=False) - - (options, args) = parser.parse_args() - # install deps for building - status = install_dependencies(args, "/usr/local") - if status: - return status - status = cmake_configure("", - options.build_type, - options.output_dir, - options.compiler, - options.cxxflags) - if status: - return status - status = make() - if status: - return status - if not options.skiptests: - status = run_tests() - if status: - return status - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 b/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 deleted file mode 100644 index 7fbcc8000ec..00000000000 --- a/src/backend/gporca/concourse/xerces-c/xerces-c-3.1.2.tar.gz.sha256 +++ /dev/null @@ -1 +0,0 @@ -743bd0a029bf8de56a587c270d97031e0099fe2b7142cef03e0da16e282655a0 From fcaa4deadd8e8fa188edb460afe098ee39a6b280 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 16 Mar 2026 16:42:26 +0800 Subject: [PATCH 023/128] Stop installing diskquota build-info file Remove generation and installation of diskquota-build-info from diskquota to avoid writing extra files into the install prefix root (e.g. $GPHOME or /usr/local/cloudberry-db). Drop the unused CMake helper cmake/BuildInfo.cmake and remove cmake/Git.cmake and its invocation now that no build-info is produced. --- gpcontrib/diskquota/CMakeLists.txt | 18 ------------- gpcontrib/diskquota/cmake/BuildInfo.cmake | 32 ----------------------- gpcontrib/diskquota/cmake/Git.cmake | 9 ------- 3 files changed, 59 deletions(-) delete mode 100644 gpcontrib/diskquota/cmake/BuildInfo.cmake delete mode 100644 gpcontrib/diskquota/cmake/Git.cmake diff --git a/gpcontrib/diskquota/CMakeLists.txt b/gpcontrib/diskquota/CMakeLists.txt index fad393cb101..face48578a6 100644 --- a/gpcontrib/diskquota/CMakeLists.txt +++ b/gpcontrib/diskquota/CMakeLists.txt @@ -12,10 +12,6 @@ endif() # generate 'compile_commands.json' set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -# Retrieve repository information -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Git.cmake) -GitHash_Get(DISKQUOTA_GIT_HASH) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Gpdb.cmake) @@ -154,19 +150,6 @@ add_custom_target(create_artifact ${CMAKE_COMMAND} -E tar czvf ${artifact_NAME} "${tgz_NAME}.tar.gz") # packing end -# Create build-info -# The diskquota-build-info shouldn't be copied to GPDB release by install_gpdb_component -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/BuildInfo.cmake) -set(build_info_PATH ${CMAKE_CURRENT_BINARY_DIR}/diskquota-build-info) -BuildInfo_Create(${build_info_PATH} - VARS - DISKQUOTA_GIT_HASH - DISKQUOTA_VERSION - GP_MAJOR_VERSION - GP_VERSION - CMAKE_BUILD_TYPE) -# Create build-info end - # Add installcheck targets add_subdirectory(tests) add_subdirectory(upgrade_test) @@ -175,4 +158,3 @@ add_subdirectory(upgrade_test) install(PROGRAMS "cmake/install_gpdb_component" DESTINATION ".") install(FILES ${diskquota_DDL} DESTINATION "share/postgresql/extension/") install(TARGETS diskquota DESTINATION "lib/postgresql/") -install(FILES ${build_info_PATH} DESTINATION ".") diff --git a/gpcontrib/diskquota/cmake/BuildInfo.cmake b/gpcontrib/diskquota/cmake/BuildInfo.cmake deleted file mode 100644 index 6e256f34502..00000000000 --- a/gpcontrib/diskquota/cmake/BuildInfo.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Create a build info file based on the given cmake variables -# For example: -# BuildInfo_Create( -# ${CMAKE_CURRENT_BINARY_DIR}/build-info -# VARS -# DISKQUOTA_GIT_HASH -# GP_MAJOR_VERSION) -# ) -# will create a build info file: -# ❯ cat build-info -# DISKQUOTA_GIT_HASH = 151ed92 -# GP_MAJOR_VERSION = 6 - -function(BuildInfo_Create path) - cmake_parse_arguments( - arg - "" - "" - "VARS" - ${ARGN}) - - # Set REGRESS test cases - foreach(key IN LISTS arg_VARS) - get_property(val VARIABLE PROPERTY ${key}) - list(APPEND info_list "${key} = ${val}") - endforeach() - file(WRITE ${path} "") - foreach(content IN LISTS info_list) - file(APPEND ${path} "${content}\n") - endforeach() -endfunction() - diff --git a/gpcontrib/diskquota/cmake/Git.cmake b/gpcontrib/diskquota/cmake/Git.cmake deleted file mode 100644 index 81a68b1f1f4..00000000000 --- a/gpcontrib/diskquota/cmake/Git.cmake +++ /dev/null @@ -1,9 +0,0 @@ -# get git hash -macro(GitHash_Get _git_hash) - find_package(Git) - execute_process( - COMMAND ${GIT_EXECUTABLE} log -1 --pretty=format:%h - OUTPUT_VARIABLE ${_git_hash} - OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -endmacro() From c828ad72a0cdadef4c406032e89948469f1107cf Mon Sep 17 00:00:00 2001 From: fairyfar Date: Mon, 16 Mar 2026 17:28:12 +0800 Subject: [PATCH 024/128] Fix syntax error in the bash script regarding LD_LIBRARY_PATH --- .github/workflows/coverity.yml | 2 +- .github/workflows/sonarqube.yml | 2 +- devops/build/automation/cloudberry/scripts/build-cloudberry.sh | 2 +- .../build/automation/cloudberry/scripts/configure-cloudberry.sh | 2 +- .../build/automation/cloudberry/scripts/unittest-cloudberry.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 4f7f74d54b2..2b6a81c91f4 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -90,7 +90,7 @@ jobs: /usr/local/cloudberry-db/lib sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db su - gpadmin -c "cd $WORKSPACE" - export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH + export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:${LD_LIBRARY_PATH:-""} export PATH=$WORKSPACE/coverity_tool/bin:$PATH ./configure --prefix=/usr/local/cloudberry-db \ --disable-external-fts \ diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index e67c2d96a54..93379d184ea 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -94,7 +94,7 @@ jobs: /usr/local/xerces-c/lib/libxerces-c-3.3.so \ /usr/local/cloudberry-db/lib sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db - export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH + export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:${LD_LIBRARY_PATH:-""} ./configure --prefix=/usr/local/cloudberry-db \ --disable-external-fts \ --enable-gpcloud \ diff --git a/devops/build/automation/cloudberry/scripts/build-cloudberry.sh b/devops/build/automation/cloudberry/scripts/build-cloudberry.sh index efa061a0f83..ca4c73d55cb 100755 --- a/devops/build/automation/cloudberry/scripts/build-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/build-cloudberry.sh @@ -71,7 +71,7 @@ init_environment "Cloudberry Build Script" "${BUILD_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:${LD_LIBRARY_PATH:-""} log_section_end "Environment Setup" # Build process diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 32a9f3d8657..2d7ad04aed8 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -131,7 +131,7 @@ log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:${LD_LIBRARY_PATH:-""} log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" diff --git a/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh b/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh index 97107ea1a9f..69536f0067f 100755 --- a/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh @@ -56,7 +56,7 @@ init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:${LD_LIBRARY_PATH:-""} log_section_end "Environment Setup" # Unittest process From a489d2a40b7dd5bcd7f48882ba7728bd49ada904 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 6 Nov 2023 06:14:13 -0800 Subject: [PATCH 025/128] Set GUC "is_superuser" in all processes that set AuthenticatedUserId. It was always false in single-user mode, in autovacuum workers, and in background workers. This had no specifically-identified security consequences, but non-core code or future work might make it security-relevant. Back-patch to v11 (all supported versions). Jelte Fennema-Nio. Reported by Jelte Fennema-Nio. --- src/backend/utils/init/miscinit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index a8e7ff0f057..83dee55ad8a 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -841,6 +841,14 @@ InitializeSessionUserIdStandalone(void) AuthenticatedUserIsSuperuser = true; SetSessionUserId(BOOTSTRAP_SUPERUSERID, true); + + /* + * XXX This should set SetConfigOption("session_authorization"), too. + * Since we don't, C code will get NULL, and current_setting() will get an + * empty string. + */ + SetConfigOption("is_superuser", "on", + PGC_INTERNAL, PGC_S_DYNAMIC_DEFAULT); } From 7eb5d85542c85610572f9973c1f37b0d7ff9df1b Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 25 Mar 2026 16:05:24 +0500 Subject: [PATCH 026/128] libpq: Bail out during SSL/GSS negotiation errors (#1633) This commit changes libpq so that errors reported by the backend during the protocol negotiation for SSL and GSS are discarded by the client, as these may include bytes that could be consumed by the client and write arbitrary bytes to a client's terminal. A failure with the SSL negotiation now leads to an error immediately reported, without a retry on any other methods allowed, like a fallback to a plaintext connection. A failure with GSS discards the error message received, and we allow a fallback as it may be possible that the error is caused by a connection attempt with a pre-11 server, GSS encryption having been introduced in v12. This was a problem only with v17 and newer versions; older versions discard the error message already in this case, assuming a failure caused by a lack of support for GSS encryption. Author: Jacob Champion Reviewed-by: Peter Eisentraut, Heikki Linnakangas, Michael Paquier Security: CVE-2024-10977 Backpatch-through: 12 Back-ported-by: reshke ====== CBDB source commit is https://git.postgresql.org/cgit/postgresql.git/commit/?h=e6c9454764d880ee30735aa8c1e05d3674722ff9 --- doc/src/sgml/protocol.sgml | 21 +++++++++++---------- src/interfaces/libpq/fe-connect.c | 15 ++++++--------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 7141f6c277a..682142724c7 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -1527,10 +1527,10 @@ SELCT 1/0; The frontend should also be prepared to handle an ErrorMessage - response to SSLRequest from the server. This would only occur if - the server predates the addition of SSL support - to PostgreSQL. (Such servers are now very ancient, - and likely do not exist in the wild anymore.) + response to SSLRequest from the server. The frontend should not display + this error message to the user/application, since the server has not been + authenticated + (CVE-2024-10977). In this case the connection must be closed, but the frontend might choose to open a fresh connection and proceed without requesting SSL. @@ -1604,12 +1604,13 @@ SELCT 1/0; The frontend should also be prepared to handle an ErrorMessage - response to GSSENCRequest from the server. This would only occur if - the server predates the addition of GSSAPI encryption - support to PostgreSQL. In this case the - connection must be closed, but the frontend might choose to open a fresh - connection and proceed without requesting GSSAPI - encryption. + response to GSSENCRequest from the server. The frontend should not display + this error message to the user/application, since the server has not been + authenticated + (CVE-2024-10977). + In this case the connection must be closed, but the frontend might choose + to open a fresh connection and proceed without requesting + GSSAPI encryption. diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 46e8540004e..0d4d2fed864 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -3168,16 +3168,13 @@ PQconnectPoll(PGconn *conn) { /* * Server failure of some sort, such as failure to - * fork a backend process. We need to process and - * report the error message, which might be formatted - * according to either protocol 2 or protocol 3. - * Rather than duplicate the code for that, we flip - * into AWAITING_RESPONSE state and let the code there - * deal with it. Note we have *not* consumed the "E" - * byte here. + * fork a backend process. Don't bother retrieving + * the error message; we should not trust it as the + * server has not been authenticated yet. */ - conn->status = CONNECTION_AWAITING_RESPONSE; - goto keep_going; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("server sent an error response during SSL exchange\n")); + goto error_return; } else { From 18c5918790a8bf66afc279d73d1653a2d8a4f9a4 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 31 Mar 2023 11:01:51 +1300 Subject: [PATCH 027/128] Parallel Hash Full Join. Full and right outer joins were not supported in the initial implementation of Parallel Hash Join because of deadlock hazards (see discussion). Therefore FULL JOIN inhibited parallelism, as the other join strategies can't do that in parallel either. Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on the inner side of one batch's hash table. For now, sidestep the deadlock problem by terminating parallelism there. The last process to arrive at that phase emits the unmatched tuples, while others detach and are free to go and work on other batches, if there are any, but otherwise they finish the join early. That unfairness is considered acceptable for now, because it's better than no parallelism at all. The build and probe phases are run in parallel, and the new scan-for-unmatched phase, while serial, is usually applied to the smaller of the two relations and is either limited by some multiple of work_mem, or it's too big and is partitioned into batches and then the situation is improved by batch-level parallelism. Author: Melanie Plageman Author: Thomas Munro Reviewed-by: Thomas Munro Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com --- src/backend/executor/nodeHash.c | 179 +++++++++++++++++++++++- src/backend/executor/nodeHashjoin.c | 95 ++++++++----- src/backend/optimizer/path/joinpath.c | 14 +- src/include/executor/hashjoin.h | 6 +- src/include/executor/nodeHash.h | 3 + src/test/regress/expected/join_hash.out | 65 ++++++++- src/test/regress/sql/join_hash.sql | 27 +++- 7 files changed, 333 insertions(+), 56 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index a236f5b4819..e3e1d5a6d87 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2395,6 +2395,69 @@ ExecPrepHashTableForUnmatched(HashJoinState *hjstate) hjstate->hj_CurTuple = NULL; } +/* + * Decide if this process is allowed to run the unmatched scan. If so, the + * batch barrier is advanced to PHJ_BATCH_SCAN and true is returned. + * Otherwise the batch is detached and false is returned. + */ +bool +ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate) +{ + HashJoinTable hashtable = hjstate->hj_HashTable; + int curbatch = hashtable->curbatch; + ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; + + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE); + + /* + * It would not be deadlock-free to wait on the batch barrier, because it + * is in PHJ_BATCH_PROBE phase, and thus processes attached to it have + * already emitted tuples. Therefore, we'll hold a wait-free election: + * only one process can continue to the next phase, and all others detach + * from this batch. They can still go any work on other batches, if there + * are any. + */ + if (!BarrierArriveAndDetachExceptLast(&batch->batch_barrier)) + { + /* This process considers the batch to be done. */ + hashtable->batches[hashtable->curbatch].done = true; + + /* Make sure any temporary files are closed. */ + sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); + sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); + + /* + * Track largest batch we've seen, which would normally happen in + * ExecHashTableDetachBatch(). + */ + hashtable->spacePeak = + Max(hashtable->spacePeak, + batch->size + sizeof(dsa_pointer_atomic) * hashtable->nbuckets); + hashtable->curbatch = -1; + return false; + } + + /* Now we are alone with this batch. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); + Assert(BarrierParticipants(&batch->batch_barrier) == 1); + + /* + * Has another process decided to give up early and command all processes + * to skip the unmatched scan? + */ + if (batch->skip_unmatched) + { + hashtable->batches[hashtable->curbatch].done = true; + ExecHashTableDetachBatch(hashtable); + return false; + } + + /* Now prepare the process local state, just as for non-parallel join. */ + ExecPrepHashTableForUnmatched(hjstate); + + return true; +} + /* * ExecScanHashTableForUnmatched * scan the hash table for unmatched inner tuples @@ -2469,6 +2532,72 @@ ExecScanHashTableForUnmatched(HashJoinState *hjstate, ExprContext *econtext) return false; } +/* + * ExecParallelScanHashTableForUnmatched + * scan the hash table for unmatched inner tuples, in parallel join + * + * On success, the inner tuple is stored into hjstate->hj_CurTuple and + * econtext->ecxt_innertuple, using hjstate->hj_HashTupleSlot as the slot + * for the latter. + */ +bool +ExecParallelScanHashTableForUnmatched(HashJoinState *hjstate, + ExprContext *econtext) +{ + HashJoinTable hashtable = hjstate->hj_HashTable; + HashJoinTuple hashTuple = hjstate->hj_CurTuple; + + for (;;) + { + /* + * hj_CurTuple is the address of the tuple last returned from the + * current bucket, or NULL if it's time to start scanning a new + * bucket. + */ + if (hashTuple != NULL) + hashTuple = ExecParallelHashNextTuple(hashtable, hashTuple); + else if (hjstate->hj_CurBucketNo < hashtable->nbuckets) + hashTuple = ExecParallelHashFirstTuple(hashtable, + hjstate->hj_CurBucketNo++); + else + break; /* finished all buckets */ + + while (hashTuple != NULL) + { + if (!HeapTupleHeaderHasMatch(HJTUPLE_MINTUPLE(hashTuple))) + { + TupleTableSlot *inntuple; + + /* insert hashtable's tuple into exec slot */ + inntuple = ExecStoreMinimalTuple(HJTUPLE_MINTUPLE(hashTuple), + hjstate->hj_HashTupleSlot, + false); /* do not pfree */ + econtext->ecxt_innertuple = inntuple; + + /* + * Reset temp memory each time; although this function doesn't + * do any qual eval, the caller will, so let's keep it + * parallel to ExecScanHashBucket. + */ + ResetExprContext(econtext); + + hjstate->hj_CurTuple = hashTuple; + return true; + } + + hashTuple = ExecParallelHashNextTuple(hashtable, hashTuple); + } + + /* allow this loop to be cancellable */ + CHECK_FOR_INTERRUPTS(); + } + + /* + * no more unmatched tuples + */ + return false; +} + /* * ExecHashTableReset * @@ -3797,6 +3926,7 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) accessor->shared = shared; accessor->preallocated = 0; accessor->done = false; + accessor->outer_eof = false; accessor->inner_tuples = sts_attach(ParallelHashJoinBatchInner(shared), hashtable->hjstate->worker_id, @@ -3842,25 +3972,62 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) { int curbatch = hashtable->curbatch; ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; + bool attached = true; /* Make sure any temporary files are closed. */ sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); - /* Detach from the batch we were last working on. */ + /* After attaching we always get at least to PHJ_BATCH_PROBE. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE || + BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); + + /* + * If we're abandoning the PHJ_BATCH_PROBE phase early without having + * reached the end of it, it means the plan doesn't want any more + * tuples, and it is happy to abandon any tuples buffered in this + * process's subplans. For correctness, we can't allow any process to + * execute the PHJ_BATCH_SCAN phase, because we will never have the + * complete set of match bits. Therefore we skip emitting unmatched + * tuples in all backends (if this is a full/right join), as if those + * tuples were all due to be emitted by this process and it has + * abandoned them too. + */ /* * CBDB_PARALLEL: Parallel Hash Left Anti Semi (Not-In) Join(parallel-aware) * If phs_lasj_has_null is true, that means we have found null when building hash table, * there were no batches to detach. */ - if (!hashtable->parallel_state->phs_lasj_has_null && BarrierArriveAndDetach(&batch->batch_barrier)) + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + !hashtable->parallel_state->phs_lasj_has_null && /* CBDB_PARALLEL */ + !hashtable->batches[curbatch].outer_eof) + { + /* + * This flag may be written to by multiple backends during + * PHJ_BATCH_PROBE phase, but will only be read in PHJ_BATCH_SCAN + * phase so requires no extra locking. + */ + batch->skip_unmatched = true; + } + + /* + * Even if we aren't doing a full/right outer join, we'll step through + * the PHJ_BATCH_SCAN phase just to maintain the invariant that + * freeing happens in PHJ_BATCH_FREE, but that'll be wait-free. + */ + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */) + attached = BarrierArriveAndDetachExceptLast(&batch->batch_barrier); + if (attached && BarrierArriveAndDetach(&batch->batch_barrier)) { /* - * Technically we shouldn't access the barrier because we're no - * longer attached, but since there is no way it's moving after - * this point it seems safe to make the following assertion. + * We are not longer attached to the batch barrier, but we're the + * process that was chosen to free resources and it's safe to + * assert the current phase. The ParallelHashJoinBatch can't go + * away underneath us while we are attached to the build barrier, + * making this access safe. */ - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_DONE); + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_FREE); /* Free shared chunks and buckets. */ while (DsaPointerIsValid(batch->chunks)) diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 88eaaa10cef..53bb29be7af 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -81,11 +81,12 @@ * aren't enough to go around. For each batch there is a separate barrier * with the following phases: * - * PHJ_BATCH_ELECTING -- initial state - * PHJ_BATCH_ALLOCATING -- one allocates buckets - * PHJ_BATCH_LOADING -- all load the hash table from disk - * PHJ_BATCH_PROBING -- all probe - * PHJ_BATCH_DONE -- end + * PHJ_BATCH_ELECT -- initial state + * PHJ_BATCH_ALLOCATE* -- one allocates buckets + * PHJ_BATCH_LOAD -- all load the hash table from disk + * PHJ_BATCH_PROBE -- all probe + * PHJ_BATCH_SCAN* -- one does full/right unmatched scan + * PHJ_BATCH_FREE* -- one frees memory * * Batch 0 is a special case, because it starts out in phase * PHJ_BATCH_PROBING; populating batch 0's hash table is done during @@ -101,10 +102,11 @@ * finished. Practically, that means that we never emit a tuple while attached * to a barrier, unless the barrier has reached a phase that means that no * process will wait on it again. We emit tuples while attached to the build - * barrier in phase PHJ_BUILD_RUNNING, and to a per-batch barrier in phase - * PHJ_BATCH_PROBING. These are advanced to PHJ_BUILD_DONE and PHJ_BATCH_DONE - * respectively without waiting, using BarrierArriveAndDetach(). The last to - * detach receives a different return value so that it knows that it's safe to + * barrier in phase PHJ_BUILD_RUN, and to a per-batch barrier in phase + * PHJ_BATCH_PROBE. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN + * respectively without waiting, using BarrierArriveAndDetach() and + * BarrierArriveAndDetachExceptLast() respectively. The last to detach + * receives a different return value so that it knows that it's safe to * clean up. Any straggler process that attaches after that phase is reached * will see that it's too late to participate or access the relevant shared * memory objects. @@ -523,8 +525,23 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) if (HJ_FILL_INNER(node)) { /* set up to scan for unmatched inner tuples */ - ExecPrepHashTableForUnmatched(node); - node->hj_JoinState = HJ_FILL_INNER_TUPLES; + if (parallel) + { + /* + * Only one process is currently allow to handle + * each batch's unmatched tuples, in a parallel + * join. + */ + if (ExecParallelPrepHashTableForUnmatched(node)) + node->hj_JoinState = HJ_FILL_INNER_TUPLES; + else + node->hj_JoinState = HJ_NEED_NEW_BATCH; + } + else + { + ExecPrepHashTableForUnmatched(node); + node->hj_JoinState = HJ_FILL_INNER_TUPLES; + } } else node->hj_JoinState = HJ_NEED_NEW_BATCH; @@ -635,25 +652,13 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) { node->hj_MatchedOuter = true; - if (parallel) - { - /* - * Full/right outer joins are currently not supported - * for parallel joins, so we don't need to set the - * match bit. Experiments show that it's worth - * avoiding the shared memory traffic on large - * systems. - */ - Assert(!HJ_FILL_INNER(node)); - } - else - { - /* - * This is really only needed if HJ_FILL_INNER(node), - * but we'll avoid the branch and just set it always. - */ + + /* + * This is really only needed if HJ_FILL_INNER(node), but + * we'll avoid the branch and just set it always. + */ + if (!HeapTupleHeaderHasMatch(HJTUPLE_MINTUPLE(node->hj_CurTuple))) HeapTupleHeaderSetMatch(HJTUPLE_MINTUPLE(node->hj_CurTuple)); - } /* In an antijoin, we never return a matched tuple */ if (node->js.jointype == JOIN_ANTI || @@ -712,7 +717,8 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) * so any unmatched inner tuples in the hashtable have to be * emitted before we continue to the next batch. */ - if (!ExecScanHashTableForUnmatched(node, econtext)) + if (!(parallel ? ExecParallelScanHashTableForUnmatched(node, econtext) + : ExecScanHashTableForUnmatched(node, econtext))) { /* no more unmatched tuples */ node->hj_JoinState = HJ_NEED_NEW_BATCH; @@ -1271,6 +1277,8 @@ ExecParallelHashJoinOuterGetTuple(PlanState *outerNode, } /* End of this batch */ + hashtable->batches[curbatch].outer_eof = true; + return NULL; } @@ -1543,15 +1551,34 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * hash table stays alive until everyone's finished * probing it, but no participant is allowed to wait at * this barrier again (or else a deadlock could occur). - * All attached participants must eventually call - * BarrierArriveAndDetach() so that the final phase - * PHJ_BATCH_DONE can be reached. + * All attached participants must eventually detach from + * the barrier and one worker must advance the phase so + * that the final phase is reached. */ ExecParallelHashTableSetCurrentBatch(hashtable, batchno); sts_begin_parallel_scan(hashtable->batches[batchno].outer_tuples); + return true; + case PHJ_BATCH_SCAN: + + /* + * In principle, we could help scan for unmatched tuples, + * since that phase is already underway (the thing we + * can't do under current deadlock-avoidance rules is wait + * for others to arrive at PHJ_BATCH_SCAN, because + * PHJ_BATCH_PROBE emits tuples, but in this case we just + * got here without waiting). That is not yet done. For + * now, we just detach and go around again. We have to + * use ExecHashTableDetachBatch() because there's a small + * chance we'll be the last to detach, and then we're + * responsible for freeing memory. + */ + ExecParallelHashTableSetCurrentBatch(hashtable, batchno); + hashtable->batches[batchno].done = true; + ExecHashTableDetachBatch(hashtable); + break; - case PHJ_BATCH_DONE: + case PHJ_BATCH_FREE: /* * Already done. Detach and go around again (if any diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index d4c2b793bb5..b5e7ef3b60c 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -2327,15 +2327,9 @@ hash_inner_and_outer(PlannerInfo *root, * able to properly guarantee uniqueness. Similarly, we can't handle * JOIN_FULL and JOIN_RIGHT, because they can produce false null * extended rows. Also, the resulting path must not be parameterized. - * We would be able to support JOIN_FULL and JOIN_RIGHT for Parallel - * Hash, since in that case we're back to a single hash table with a - * single set of match bits for each batch, but that will require - * figuring out a deadlock-free way to wait for the probe to finish. */ if (joinrel->consider_parallel && save_jointype != JOIN_UNIQUE_OUTER && - save_jointype != JOIN_FULL && - save_jointype != JOIN_RIGHT && outerrel->partial_pathlist != NIL && bms_is_empty(joinrel->lateral_relids)) { @@ -2372,9 +2366,13 @@ hash_inner_and_outer(PlannerInfo *root, * total inner path will also be parallel-safe, but if not, we'll * have to search for the cheapest safe, unparameterized inner * path. If doing JOIN_UNIQUE_INNER, we can't use any alternative - * inner path. + * inner path. If full or right join, we can't use parallelism + * (building the hash table in each backend) because no one + * process has all the match bits. */ - if (cheapest_total_inner->parallel_safe) + if (save_jointype == JOIN_FULL || save_jointype == JOIN_RIGHT) + cheapest_safe_inner = NULL; + else if (cheapest_total_inner->parallel_safe) cheapest_safe_inner = cheapest_total_inner; else if (save_jointype != JOIN_UNIQUE_INNER) cheapest_safe_inner = diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index e324e67d914..9e243c47847 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -195,6 +195,7 @@ typedef struct ParallelHashJoinBatch size_t ntuples; /* number of tuples loaded */ size_t old_ntuples; /* number of tuples before repartitioning */ bool space_exhausted; + bool skip_unmatched; /* whether to abandon unmatched scan */ /* * Variable-sized SharedTuplestore objects follow this struct in memory. @@ -239,7 +240,7 @@ typedef struct ParallelHashJoinBatchAccessor size_t estimated_size; /* size of partition on disk */ size_t old_ntuples; /* how many tuples before repartitioning? */ bool at_least_one_chunk; /* has this backend allocated a chunk? */ - + bool outer_eof; /* has this process hit end of batch? */ bool done; /* flag to remember that a batch is done */ SharedTuplestoreAccessor *inner_tuples; SharedTuplestoreAccessor *outer_tuples; @@ -306,7 +307,8 @@ typedef struct ParallelHashJoinState #define PHJ_BATCH_ALLOCATING 1 #define PHJ_BATCH_LOADING 2 #define PHJ_BATCH_PROBING 3 -#define PHJ_BATCH_DONE 4 +#define PHJ_BATCH_SCAN 4 +#define PHJ_BATCH_FREE 5 /* The phases of batch growth while hashing, for grow_batches_barrier. */ #define PHJ_GROW_BATCHES_ELECTING 0 diff --git a/src/include/executor/nodeHash.h b/src/include/executor/nodeHash.h index 993de4519b5..36549376ef9 100644 --- a/src/include/executor/nodeHash.h +++ b/src/include/executor/nodeHash.h @@ -64,9 +64,12 @@ extern bool ExecScanHashBucket(HashState *hashState, HashJoinState *hjstate, extern bool ExecParallelScanHashBucket(HashState *hashState, HashJoinState *hjstate, ExprContext *econtext); extern void ExecPrepHashTableForUnmatched(HashJoinState *hjstate); +extern bool ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate); extern bool ExecScanHashTableForUnmatched(HashJoinState *hjstate, ExprContext *econtext); extern void ExecHashTableReset(HashState *hashState, HashJoinTable hashtable); +extern bool ExecParallelScanHashTableForUnmatched(HashJoinState *hjstate, + ExprContext *econtext); extern void ExecHashTableResetMatchFlags(HashJoinTable hashtable); extern void ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew, uint64 operatorMemKB, diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 5171a7d9cf3..250704efbd7 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -315,6 +315,13 @@ $$); t | f (1 row) +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); + count +------- + 20000 +(1 row) + rollback to settings; -- The "bad" case: during execution we need to increase number of -- batches; in this case we plan for 1 batch, and increase at least a @@ -816,8 +823,9 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); @@ -841,7 +849,32 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); + QUERY PLAN +------------------------------------------------------------- + Finalize Aggregate + -> Gather + Workers Planned: 2 + -> Partial Aggregate + -> Parallel Hash Full Join + Hash Cond: (r.id = s.id) + -> Parallel Seq Scan on simple r + -> Parallel Hash + -> Parallel Seq Scan on simple s +(9 rows) + +select count(*) from simple r full outer join simple s using (id); + count +------- + 20000 +(1 row) + +rollback to settings; +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; @@ -869,8 +902,9 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); @@ -895,6 +929,31 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); 120000 (1 row) +rollback to settings; +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + QUERY PLAN +------------------------------------------------------------- + Finalize Aggregate + -> Gather + Workers Planned: 2 + -> Partial Aggregate + -> Parallel Hash Full Join + Hash Cond: ((0 - s.id) = r.id) + -> Parallel Seq Scan on simple s + -> Parallel Hash + -> Parallel Seq Scan on simple r +(9 rows) + +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + count +------- + 40000 +(1 row) + rollback to settings; -- exercise special code paths for huge tuples (note use of non-strict -- expression and left join required to get the detoasted tuple into diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 325068e9d23..01961d1ce6e 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -191,6 +191,8 @@ select original > 1 as initially_multibatch, final > original as increased_batch $$ select count(*) from simple r join simple s using (id); $$); +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); rollback to settings; -- The "bad" case: during execution we need to increase number of @@ -438,15 +440,24 @@ explain (costs off) select count(*) from simple r full outer join simple s using (id); rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); select count(*) from simple r full outer join simple s using (id); rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); +select count(*) from simple r full outer join simple s using (id); +rollback to settings; + +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; @@ -456,14 +467,24 @@ explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); +rollback to settings; + + -- exercise special code paths for huge tuples (note use of non-strict -- expression and left join required to get the detoasted tuple into -- the hash table) From 992e34a56e89d41ea8d4a916e3e9a5a6f005e31c Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 14 Apr 2023 10:52:58 +1200 Subject: [PATCH 028/128] Fix PHJ match bit initialization. Hash join tuples reuse the HOT status bit to indicate match status during hash join execution. Correct reuse requires clearing the bit in all tuples. Serial hash join and parallel multi-batch hash join do so upon inserting the tuple into the hashtable. Single batch parallel hash join and batch 0 of unexpected multi-batch hash joins forgot to do this. It hadn't come up before because hashtable tuple match bits are only used for right and full outer joins and parallel ROJ and FOJ were unsupported. 11c2d6fdf5 introduced support for parallel ROJ/FOJ but neglected to ensure the match bits were reset. Author: Melanie Plageman Reported-by: Richard Guo Discussion: https://postgr.es/m/flat/CAMbWs48Nde1Mv%3DBJv6_vXmRKHMuHZm2Q_g4F6Z3_pn%2B3EV6BGQ%40mail.gmail.com --- src/backend/executor/nodeHash.c | 1 + src/test/regress/expected/join_hash.out | 37 +++++++++++++++++++++++++ src/test/regress/sql/join_hash.sql | 27 ++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index e3e1d5a6d87..d4a4fbc84f2 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2011,6 +2011,7 @@ ExecParallelHashTableInsert(HashJoinTable hashtable, /* Store the hash value in the HashJoinTuple header. */ hashTuple->hashvalue = hashvalue; memcpy(HJTUPLE_MINTUPLE(hashTuple), tuple, tuple->t_len); + HeapTupleHeaderClearMatch(HJTUPLE_MINTUPLE(hashTuple)); /* Push it onto the front of the bucket's list */ ExecParallelHashPushTuple(&hashtable->buckets.shared[bucketno], diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 250704efbd7..b1f780ff7b8 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -1031,6 +1031,43 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b (7 rows) rollback to settings; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +SAVEPOINT settings; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +CREATE TABLE hjtest_matchbits_t2(id int); +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +ROLLBACK TO settings; rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 01961d1ce6e..0858e14040f 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -539,6 +539,33 @@ rollback to settings; rollback; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +SAVEPOINT settings; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +CREATE TABLE hjtest_matchbits_t2(id int); +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +ROLLBACK TO settings; + +rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's From d12bef2b226c1c5c040bf3c60acce52cdaca2f33 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 12 Jun 2023 12:19:46 +0900 Subject: [PATCH 029/128] Fix instability in regression test for Parallel Hash Full Join As reported by buildfarm member conchuela, one of the regression tests added by 558c9d7 is having some ordering issues. This commit adds an ORDER BY clause to make the output more stable for the problematic query. Fix suggested by Tom Lane. The plan of the query updated still uses a parallel hash full join. Author: Melanie Plageman Discussion: https://postgr.es/m/623596.1684541098@sss.pgh.pa.us --- src/test/regress/expected/join_hash.out | 3 ++- src/test/regress/sql/join_hash.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index b1f780ff7b8..28f4558ec04 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -1047,7 +1047,8 @@ INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't -- correctly emit the NULL-extended unmatching tuple in full hash join. UPDATE hjtest_matchbits_t2 set id = 2; -SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; id | id ----+---- 1 | diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 0858e14040f..0115489a6b9 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -555,7 +555,8 @@ INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't -- correctly emit the NULL-extended unmatching tuple in full hash join. UPDATE hjtest_matchbits_t2 set id = 2; -SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; -- Test serial full hash join. -- Resetting parallel_setup_cost should force a serial plan. -- Just to be safe, however, set enable_parallel_hash to off, as parallel full From 337ef46086b7a4821d9520889133b1c74213e631 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 24 Mar 2026 13:00:49 +0800 Subject: [PATCH 030/128] Parallel Hash Full Join and Right Join PostgreSQL originally excluded FULL and RIGHT outer joins from parallel hash join because of deadlock hazards in the per-batch barrier protocol. PG 14 resolved this by introducing a dedicated PHJ_BATCH_SCAN phase: one elected worker emits unmatched inner-side rows after probing, while the others detach and move on. In CBDB, distributed execution adds a second dimension: after a full outer join the unmatched NULL-filled rows may come from any segment, so the result carries a HashedOJ locus rather than a plain Hashed locus. This change teaches the parallel planner about that: - FULL JOIN and RIGHT JOIN are now valid parallel join types in the distributed planner. Previously they were unconditionally rejected, forcing serial execution across all segments. - The HashedOJ locus produced by a parallel full join now carries parallel_workers, so operators above the join (aggregates, further joins) can remain parallel. - A crash that could occur when a parallel LASJ_NOTIN (NOT IN) join encountered NULL inner keys is fixed. The worker would exit early but the batch barrier, which was never attached to, would be touched on shutdown causing an assertion failure. Example plans (3 segments, parallel_workers=2): -- FULL JOIN: result locus is HashedOJ with Parallel Workers: 2 EXPLAIN(costs off, locus) SELECT count(*) FROM t1 FULL JOIN t2 USING (id); Finalize Aggregate Locus: Entry -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate Locus: HashedOJ Parallel Workers: 2 -> Parallel Hash Full Join Locus: HashedOJ Parallel Workers: 2 Hash Cond: (t1.id = t2.id) -> Parallel Seq Scan on t1 Locus: HashedWorkers -> Parallel Hash -> Parallel Seq Scan on t2 Locus: HashedWorkers -- RIGHT JOIN: when t1 is larger the planner hashes the smaller t2 -- and probes with t1; result locus HashedWorkers EXPLAIN(costs off, locus) SELECT count(*) FROM t1 RIGHT JOIN t2 USING (id); Finalize Aggregate Locus: Entry -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate Locus: HashedWorkers Parallel Workers: 2 -> Parallel Hash Right Join Locus: HashedWorkers Parallel Workers: 2 Hash Cond: (t1.id = t2.id) -> Parallel Seq Scan on t1 Locus: HashedWorkers -> Parallel Hash -> Parallel Seq Scan on t2 Locus: HashedWorkers Performance (3 segments x 2 parallel workers, 6M rows each, 50% overlap): FULL JOIN parallel: 4040 ms serial: 6347 ms speedup: 1.57x RIGHT JOIN parallel: 3039 ms serial: 5568 ms speedup: 1.83x --- src/backend/cdb/cdbpath.c | 5 +++-- src/backend/cdb/cdbpathlocus.c | 28 +++++++++++++++++++++------- src/backend/executor/nodeHash.c | 19 ++++++++++--------- src/backend/executor/nodeHashjoin.c | 6 +++--- src/include/cdb/cdbpathlocus.h | 4 ++-- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/backend/cdb/cdbpath.c b/src/backend/cdb/cdbpath.c index 9e3697a3b03..e9d7dac9895 100644 --- a/src/backend/cdb/cdbpath.c +++ b/src/backend/cdb/cdbpath.c @@ -3112,8 +3112,9 @@ cdbpath_motion_for_parallel_join(PlannerInfo *root, case JOIN_UNIQUE_INNER: case JOIN_RIGHT: case JOIN_FULL: - /* Join types are not supported in parallel yet. */ - goto fail; + outer.ok_to_replicate = false; + inner.ok_to_replicate = false; + break; case JOIN_DEDUP_SEMI: if (!enable_parallel_dedup_semi_join) goto fail; diff --git a/src/backend/cdb/cdbpathlocus.c b/src/backend/cdb/cdbpathlocus.c index 29930085429..dddae1aa64c 100644 --- a/src/backend/cdb/cdbpathlocus.c +++ b/src/backend/cdb/cdbpathlocus.c @@ -119,6 +119,11 @@ cdbpathlocus_equal(CdbPathLocus a, CdbPathLocus b) list_length(a.distkey) != list_length(b.distkey)) return false; + /* + * CBDB_PARALLEL: What if both a and b are HashedOJ with parallel workers > 0 ? + * Are they equal in practice? + */ + if ((CdbPathLocus_IsHashed(a) || CdbPathLocus_IsHashedOJ(a)) && (CdbPathLocus_IsHashed(b) || CdbPathLocus_IsHashedOJ(b))) return cdbpath_distkey_equal(a.distkey, b.distkey); @@ -544,7 +549,7 @@ cdbpathlocus_from_subquery(struct PlannerInfo *root, else { Assert(CdbPathLocus_IsHashedOJ(subpath->locus)); - CdbPathLocus_MakeHashedOJ(&locus, distkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&locus, distkeys, numsegments, subpath->locus.parallel_workers); } } else @@ -711,7 +716,7 @@ cdbpathlocus_pull_above_projection(struct PlannerInfo *root, CdbPathLocus_MakeHashedWorkers(&newlocus, newdistkeys, numsegments, locus.parallel_workers); } else - CdbPathLocus_MakeHashedOJ(&newlocus, newdistkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&newlocus, newdistkeys, numsegments, locus.parallel_workers); return newlocus; } else @@ -880,7 +885,7 @@ cdbpathlocus_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b) newdistkeys = lappend(newdistkeys, newdistkey); } - CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments, 0 /* Both are 0 parallel here*/); } Assert(cdbpathlocus_is_valid(resultlocus)); return resultlocus; @@ -1236,8 +1241,14 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo Assert(cdbpathlocus_is_valid(a)); Assert(cdbpathlocus_is_valid(b)); - /* Do both input rels have same locus? */ - if (cdbpathlocus_equal(a, b)) + /* + * Do both input rels have same locus? + * CBDB_PARALLEL: for FULL JOIN, it could be different even both + * are same loucs. Because the NULL values could be on any segments + * after join. + */ + + if (jointype != JOIN_FULL && cdbpathlocus_equal(a, b)) return a; /* @@ -1412,8 +1423,9 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo * If inner is hashed workers, and outer is hashed. Join locus will be hashed. * If outer is hashed workers, and inner is hashed. Join locus will be hashed workers. * Seems we should just return outer locus anyway. + * Things changed since we have parallel full join now. */ - if (parallel_aware) + if (parallel_aware && jointype != JOIN_FULL) return a; numsegments = CdbPathLocus_NumSegments(a); @@ -1469,7 +1481,9 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo newdistkeys = lappend(newdistkeys, newdistkey); } - CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments); + Assert(CdbPathLocus_NumParallelWorkers(a) == CdbPathLocus_NumParallelWorkers(b)); + + CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments, CdbPathLocus_NumParallelWorkers(a)); } Assert(cdbpathlocus_is_valid(resultlocus)); return resultlocus; diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index d4a4fbc84f2..e59a7c7ccc3 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2408,11 +2408,11 @@ ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate) int curbatch = hashtable->curbatch; ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE); + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING); /* * It would not be deadlock-free to wait on the batch barrier, because it - * is in PHJ_BATCH_PROBE phase, and thus processes attached to it have + * is in PHJ_BATCH_PROBING phase, and thus processes attached to it have * already emitted tuples. Therefore, we'll hold a wait-free election: * only one process can continue to the next phase, and all others detach * from this batch. They can still go any work on other batches, if there @@ -3979,12 +3979,12 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); - /* After attaching we always get at least to PHJ_BATCH_PROBE. */ - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE || + /* After attaching we always get at least to PHJ_BATCH_PROBING. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING || BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); /* - * If we're abandoning the PHJ_BATCH_PROBE phase early without having + * If we're abandoning the PHJ_BATCH_PROBING phase early without having * reached the end of it, it means the plan doesn't want any more * tuples, and it is happy to abandon any tuples buffered in this * process's subplans. For correctness, we can't allow any process to @@ -3999,13 +3999,13 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) * If phs_lasj_has_null is true, that means we have found null when building hash table, * there were no batches to detach. */ - if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING && !hashtable->parallel_state->phs_lasj_has_null && /* CBDB_PARALLEL */ !hashtable->batches[curbatch].outer_eof) { /* * This flag may be written to by multiple backends during - * PHJ_BATCH_PROBE phase, but will only be read in PHJ_BATCH_SCAN + * PHJ_BATCH_PROBING phase, but will only be read in PHJ_BATCH_SCAN * phase so requires no extra locking. */ batch->skip_unmatched = true; @@ -4016,10 +4016,11 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) * the PHJ_BATCH_SCAN phase just to maintain the invariant that * freeing happens in PHJ_BATCH_FREE, but that'll be wait-free. */ - if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING && !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */) attached = BarrierArriveAndDetachExceptLast(&batch->batch_barrier); - if (attached && BarrierArriveAndDetach(&batch->batch_barrier)) + if (attached && !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */ && + BarrierArriveAndDetach(&batch->batch_barrier)) { /* * We are not longer attached to the batch barrier, but we're the diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 53bb29be7af..9981ed8f7ae 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -84,7 +84,7 @@ * PHJ_BATCH_ELECT -- initial state * PHJ_BATCH_ALLOCATE* -- one allocates buckets * PHJ_BATCH_LOAD -- all load the hash table from disk - * PHJ_BATCH_PROBE -- all probe + * PHJ_BATCH_PROBING -- all probe * PHJ_BATCH_SCAN* -- one does full/right unmatched scan * PHJ_BATCH_FREE* -- one frees memory * @@ -103,7 +103,7 @@ * to a barrier, unless the barrier has reached a phase that means that no * process will wait on it again. We emit tuples while attached to the build * barrier in phase PHJ_BUILD_RUN, and to a per-batch barrier in phase - * PHJ_BATCH_PROBE. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN + * PHJ_BATCH_PROBING. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN * respectively without waiting, using BarrierArriveAndDetach() and * BarrierArriveAndDetachExceptLast() respectively. The last to detach * receives a different return value so that it knows that it's safe to @@ -1566,7 +1566,7 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * since that phase is already underway (the thing we * can't do under current deadlock-avoidance rules is wait * for others to arrive at PHJ_BATCH_SCAN, because - * PHJ_BATCH_PROBE emits tuples, but in this case we just + * PHJ_BATCH_PROBING emits tuples, but in this case we just * got here without waiting). That is not yet done. For * now, we just detach and go around again. We have to * use ExecHashTableDetachBatch() because there's a small diff --git a/src/include/cdb/cdbpathlocus.h b/src/include/cdb/cdbpathlocus.h index 0f71ba55dfb..9f5a8227e68 100644 --- a/src/include/cdb/cdbpathlocus.h +++ b/src/include/cdb/cdbpathlocus.h @@ -292,13 +292,13 @@ typedef struct CdbPathLocus _locus->parallel_workers = (parallel_workers_); \ Assert(cdbpathlocus_is_valid(*_locus)); \ } while (0) -#define CdbPathLocus_MakeHashedOJ(plocus, distkey_, numsegments_) \ +#define CdbPathLocus_MakeHashedOJ(plocus, distkey_, numsegments_, parallel_workers_) \ do { \ CdbPathLocus *_locus = (plocus); \ _locus->locustype = CdbLocusType_HashedOJ; \ _locus->numsegments = (numsegments_); \ _locus->distkey = (distkey_); \ - _locus->parallel_workers = 0; \ + _locus->parallel_workers = (parallel_workers_); \ Assert(cdbpathlocus_is_valid(*_locus)); \ } while (0) #define CdbPathLocus_MakeHashedWorkers(plocus, distkey_, numsegments_, parallel_workers_) \ From 1259af580eab9e34f39bca298483168b1444e8a3 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 24 Mar 2026 16:33:27 +0800 Subject: [PATCH 031/128] tests: add Parallel Hash Full/Right Join regression cases cbdb_parallel.sql: add a new test block covering: - Parallel Hash Full Join (HashedWorkers FULL JOIN HashedWorkers produces HashedOJ with parallel_workers=2) - Parallel Hash Right Join (pj_t1 is 3x larger than pj_t2, so the planner hashes the smaller pj_t2 and probes with pj_t1; result locus HashedWorkers) - Correctness checks: count(*) matches serial execution - Locus propagation: HashedOJ(parallel) followed by INNER JOIN produces HashedOJ; followed by FULL JOIN produces HashedOJ join_hash.sql/out: CBDB-specific adaptations for the upstream parallel full join test -- disable parallel mode for tests that require serial plans, fix SAVEPOINT inside a parallel worker context, and update expected output to match CBDB plan shapes. --- .../pax_storage/expected/cbdb_parallel.out | 183 ++++++--- src/test/regress/expected/cbdb_parallel.out | 370 +++++++++++++----- src/test/regress/expected/join_hash.out | 83 ++-- .../regress/expected/join_hash_optimizer.out | 204 +++++++--- src/test/regress/sql/cbdb_parallel.sql | 50 +++ src/test/regress/sql/join_hash.sql | 6 + 6 files changed, 668 insertions(+), 228 deletions(-) diff --git a/contrib/pax_storage/expected/cbdb_parallel.out b/contrib/pax_storage/expected/cbdb_parallel.out index db583090026..ec6ceba7e3c 100644 --- a/contrib/pax_storage/expected/cbdb_parallel.out +++ b/contrib/pax_storage/expected/cbdb_parallel.out @@ -41,13 +41,29 @@ set gp_appendonly_insert_files = 4; begin; set local enable_parallel = on; create table test_131_ao1(x int, y int) using ao_row with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao2(x int, y int) using ao_row with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao3(x int, y int) using ao_row with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao4(x int, y int) using ao_row with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco1(x int, y int) using ao_column with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco2(x int, y int) using ao_column with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco3(x int, y int) using ao_column with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco4(x int, y int) using ao_column with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. select relname, reloptions from pg_catalog.pg_class where relname like 'test_131_ao%'; relname | reloptions ----------------+---------------------- @@ -155,8 +171,14 @@ explain(locus, costs off) select count(*) from test_131_aoco3, test_131_aoco4 wh abort; create table ao1(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table ao2(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table aocs1(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. begin; -- encourage use of parallel plans set local min_parallel_table_scan_size = 0; @@ -367,6 +389,8 @@ abort; begin; set local max_parallel_workers_per_gather = 2; create table t1(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table rt1(a int, b int) with(parallel_workers=2) distributed replicated; create table rt2(a int, b int) distributed replicated; create table rt3(a int, b int) distributed replicated; @@ -599,6 +623,8 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -606,8 +632,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 6 | 7 | 5 | 6 | 6 | 7 7 | 8 | 6 | 7 | 7 | 8 10 | 11 | 9 | 10 | 10 | 11 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 (19 rows) -- parallel hash join @@ -650,13 +674,6 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 5 | 6 | 6 | 7 - 6 | 7 | 6 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 9 | 10 | 10 | 11 - 10 | 11 | 10 | 10 | 10 | 11 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 2 | 3 | 3 | 4 3 | 4 | 3 | 3 | 3 | 4 @@ -669,6 +686,13 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 9 | 10 | 8 | 9 | 9 | 10 1 | 2 | 1 | 1 | 1 | 2 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 5 | 6 | 6 | 7 + 6 | 7 | 6 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 9 | 10 | 10 | 11 + 10 | 11 | 10 | 10 | 10 | 11 (19 rows) -- @@ -702,6 +726,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt3 on select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -712,8 +738,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -779,6 +803,8 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; (19 rows) create table t2(a int, b int) with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table rt4(a int, b int) with(parallel_workers=2) distributed replicated; insert into t2 select i, i+1 from generate_series(1, 10) i; insert into rt4 select i, i+1 from generate_series(1, 10000) i; @@ -788,16 +814,16 @@ set local enable_parallel = off; select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 - 6 | 5 | 5 - 7 | 6 | 6 - 10 | 9 | 9 - 11 | 10 | 10 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 + 2 | 1 | 1 + 6 | 5 | 5 + 7 | 6 | 6 + 10 | 9 | 9 + 11 | 10 | 10 (10 rows) set local enable_parallel = on; @@ -828,19 +854,21 @@ explain(locus, costs off) select * from rt4 join t2 using(b); select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 + 6 | 5 | 5 + 7 | 6 | 6 + 10 | 9 | 9 + 11 | 10 | 10 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 - 6 | 5 | 5 - 7 | 6 | 6 - 10 | 9 | 9 - 11 | 10 | 10 + 2 | 1 | 1 (10 rows) create table t3(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t3 select i, i+1 from generate_series(1, 9000) i; analyze t3; set local enable_parallel = off; @@ -919,10 +947,10 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 2 | 3 | 3 4 | 5 | 5 + 3 | 4 | 4 5 | 6 | 6 (5 rows) @@ -931,11 +959,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 3 | 4 | 4 - 1 | 2 | 2 - 4 | 5 | 5 5 | 6 | 6 + 4 | 5 | 5 + 1 | 2 | 2 + 2 | 3 | 3 (5 rows) abort; @@ -976,11 +1004,11 @@ explain(locus, costs off) select * from t_replica_workers_2 right join t_random_ select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- - 5 | 6 | 6 1 | 2 | 2 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 + 5 | 6 | 6 (5 rows) -- non parallel results @@ -1028,14 +1056,14 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker Locus: Strewn Parallel Workers: 2 Optimizer: Postgres query optimizer -(16 rows) +(15 rows) select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1045,9 +1073,9 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1059,7 +1087,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(b int, a int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10) i; insert into t2 select i, i+1 from generate_series(1, 5) i; analyze t1; @@ -1071,17 +1103,17 @@ explain(costs off) select * from t1 right join t2 on t1.b = t2.a; QUERY PLAN ------------------------------------------------------------------ Gather Motion 9:1 (slice1; segments: 9) - -> Parallel Hash Left Join - Hash Cond: (t2.a = t1.b) - -> Redistribute Motion 6:9 (slice2; segments: 6) - Hash Key: t2.a + -> Parallel Hash Right Join + Hash Cond: (t1.b = t2.a) + -> Redistribute Motion 9:9 (slice2; segments: 9) + Hash Key: t1.b Hash Module: 3 - -> Parallel Seq Scan on t2 + -> Parallel Seq Scan on t1 -> Parallel Hash - -> Redistribute Motion 9:9 (slice3; segments: 9) - Hash Key: t1.b + -> Redistribute Motion 6:9 (slice3; segments: 6) + Hash Key: t2.a Hash Module: 3 - -> Parallel Seq Scan on t1 + -> Parallel Seq Scan on t2 Optimizer: Postgres query optimizer (13 rows) @@ -1091,7 +1123,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i%10, i from generate_series(1, 5) i; insert into t1 values (100000); insert into t2 select i%10, i from generate_series(1, 100000) i; @@ -1100,34 +1136,34 @@ analyze t2; set local enable_parallel = on; -- parallel hash join with shared table, SinglQE as outer partial path. explain(locus, costs off) select * from (select count(*) as a from t2) t2 left join t1 on t1.a = t2.a; - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Gather Motion 6:1 (slice1; segments: 6) Locus: Entry - -> Parallel Hash Left Join - Locus: Hashed + -> Parallel Hash Right Join + Locus: HashedWorkers Parallel Workers: 2 - Hash Cond: ((count(*)) = t1.a) - -> Redistribute Motion 1:6 (slice2; segments: 1) - Locus: Hashed + Hash Cond: (t1.a = (count(*))) + -> Parallel Seq Scan on t1 + Locus: HashedWorkers Parallel Workers: 2 - Hash Key: (count(*)) - Hash Module: 3 - -> Finalize Aggregate - Locus: SingleQE - -> Gather Motion 6:1 (slice3; segments: 6) - Locus: SingleQE - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t2 - Locus: HashedWorkers - Parallel Workers: 2 -> Parallel Hash Locus: Hashed - -> Parallel Seq Scan on t1 - Locus: HashedWorkers + -> Redistribute Motion 1:6 (slice2; segments: 1) + Locus: Hashed Parallel Workers: 2 + Hash Key: (count(*)) + Hash Module: 3 + -> Finalize Aggregate + Locus: SingleQE + -> Gather Motion 6:1 (slice3; segments: 6) + Locus: SingleQE + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t2 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (27 rows) @@ -1323,12 +1359,18 @@ begin; create table rt1(a int, b int) distributed replicated; create table rt2(a int, b int) with (parallel_workers = 0) distributed replicated; create table t1(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(a int, b int) with (parallel_workers = 0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10000) i; insert into t2 select i, i+1 from generate_series(1, 10000) i; insert into rt1 select i, i+1 from generate_series(1, 10000) i; insert into rt2 select i, i+1 from generate_series(1, 10000) i; CREATE TABLE sq1 AS SELECT a, b FROM t1 WHERE gp_segment_id = 0; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. set local optimizer=off; set local enable_parallel=on; set local min_parallel_table_scan_size to 0; @@ -1385,7 +1427,7 @@ explain (locus, costs off) select * from rt1 union all select * from t1; -> Result Locus: Strewn Parallel Workers: 2 - One-Time Filter: (gp_execution_segment() = 1) + One-Time Filter: (gp_execution_segment() = 0) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 2 @@ -1409,7 +1451,7 @@ explain (locus, costs off) select * from rt1 union all select * from t2; -> Result Locus: Strewn Parallel Workers: 2 - One-Time Filter: (gp_execution_segment() = 1) + One-Time Filter: (gp_execution_segment() = 0) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 2 @@ -1482,6 +1524,8 @@ abort; -- begin; create table t1(c1 int, c2 int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 100000) i; analyze t1; set local optimizer = off; @@ -1549,6 +1593,8 @@ abort; -- begin; create table t1(c1 int, c2 int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 100000) i; analyze t1; set local optimizer = off; @@ -1768,6 +1814,8 @@ set local optimizer = off; set local enable_parallel = on; -- ao table create table ao (a INT, b INT) using ao_row; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into ao select i as a, i as b from generate_series(1, 100) AS i; alter table ao set (parallel_workers = 2); explain(costs off) select count(*) from ao; @@ -1789,6 +1837,8 @@ select count(*) from ao; alter table ao reset (parallel_workers); -- aocs table create table aocs (a INT, b INT) using ao_column; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into aocs select i as a, i as b from generate_series(1, 100) AS i; alter table aocs set (parallel_workers = 2); explain(costs off) select count(*) from aocs; @@ -1862,9 +1912,14 @@ select * from abort; begin; create table pagg_tab (a int, b int, c text, d int) partition by list(c); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table pagg_tab_p1 partition of pagg_tab for values in ('0000', '0001', '0002', '0003', '0004'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p2 partition of pagg_tab for values in ('0005', '0006', '0007', '0008'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p3 partition of pagg_tab for values in ('0009', '0010', '0011'); +NOTICE: table has parent, setting distribution columns to match parent table insert into pagg_tab select i % 20, i % 30, to_char(i % 12, 'FM0000'), i % 30 from generate_series(0, 2999) i; analyze pagg_tab; set local enable_parallel to off; @@ -1939,7 +1994,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(b int, a int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10) i; insert into t2 select i, i+1 from generate_series(1, 5) i; analyze t1; @@ -2329,6 +2388,8 @@ abort; -- prepare, execute locus is null begin; create table t1(c1 int, c2 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze t1; prepare t1_count(integer) as select count(*) from t1; explain(locus, costs off) execute t1_count(1); diff --git a/src/test/regress/expected/cbdb_parallel.out b/src/test/regress/expected/cbdb_parallel.out index 35e90eebfa1..af975de50f4 100644 --- a/src/test/regress/expected/cbdb_parallel.out +++ b/src/test/regress/expected/cbdb_parallel.out @@ -112,8 +112,8 @@ set local enable_parallel_dedup_semi_reverse_join = on; set local enable_parallel_dedup_semi_join = on; explain (costs off) select sum(foo.a) from foo where exists (select 1 from bar where foo.a = bar.b); - QUERY PLAN ------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate @@ -1032,6 +1032,15 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 6 | 6 | 6 | 7 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 10 | 10 | 10 | 11 + 6 | 7 | 5 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 10 | 11 | 9 | 10 | 10 | 11 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1042,15 +1051,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 6 | 6 | 6 | 7 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 10 | 10 | 10 | 11 - 6 | 7 | 5 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 10 | 11 | 9 | 10 | 10 | 11 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 (19 rows) -- parallel hash join @@ -1093,13 +1093,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 5 | 6 | 6 | 7 - 6 | 7 | 6 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 9 | 10 | 10 | 11 - 10 | 11 | 10 | 10 | 10 | 11 + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 2 | 3 | 3 | 4 3 | 4 | 3 | 3 | 3 | 4 @@ -1110,8 +1105,13 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 8 | 9 | 7 | 8 | 8 | 9 8 | 9 | 8 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 5 | 6 | 6 | 7 + 6 | 7 | 6 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 9 | 10 | 10 | 11 + 10 | 11 | 10 | 10 | 10 | 11 (19 rows) -- @@ -1145,6 +1145,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt3 on select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1155,8 +1157,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -1201,14 +1201,11 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 10 | 11 | 10 | 10 | 10 | 11 - 6 | 7 | 5 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 10 | 11 | 9 | 10 | 10 | 11 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1219,6 +1216,9 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 + 6 | 7 | 5 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 10 | 11 | 9 | 10 | 10 | 11 (19 rows) create table t2(a int, b int) with(parallel_workers=0); @@ -1271,12 +1271,12 @@ explain(locus, costs off) select * from rt4 join t2 using(b); select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 + 2 | 1 | 1 6 | 5 | 5 7 | 6 | 6 10 | 9 | 9 @@ -1362,9 +1362,9 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1374,9 +1374,9 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1419,9 +1419,9 @@ explain(locus, costs off) select * from t_replica_workers_2 right join t_random_ select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- + 2 | 3 | 3 5 | 6 | 6 1 | 2 | 2 - 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 (5 rows) @@ -1431,11 +1431,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- + 5 | 6 | 6 1 | 2 | 2 - 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 - 5 | 6 | 6 + 2 | 3 | 3 (5 rows) abort; @@ -1471,13 +1471,13 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker Locus: Strewn Parallel Workers: 2 Optimizer: Postgres query optimizer -(16 rows) +(15 rows) select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 + 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 5 | 6 | 6 @@ -1488,11 +1488,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 1 | 2 | 2 3 | 4 | 4 4 | 5 | 5 5 | 6 | 6 + 1 | 2 | 2 + 2 | 3 | 3 (5 rows) abort; @@ -1510,28 +1510,28 @@ analyze t1; analyze rt1; set local enable_parallel = on; explain(locus, costs off) select * from (select count(*) as a from t1) t1 left join rt1 on rt1.a = t1.a; - QUERY PLAN ------------------------------------------------------- - Parallel Hash Left Join + QUERY PLAN +------------------------------------------------------------ + Parallel Hash Right Join Locus: Entry - Hash Cond: ((count(*)) = rt1.a) - -> Finalize Aggregate + Hash Cond: (rt1.a = (count(*))) + -> Gather Motion 2:1 (slice1; segments: 2) Locus: Entry - -> Gather Motion 6:1 (slice1; segments: 6) - Locus: Entry - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t1 - Locus: HashedWorkers - Parallel Workers: 2 + -> Parallel Seq Scan on rt1 + Locus: SegmentGeneralWorkers + Parallel Workers: 2 -> Parallel Hash Locus: Entry - -> Gather Motion 2:1 (slice2; segments: 2) + -> Finalize Aggregate Locus: Entry - -> Parallel Seq Scan on rt1 - Locus: SegmentGeneralWorkers - Parallel Workers: 2 + -> Gather Motion 6:1 (slice2; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t1 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (21 rows) @@ -1661,17 +1661,17 @@ explain(costs off) select * from t1 right join t2 on t1.b = t2.a; QUERY PLAN ------------------------------------------------------------------ Gather Motion 9:1 (slice1; segments: 9) - -> Parallel Hash Left Join - Hash Cond: (t2.a = t1.b) - -> Redistribute Motion 6:9 (slice2; segments: 6) - Hash Key: t2.a + -> Parallel Hash Right Join + Hash Cond: (t1.b = t2.a) + -> Redistribute Motion 9:9 (slice2; segments: 9) + Hash Key: t1.b Hash Module: 3 - -> Parallel Seq Scan on t2 + -> Parallel Seq Scan on t1 -> Parallel Hash - -> Redistribute Motion 9:9 (slice3; segments: 9) - Hash Key: t1.b + -> Redistribute Motion 6:9 (slice3; segments: 6) + Hash Key: t2.a Hash Module: 3 - -> Parallel Seq Scan on t1 + -> Parallel Seq Scan on t2 Optimizer: Postgres query optimizer (13 rows) @@ -1690,34 +1690,34 @@ analyze t2; set local enable_parallel = on; -- parallel hash join with shared table, SinglQE as outer partial path. explain(locus, costs off) select * from (select count(*) as a from t2) t2 left join t1 on t1.a = t2.a; - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Gather Motion 6:1 (slice1; segments: 6) Locus: Entry - -> Parallel Hash Left Join - Locus: Hashed + -> Parallel Hash Right Join + Locus: HashedWorkers Parallel Workers: 2 - Hash Cond: ((count(*)) = t1.a) - -> Redistribute Motion 1:6 (slice2; segments: 1) - Locus: Hashed + Hash Cond: (t1.a = (count(*))) + -> Parallel Seq Scan on t1 + Locus: HashedWorkers Parallel Workers: 2 - Hash Key: (count(*)) - Hash Module: 3 - -> Finalize Aggregate - Locus: SingleQE - -> Gather Motion 6:1 (slice3; segments: 6) - Locus: SingleQE - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t2 - Locus: HashedWorkers - Parallel Workers: 2 -> Parallel Hash Locus: Hashed - -> Parallel Seq Scan on t1 - Locus: HashedWorkers + -> Redistribute Motion 1:6 (slice2; segments: 1) + Locus: Hashed Parallel Workers: 2 + Hash Key: (count(*)) + Hash Module: 3 + -> Finalize Aggregate + Locus: SingleQE + -> Gather Motion 6:1 (slice3; segments: 6) + Locus: SingleQE + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t2 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (27 rows) @@ -1975,7 +1975,7 @@ explain (locus, costs off) select * from rt1 union all select * from t1; -> Result Locus: Strewn Parallel Workers: 3 - One-Time Filter: (gp_execution_segment() = 0) + One-Time Filter: (gp_execution_segment() = 1) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 3 @@ -1999,7 +1999,7 @@ explain (locus, costs off) select * from rt1 union all select * from t2; -> Result Locus: Strewn Parallel Workers: 3 - One-Time Filter: (gp_execution_segment() = 0) + One-Time Filter: (gp_execution_segment() = 1) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 3 @@ -2296,8 +2296,8 @@ analyze t1; analyze t2; analyze t3_null; explain(costs off) select sum(t1.c1) from t1 where c1 not in (select c2 from t2); - QUERY PLAN ------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate @@ -2317,8 +2317,8 @@ select sum(t1.c1) from t1 where c1 not in (select c2 from t2); (1 row) explain(costs off) select * from t1 where c1 not in (select c2 from t3_null); - QUERY PLAN ------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------- Gather Motion 6:1 (slice1; segments: 6) -> Parallel Hash Left Anti Semi (Not-In) Join Hash Cond: (t1.c1 = t3_null.c2) @@ -2457,8 +2457,11 @@ abort; begin; create table pagg_tab (a int, b int, c text, d int) partition by list(c); create table pagg_tab_p1 partition of pagg_tab for values in ('0000', '0001', '0002', '0003', '0004'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p2 partition of pagg_tab for values in ('0005', '0006', '0007', '0008'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p3 partition of pagg_tab for values in ('0009', '0010', '0011'); +NOTICE: table has parent, setting distribution columns to match parent table insert into pagg_tab select i % 20, i % 30, to_char(i % 12, 'FM0000'), i % 30 from generate_series(0, 2999) i; analyze pagg_tab; set local enable_parallel to off; @@ -2972,7 +2975,7 @@ create table t2_anti(a int, b int) with(parallel_workers=2) distributed by (b); insert into t2_anti values(generate_series(5, 10)); explain(costs off, verbose) select t1_anti.a, t1_anti.b from t1_anti left join t2_anti on t1_anti.a = t2_anti.a where t2_anti.a is null; - QUERY PLAN + QUERY PLAN ------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) Output: t1_anti.a, t1_anti.b @@ -3068,8 +3071,8 @@ select t1_anti.a, t1_anti.b from t1_anti left join t2_anti on t1_anti.a = t2_ant ---+--- 3 | 4 | - 1 | 2 | + 1 | (4 rows) abort; @@ -3098,7 +3101,7 @@ insert into t_distinct_0 select * from t_distinct_0; analyze t_distinct_0; explain(costs off) select distinct a from t_distinct_0; - QUERY PLAN + QUERY PLAN ------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) -> HashAggregate @@ -3232,8 +3235,6 @@ select distinct a, b from t_distinct_0; drop table if exists t_distinct_1; NOTICE: table "t_distinct_1" does not exist, skipping create table t_distinct_1(a int, b int) using ao_column; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t_distinct_1 select * from t_distinct_0; analyze t_distinct_1; set enable_parallel = off; @@ -3520,10 +3521,7 @@ WHERE e.salary > ( -- Test https://github.com/apache/cloudberry/issues/1376 -- create table t1(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2 (like t1); -NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table set gp_cte_sharing = on; explain(locus, costs off) with x as (select a, count(*) as b from t1 group by a union all @@ -3571,8 +3569,184 @@ explain(locus, costs off) with x as reset gp_cte_sharing; reset enable_parallel; reset min_parallel_table_scan_size; +-- +-- Parallel Hash Full/Right Join +-- +begin; +create table pj_t1(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t2(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t3(id int, v int) with(parallel_workers=0) distributed by (id); +-- pj_t1 is 3x larger than pj_t2 so the planner hashes the smaller pj_t2 +-- and probes with pj_t1, producing a genuine Parallel Hash Right Join plan. +insert into pj_t1 select i, i from generate_series(1,30000)i; +insert into pj_t2 select i, i from generate_series(25001,35000)i; +insert into pj_t3 select i, i from generate_series(1,10000)i; +analyze pj_t1; +analyze pj_t2; +analyze pj_t3; +set local enable_parallel = on; +set local min_parallel_table_scan_size = 0; +-- 12_P_12_10: Parallel Hash Full Join: HashedWorkers FULL JOIN HashedWorkers -> HashedOJ(parallel) +explain(costs off, locus) +select count(*) from pj_t1 full join pj_t2 using (id); + QUERY PLAN +---------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 6:1 (slice1; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + Parallel Workers: 2 + -> Parallel Hash Full Join + Locus: HashedOJ + Parallel Workers: 2 + Hash Cond: (pj_t1.id = pj_t2.id) + -> Parallel Seq Scan on pj_t1 + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash + Locus: Hashed + -> Parallel Seq Scan on pj_t2 + Locus: HashedWorkers + Parallel Workers: 2 + Optimizer: Postgres query optimizer +(20 rows) + +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 full join pj_t2 using (id); + count +------- + 35000 +(1 row) + +set local enable_parallel = on; +select count(*) from pj_t1 full join pj_t2 using (id); + count +------- + 35000 +(1 row) + +-- Parallel Hash Right Join: pj_t1 (30K) is larger, so the planner hashes the smaller pj_t2 +-- (10K) as the build side and probes with pj_t1; result locus HashedWorkers(parallel) +explain(costs off, locus) +select count(*) from pj_t1 right join pj_t2 using (id); + QUERY PLAN +---------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 6:1 (slice1; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash Right Join + Locus: HashedWorkers + Parallel Workers: 2 + Hash Cond: (pj_t1.id = pj_t2.id) + -> Parallel Seq Scan on pj_t1 + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash + Locus: Hashed + -> Parallel Seq Scan on pj_t2 + Locus: HashedWorkers + Parallel Workers: 2 + Optimizer: Postgres query optimizer +(20 rows) + +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 right join pj_t2 using (id); + count +------- + 10000 +(1 row) + +set local enable_parallel = on; +select count(*) from pj_t1 right join pj_t2 using (id); + count +------- + 10000 +(1 row) + +-- Locus propagation: HashedOJ(parallel) followed by INNER JOIN with Hashed(serial) +-- The full join result (HashedOJ,parallel=2) is joined with pj_t3 (Hashed,serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj inner join pj_t3 using (id); + QUERY PLAN +--------------------------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 3:1 (slice1; segments: 3) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + -> Hash Join + Locus: HashedOJ + Hash Cond: (COALESCE(pj_t1.id, pj_t2.id) = pj_t3.id) + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (pj_t1.id = pj_t2.id) + -> Seq Scan on pj_t1 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t2 + Locus: Hashed + -> Hash + Locus: Replicated + -> Broadcast Motion 3:3 (slice2; segments: 3) + Locus: Replicated + -> Seq Scan on pj_t3 + Locus: Hashed + Optimizer: Postgres query optimizer +(25 rows) + +-- Locus propagation: HashedOJ(parallel) followed by FULL JOIN with Hashed(serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj full join pj_t3 using (id); + QUERY PLAN +-------------------------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 3:1 (slice1; segments: 3) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (COALESCE(pj_t1.id, pj_t2.id) = pj_t3.id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Locus: Hashed + Hash Key: COALESCE(pj_t1.id, pj_t2.id) + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (pj_t1.id = pj_t2.id) + -> Seq Scan on pj_t1 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t2 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t3 + Locus: Hashed + Optimizer: Postgres query optimizer +(26 rows) + +abort; -- start_ignore drop schema test_parallel cascade; +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table t_distinct_0 +drop cascades to table t_distinct_1 +drop cascades to table departments +drop cascades to table employees +drop cascades to table t1 +drop cascades to table t2 -- end_ignore reset gp_appendonly_insert_files; reset force_parallel_mode; diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 28f4558ec04..e5f74c18d28 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -10,6 +10,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or -- buckets) will be required because it can vary, but we can in some @@ -58,12 +61,16 @@ $$; -- estimated size. create table simple as select generate_series(1, 60000) AS id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table simple set (parallel_workers = 2); analyze simple; -- Make a relation whose size we will under-estimate. We want stats -- to say 1000 rows, but actually there are 20,000 rows. create table bigger_than_it_looks as select generate_series(1, 60000) as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table bigger_than_it_looks set (autovacuum_enabled = 'false'); WARNING: autovacuum is not supported in Cloudberry alter table bigger_than_it_looks set (parallel_workers = 2); @@ -73,6 +80,8 @@ update pg_class set reltuples = 1000 where relname = 'bigger_than_it_looks'; -- kind of skew that breaks our batching scheme. We want stats to say -- 2 rows, but actually there are 20,000 rows with the same key. create table extremely_skewed (id int, t text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table extremely_skewed set (autovacuum_enabled = 'false'); WARNING: autovacuum is not supported in Cloudberry alter table extremely_skewed set (parallel_workers = 2); @@ -85,6 +94,8 @@ update pg_class where relname = 'extremely_skewed'; -- Make a relation with a couple of enormous tuples. create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table wide set (parallel_workers = 2); ANALYZE wide; -- The "optimal" case: the hash table fits in memory; we plan for 1 @@ -319,7 +330,7 @@ $$); select count(*) from simple r full outer join simple s using (id); count ------- - 20000 + 60000 (1 row) rollback to settings; @@ -574,9 +585,13 @@ rollback to settings; -- Exercise rescans. We'll turn off parallel_leader_participation so -- that we can check that instrumentation comes back correctly. create table join_foo as select generate_series(1, 3) as id, 'xxxxx'::text as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze join_foo; alter table join_foo set (parallel_workers = 0); create table join_bar as select generate_series(1, 20000) as id, 'xxxxx'::text as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze join_bar; alter table join_bar set (parallel_workers = 2); -- multi-batch with rescan, parallel-oblivious @@ -854,23 +869,23 @@ savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); - QUERY PLAN -------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------- Finalize Aggregate - -> Gather - Workers Planned: 2 + -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate - -> Parallel Hash Full Join + -> Hash Full Join Hash Cond: (r.id = s.id) - -> Parallel Seq Scan on simple r - -> Parallel Hash - -> Parallel Seq Scan on simple s + -> Seq Scan on simple r + -> Hash + -> Seq Scan on simple s + Optimizer: Postgres query optimizer (9 rows) select count(*) from simple r full outer join simple s using (id); count ------- - 20000 + 60000 (1 row) rollback to settings; @@ -935,23 +950,25 @@ savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); - QUERY PLAN -------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Finalize Aggregate - -> Gather - Workers Planned: 2 + -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate - -> Parallel Hash Full Join + -> Hash Full Join Hash Cond: ((0 - s.id) = r.id) - -> Parallel Seq Scan on simple s - -> Parallel Hash - -> Parallel Seq Scan on simple r -(9 rows) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: (0 - s.id) + -> Seq Scan on simple s + -> Hash + -> Seq Scan on simple r + Optimizer: Postgres query optimizer +(11 rows) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); - count -------- - 40000 + count +-------- + 120000 (1 row) rollback to settings; @@ -1013,7 +1030,11 @@ rollback to settings; savepoint settings; set max_parallel_workers_per_gather = 0; create table join_hash_t_small(a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table join_hash_t_big(b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into join_hash_t_small select i%100 from generate_series(0, 3000)i; insert into join_hash_t_big select i%100000 from generate_series(1, 100000)i ; analyze join_hash_t_small; @@ -1031,17 +1052,25 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b (7 rows) rollback to settings; +rollback; -- Hash join reuses the HOT status bit to indicate match status. This can only -- be guaranteed to produce correct results if all the hash join tuple match -- bits are reset before reuse. This is done upon loading them into the -- hashtable. +begin; SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; SET enable_parallel_hash = on; SET min_parallel_table_scan_size = 0; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; CREATE TABLE hjtest_matchbits_t1(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE hjtest_matchbits_t2(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO hjtest_matchbits_t1 VALUES (1); INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't @@ -1064,8 +1093,8 @@ SET enable_parallel_hash = off; SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; id | id ----+---- - 1 | | 2 + 1 | (2 rows) ROLLBACK TO settings; @@ -1085,7 +1114,11 @@ BEGIN; SET LOCAL enable_sort = OFF; -- avoid mergejoins SET LOCAL from_collapse_limit = 1; -- allows easy changing of join order CREATE TABLE hjtest_1 (a text, b int, id int, c bool); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE hjtest_2 (a bool, id int, b text, c int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 2, 1, false); -- matches INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 1, 2, false); -- fails id join condition INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 20, 1, false); -- fails < 50 @@ -1142,8 +1175,8 @@ WHERE SubPlan 2 -> Result Output: (hjtest_1.b * 5) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'off' Optimizer: Postgres query optimizer - Settings: enable_sort=off, from_collapse_limit=1 (38 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 @@ -1206,8 +1239,8 @@ WHERE SubPlan 3 -> Result Output: (hjtest_2.c * 5) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'off' Optimizer: Postgres query optimizer - Settings: enable_sort=off, from_collapse_limit=1 (38 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 diff --git a/src/test/regress/expected/join_hash_optimizer.out b/src/test/regress/expected/join_hash_optimizer.out index 053d0ef4898..1835bfa4f31 100644 --- a/src/test/regress/expected/join_hash_optimizer.out +++ b/src/test/regress/expected/join_hash_optimizer.out @@ -10,6 +10,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or -- buckets) will be required because it can vary, but we can in some @@ -115,7 +118,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -156,7 +159,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -197,7 +200,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -241,7 +244,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -283,7 +286,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -325,7 +328,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -344,6 +347,13 @@ $$); t | f (1 row) +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); + count +------- + 60000 +(1 row) + rollback to settings; -- The "bad" case: during execution we need to increase number of -- batches; in this case we plan for 1 batch, and increase at least a @@ -356,8 +366,8 @@ set local work_mem = '128kB'; set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_mem explain (costs off) select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -367,8 +377,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); count @@ -395,8 +405,8 @@ set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_m set local enable_parallel_hash = off; explain (costs off) select count(*) from simple r join bigger_than_it_looks s using (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -406,8 +416,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) from simple r join bigger_than_it_looks s using (id); count @@ -434,8 +444,8 @@ set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_m set local enable_parallel_hash = on; explain (costs off) select count(*) from simple r join bigger_than_it_looks s using (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -445,8 +455,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) from simple r join bigger_than_it_looks s using (id); count @@ -490,7 +500,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -534,7 +544,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -578,7 +588,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -643,8 +653,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -662,7 +672,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -701,8 +711,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -720,7 +730,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -760,8 +770,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -779,7 +789,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -818,8 +828,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -837,7 +847,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -891,8 +901,9 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); @@ -920,7 +931,36 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); + QUERY PLAN +------------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Hash Full Join + Hash Cond: (r.id = s.id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: r.id + -> Seq Scan on simple r + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: s.id + -> Seq Scan on simple s + Optimizer: GPORCA +(13 rows) + +select count(*) from simple r full outer join simple s using (id); + count +------- + 60000 +(1 row) + +rollback to settings; +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; @@ -950,7 +990,37 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join +savepoint settings; +set enable_parallel_hash = off; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + QUERY PLAN +------------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Hash Full Join + Hash Cond: (r.id = (0 - s.id)) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: r.id + -> Seq Scan on simple r + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: (0 - s.id) + -> Seq Scan on simple s + Optimizer: GPORCA +(13 rows) + +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + count +-------- + 120000 +(1 row) + +rollback to settings; +-- parallelism is possible with parallel-aware full hash join savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) @@ -1012,7 +1082,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: wide_1.id -> Seq Scan on wide wide_1 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select length(max(s.t)) @@ -1060,11 +1130,57 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b -> Seq Scan on join_hash_t_big -> Hash -> Seq Scan on join_hash_t_small - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (7 rows) rollback to settings; rollback; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +begin; +SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE hjtest_matchbits_t2(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + | 2 + 1 | +(2 rows) + +ROLLBACK TO settings; +rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's -- need to reference Hash's outer plan (which is below HashJoin's @@ -1154,9 +1270,9 @@ WHERE Filter: (((hjtest_1.b * 5)) < 50) -> Result Output: (hjtest_1.b * 5) - Settings: enable_sort = 'off', from_collapse_limit = '1' - Optimizer: Pivotal Optimizer (GPORCA) -(49 rows) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'on' + Optimizer: GPORCA +(51 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 FROM hjtest_1, hjtest_2 @@ -1231,9 +1347,9 @@ WHERE Filter: (((hjtest_1.b * 5)) < 50) -> Result Output: (hjtest_1.b * 5) - Settings: enable_sort = 'off', from_collapse_limit = '1' - Optimizer: Pivotal Optimizer (GPORCA) -(49 rows) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'on' + Optimizer: GPORCA +(51 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 FROM hjtest_2, hjtest_1 diff --git a/src/test/regress/sql/cbdb_parallel.sql b/src/test/regress/sql/cbdb_parallel.sql index f9d01dd8a00..08e7aa198f9 100644 --- a/src/test/regress/sql/cbdb_parallel.sql +++ b/src/test/regress/sql/cbdb_parallel.sql @@ -1149,6 +1149,56 @@ reset gp_cte_sharing; reset enable_parallel; reset min_parallel_table_scan_size; +-- +-- Parallel Hash Full/Right Join +-- +begin; +create table pj_t1(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t2(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t3(id int, v int) with(parallel_workers=0) distributed by (id); + +-- pj_t1 is 3x larger than pj_t2 so the planner hashes the smaller pj_t2 +-- and probes with pj_t1, producing a genuine Parallel Hash Right Join plan. +insert into pj_t1 select i, i from generate_series(1,30000)i; +insert into pj_t2 select i, i from generate_series(25001,35000)i; +insert into pj_t3 select i, i from generate_series(1,10000)i; +analyze pj_t1; +analyze pj_t2; +analyze pj_t3; + +set local enable_parallel = on; +set local min_parallel_table_scan_size = 0; + +-- 12_P_12_10: Parallel Hash Full Join: HashedWorkers FULL JOIN HashedWorkers -> HashedOJ(parallel) +explain(costs off, locus) +select count(*) from pj_t1 full join pj_t2 using (id); +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 full join pj_t2 using (id); +set local enable_parallel = on; +select count(*) from pj_t1 full join pj_t2 using (id); + +-- Parallel Hash Right Join: pj_t1 (30K) is larger, so the planner hashes the smaller pj_t2 +-- (10K) as the build side and probes with pj_t1; result locus HashedWorkers(parallel) +explain(costs off, locus) +select count(*) from pj_t1 right join pj_t2 using (id); +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 right join pj_t2 using (id); +set local enable_parallel = on; +select count(*) from pj_t1 right join pj_t2 using (id); + +-- Locus propagation: HashedOJ(parallel) followed by INNER JOIN with Hashed(serial) +-- The full join result (HashedOJ,parallel=2) is joined with pj_t3 (Hashed,serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj inner join pj_t3 using (id); + +-- Locus propagation: HashedOJ(parallel) followed by FULL JOIN with Hashed(serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj full join pj_t3 using (id); + +abort; + -- start_ignore drop schema test_parallel cascade; -- end_ignore diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 0115489a6b9..2978e155ecd 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -13,6 +13,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or @@ -543,7 +546,10 @@ rollback; -- be guaranteed to produce correct results if all the hash join tuple match -- bits are reset before reuse. This is done upon loading them into the -- hashtable. +begin; SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; SET enable_parallel_hash = on; SET min_parallel_table_scan_size = 0; SET parallel_setup_cost = 0; From f7ef9a1675cd820f358e567e2a8afc738d0fd75c Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:54:48 +0300 Subject: [PATCH 032/128] Widen MotionLayerState stat counters from uint32 to uint64 (#1647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MotionLayerState accumulates stats across all MotionNodeEntry instances. Per-node entries already use uint64. The global sum ≥ any individual node, so it overflows first — at 4GB. Fix by widening to uint64. Also fix the debug elog() format specifiers to match. --- src/backend/cdb/motion/cdbmotion.c | 4 ++-- src/include/cdb/cdbinterconnect.h | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/cdb/motion/cdbmotion.c b/src/backend/cdb/motion/cdbmotion.c index 24b112cd1bd..ad3b1c49a65 100644 --- a/src/backend/cdb/motion/cdbmotion.c +++ b/src/backend/cdb/motion/cdbmotion.c @@ -133,8 +133,8 @@ RemoveMotionLayer(MotionLayerState *mlStates) /* Emit statistics to log */ if (gp_log_interconnect >= GPVARS_VERBOSITY_VERBOSE) elog(LOG, "RemoveMotionLayer(): dumping stats\n" - " Sent: %9u chunks %9u total bytes %9u tuple bytes\n" - " Received: %9u chunks %9u total bytes %9u tuple bytes; " + " Sent: %9" INT64_MODIFIER "u chunks %9" INT64_MODIFIER "u total bytes %9" INT64_MODIFIER "u tuple bytes\n" + " Received: %9" INT64_MODIFIER "u chunks %9" INT64_MODIFIER "u total bytes %9" INT64_MODIFIER "u tuple bytes; " "%9u chunkproc calls\n", mlStates->stat_total_chunks_sent, mlStates->stat_total_bytes_sent, diff --git a/src/include/cdb/cdbinterconnect.h b/src/include/cdb/cdbinterconnect.h index 5204d4c1b94..c6c64de9590 100644 --- a/src/include/cdb/cdbinterconnect.h +++ b/src/include/cdb/cdbinterconnect.h @@ -154,13 +154,13 @@ typedef struct MotionLayerState /* * GLOBAL MOTION-LAYER STATISTICS */ - uint32 stat_total_chunks_sent; /* Tuple-chunks sent. */ - uint32 stat_total_bytes_sent; /* Bytes sent, including headers. */ - uint32 stat_tuple_bytes_sent; /* Bytes of pure tuple-data sent. */ + uint64 stat_total_chunks_sent; /* Tuple-chunks sent. */ + uint64 stat_total_bytes_sent; /* Bytes sent, including headers. */ + uint64 stat_tuple_bytes_sent; /* Bytes of pure tuple-data sent. */ - uint32 stat_total_chunks_recvd;/* Tuple-chunks received. */ - uint32 stat_total_bytes_recvd; /* Bytes received, including headers. */ - uint32 stat_tuple_bytes_recvd; /* Bytes of pure tuple-data received. */ + uint64 stat_total_chunks_recvd;/* Tuple-chunks received. */ + uint64 stat_total_bytes_recvd; /* Bytes received, including headers. */ + uint64 stat_tuple_bytes_recvd; /* Bytes of pure tuple-data received. */ uint32 stat_total_chunkproc_calls; /* Calls to processIncomingChunks() */ From 7deca90a45f484333c2079edecf834d98555f841 Mon Sep 17 00:00:00 2001 From: fairyfar Date: Sun, 29 Mar 2026 09:18:20 +0800 Subject: [PATCH 033/128] Fix the issue of duplicate counting of num_executed in gp_toolkit.gp_resgroup_status This is a defect of the original GPDB. When enabling resource group management and the transaction switches from "Assign" to "Bypass" state, the "num_executed" counter is repeatedly counted. --- src/backend/utils/resgroup/resgroup.c | 1 - .../expected/resgroup/resgroup_bypass.out | 18 ++++++++++++++++++ .../sql/resgroup/resgroup_bypass.sql | 9 +++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/resgroup/resgroup.c b/src/backend/utils/resgroup/resgroup.c index e474e106490..88f9acea9a0 100644 --- a/src/backend/utils/resgroup/resgroup.c +++ b/src/backend/utils/resgroup/resgroup.c @@ -3700,7 +3700,6 @@ check_and_unassign_from_resgroup(PlannedStmt* stmt) } while (!groupIncBypassedRef(&groupInfo)); bypassedGroup = groupInfo.group; - bypassedGroup->totalExecuted++; pgstat_report_resgroup(bypassedGroup->groupId); bypassedSlot.group = groupInfo.group; bypassedSlot.groupId = groupInfo.groupId; diff --git a/src/test/isolation2/expected/resgroup/resgroup_bypass.out b/src/test/isolation2/expected/resgroup/resgroup_bypass.out index 5cff41d745f..878c759c306 100644 --- a/src/test/isolation2/expected/resgroup/resgroup_bypass.out +++ b/src/test/isolation2/expected/resgroup/resgroup_bypass.out @@ -276,6 +276,24 @@ SELECT gp_inject_fault('func_init_plan_end', 'reset', 1); 1q: ... 2q: ... +-- verify the increment of num_executed in gp_toolkit.gp_resgroup_status +1: SET ROLE role_bypass; +SET +1: SELECT num_executed INTO temporary temp_num1 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +SELECT 1 +1: SELECT num_executed INTO temporary temp_num2 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +SELECT 1 +1: SELECT temp_num2.num_executed - temp_num1.num_executed AS delta FROM temp_num1, temp_num2; + delta +------- + 1 +(1 row) +1: DROP TABLE temp_num1; +DROP +1: DROP TABLE temp_num2; +DROP +1q: ... + -- cleanup -- start_ignore DROP TABLE t_bypass; diff --git a/src/test/isolation2/sql/resgroup/resgroup_bypass.sql b/src/test/isolation2/sql/resgroup/resgroup_bypass.sql index 19c50771f75..01d9d60cbd0 100644 --- a/src/test/isolation2/sql/resgroup/resgroup_bypass.sql +++ b/src/test/isolation2/sql/resgroup/resgroup_bypass.sql @@ -133,6 +133,15 @@ SELECT gp_inject_fault('func_init_plan_end', 'reset', 1); 1q: 2q: +-- verify the increment of num_executed in gp_toolkit.gp_resgroup_status +1: SET ROLE role_bypass; +1: SELECT num_executed INTO temporary temp_num1 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +1: SELECT num_executed INTO temporary temp_num2 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +1: SELECT temp_num2.num_executed - temp_num1.num_executed AS delta FROM temp_num1, temp_num2; +1: DROP TABLE temp_num1; +1: DROP TABLE temp_num2; +1q: + -- cleanup -- start_ignore DROP TABLE t_bypass; From 6afa5b775a32559aac78f7badc215703b814e8d1 Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Wed, 1 Apr 2026 05:43:21 -0700 Subject: [PATCH 034/128] Fix ORCA choosing wrong column type for CTAS with UNION ALL (#1431) (#1645) * Fix ORCA choosing wrong column type for CTAS with UNION ALL (#1431) When removing redundant Result nodes in the ORCA post-processing, push_down_expr_mutator replaces parent Var nodes with child expressions. It already propagates typmod for Const child expressions, but missed the case where the child expression is also a Var. This caused the correctly resolved common typmod (e.g. -1 for varchar without length) to be overwritten by the child's original typmod (e.g. varchar(1)), resulting in wrong column types in the created table. * Add regression test for CTAS with UNION ALL typmod fix (#1431) Verify that ORCA produces the correct column type (character varying without length limit) when creating a table from UNION ALL of branches with different varchar lengths. --------- Co-authored-by: reshke --- src/backend/optimizer/plan/orca.c | 4 + src/test/regress/expected/union_gp.out | 30 ++++ .../regress/expected/union_gp_optimizer.out | 134 ++++++++++++------ src/test/regress/sql/union_gp.sql | 25 +++- 4 files changed, 147 insertions(+), 46 deletions(-) diff --git a/src/backend/optimizer/plan/orca.c b/src/backend/optimizer/plan/orca.c index 97f63f7a334..514385cc2e9 100644 --- a/src/backend/optimizer/plan/orca.c +++ b/src/backend/optimizer/plan/orca.c @@ -545,6 +545,10 @@ push_down_expr_mutator(Node *node, List *child_tlist) { ((Const *) child_tle->expr)->consttypmod = ((Var *) node)->vartypmod; } + else if (IsA(child_tle->expr, Var)) + { + ((Var *) child_tle->expr)->vartypmod = ((Var *) node)->vartypmod; + } return (Node *) child_tle->expr; } diff --git a/src/test/regress/expected/union_gp.out b/src/test/regress/expected/union_gp.out index 5bdae3e887c..d134f223502 100644 --- a/src/test/regress/expected/union_gp.out +++ b/src/test/regress/expected/union_gp.out @@ -2342,6 +2342,36 @@ with result as (update r_1240 set a = a +1 where a < 5 returning *) select * fro drop table r_1240; drop table p1_1240; -- +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. +-- +create table union_ctas_t1(id int, name varchar(1)); +create table union_ctas_t2(id int, name varchar(2)); +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; + atttypmod +----------- + -1 +(1 row) + +-- data should not be truncated +select * from union_ctas_result order by name; + id | name +----+------ + 1 | a + 1 | ab +(2 rows) + +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; +-- -- Clean up -- DROP TABLE IF EXISTS T_a1 CASCADE; diff --git a/src/test/regress/expected/union_gp_optimizer.out b/src/test/regress/expected/union_gp_optimizer.out index 8ff8655591d..8704f0fe7a7 100644 --- a/src/test/regress/expected/union_gp_optimizer.out +++ b/src/test/regress/expected/union_gp_optimizer.out @@ -1,7 +1,7 @@ -- Additional GPDB-added tests for UNION SET optimizer_trace_fallback=on; create temp table t_union1 (a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. select distinct a, null::integer as c from t_union1 union select a, b from t_union1; a | c @@ -44,8 +44,8 @@ LINE 1: select 1 intersect (select 1, 2 union all select 3, 4); select 1 a, row_number() over (partition by 'a') union all (select 1 a , 2 b); a | row_number ---+------------ - 1 | 2 1 | 1 + 1 | 2 (2 rows) -- This should preserve domain types @@ -104,8 +104,7 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support (1 row) CREATE TABLE union_ctas (a, b) AS SELECT 1, 2 UNION SELECT 1, 1 UNION SELECT 1, 1; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. SELECT * FROM union_ctas; a | b ---+--- @@ -116,11 +115,9 @@ SELECT * FROM union_ctas; DROP TABLE union_ctas; -- MPP-21075: push quals below union CREATE TABLE union_quals1 (a, b) AS SELECT i, i%2 from generate_series(1,10) i; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. CREATE TABLE union_quals2 (a, b) AS SELECT i%2, i from generate_series(1,10) i; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. SELECT * FROM (SELECT a, b from union_quals1 UNION SELECT b, a from union_quals2) as foo(a,b) where a > b order by a; a | b ----+--- @@ -225,7 +222,7 @@ select distinct a from (select distinct 'A' from (select 'C' from (select disti -- on a single QE. -- CREATE TABLE test1 (id int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into test1 values (1); CREATE EXTERNAL WEB TABLE test2 (id int) EXECUTE 'echo 2' ON COORDINATOR FORMAT 'csv'; @@ -234,8 +231,8 @@ union (SELECT 'test2' as branch, id FROM test2); branch | id --------+---- - test1 | 1 test2 | 2 + test1 | 1 (2 rows) explain (SELECT 'test1' as branch, id FROM test1 LIMIT 1) @@ -243,10 +240,10 @@ union (SELECT 'test2' as branch, id FROM test2); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------ - Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..984.78 rows=1125 width=12) - -> HashAggregate (cost=0.00..984.73 rows=375 width=12) + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..985.86 rows=1125 width=12) + -> HashAggregate (cost=0.00..985.81 rows=375 width=12) Group Key: ('test1'::text), test1.id - -> Append (cost=0.00..984.65 rows=334 width=12) + -> Append (cost=0.00..985.73 rows=334 width=12) -> Redistribute Motion 1:3 (slice2) (cost=0.00..431.00 rows=1 width=12) Hash Key: ('test1'::text), test1.id -> GroupAggregate (cost=0.00..431.00 rows=1 width=12) @@ -257,16 +254,16 @@ union -> Result (cost=0.00..431.00 rows=1 width=12) -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) -> Seq Scan on test1 (cost=0.00..431.00 rows=1 width=4) - -> HashAggregate (cost=0.00..553.64 rows=334 width=12) - Group Key: ('test2'::text), test2.id - -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..553.56 rows=334 width=12) - Hash Key: ('test2'::text), test2.id - -> Streaming HashAggregate (cost=0.00..553.55 rows=334 width=12) - Group Key: 'test2'::text, test2.id - -> Result (cost=0.00..471.53 rows=333334 width=12) - -> Redistribute Motion 1:3 (slice5) (cost=0.00..467.53 rows=333334 width=4) - -> Foreign Scan on test2 (cost=0.00..449.70 rows=1000000 width=4) - Optimizer: Pivotal Optimizer (GPORCA) + -> HashAggregate (cost=0.00..554.73 rows=334 width=12) + Group Key: ('test2'::text), id + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..554.64 rows=334 width=12) + Hash Key: ('test2'::text), id + -> Streaming HashAggregate (cost=0.00..554.63 rows=334 width=12) + Group Key: 'test2'::text, id + -> Result (cost=0.00..473.73 rows=333334 width=12) + -> Redistribute Motion 1:3 (slice5) (cost=0.00..469.73 rows=333334 width=4) + -> Foreign Scan on test2 (cost=0.00..451.90 rows=1000000 width=4) + Optimizer: GPORCA (24 rows) -- @@ -320,8 +317,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN --------------------------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) (cost=1922.00..1922.00 rows=172200 width=8) - -> Append (cost=0.00..1922.00 rows=172200 width=8) + Gather Motion 1:1 (slice1; segments: 1) (cost=2783.00..2783.00 rows=172200 width=8) + -> Append (cost=0.00..2783.00 rows=172200 width=8) -> Seq Scan on rep2 (cost=0.00..961.00 rows=86100 width=8) -> Seq Scan on rep3 (cost=0.00..961.00 rows=86100 width=8) Optimizer: Postgres query optimizer @@ -353,7 +350,7 @@ INSERT INTO T_a1 SELECT i, i%5 from generate_series(1,10) i; CREATE TABLE T_b2 (b1 int, b2 int) DISTRIBUTED BY(b2); INSERT INTO T_b2 SELECT i, i%5 from generate_series(1,20) i; CREATE TABLE T_random (c1 int, c2 int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO T_random SELECT i, i%5 from generate_series(1,30) i; --start_ignore @@ -2079,14 +2076,18 @@ insert into t1_ncols values (1, 11, 'one', '2001-01-01'); insert into t2_ncols values (2, 22, 'two', '2002-02-02'); insert into t2_ncols values (4, 44, 'four','2004-04-04'); select b from t1_ncols union all select a from t2_ncols; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. b ---- - 4 - 2 11 + 2 + 4 (3 rows) select a+100, b, d from t1_ncols union select b, a+200, d from t2_ncols order by 1; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. ?column? | b | d ----------+-----+------------ 22 | 202 | 02-02-2002 @@ -2095,15 +2096,19 @@ select a+100, b, d from t1_ncols union select b, a+200, d from t2_ncols order by (3 rows) select c, a from v1_ncols; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. c | a ------+--- one | 1 - four | 4 two | 2 + four | 4 (3 rows) with cte1(aa, b, c, d) as (select a*100, b, c, d from t1_ncols union select * from t2_ncols) select x.aa/100 aaa, x.c, y.c from cte1 x join cte1 y on x.aa=y.aa; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. aaa | c | c -----+------+------ 0 | two | two @@ -2122,13 +2127,13 @@ NOTICE: schema "union_schema" does not exist, skipping -- end_ignore create schema union_schema; create table union_schema.t1(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table union_schema.t2(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table union_schema.t3(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. set allow_system_table_mods = on; update gp_distribution_policy set numsegments = 1 @@ -2188,8 +2193,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN ----------------------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=1.23..1472.30 rows=86130 width=8) - -> Append (cost=1.23..323.90 rows=28710 width=8) + Gather Motion 3:1 (slice1; segments: 3) (cost=1.23..1615.85 rows=86130 width=8) + -> Append (cost=1.23..467.45 rows=28710 width=8) -> Hash Join (cost=1.23..2.80 rows=10 width=8) Hash Cond: (t2.b = t1.a) -> Redistribute Motion 2:3 (slice2; segments: 2) (cost=0.00..1.40 rows=20 width=4) @@ -2208,6 +2213,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data a | b | a | b ----+----+----+---- + 1 | 1 | 1 | 1 + 5 | 5 | 5 | 5 2 | 2 | 2 | 2 3 | 3 | 3 | 3 4 | 4 | 4 | 4 @@ -2216,8 +2223,6 @@ DETAIL: Unknown error: Partially Distributed Data 8 | 8 | 8 | 8 9 | 9 | 9 | 9 10 | 10 | 10 | 10 - 1 | 1 | 1 | 1 - 5 | 5 | 5 | 5 (10 rows) select union_schema.t1.a, union_schema.t2.b @@ -2229,6 +2234,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data a | b ----+---- + 1 | 1 + 5 | 5 2 | 2 3 | 3 4 | 4 @@ -2237,8 +2244,6 @@ DETAIL: Unknown error: Partially Distributed Data 8 | 8 9 | 9 10 | 10 - 1 | 1 - 5 | 5 (10 rows) truncate union_schema.t1, union_schema.t2; @@ -2276,8 +2281,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=1.32..1472.20 rows=86130 width=8) - -> Append (cost=1.32..323.80 rows=28710 width=8) + Gather Motion 3:1 (slice1; segments: 3) (cost=1.32..1615.75 rows=86130 width=8) + -> Append (cost=1.32..467.35 rows=28710 width=8) -> Hash Join (cost=1.32..2.70 rows=10 width=8) Hash Cond: (t1.a = t2.b) -> Seq Scan on t1 (cost=0.00..1.20 rows=20 width=4) @@ -2340,6 +2345,8 @@ reset allow_system_table_mods; create table rep (a int) distributed replicated; insert into rep select i from generate_series (1, 10) i; create table dist (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into dist select i from generate_series (1, 1000) i; analyze dist; analyze rep; @@ -2352,7 +2359,7 @@ explain select a from rep union all select a from dist; Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.03 rows=1010 width=4) -> Append (cost=0.00..862.01 rows=337 width=4) -> Result (cost=0.00..431.00 rows=4 width=4) - One-Time Filter: (gp_execution_segment() = 2) + One-Time Filter: (gp_execution_segment() = 0) -> Seq Scan on rep (cost=0.00..431.00 rows=10 width=4) -> Seq Scan on dist (cost=0.00..431.01 rows=334 width=4) Optimizer: GPORCA @@ -2368,12 +2375,12 @@ analyze rand; explain select i from generate_series(1,1000) i union all select a from rand; QUERY PLAN ---------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.28 rows=11000 width=4) + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.29 rows=11000 width=4) -> Append (cost=0.00..431.12 rows=3667 width=4) -> Result (cost=0.00..0.01 rows=334 width=4) - One-Time Filter: (gp_execution_segment() = 2) + One-Time Filter: (gp_execution_segment() = 0) -> Function Scan on generate_series (cost=0.00..0.00 rows=334 width=4) - -> Seq Scan on rand (cost=0.00..431.06 rows=3334 width=4) + -> Seq Scan on rand (cost=0.00..431.07 rows=3334 width=4) Optimizer: GPORCA (7 rows) @@ -2460,7 +2467,7 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support -> Gather Motion 3:1 (slice2; segments: 3) -> Subquery Scan on "*SELECT* 2" -> Seq Scan on p1_1240 - Optimizer: Postgres-based planner + Optimizer: Postgres query optimizer (11 rows) with result as (update r_1240 set a = a +1 where a < 5 returning *) select * from result except select * from p1_1240; @@ -2475,6 +2482,43 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support drop table r_1240; drop table p1_1240; -- +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. +-- +create table union_ctas_t1(id int, name varchar(1)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create table union_ctas_t2(id int, name varchar(2)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; +INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner +DETAIL: Falling back to Postgres-based planner because GPORCA does not support the following feature: Queries on master-only tables + atttypmod +----------- + -1 +(1 row) + +-- data should not be truncated +select * from union_ctas_result order by name; + id | name +----+------ + 1 | a + 1 | ab +(2 rows) + +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; +-- -- Clean up -- DROP TABLE IF EXISTS T_a1 CASCADE; diff --git a/src/test/regress/sql/union_gp.sql b/src/test/regress/sql/union_gp.sql index e7cac952704..9e9e5c3a815 100644 --- a/src/test/regress/sql/union_gp.sql +++ b/src/test/regress/sql/union_gp.sql @@ -721,9 +721,32 @@ drop table r_1240; drop table p1_1240; -- --- Clean up +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. -- +create table union_ctas_t1(id int, name varchar(1)); +create table union_ctas_t2(id int, name varchar(2)); +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); + +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); + +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; + +-- data should not be truncated +select * from union_ctas_result order by name; +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; + +-- +-- Clean up +-- DROP TABLE IF EXISTS T_a1 CASCADE; DROP TABLE IF EXISTS T_b2 CASCADE; DROP TABLE IF EXISTS T_random CASCADE; From b8e6a9dd0eaaa67edb824bd59c0d1a9404f6ef17 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 1 Apr 2026 20:48:48 +0800 Subject: [PATCH 035/128] Revert "Fix COPY TO returning 0 rows during concurrent reorganize" This reverts commit f97979911465650b6626eeffa49af9429a11d2c6. --- .../pax/copy_to_concurrent_reorganize.out | 289 ------ .../src/test/isolation2/isolation2_schedule | 1 - .../sql/pax/copy_to_concurrent_reorganize.sql | 170 ---- src/backend/commands/copy.c | 81 -- src/backend/commands/copyto.c | 37 - .../copy_to_concurrent_reorganize.out | 918 ------------------ src/test/isolation2/isolation2_schedule | 1 - .../sql/copy_to_concurrent_reorganize.sql | 561 ----------- 8 files changed, 2058 deletions(-) delete mode 100644 contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out delete mode 100644 contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql delete mode 100644 src/test/isolation2/expected/copy_to_concurrent_reorganize.out delete mode 100644 src/test/isolation2/sql/copy_to_concurrent_reorganize.sql diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out b/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out deleted file mode 100644 index b4beed7d035..00000000000 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out +++ /dev/null @@ -1,289 +0,0 @@ --- Test: PAX table — relation-based COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- Same as test 2.1 in the main isolation2 suite but for PAX storage. - -CREATE TABLE copy_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_pax_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_pax_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_pax_test TO '/tmp/copy_reorg_pax_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_pax_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_pax_verify FROM '/tmp/copy_reorg_pax_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_pax_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_pax_verify; -DROP -DROP TABLE copy_reorg_pax_test; -DROP - --- ============================================================ --- Test 2.2c: PAX — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_pax_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_pax_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_pax_test) TO '/tmp/copy_query_reorg_pax_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_pax_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_pax_verify FROM '/tmp/copy_query_reorg_pax_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_pax_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_pax_verify; -DROP -DROP TABLE copy_query_reorg_pax_test; -DROP - --- ============================================================ --- Test 2.3c: PAX — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_pax (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (1) TO (501); -CREATE -CREATE TABLE copy_part_child2_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (501) TO (1001); -CREATE -INSERT INTO copy_part_parent_pax SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_pax; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_pax SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_pax TO '/tmp/copy_part_parent_pax.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_pax%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_part_pax_verify FROM '/tmp/copy_part_parent_pax.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_pax_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_pax_verify; -DROP -DROP TABLE copy_part_parent_pax; -DROP - --- ============================================================ --- Test 2.4c: PAX — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2c — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_pax_lookup (cat INT) DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_pax_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_pax_main (a INT, category INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_pax_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_pax_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_pax ON copy_rls_pax_main USING (category IN (SELECT cat from copy_rls_pax_lookup)); -CREATE - -CREATE ROLE copy_rls_pax_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_pax_testuser; -GRANT -GRANT ALL ON copy_rls_pax_main TO copy_rls_pax_testuser; -GRANT -GRANT ALL ON copy_rls_pax_lookup TO copy_rls_pax_testuser; -GRANT - -SELECT count(*) FROM copy_rls_pax_main; - count -------- - 1000 -(1 row) - -2: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_pax_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_pax_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_pax_verify (a INT, category INT) DISTRIBUTED BY (a); -CREATE -COPY copy_rls_pax_verify FROM '/tmp/copy_rls_pax_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_pax_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_pax_verify; -DROP -DROP POLICY p_rls_pax ON copy_rls_pax_main; -DROP -DROP TABLE copy_rls_pax_main; -DROP -DROP TABLE copy_rls_pax_lookup; -DROP -DROP ROLE copy_rls_pax_testuser; -DROP - --- ============================================================ --- Test 2.5c: PAX — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_pax_src (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_pax_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_pax_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_pax_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_pax_dst AS SELECT * FROM ctas_reorg_pax_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_pax_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_pax_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_pax_dst; -DROP -DROP TABLE ctas_reorg_pax_src; -DROP - --- NOTE: Test 2.6c (PAX variant of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/contrib/pax_storage/src/test/isolation2/isolation2_schedule b/contrib/pax_storage/src/test/isolation2/isolation2_schedule index fa163aa96b6..72fa06f5204 100644 --- a/contrib/pax_storage/src/test/isolation2/isolation2_schedule +++ b/contrib/pax_storage/src/test/isolation2/isolation2_schedule @@ -157,7 +157,6 @@ test: pax/vacuum_while_vacuum # test: uao/bad_buffer_on_temp_ao_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop -test: pax/copy_to_concurrent_reorganize # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql b/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql deleted file mode 100644 index 05ef25852e9..00000000000 --- a/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql +++ /dev/null @@ -1,170 +0,0 @@ --- Test: PAX table — relation-based COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- Same as test 2.1 in the main isolation2 suite but for PAX storage. - -CREATE TABLE copy_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_pax_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_pax_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_pax_test TO '/tmp/copy_reorg_pax_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_pax_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_reorg_pax_verify FROM '/tmp/copy_reorg_pax_test.csv'; -SELECT count(*) FROM copy_reorg_pax_verify; - --- Cleanup -DROP TABLE copy_reorg_pax_verify; -DROP TABLE copy_reorg_pax_test; - --- ============================================================ --- Test 2.2c: PAX — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_pax_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_pax_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_pax_test) TO '/tmp/copy_query_reorg_pax_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_pax_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_query_reorg_pax_verify FROM '/tmp/copy_query_reorg_pax_test.csv'; -SELECT count(*) FROM copy_query_reorg_pax_verify; - -DROP TABLE copy_query_reorg_pax_verify; -DROP TABLE copy_query_reorg_pax_test; - --- ============================================================ --- Test 2.3c: PAX — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_pax (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (1) TO (501); -CREATE TABLE copy_part_child2_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (501) TO (1001); -INSERT INTO copy_part_parent_pax SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_pax; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_pax SET WITH (reorganize=true); - -2&: COPY copy_part_parent_pax TO '/tmp/copy_part_parent_pax.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_pax%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_part_pax_verify FROM '/tmp/copy_part_parent_pax.csv'; -SELECT count(*) FROM copy_part_pax_verify; - -DROP TABLE copy_part_pax_verify; -DROP TABLE copy_part_parent_pax; - --- ============================================================ --- Test 2.4c: PAX — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2c — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_pax_lookup (cat INT) DISTRIBUTED BY (cat); -INSERT INTO copy_rls_pax_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_pax_main (a INT, category INT) DISTRIBUTED BY (a); -INSERT INTO copy_rls_pax_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_pax_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_pax ON copy_rls_pax_main USING (category IN (SELECT cat from copy_rls_pax_lookup)); - -CREATE ROLE copy_rls_pax_testuser; -GRANT pg_write_server_files TO copy_rls_pax_testuser; -GRANT ALL ON copy_rls_pax_main TO copy_rls_pax_testuser; -GRANT ALL ON copy_rls_pax_lookup TO copy_rls_pax_testuser; - -SELECT count(*) FROM copy_rls_pax_main; - -2: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_pax_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_pax_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_pax_verify (a INT, category INT) DISTRIBUTED BY (a); -COPY copy_rls_pax_verify FROM '/tmp/copy_rls_pax_main.csv'; -SELECT count(*) FROM copy_rls_pax_verify; - -DROP TABLE copy_rls_pax_verify; -DROP POLICY p_rls_pax ON copy_rls_pax_main; -DROP TABLE copy_rls_pax_main; -DROP TABLE copy_rls_pax_lookup; -DROP ROLE copy_rls_pax_testuser; - --- ============================================================ --- Test 2.5c: PAX — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_pax_src (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_pax_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_pax_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_pax_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_pax_dst AS SELECT * FROM ctas_reorg_pax_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_pax_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_pax_dst; - -DROP TABLE ctas_reorg_pax_dst; -DROP TABLE ctas_reorg_pax_src; - --- NOTE: Test 2.6c (PAX variant of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index c9d2ac4f968..4ccd3798067 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -58,7 +58,6 @@ #include "catalog/catalog.h" #include "catalog/gp_matview_aux.h" #include "catalog/namespace.h" -#include "catalog/pg_inherits.h" #include "catalog/pg_extprotocol.h" #include "cdb/cdbappendonlyam.h" #include "cdb/cdbaocsam.h" @@ -137,37 +136,6 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, { /* Open and lock the relation, using the appropriate lock type. */ rel = table_openrv(stmt->relation, lockmode); - - /* - * For COPY TO, refresh the active snapshot after acquiring the lock. - * - * The snapshot was originally pushed by PortalRunUtility() before - * DoCopy() was called, which means it was taken before we acquired - * the lock on the relation. If we had to wait for a conflicting lock - * (e.g., AccessExclusiveLock held by a concurrent ALTER TABLE ... - * SET WITH (reorganize=true)), the snapshot may predate the - * concurrent transaction's commit. After the lock is granted, scanning - * with such a stale snapshot would miss all tuples written by the - * concurrent transaction, resulting in COPY returning zero rows. - * - * This mirrors the approach used by exec_simple_query() for SELECT - * statements, which pops the parse/analyze snapshot and takes a fresh - * one in PortalStart() after locks have been acquired (see the comment - * at postgres.c:1859-1867). It is also consistent with how VACUUM and - * CLUSTER manage their own snapshots internally. - * - * In REPEATABLE READ or SERIALIZABLE mode, GetTransactionSnapshot() - * returns the same transaction-level snapshot regardless, making this - * a harmless no-op. - * - * We only do this for COPY TO (!is_from) because COPY FROM inserts - * data and does not scan existing tuples with a snapshot. - */ - if (!is_from && ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } } /* @@ -304,55 +272,6 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, errmsg("COPY FROM not supported with row-level security"), errhint("Use INSERT statements instead."))); - /* - * For partitioned table COPY TO: eagerly acquire AccessShareLock - * on all child partitions before refreshing the snapshot. - * - * When COPY is performed on a partitioned table, the parent - * relation's AccessShareLock is acquired above (via table_openrv) - * and Method A already refreshed the snapshot. However, the - * parent's AccessShareLock does NOT conflict with an - * AccessExclusiveLock held on a child partition by a concurrent - * reorganize. As a result, Method A's snapshot may still predate - * the child's reorganize commit. - * - * Child partition locks are acquired later, deep inside - * ExecutorStart() via ExecInitAppend(), by which time the snapshot - * has already been embedded in the QueryDesc via - * PushCopiedSnapshot() in BeginCopy(). Even a second snapshot - * refresh in BeginCopy() (after AcquireRewriteLocks) would not - * help, because AcquireRewriteLocks only locks the parent (child - * partitions are not in the initial range table of - * "SELECT * FROM parent"). - * - * The fix: call find_all_inheritors() with AccessShareLock to - * acquire locks on every child partition NOW, before building the - * query. If a child partition's reorganize holds - * AccessExclusiveLock, this call blocks until that transaction - * commits. Once it returns, all child-level reorganize operations - * have committed, and a fresh snapshot taken here will see all - * reorganized child data. - * - * find_all_inheritors() acquires locks that persist to end of - * transaction. The executor will re-acquire them during scan - * initialization, which is a lock-manager no-op. - */ - if (!is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - { - List *part_oids; - - part_oids = find_all_inheritors(RelationGetRelid(rel), - AccessShareLock, NULL); - list_free(part_oids); - - /* Refresh snapshot: all child partition locks now held */ - if (ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } - } - /* * Build target list * diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 88e61305250..871a973235e 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -1198,43 +1198,6 @@ BeginCopy(ParseState *pstate, Assert(query->utilityStmt == NULL); - /* - * Refresh the active snapshot after pg_analyze_and_rewrite() has - * acquired all necessary relation locks via AcquireRewriteLocks(). - * - * The snapshot in use was pushed by PortalRunUtility() before DoCopy() - * was called -- before any table locks were acquired. If - * AcquireRewriteLocks() had to wait for a conflicting - * AccessExclusiveLock (e.g., held by a concurrent ALTER TABLE ... - * SET WITH (reorganize=true)), the lock wait is now over and the - * reorganize transaction has committed. The snapshot taken before the - * wait does not reflect that commit: after reorganize completes, - * swap_relation_files() has replaced the physical storage, so old - * tuples no longer exist and the new tuples have xmin = reorganize_xid - * which is not yet visible in the pre-wait snapshot. Scanning with - * the stale snapshot returns 0 rows -- a violation of transaction - * atomicity (the reader must see either all old rows or all new rows). - * - * By refreshing the snapshot here -- after all locks are acquired -- - * we guarantee that the query will see the committed post-reorganize - * data. - * - * This applies to: - * - Pure query-based COPY TO: COPY (SELECT ...) TO - * - RLS table COPY TO: converted to query-based in DoCopy(); the - * RLS policy references an external lookup table whose lock is - * acquired by AcquireRewriteLocks(). - * - * In REPEATABLE READ or SERIALIZABLE isolation, - * GetTransactionSnapshot() returns the same transaction-level - * snapshot, making this a harmless no-op. - */ - if (ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } - /* * Similarly the grammar doesn't enforce the presence of a RETURNING * clause, but this is required here. diff --git a/src/test/isolation2/expected/copy_to_concurrent_reorganize.out b/src/test/isolation2/expected/copy_to_concurrent_reorganize.out deleted file mode 100644 index 0a7dfd38801..00000000000 --- a/src/test/isolation2/expected/copy_to_concurrent_reorganize.out +++ /dev/null @@ -1,918 +0,0 @@ --- Test: COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- --- Tests 2.1: Core fix (relation-based COPY TO) --- Tests 2.2-2.5: Extended fixes for query-based, partitioned, RLS, and CTAS paths - --- ============================================================ --- Test 2.1: relation-based COPY TO + concurrent reorganize --- Reproduces issue #1545: COPY TO should return correct row count --- after waiting for reorganize to release AccessExclusiveLock. --- ============================================================ - -CREATE TABLE copy_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock --- At this point PortalRunUtility has already acquired a snapshot (before reorganize commits), --- then DoCopy tries to acquire the lock and blocks. -2&: COPY copy_reorg_test TO '/tmp/copy_reorg_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_verify FROM '/tmp/copy_reorg_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_verify; -DROP -DROP TABLE copy_reorg_test; -DROP - --- ============================================================ --- Test 2.2: query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after pg_analyze_and_rewrite() --- acquires all relation locks via AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_test; - count -------- - 1000 -(1 row) - --- Session 1: reorganize holds AccessExclusiveLock -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_test SET WITH (reorganize=true); -ALTER - --- Session 2: query-based COPY TO blocks (lock acquired in pg_analyze_and_rewrite -> AcquireRewriteLocks) -2&: COPY (SELECT * FROM copy_query_reorg_test) TO '/tmp/copy_query_reorg_test.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_query_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_verify FROM '/tmp/copy_query_reorg_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_query_reorg_verify; -DROP -DROP TABLE copy_query_reorg_test; -DROP - --- ============================================================ --- Test 2.3: partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to eagerly lock all child --- partitions before refreshing the snapshot, ensuring the snapshot sees all --- child reorganize commits before the query is built. --- ============================================================ - -CREATE TABLE copy_part_parent (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1 PARTITION OF copy_part_parent FOR VALUES FROM (1) TO (501); -CREATE -CREATE TABLE copy_part_child2 PARTITION OF copy_part_parent FOR VALUES FROM (501) TO (1001); -CREATE -INSERT INTO copy_part_parent SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent; - count -------- - 1000 -(1 row) - --- Session 1: reorganize the child partition -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1 SET WITH (reorganize=true); -ALTER - --- Session 2: COPY parent TO (internally converted to query-based, child lock acquired in analyze phase) -2&: COPY copy_part_parent TO '/tmp/copy_part_parent.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_part_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_part_verify FROM '/tmp/copy_part_parent.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_part_verify; -DROP -DROP TABLE copy_part_parent; -DROP - --- ============================================================ --- Test 2.4: RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2 — BeginCopy() refreshes snapshot after AcquireRewriteLocks() --- which also acquires the lock on the RLS policy's lookup table. --- ============================================================ - -CREATE TABLE copy_rls_lookup (cat INT) DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_main (a INT, category INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls ON copy_rls_main USING (category IN (SELECT cat FROM copy_rls_lookup)); -CREATE - --- Create non-superuser to trigger RLS (needs pg_write_server_files to COPY TO file) -CREATE ROLE copy_rls_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_testuser; -GRANT -GRANT ALL ON copy_rls_main TO copy_rls_testuser; -GRANT -GRANT ALL ON copy_rls_lookup TO copy_rls_testuser; -GRANT - -SELECT count(*) FROM copy_rls_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; -SET 400 - --- Session 1: reorganize the lookup table -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_lookup SET WITH (reorganize=true); -ALTER - --- Session 2: COPY TO as non-superuser (RLS active, internally converted to query-based) -2&: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -SET 400 - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; -RESET - --- Verify: should match baseline count (400 rows filtered by RLS) -RESET ROLE; -RESET -CREATE TABLE copy_rls_verify (a INT, category INT) DISTRIBUTED BY (a); -CREATE -COPY copy_rls_verify FROM '/tmp/copy_rls_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_verify; - count -------- - 400 -(1 row) - --- Cleanup -DROP TABLE copy_rls_verify; -DROP -DROP POLICY p_rls ON copy_rls_main; -DROP -DROP TABLE copy_rls_main; -DROP -DROP TABLE copy_rls_lookup; -DROP -DROP ROLE copy_rls_testuser; -DROP - --- ============================================================ --- Test 2.5: CTAS + concurrent reorganize --- Fixed as a side effect: CTAS goes through pg_analyze_and_rewrite() + --- AcquireRewriteLocks(), so the snapshot refresh in BeginCopy() also fixes it. --- ============================================================ - -CREATE TABLE ctas_reorg_src (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_src; - count -------- - 1000 -(1 row) - --- Session 1: reorganize -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_src SET WITH (reorganize=true); -ALTER - --- Session 2: CTAS should block (lock acquired in executor or analyze phase) -2&: CREATE TABLE ctas_reorg_dst AS SELECT * FROM ctas_reorg_src DISTRIBUTED BY (a); - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -CREATE 1000 - --- Verify row count after CTAS completes -SELECT count(*) FROM ctas_reorg_dst; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE ctas_reorg_dst; -DROP -DROP TABLE ctas_reorg_src; -DROP - --- NOTE: Test 2.6 (change distribution key + query-based COPY TO) removed because --- ALTER TABLE SET DISTRIBUTED BY + concurrent query-based COPY TO causes a server --- crash (pre-existing Cloudberry bug, not related to this fix). - --- ============================================================ --- Test 2.1a: AO row table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized row-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_row_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_ao_row_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_row_test TO '/tmp/copy_reorg_ao_row_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_ao_row_verify FROM '/tmp/copy_reorg_ao_row_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_ao_row_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_ao_row_verify; -DROP -DROP TABLE copy_reorg_ao_row_test; -DROP - --- ============================================================ --- Test 2.1b: AO column table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized column-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_col_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_ao_col_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_col_test TO '/tmp/copy_reorg_ao_col_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_ao_col_verify FROM '/tmp/copy_reorg_ao_col_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_ao_col_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_ao_col_verify; -DROP -DROP TABLE copy_reorg_ao_col_test; -DROP - --- ============================================================ --- Test 2.2a: AO row — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_ao_row_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_ao_row_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_ao_row_test) TO '/tmp/copy_query_reorg_ao_row_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_ao_row_verify FROM '/tmp/copy_query_reorg_ao_row_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_ao_row_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_ao_row_verify; -DROP -DROP TABLE copy_query_reorg_ao_row_test; -DROP - --- ============================================================ --- Test 2.2b: AO column — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_ao_col_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_ao_col_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_ao_col_test) TO '/tmp/copy_query_reorg_ao_col_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_ao_col_verify FROM '/tmp/copy_query_reorg_ao_col_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_ao_col_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_ao_col_verify; -DROP -DROP TABLE copy_query_reorg_ao_col_test; -DROP - --- ============================================================ --- Test 2.3a: AO row — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_row (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (1) TO (501) USING ao_row; -CREATE -CREATE TABLE copy_part_child2_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (501) TO (1001) USING ao_row; -CREATE -INSERT INTO copy_part_parent_ao_row SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_ao_row; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_ao_row SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_ao_row TO '/tmp/copy_part_parent_ao_row.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_ao_row%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_part_ao_row_verify FROM '/tmp/copy_part_parent_ao_row.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_ao_row_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_ao_row_verify; -DROP -DROP TABLE copy_part_parent_ao_row; -DROP - --- ============================================================ --- Test 2.3b: AO column — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_col (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (1) TO (501) USING ao_column; -CREATE -CREATE TABLE copy_part_child2_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (501) TO (1001) USING ao_column; -CREATE -INSERT INTO copy_part_parent_ao_col SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_ao_col; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_ao_col SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_ao_col TO '/tmp/copy_part_parent_ao_col.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_ao_col%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_part_ao_col_verify FROM '/tmp/copy_part_parent_ao_col.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_ao_col_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_ao_col_verify; -DROP -DROP TABLE copy_part_parent_ao_col; -DROP - --- ============================================================ --- Test 2.4a: AO row — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_row_lookup (cat INT) USING ao_row DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_ao_row_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_ao_row_main (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_ao_row_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_ao_row_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_ao_row ON copy_rls_ao_row_main USING (category IN (SELECT cat FROM copy_rls_ao_row_lookup)); -CREATE - -CREATE ROLE copy_rls_ao_row_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_ao_row_testuser; -GRANT -GRANT ALL ON copy_rls_ao_row_main TO copy_rls_ao_row_testuser; -GRANT -GRANT ALL ON copy_rls_ao_row_lookup TO copy_rls_ao_row_testuser; -GRANT - -SELECT count(*) FROM copy_rls_ao_row_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_ao_row_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_ao_row_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_ao_row_verify (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_rls_ao_row_verify FROM '/tmp/copy_rls_ao_row_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_ao_row_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_ao_row_verify; -DROP -DROP POLICY p_rls_ao_row ON copy_rls_ao_row_main; -DROP -DROP TABLE copy_rls_ao_row_main; -DROP -DROP TABLE copy_rls_ao_row_lookup; -DROP -DROP ROLE copy_rls_ao_row_testuser; -DROP - --- ============================================================ --- Test 2.4b: AO column — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_col_lookup (cat INT) USING ao_column DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_ao_col_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_ao_col_main (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_ao_col_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_ao_col_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_ao_col ON copy_rls_ao_col_main USING (category IN (SELECT cat FROM copy_rls_ao_col_lookup)); -CREATE - -CREATE ROLE copy_rls_ao_col_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_ao_col_testuser; -GRANT -GRANT ALL ON copy_rls_ao_col_main TO copy_rls_ao_col_testuser; -GRANT -GRANT ALL ON copy_rls_ao_col_lookup TO copy_rls_ao_col_testuser; -GRANT - -SELECT count(*) FROM copy_rls_ao_col_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_ao_col_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_ao_col_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_ao_col_verify (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_rls_ao_col_verify FROM '/tmp/copy_rls_ao_col_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_ao_col_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_ao_col_verify; -DROP -DROP POLICY p_rls_ao_col ON copy_rls_ao_col_main; -DROP -DROP TABLE copy_rls_ao_col_main; -DROP -DROP TABLE copy_rls_ao_col_lookup; -DROP -DROP ROLE copy_rls_ao_col_testuser; -DROP - --- ============================================================ --- Test 2.5a: AO row — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_row_src (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_ao_row_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_ao_row_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_ao_row_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_ao_row_dst AS SELECT * FROM ctas_reorg_ao_row_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_row_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_ao_row_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_ao_row_dst; -DROP -DROP TABLE ctas_reorg_ao_row_src; -DROP - --- ============================================================ --- Test 2.5b: AO column — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_col_src (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_ao_col_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_ao_col_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_ao_col_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_ao_col_dst AS SELECT * FROM ctas_reorg_ao_col_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_col_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_ao_col_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_ao_col_dst; -DROP -DROP TABLE ctas_reorg_ao_col_src; -DROP - --- NOTE: Tests 2.6a/2.6b (AO variants of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 4a0f9dc6925..d9d33ad76e4 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -152,7 +152,6 @@ test: uao/fast_analyze_row test: uao/create_index_allows_readonly_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop -test: copy_to_concurrent_reorganize # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql b/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql deleted file mode 100644 index 3473193d142..00000000000 --- a/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql +++ /dev/null @@ -1,561 +0,0 @@ --- Test: COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- --- Tests 2.1: Core fix (relation-based COPY TO) --- Tests 2.2-2.5: Extended fixes for query-based, partitioned, RLS, and CTAS paths - --- ============================================================ --- Test 2.1: relation-based COPY TO + concurrent reorganize --- Reproduces issue #1545: COPY TO should return correct row count --- after waiting for reorganize to release AccessExclusiveLock. --- ============================================================ - -CREATE TABLE copy_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_reorg_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock --- At this point PortalRunUtility has already acquired a snapshot (before reorganize commits), --- then DoCopy tries to acquire the lock and blocks. -2&: COPY copy_reorg_test TO '/tmp/copy_reorg_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_reorg_verify FROM '/tmp/copy_reorg_test.csv'; -SELECT count(*) FROM copy_reorg_verify; - --- Cleanup -DROP TABLE copy_reorg_verify; -DROP TABLE copy_reorg_test; - --- ============================================================ --- Test 2.2: query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after pg_analyze_and_rewrite() --- acquires all relation locks via AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_test; - --- Session 1: reorganize holds AccessExclusiveLock -1: BEGIN; -1: ALTER TABLE copy_query_reorg_test SET WITH (reorganize=true); - --- Session 2: query-based COPY TO blocks (lock acquired in pg_analyze_and_rewrite -> AcquireRewriteLocks) -2&: COPY (SELECT * FROM copy_query_reorg_test) TO '/tmp/copy_query_reorg_test.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_query_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_query_reorg_verify FROM '/tmp/copy_query_reorg_test.csv'; -SELECT count(*) FROM copy_query_reorg_verify; - --- Cleanup -DROP TABLE copy_query_reorg_verify; -DROP TABLE copy_query_reorg_test; - --- ============================================================ --- Test 2.3: partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to eagerly lock all child --- partitions before refreshing the snapshot, ensuring the snapshot sees all --- child reorganize commits before the query is built. --- ============================================================ - -CREATE TABLE copy_part_parent (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1 PARTITION OF copy_part_parent FOR VALUES FROM (1) TO (501); -CREATE TABLE copy_part_child2 PARTITION OF copy_part_parent FOR VALUES FROM (501) TO (1001); -INSERT INTO copy_part_parent SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent; - --- Session 1: reorganize the child partition -1: BEGIN; -1: ALTER TABLE copy_part_child1 SET WITH (reorganize=true); - --- Session 2: COPY parent TO (internally converted to query-based, child lock acquired in analyze phase) -2&: COPY copy_part_parent TO '/tmp/copy_part_parent.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_part_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_part_verify FROM '/tmp/copy_part_parent.csv'; -SELECT count(*) FROM copy_part_verify; - --- Cleanup -DROP TABLE copy_part_verify; -DROP TABLE copy_part_parent; - --- ============================================================ --- Test 2.4: RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2 — BeginCopy() refreshes snapshot after AcquireRewriteLocks() --- which also acquires the lock on the RLS policy's lookup table. --- ============================================================ - -CREATE TABLE copy_rls_lookup (cat INT) DISTRIBUTED BY (cat); -INSERT INTO copy_rls_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_main (a INT, category INT) DISTRIBUTED BY (a); -INSERT INTO copy_rls_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls ON copy_rls_main USING (category IN (SELECT cat FROM copy_rls_lookup)); - --- Create non-superuser to trigger RLS (needs pg_write_server_files to COPY TO file) -CREATE ROLE copy_rls_testuser; -GRANT pg_write_server_files TO copy_rls_testuser; -GRANT ALL ON copy_rls_main TO copy_rls_testuser; -GRANT ALL ON copy_rls_lookup TO copy_rls_testuser; - -SELECT count(*) FROM copy_rls_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Session 1: reorganize the lookup table -1: BEGIN; -1: ALTER TABLE copy_rls_lookup SET WITH (reorganize=true); - --- Session 2: COPY TO as non-superuser (RLS active, internally converted to query-based) -2&: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_main%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; - --- Verify: should match baseline count (400 rows filtered by RLS) -RESET ROLE; -CREATE TABLE copy_rls_verify (a INT, category INT) DISTRIBUTED BY (a); -COPY copy_rls_verify FROM '/tmp/copy_rls_main.csv'; -SELECT count(*) FROM copy_rls_verify; - --- Cleanup -DROP TABLE copy_rls_verify; -DROP POLICY p_rls ON copy_rls_main; -DROP TABLE copy_rls_main; -DROP TABLE copy_rls_lookup; -DROP ROLE copy_rls_testuser; - --- ============================================================ --- Test 2.5: CTAS + concurrent reorganize --- Fixed as a side effect: CTAS goes through pg_analyze_and_rewrite() + --- AcquireRewriteLocks(), so the snapshot refresh in BeginCopy() also fixes it. --- ============================================================ - -CREATE TABLE ctas_reorg_src (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_src; - --- Session 1: reorganize -1: BEGIN; -1: ALTER TABLE ctas_reorg_src SET WITH (reorganize=true); - --- Session 2: CTAS should block (lock acquired in executor or analyze phase) -2&: CREATE TABLE ctas_reorg_dst AS SELECT * FROM ctas_reorg_src DISTRIBUTED BY (a); - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_dst%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify row count after CTAS completes -SELECT count(*) FROM ctas_reorg_dst; - --- Cleanup -DROP TABLE ctas_reorg_dst; -DROP TABLE ctas_reorg_src; - --- NOTE: Test 2.6 (change distribution key + query-based COPY TO) removed because --- ALTER TABLE SET DISTRIBUTED BY + concurrent query-based COPY TO causes a server --- crash (pre-existing Cloudberry bug, not related to this fix). - --- ============================================================ --- Test 2.1a: AO row table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized row-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_row_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_ao_row_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_row_test TO '/tmp/copy_reorg_ao_row_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_reorg_ao_row_verify FROM '/tmp/copy_reorg_ao_row_test.csv'; -SELECT count(*) FROM copy_reorg_ao_row_verify; - --- Cleanup -DROP TABLE copy_reorg_ao_row_verify; -DROP TABLE copy_reorg_ao_row_test; - --- ============================================================ --- Test 2.1b: AO column table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized column-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_col_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_ao_col_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_col_test TO '/tmp/copy_reorg_ao_col_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_reorg_ao_col_verify FROM '/tmp/copy_reorg_ao_col_test.csv'; -SELECT count(*) FROM copy_reorg_ao_col_verify; - --- Cleanup -DROP TABLE copy_reorg_ao_col_verify; -DROP TABLE copy_reorg_ao_col_test; - --- ============================================================ --- Test 2.2a: AO row — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_ao_row_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_ao_row_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_ao_row_test) TO '/tmp/copy_query_reorg_ao_row_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_query_reorg_ao_row_verify FROM '/tmp/copy_query_reorg_ao_row_test.csv'; -SELECT count(*) FROM copy_query_reorg_ao_row_verify; - -DROP TABLE copy_query_reorg_ao_row_verify; -DROP TABLE copy_query_reorg_ao_row_test; - --- ============================================================ --- Test 2.2b: AO column — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_ao_col_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_ao_col_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_ao_col_test) TO '/tmp/copy_query_reorg_ao_col_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_query_reorg_ao_col_verify FROM '/tmp/copy_query_reorg_ao_col_test.csv'; -SELECT count(*) FROM copy_query_reorg_ao_col_verify; - -DROP TABLE copy_query_reorg_ao_col_verify; -DROP TABLE copy_query_reorg_ao_col_test; - --- ============================================================ --- Test 2.3a: AO row — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_row (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (1) TO (501) USING ao_row; -CREATE TABLE copy_part_child2_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (501) TO (1001) USING ao_row; -INSERT INTO copy_part_parent_ao_row SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_ao_row; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_ao_row SET WITH (reorganize=true); - -2&: COPY copy_part_parent_ao_row TO '/tmp/copy_part_parent_ao_row.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_ao_row%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_part_ao_row_verify FROM '/tmp/copy_part_parent_ao_row.csv'; -SELECT count(*) FROM copy_part_ao_row_verify; - -DROP TABLE copy_part_ao_row_verify; -DROP TABLE copy_part_parent_ao_row; - --- ============================================================ --- Test 2.3b: AO column — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_col (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (1) TO (501) USING ao_column; -CREATE TABLE copy_part_child2_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (501) TO (1001) USING ao_column; -INSERT INTO copy_part_parent_ao_col SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_ao_col; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_ao_col SET WITH (reorganize=true); - -2&: COPY copy_part_parent_ao_col TO '/tmp/copy_part_parent_ao_col.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_ao_col%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_part_ao_col_verify FROM '/tmp/copy_part_parent_ao_col.csv'; -SELECT count(*) FROM copy_part_ao_col_verify; - -DROP TABLE copy_part_ao_col_verify; -DROP TABLE copy_part_parent_ao_col; - --- ============================================================ --- Test 2.4a: AO row — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_row_lookup (cat INT) USING ao_row DISTRIBUTED BY (cat); -INSERT INTO copy_rls_ao_row_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_ao_row_main (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_rls_ao_row_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_ao_row_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_ao_row ON copy_rls_ao_row_main USING (category IN (SELECT cat FROM copy_rls_ao_row_lookup)); - -CREATE ROLE copy_rls_ao_row_testuser; -GRANT pg_write_server_files TO copy_rls_ao_row_testuser; -GRANT ALL ON copy_rls_ao_row_main TO copy_rls_ao_row_testuser; -GRANT ALL ON copy_rls_ao_row_lookup TO copy_rls_ao_row_testuser; - -SELECT count(*) FROM copy_rls_ao_row_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_ao_row_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_ao_row_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_ao_row_verify (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_rls_ao_row_verify FROM '/tmp/copy_rls_ao_row_main.csv'; -SELECT count(*) FROM copy_rls_ao_row_verify; - -DROP TABLE copy_rls_ao_row_verify; -DROP POLICY p_rls_ao_row ON copy_rls_ao_row_main; -DROP TABLE copy_rls_ao_row_main; -DROP TABLE copy_rls_ao_row_lookup; -DROP ROLE copy_rls_ao_row_testuser; - --- ============================================================ --- Test 2.4b: AO column — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_col_lookup (cat INT) USING ao_column DISTRIBUTED BY (cat); -INSERT INTO copy_rls_ao_col_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_ao_col_main (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_rls_ao_col_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_ao_col_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_ao_col ON copy_rls_ao_col_main USING (category IN (SELECT cat FROM copy_rls_ao_col_lookup)); - -CREATE ROLE copy_rls_ao_col_testuser; -GRANT pg_write_server_files TO copy_rls_ao_col_testuser; -GRANT ALL ON copy_rls_ao_col_main TO copy_rls_ao_col_testuser; -GRANT ALL ON copy_rls_ao_col_lookup TO copy_rls_ao_col_testuser; - -SELECT count(*) FROM copy_rls_ao_col_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_ao_col_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_ao_col_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_ao_col_verify (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_rls_ao_col_verify FROM '/tmp/copy_rls_ao_col_main.csv'; -SELECT count(*) FROM copy_rls_ao_col_verify; - -DROP TABLE copy_rls_ao_col_verify; -DROP POLICY p_rls_ao_col ON copy_rls_ao_col_main; -DROP TABLE copy_rls_ao_col_main; -DROP TABLE copy_rls_ao_col_lookup; -DROP ROLE copy_rls_ao_col_testuser; - --- ============================================================ --- Test 2.5a: AO row — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_row_src (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_ao_row_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_ao_row_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_ao_row_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_ao_row_dst AS SELECT * FROM ctas_reorg_ao_row_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_row_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_ao_row_dst; - -DROP TABLE ctas_reorg_ao_row_dst; -DROP TABLE ctas_reorg_ao_row_src; - --- ============================================================ --- Test 2.5b: AO column — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_col_src (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_ao_col_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_ao_col_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_ao_col_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_ao_col_dst AS SELECT * FROM ctas_reorg_ao_col_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_col_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_ao_col_dst; - -DROP TABLE ctas_reorg_ao_col_dst; -DROP TABLE ctas_reorg_ao_col_src; - --- NOTE: Tests 2.6a/2.6b (AO variants of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). From 49a0f598833b8bdb48c73af5f5ca739322764414 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 23 Mar 2023 15:15:26 +0300 Subject: [PATCH 036/128] [yagp_hooks_collector] Add extension skeleton with GRPC transport Add yagp_hooks_collector, a shared-preload module that hooks into ExecutorStart and ExecutorFinish to capture query lifecycle events. Includes Makefile with protobuf code generation, GRPC-based delivery, QueryInfo generation (query text, plan text, query_id, plan_id, session metadata), and basic protobuf message filling. --- .gitignore | 7 +- Makefile | 2 - protos/yagpcc_metrics.proto | 130 +++ protos/yagpcc_plan.proto | 570 +++++++++++++ protos/yagpcc_set_service.proto | 45 + sql/yagp-hooks-collector--1.0.sql | 2 + sql/yagp-hooks-collector--unpackaged--1.0.sql | 2 + src/EventSender.cpp | 189 +++++ src/EventSender.h | 19 + src/GrpcConnector.cpp | 55 ++ src/GrpcConnector.h | 15 + src/hook_wrappers.cpp | 67 ++ src/hook_wrappers.h | 12 + src/stat_statements_parser/README.MD | 1 + .../pg_stat_statements_ya_parser.c | 771 ++++++++++++++++++ .../pg_stat_statements_ya_parser.h | 15 + src/yagp_hooks_collector.c | 22 + yagp-hooks-collector.control | 5 + 18 files changed, 1926 insertions(+), 3 deletions(-) create mode 100644 protos/yagpcc_metrics.proto create mode 100644 protos/yagpcc_plan.proto create mode 100644 protos/yagpcc_set_service.proto create mode 100644 sql/yagp-hooks-collector--1.0.sql create mode 100644 sql/yagp-hooks-collector--unpackaged--1.0.sql create mode 100644 src/EventSender.cpp create mode 100644 src/EventSender.h create mode 100644 src/GrpcConnector.cpp create mode 100644 src/GrpcConnector.h create mode 100644 src/hook_wrappers.cpp create mode 100644 src/hook_wrappers.h create mode 100644 src/stat_statements_parser/README.MD create mode 100644 src/stat_statements_parser/pg_stat_statements_ya_parser.c create mode 100644 src/stat_statements_parser/pg_stat_statements_ya_parser.h create mode 100644 src/yagp_hooks_collector.c create mode 100644 yagp-hooks-collector.control diff --git a/.gitignore b/.gitignore index 5c21989c4ab..29b40ee096c 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,9 @@ lib*.pc /compile_commands.json /tmp_install/ /.cache/ -/install/ \ No newline at end of file +/install/ +*.o +*.so +src/protos/ +.vscode +compile_commands.json diff --git a/Makefile b/Makefile index e9ab3fbf2d4..15c5dabb70e 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,12 @@ # to build Postgres with a different make, we have this make file # that, as a service, will look for a GNU make and invoke it, or show # an error message if none could be found. - # If the user were using GNU make now, this file would not get used # because GNU make uses a make file named "GNUmakefile" in preference # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. - # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. all: diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto new file mode 100644 index 00000000000..b7e255484c7 --- /dev/null +++ b/protos/yagpcc_metrics.proto @@ -0,0 +1,130 @@ +syntax = "proto3"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCM"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; + +enum QueryStatus { + QUERY_STATUS_UNSPECIFIED = 0; + QUERY_STATUS_SUBMIT = 1; + QUERY_STATUS_START = 2; + QUERY_STATUS_DONE = 3; + QUERY_STATUS_QUERY_DONE = 4; + QUERY_STATUS_ERROR = 5; + QUERY_STATUS_CANCELLING = 6; + QUERY_STATUS_CANCELED = 7; + QUERY_STATUS_END = 8; +} + +enum PlanNodeStatus { + PLAN_NODE_STATUS_UNSPECIFIED = 0; + PLAN_NODE_STATUS_INITIALIZED = 1; + PLAN_NODE_STATUS_EXECUTING = 2; + PLAN_NODE_STATUS_FINISHED = 3; +} + +message QueryInfo { + PlanGenerator generator = 1; + uint64 query_id = 2; + uint64 plan_id = 3; + string queryText = 4; + string planText = 5; + SessionInfo sessionInfo = 6; +} + +enum PlanGenerator +{ + PLAN_GENERATOR_UNSPECIFIED = 0; + PLAN_GENERATOR_PLANNER = 1; /* plan produced by the planner*/ + PLAN_GENERATOR_OPTIMIZER = 2; /* plan produced by the optimizer*/ +} + +message GPMetrics { + SystemStat systemStat = 1; + MetricInstrumentation instrumentation = 2; + SpillInfo spill = 3; +} + +message QueryInfoHeader { + int32 pid = 1; + GpId gpIdentity = 2; + + int32 tmid = 3; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ + int32 ssid = 4; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ + int32 ccnt = 5; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ + int32 sliceid = 6; /* slice identificator, 0 means general info for the whole query */ +} + +message GpId { + int32 dbid = 1; /* the dbid of this database */ + int32 segindex = 2; /* content indicator: -1 for entry database, + * 0, ..., n-1 for segment database * + * a primary and its mirror have the same segIndex */ + GpRole gp_role = 3; + GpRole gp_session_role = 4; +} + +enum GpRole +{ + GP_ROLE_UNSPECIFIED = 0; + GP_ROLE_UTILITY = 1; /* Operating as a simple database engine */ + GP_ROLE_DISPATCH = 2; /* Operating as the parallel query dispatcher */ + GP_ROLE_EXECUTE = 3; /* Operating as a parallel query executor */ + GP_ROLE_UNDEFINED = 4; /* Should never see this role in use */ +} + +message SessionInfo { + string sql = 1; + string userName = 2; + string databaseName = 3; + string resourceGroup = 4; + string applicationName = 5; +} + +message SystemStat { + /* CPU stat*/ + double runningTimeSeconds = 1; + double userTimeSeconds = 2; + double kernelTimeSeconds = 3; + + /* Memory stat */ + uint64 vsize = 4; + uint64 rss = 5; + uint64 VmSizeKb = 6; + uint64 VmPeakKb = 7; + + /* Storage stat */ + uint64 rchar = 8; + uint64 wchar = 9; + uint64 syscr = 10; + uint64 syscw = 11; + uint64 read_bytes = 12; + uint64 write_bytes = 13; + uint64 cancelled_write_bytes = 14; +} + +message MetricInstrumentation { + uint64 ntuples = 1; /* Total tuples produced */ + uint64 nloops = 2; /* # of run cycles for this node */ + uint64 tuplecount = 3; /* Tuples emitted so far this cycle */ + double firsttuple = 4; /* Time for first tuple of this cycle */ + double startup = 5; /* Total startup time (in seconds) */ + double total = 6; /* Total total time (in seconds) */ + uint64 shared_blks_hit = 7; /* shared blocks stats*/ + uint64 shared_blks_read = 8; + uint64 shared_blks_dirtied = 9; + uint64 shared_blks_written = 10; + uint64 local_blks_hit = 11; /* data read from disks */ + uint64 local_blks_read = 12; + uint64 local_blks_dirtied = 13; + uint64 local_blks_written = 14; + uint64 temp_blks_read = 15; /* temporary tables read stat */ + uint64 temp_blks_written = 16; + double blk_read_time = 17; /* measured read/write time */ + double blk_write_time = 18; +} + +message SpillInfo { + int32 fileCount = 1; + int64 totalBytes = 2; +} diff --git a/protos/yagpcc_plan.proto b/protos/yagpcc_plan.proto new file mode 100644 index 00000000000..962fab4bbdd --- /dev/null +++ b/protos/yagpcc_plan.proto @@ -0,0 +1,570 @@ +syntax = "proto3"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCP"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; + +message MetricPlan { + GpdbNodeType type = 1; + + int32 plan_node_id = 2; + int32 parent_plan_node_id = 3; // Valid only for QueryInfoMetricQuerySubmit + + double startup_cost = 4; /* cost expended before fetching any tuples */ + double total_cost = 5; /* total cost (assuming all tuples fetched) */ + double plan_rows = 6; /* number of rows plan is expected to emit */ + int32 plan_width = 7; /* average row width in bytes */ + + int32 arg1 = 8; // for some nodes it's additional opperand type + int32 arg2 = 9; // for some nodes it's additional opperand type + + MetricMotionInfo motion_info = 10; + MetricRelationInfo relation_info = 11; + + string scan_index_name = 12; + ScanDirection scan_direction = 13; + MetricSliceInfo slice_info = 14; + string statement = 15; +} + +message MetricMotionInfo { + MotionType type = 1; + bool isBroadcast = 2; + CdbLocusType locusType = 3; + + int32 sliceId = 4; + int32 parentSliceId = 5; +} + +message MetricRelationInfo { + int32 oid = 1; + string name = 2; + string schema = 3; + string alias = 4; + int32 dynamicScanId = 5; +} + +message MetricSliceInfo { + int32 slice = 1; + int32 segments = 2; + GangType gangType = 3; + int32 gang = 4; +} + +enum ScanDirection +{ + SCAN_DIRECTION_UNSPECIFIED = 0; + SCAN_DIRECTION_BACKWARD = 1; + SCAN_DIRECTION_FORWARD = 2; +} + +/* GangType enumeration is used in several structures related to CDB + * slice plan support. + */ +enum GangType +{ + GANG_TYPE_UNSPECIFIED = 0; + GANG_TYPE_UNALLOCATED = 1; /* a root slice executed by the qDisp */ + GANG_TYPE_ENTRYDB_READER = 2; /* a 1-gang with read access to the entry db */ + GANG_TYPE_SINGLETON_READER = 3; /* a 1-gang to read the segment dbs */ + GANG_TYPE_PRIMARY_READER = 4; /* a 1-gang or N-gang to read the segment dbs */ + GANG_TYPE_PRIMARY_WRITER = 5; /* the N-gang that can update the segment dbs */ +} + + +enum CdbLocusType +{ + CDB_LOCUS_TYPE_UNSPECIFIED = 0; + CDB_LOCUS_TYPE_ENTRY = 1; /* a single backend process on the entry db: + * usually the qDisp itself, but could be a + * qExec started by the entry postmaster. + */ + + CDB_LOCUS_TYPE_SINGLE_QE = 2; /* a single backend process on any db: the + * qDisp itself, or a qExec started by a + * segment postmaster or the entry postmaster. + */ + + CDB_LOCUS_TYPE_GENERAL = 3; /* compatible with any locus (data is + * self-contained in the query plan or + * generally available in any qExec or qDisp) */ + + CDB_LOCUS_TYPE_SEGMENT_GENERAL = 4; /* generally available in any qExec, but not + * available in qDisp */ + + CDB_LOCUS_TYPE_REPLICATED = 5; /* replicated over all qExecs of an N-gang */ + CDB_LOCUS_TYPE_HASHED = 6; /* hash partitioned over all qExecs of N-gang */ + CDB_LOCUS_TYPE_HASHED_OJ = 7; /* result of hash partitioned outer join, NULLs can be anywhere */ + CDB_LOCUS_TYPE_STREWN = 8; /* partitioned on no known function */ + CDB_LOCUS_TYPE_END = 9; /* = last valid CdbLocusType + 1 */ +} + +enum MotionType +{ + MOTION_TYPE_UNSPECIFIED = 0; + MOTION_TYPE_HASH = 1; // Use hashing to select a segindex destination + MOTION_TYPE_FIXED = 2; // Send tuples to a fixed set of segindexes + MOTION_TYPE_EXPLICIT = 3; // Send tuples to the segment explicitly specified in their segid column +} + +enum GpdbNodeType { + GPDB_NODE_TYPE_UNSPECIFIED = 0; + INDEX_INFO = 1; + EXPR_CONTEXT = 2; + PROJECTION_INFO = 3; + JUNK_FILTER = 4; + RESULT_REL_INFO = 5; + E_STATE = 6; + TUPLE_TABLE_SLOT = 7; + CDB_PROCESS = 8; + SLICE = 9; + SLICE_TABLE = 10; + CURSOR_POS_INFO = 11; + SHARE_NODE_ENTRY = 12; + PARTITION_STATE = 13; + QUERY_DISPATCH_DESC = 14; + OID_ASSIGNMENT = 15; + PLAN = 16; + SCAN = 17; + JOIN = 18; + RESULT = 19; + MODIFY_TABLE = 20; + APPEND = 21; + MERGE_APPEND = 22; + RECURSIVE_UNION = 23; + SEQUENCE = 24; + BITMAP_AND = 25; + BITMAP_OR = 26; + SEQ_SCAN = 27; + DYNAMIC_SEQ_SCAN = 28; + EXTERNAL_SCAN = 29; + INDEX_SCAN = 30; + DYNAMIC_INDEX_SCAN = 31; + INDEX_ONLY_SCAN = 32; + BITMAP_INDEX_SCAN = 33; + DYNAMIC_BITMAP_INDEX_SCAN = 34; + BITMAP_HEAP_SCAN = 35; + DYNAMIC_BITMAP_HEAP_SCAN = 36; + TID_SCAN = 37; + SUBQUERY_SCAN = 38; + FUNCTION_SCAN = 39; + TABLE_FUNCTION_SCAN = 40; + VALUES_SCAN = 41; + CTE_SCAN = 42; + WORK_TABLE_SCAN = 43; + FOREIGN_SCAN = 44; + NEST_LOOP = 45; + MERGE_JOIN = 46; + HASH_JOIN = 47; + MATERIAL = 48; + SORT = 49; + AGG = 50; + WINDOW_AGG = 51; + UNIQUE = 52; + HASH = 53; + SET_OP = 54; + LOCK_ROWS = 55; + LIMIT = 56; + MOTION = 57; + SHARE_INPUT_SCAN = 58; + REPEAT = 59; + DML = 60; + SPLIT_UPDATE = 61; + ROW_TRIGGER = 62; + ASSERT_OP = 63; + PARTITION_SELECTOR = 64; + PLAN_END = 65; + NEST_LOOP_PARAM = 66; + PLAN_ROW_MARK = 67; + PLAN_INVAL_ITEM = 68; + PLAN_STATE = 69; + SCAN_STATE = 70; + JOIN_STATE = 71; + RESULT_STATE = 72; + MODIFY_TABLE_STATE = 73; + APPEND_STATE = 74; + MERGE_APPEND_STATE = 75; + RECURSIVE_UNION_STATE = 76; + SEQUENCE_STATE = 77; + BITMAP_AND_STATE = 78; + BITMAP_OR_STATE = 79; + SEQ_SCAN_STATE = 80; + DYNAMIC_SEQ_SCAN_STATE = 81; + EXTERNAL_SCAN_STATE = 82; + INDEX_SCAN_STATE = 83; + DYNAMIC_INDEX_SCAN_STATE = 84; + INDEX_ONLY_SCAN_STATE = 85; + BITMAP_INDEX_SCAN_STATE = 86; + DYNAMIC_BITMAP_INDEX_SCAN_STATE = 87; + BITMAP_HEAP_SCAN_STATE = 88; + DYNAMIC_BITMAP_HEAP_SCAN_STATE = 89; + TID_SCAN_STATE = 90; + SUBQUERY_SCAN_STATE = 91; + FUNCTION_SCAN_STATE = 92; + TABLE_FUNCTION_STATE = 93; + VALUES_SCAN_STATE = 94; + CTE_SCAN_STATE = 95; + WORK_TABLE_SCAN_STATE = 96; + FOREIGN_SCAN_STATE = 97; + NEST_LOOP_STATE = 98; + MERGE_JOIN_STATE = 99; + HASH_JOIN_STATE = 100; + MATERIAL_STATE = 101; + SORT_STATE = 102; + AGG_STATE = 103; + WINDOW_AGG_STATE = 104; + UNIQUE_STATE = 105; + HASH_STATE = 106; + SET_OP_STATE = 107; + LOCK_ROWS_STATE = 108; + LIMIT_STATE = 109; + MOTION_STATE = 110; + SHARE_INPUT_SCAN_STATE = 111; + REPEAT_STATE = 112; + DML_STATE = 113; + SPLIT_UPDATE_STATE = 114; + ROW_TRIGGER_STATE = 115; + ASSERT_OP_STATE = 116; + PARTITION_SELECTOR_STATE = 117; + TUPLE_DESC_NODE = 118; + SERIALIZED_PARAM_EXTERN_DATA = 119; + ALIAS = 120; + RANGE_VAR = 121; + EXPR = 122; + VAR = 123; + CONST = 124; + PARAM = 125; + AGGREF = 126; + WINDOW_FUNC = 127; + ARRAY_REF = 128; + FUNC_EXPR = 129; + NAMED_ARG_EXPR = 130; + OP_EXPR = 131; + DISTINCT_EXPR = 132; + NULL_IF_EXPR = 133; + SCALAR_ARRAY_OP_EXPR = 134; + BOOL_EXPR = 135; + SUB_LINK = 136; + SUB_PLAN = 137; + ALTERNATIVE_SUB_PLAN = 138; + FIELD_SELECT = 139; + FIELD_STORE = 140; + RELABEL_TYPE = 141; + COERCE_VIA_IO = 142; + ARRAY_COERCE_EXPR = 143; + CONVERT_ROWTYPE_EXPR = 144; + COLLATE_EXPR = 145; + CASE_EXPR = 146; + CASE_WHEN = 147; + CASE_TEST_EXPR = 148; + ARRAY_EXPR = 149; + ROW_EXPR = 150; + ROW_COMPARE_EXPR = 151; + COALESCE_EXPR = 152; + MIN_MAX_EXPR = 153; + XML_EXPR = 154; + NULL_TEST = 155; + BOOLEAN_TEST = 156; + COERCE_TO_DOMAIN = 157; + COERCE_TO_DOMAIN_VALUES = 158; + SET_TO_DEFAULT = 159; + CURRENT_OF_EXPR = 160; + TARGET_ENTRY = 161; + RANGE_TBL_REF = 162; + JOIN_EXPR = 163; + FROM_EXPR = 164; + INTO_CLAUSE = 165; + COPY_INTO_CLAUSE = 166; + REFRESH_CLAUSE = 167; + FLOW = 168; + GROUPING = 169; + GROUP_ID = 170; + DISTRIBUTED_BY = 171; + DML_ACTION_EXPR = 172; + PART_SELECTED_EXPR = 173; + PART_DEFAULT_EXPR = 174; + PART_BOUND_EXPR = 175; + PART_BOUND_INCLUSION_EXPR = 176; + PART_BOUND_OPEN_EXPR = 177; + PART_LIST_RULE_EXPR = 178; + PART_LIST_NULL_TEST_EXPR = 179; + TABLE_OID_INFO = 180; + EXPR_STATE = 181; + GENERIC_EXPR_STATE = 182; + WHOLE_ROW_VAR_EXPR_STATE = 183; + AGGREF_EXPR_STATE = 184; + WINDOW_FUNC_EXPR_STATE = 185; + ARRAY_REF_EXPR_STATE = 186; + FUNC_EXPR_STATE = 187; + SCALAR_ARRAY_OP_EXPR_STATE = 188; + BOOL_EXPR_STATE = 189; + SUB_PLAN_STATE = 190; + ALTERNATIVE_SUB_PLAN_STATE = 191; + FIELD_SELECT_STATE = 192; + FIELD_STORE_STATE = 193; + COERCE_VIA_IO_STATE = 194; + ARRAY_COERCE_EXPR_STATE = 195; + CONVERT_ROWTYPE_EXPR_STATE = 196; + CASE_EXPR_STATE = 197; + CASE_WHEN_STATE = 198; + ARRAY_EXPR_STATE = 199; + ROW_EXPR_STATE = 200; + ROW_COMPARE_EXPR_STATE = 201; + COALESCE_EXPR_STATE = 202; + MIN_MAX_EXPR_STATE = 203; + XML_EXPR_STATE = 204; + NULL_TEST_STATE = 205; + COERCE_TO_DOMAIN_STATE = 206; + DOMAIN_CONSTRAINT_STATE = 207; + GROUPING_FUNC_EXPR_STATE = 208; + PART_SELECTED_EXPR_STATE = 209; + PART_DEFAULT_EXPR_STATE = 210; + PART_BOUND_EXPR_STATE = 211; + PART_BOUND_INCLUSION_EXPR_STATE = 212; + PART_BOUND_OPEN_EXPR_STATE = 213; + PART_LIST_RULE_EXPR_STATE = 214; + PART_LIST_NULL_TEST_EXPR_STATE = 215; + PLANNER_INFO = 216; + PLANNER_GLOBAL = 217; + REL_OPT_INFO = 218; + INDEX_OPT_INFO = 219; + PARAM_PATH_INFO = 220; + PATH = 221; + APPEND_ONLY_PATH = 222; + AOCS_PATH = 223; + EXTERNAL_PATH = 224; + INDEX_PATH = 225; + BITMAP_HEAP_PATH = 226; + BITMAP_AND_PATH = 227; + BITMAP_OR_PATH = 228; + NEST_PATH = 229; + MERGE_PATH = 230; + HASH_PATH = 231; + TID_PATH = 232; + FOREIGN_PATH = 233; + APPEND_PATH = 234; + MERGE_APPEND_PATH = 235; + RESULT_PATH = 236; + MATERIAL_PATH = 237; + UNIQUE_PATH = 238; + PROJECTION_PATH = 239; + EQUIVALENCE_CLASS = 240; + EQUIVALENCE_MEMBER = 241; + PATH_KEY = 242; + RESTRICT_INFO = 243; + PLACE_HOLDER_VAR = 244; + SPECIAL_JOIN_INFO = 245; + LATERAL_JOIN_INFO = 246; + APPEND_REL_INFO = 247; + PLACE_HOLDER_INFO = 248; + MIN_MAX_AGG_INFO = 249; + PARTITION = 250; + PARTITION_RULE = 251; + PARTITION_NODE = 252; + PG_PART_RULE = 253; + SEGFILE_MAP_NODE = 254; + PLANNER_PARAM_ITEM = 255; + CDB_MOTION_PATH = 256; + PARTITION_SELECTOR_PATH = 257; + CDB_REL_COLUMN_INFO = 258; + DISTRIBUTION_KEY = 259; + MEMORY_CONTEXT = 260; + ALLOC_SET_CONTEXT = 261; + MEMORY_ACCOUNT = 262; + VALUE = 263; + INTEGER = 264; + FLOAT = 265; + STRING = 266; + BIT_STRING = 267; + NULL_VALUE = 268; + LIST = 269; + INT_LIST = 270; + OID_LIST = 271; + QUERY = 272; + PLANNED_STMT = 273; + INSERT_STMT = 274; + DELETE_STMT = 275; + UPDATE_STMT = 276; + SELECT_STMT = 277; + ALTER_TABLE_STMT = 278; + ALTER_TABLE_CMD = 279; + ALTER_DOMAIN_STMT = 280; + SET_OPERATION_STMT = 281; + GRANT_STMT = 282; + GRANT_ROLE_STMT = 283; + ALTER_DEFAULT_PRIVILEGES_STMT = 284; + CLOSE_PORTAL_STMT = 285; + CLUSTER_STMT = 286; + COPY_STMT = 287; + CREATE_STMT = 288; + SINGLE_ROW_ERROR_DESC = 289; + EXT_TABLE_TYPE_DESC = 290; + CREATE_EXTERNAL_STMT = 291; + DEFINE_STMT = 292; + DROP_STMT = 293; + TRUNCATE_STMT = 294; + COMMENT_STMT = 295; + FETCH_STMT = 296; + INDEX_STMT = 297; + CREATE_FUNCTION_STMT = 298; + ALTER_FUNCTION_STMT = 299; + DO_STMT = 300; + RENAME_STMT = 301; + RULE_STMT = 302; + NOTIFY_STMT = 303; + LISTEN_STMT = 304; + UNLISTEN_STMT = 305; + TRANSACTION_STMT = 306; + VIEW_STMT = 307; + LOAD_STMT = 308; + CREATE_DOMAIN_STMT = 309; + CREATEDB_STMT = 310; + DROPDB_STMT = 311; + VACUUM_STMT = 312; + EXPLAIN_STMT = 313; + CREATE_TABLE_AS_STMT = 314; + CREATE_SEQ_STMT = 315; + ALTER_SEQ_STMT = 316; + VARIABLE_SET_STMT = 317; + VARIABLE_SHOW_STMT = 318; + DISCARD_STMT = 319; + CREATE_TRIG_STMT = 320; + CREATE_P_LANG_STMT = 321; + CREATE_ROLE_STMT = 322; + ALTER_ROLE_STMT = 323; + DROP_ROLE_STMT = 324; + CREATE_QUEUE_STMT = 325; + ALTER_QUEUE_STMT = 326; + DROP_QUEUE_STMT = 327; + CREATE_RESOURCE_GROUP_STMT = 328; + DROP_RESOURCE_GROUP_STMT = 329; + ALTER_RESOURCE_GROUP_STMT = 330; + LOCK_STMT = 331; + CONSTRAINTS_SET_STMT = 332; + REINDEX_STMT = 333; + CHECK_POINT_STMT = 334; + CREATE_SCHEMA_STMT = 335; + ALTER_DATABASE_STMT = 336; + ALTER_DATABASE_SET_STMT = 337; + ALTER_ROLE_SET_STMT = 338; + CREATE_CONVERSION_STMT = 339; + CREATE_CAST_STMT = 340; + CREATE_OP_CLASS_STMT = 341; + CREATE_OP_FAMILY_STMT = 342; + ALTER_OP_FAMILY_STMT = 343; + PREPARE_STMT = 344; + EXECUTE_STMT = 345; + DEALLOCATE_STMT = 346; + DECLARE_CURSOR_STMT = 347; + CREATE_TABLE_SPACE_STMT = 348; + DROP_TABLE_SPACE_STMT = 349; + ALTER_OBJECT_SCHEMA_STMT = 350; + ALTER_OWNER_STMT = 351; + DROP_OWNED_STMT = 352; + REASSIGN_OWNED_STMT = 353; + COMPOSITE_TYPE_STMT = 354; + CREATE_ENUM_STMT = 355; + CREATE_RANGE_STMT = 356; + ALTER_ENUM_STMT = 357; + ALTER_TS_DICTIONARY_STMT = 358; + ALTER_TS_CONFIGURATION_STMT = 359; + CREATE_FDW_STMT = 360; + ALTER_FDW_STMT = 361; + CREATE_FOREIGN_SERVER_STMT = 362; + ALTER_FOREIGN_SERVER_STMT = 363; + CREATE_USER_MAPPING_STMT = 364; + ALTER_USER_MAPPING_STMT = 365; + DROP_USER_MAPPING_STMT = 366; + ALTER_TABLE_SPACE_OPTIONS_STMT = 367; + ALTER_TABLE_MOVE_ALL_STMT = 368; + SEC_LABEL_STMT = 369; + CREATE_FOREIGN_TABLE_STMT = 370; + CREATE_EXTENSION_STMT = 371; + ALTER_EXTENSION_STMT = 372; + ALTER_EXTENSION_CONTENTS_STMT = 373; + CREATE_EVENT_TRIG_STMT = 374; + ALTER_EVENT_TRIG_STMT = 375; + REFRESH_MAT_VIEW_STMT = 376; + REPLICA_IDENTITY_STMT = 377; + ALTER_SYSTEM_STMT = 378; + PARTITION_BY = 379; + PARTITION_ELEM = 380; + PARTITION_RANGE_ITEM = 381; + PARTITION_BOUND_SPEC = 382; + PARTITION_SPEC = 383; + PARTITION_VALUES_SPEC = 384; + ALTER_PARTITION_ID = 385; + ALTER_PARTITION_CMD = 386; + INHERIT_PARTITION_CMD = 387; + CREATE_FILE_SPACE_STMT = 388; + FILE_SPACE_ENTRY = 389; + DROP_FILE_SPACE_STMT = 390; + TABLE_VALUE_EXPR = 391; + DENY_LOGIN_INTERVAL = 392; + DENY_LOGIN_POINT = 393; + ALTER_TYPE_STMT = 394; + SET_DISTRIBUTION_CMD = 395; + EXPAND_STMT_SPEC = 396; + A_EXPR = 397; + COLUMN_REF = 398; + PARAM_REF = 399; + A_CONST = 400; + FUNC_CALL = 401; + A_STAR = 402; + A_INDICES = 403; + A_INDIRECTION = 404; + A_ARRAY_EXPR = 405; + RES_TARGET = 406; + TYPE_CAST = 407; + COLLATE_CLAUSE = 408; + SORT_BY = 409; + WINDOW_DEF = 410; + RANGE_SUBSELECT = 411; + RANGE_FUNCTION = 412; + TYPE_NAME = 413; + COLUMN_DEF = 414; + INDEX_ELEM = 415; + CONSTRAINT = 416; + DEF_ELEM = 417; + RANGE_TBL_ENTRY = 418; + RANGE_TBL_FUNCTION = 419; + WITH_CHECK_OPTION = 420; + GROUPING_CLAUSE = 421; + GROUPING_FUNC = 422; + SORT_GROUP_CLAUSE = 423; + WINDOW_CLAUSE = 424; + PRIV_GRANTEE = 425; + FUNC_WITH_ARGS = 426; + ACCESS_PRIV = 427; + CREATE_OP_CLASS_ITEM = 428; + TABLE_LIKE_CLAUSE = 429; + FUNCTION_PARAMETER = 430; + LOCKING_CLAUSE = 431; + ROW_MARK_CLAUSE = 432; + XML_SERIALIZE = 433; + WITH_CLAUSE = 434; + COMMON_TABLE_EXPR = 435; + COLUMN_REFERENCE_STORAGE_DIRECTIVE = 436; + IDENTIFY_SYSTEM_CMD = 437; + BASE_BACKUP_CMD = 438; + CREATE_REPLICATION_SLOT_CMD = 439; + DROP_REPLICATION_SLOT_CMD = 440; + START_REPLICATION_CMD = 441; + TIME_LINE_HISTORY_CMD = 442; + TRIGGER_DATA = 443; + EVENT_TRIGGER_DATA = 444; + RETURN_SET_INFO = 445; + WINDOW_OBJECT_DATA = 446; + TID_BITMAP = 447; + INLINE_CODE_BLOCK = 448; + FDW_ROUTINE = 449; + STREAM_BITMAP = 450; + FORMATTER_DATA = 451; + EXT_PROTOCOL_DATA = 452; + EXT_PROTOCOL_VALIDATOR_DATA = 453; + SELECTED_PARTS = 454; + COOKED_CONSTRAINT = 455; + CDB_EXPLAIN_STAT_HDR = 456; + GP_POLICY = 457; + RETRIEVE_STMT = 458; +} diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto new file mode 100644 index 00000000000..0bef72891ee --- /dev/null +++ b/protos/yagpcc_set_service.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +import "protos/yagpcc_metrics.proto"; +import "protos/yagpcc_plan.proto"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCAS"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; + +service SetQueryInfo { + rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} + + rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} +} + +message MetricResponse { + MetricResponseStatusCode error_code = 1; + string error_text = 2; +} + +enum MetricResponseStatusCode { + METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; + METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; + METRIC_RESPONSE_STATUS_CODE_ERROR = 2; +} + +message SetQueryReq { + QueryStatus query_status = 1; + google.protobuf.Timestamp datetime = 2; + + QueryInfoHeader header = 3; + QueryInfo query_info = 4; + GPMetrics query_metrics = 5; + repeated MetricPlan plan_tree = 6; +} + +message SetPlanNodeReq { + PlanNodeStatus node_status = 1; + google.protobuf.Timestamp datetime = 2; + QueryInfoHeader header = 3; + GPMetrics node_metrics = 4; + MetricPlan plan_node = 5; +} diff --git a/sql/yagp-hooks-collector--1.0.sql b/sql/yagp-hooks-collector--1.0.sql new file mode 100644 index 00000000000..f9ab15fb400 --- /dev/null +++ b/sql/yagp-hooks-collector--1.0.sql @@ -0,0 +1,2 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "yagp-hooks-collector"''' to load this file. \quit diff --git a/sql/yagp-hooks-collector--unpackaged--1.0.sql b/sql/yagp-hooks-collector--unpackaged--1.0.sql new file mode 100644 index 00000000000..0441c97bd84 --- /dev/null +++ b/sql/yagp-hooks-collector--unpackaged--1.0.sql @@ -0,0 +1,2 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "uuid-cb" FROM unpackaged''' to load this file. \quit diff --git a/src/EventSender.cpp b/src/EventSender.cpp new file mode 100644 index 00000000000..bb4765adeb1 --- /dev/null +++ b/src/EventSender.cpp @@ -0,0 +1,189 @@ +#include "EventSender.h" +#include "GrpcConnector.h" +#include "protos/yagpcc_set_service.pb.h" +#include + +extern "C" +{ +#include "postgres.h" +#include "utils/metrics_utils.h" +#include "utils/elog.h" +#include "executor/executor.h" +#include "commands/explain.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" + +#include "cdb/cdbvars.h" +#include "cdb/cdbexplain.h" + +#include "tcop/utility.h" +#include "pg_stat_statements_ya_parser.h" +} + +namespace +{ +std::string* get_user_name() +{ + const char *username = GetConfigOption("session_authorization", false, false); + return username ? new std::string(username) : nullptr; +} + +std::string* get_db_name() +{ + char *dbname = get_database_name(MyDatabaseId); + std::string* result = dbname ? new std::string(dbname) : nullptr; + pfree(dbname); + return result; +} + +std::string* get_rg_name() +{ + auto userId = GetUserId(); + if (!OidIsValid(userId)) + return nullptr; + auto groupId = GetResGroupIdForRole(userId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + pfree(rgname); + return new std::string(rgname); +} + +std::string* get_app_name() +{ + return application_name ? new std::string(application_name) : nullptr; +} + +int get_cur_slice_id(QueryDesc *desc) +{ + if (!desc->estate) + { + return 0; + } + return LocallyExecutingSliceIndex(desc->estate); +} + +google::protobuf::Timestamp current_ts() +{ + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_header(yagpcc::QueryInfoHeader *header, QueryDesc *queryDesc) +{ + header->set_pid(MyProcPid); + auto gpId = header->mutable_gpidentity(); + gpId->set_dbid(GpIdentity.dbid); + gpId->set_segindex(GpIdentity.segindex); + gpId->set_gp_role(static_cast(Gp_role)); + gpId->set_gp_session_role(static_cast(Gp_session_role)); + header->set_ssid(gp_session_id); + header->set_ccnt(gp_command_count); + header->set_sliceid(get_cur_slice_id(queryDesc)); + int32 tmid = 0; + gpmon_gettmid(&tmid); + header->set_tmid(tmid); +} + +void set_session_info(yagpcc::SessionInfo *si, QueryDesc *queryDesc) +{ + if (queryDesc->sourceText) + *si->mutable_sql() = std::string(queryDesc->sourceText); + si->set_allocated_applicationname(get_app_name()); + si->set_allocated_databasename(get_db_name()); + si->set_allocated_resourcegroup(get_rg_name()); + si->set_allocated_username(get_user_name()); +} + +ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) +{ + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, queryDesc); + ExplainEndOutput(&es); + return es; +} + +void set_plan_text(std::string *plan_text, QueryDesc *queryDesc) +{ + auto es = get_explain_state(queryDesc, true); + *plan_text = std::string(es.str->data, es.str->len); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) +{ + set_session_info(qi->mutable_sessioninfo(), queryDesc); + if (queryDesc->sourceText) + *qi->mutable_querytext() = queryDesc->sourceText; + if (queryDesc->plannedstmt) + { + qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plantext(), queryDesc); + qi->set_plan_id(get_plan_id(queryDesc)); + qi->set_query_id(queryDesc->plannedstmt->queryId); + } +} +} // namespace + +void EventSender::ExecutorStart(QueryDesc *queryDesc, int /* eflags*/) +{ + elog(DEBUG1, "Query %s start recording", queryDesc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *req.mutable_datetime() = current_ts(); + set_header(req.mutable_header(), queryDesc); + set_query_info(req.mutable_query_info(), queryDesc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) + { + elog(WARNING, "Query %s start reporting failed with an error %s", + queryDesc->sourceText, result.error_text().c_str()); + } + else + { + elog(DEBUG1, "Query %s start successful", queryDesc->sourceText); + } +} + +void EventSender::ExecutorFinish(QueryDesc *queryDesc) +{ + elog(DEBUG1, "Query %s finish recording", queryDesc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + *req.mutable_datetime() = current_ts(); + set_header(req.mutable_header(), queryDesc); + set_query_info(req.mutable_query_info(), queryDesc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) + { + elog(WARNING, "Query %s finish reporting failed with an error %s", + queryDesc->sourceText, result.error_text().c_str()); + } + else + { + elog(DEBUG1, "Query %s finish successful", queryDesc->sourceText); + } +} + +EventSender *EventSender::instance() +{ + static EventSender sender; + return &sender; +} + +EventSender::EventSender() +{ + connector = std::make_unique(); +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h new file mode 100644 index 00000000000..70868f6c757 --- /dev/null +++ b/src/EventSender.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +class GrpcConnector; + +struct QueryDesc; + +class EventSender +{ +public: + void ExecutorStart(QueryDesc *queryDesc, int eflags); + void ExecutorFinish(QueryDesc *queryDesc); + static EventSender *instance(); + +private: + EventSender(); + std::unique_ptr connector; +}; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp new file mode 100644 index 00000000000..7329f392010 --- /dev/null +++ b/src/GrpcConnector.cpp @@ -0,0 +1,55 @@ +#include "GrpcConnector.h" +#include "yagpcc_set_service.grpc.pb.h" + +#include +#include +#include + +class GrpcConnector::Impl +{ +public: + Impl() + { + GOOGLE_PROTOBUF_VERIFY_VERSION; + this->stub = yagpcc::SetQueryInfo::NewStub(grpc::CreateChannel( + SOCKET_FILE, grpc::InsecureChannelCredentials())); + } + + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) + { + yagpcc::MetricResponse response; + grpc::ClientContext context; + auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(50); + context.set_deadline(deadline); + + grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); + + if (!status.ok()) + { + response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + } + + return response; + } + +private: + const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; + const std::string TCP_ADDRESS = "127.0.0.1:1432"; + std::unique_ptr stub; +}; + +GrpcConnector::GrpcConnector() +{ + impl = new Impl(); +} + +GrpcConnector::~GrpcConnector() +{ + delete impl; +} + +yagpcc::MetricResponse GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) +{ + return impl->set_metric_query(req); +} \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h new file mode 100644 index 00000000000..dc0f21706a3 --- /dev/null +++ b/src/GrpcConnector.h @@ -0,0 +1,15 @@ +#pragma once + +#include "yagpcc_set_service.pb.h" + +class GrpcConnector +{ +public: + GrpcConnector(); + ~GrpcConnector(); + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + +private: + class Impl; + Impl *impl; +}; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp new file mode 100644 index 00000000000..9f3200c006f --- /dev/null +++ b/src/hook_wrappers.cpp @@ -0,0 +1,67 @@ +#include "hook_wrappers.h" +#include "EventSender.h" + +extern "C" +{ +#include "postgres.h" +#include "utils/metrics_utils.h" +#include "utils/elog.h" +#include "executor/executor.h" + +#include "cdb/cdbvars.h" +#include "cdb/cdbexplain.h" + +#include "tcop/utility.h" +} + +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" + +static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; +static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; + +static void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags); +static void ya_ExecutorFinish_hook(QueryDesc *queryDesc); + +#define REPLACE_HOOK(hookName) \ + previous_##hookName = hookName; \ + hookName = ya_##hookName; + +void hooks_init() +{ + REPLACE_HOOK(ExecutorStart_hook); + REPLACE_HOOK(ExecutorFinish_hook); + stat_statements_parser_init(); +} + +void hooks_deinit() +{ + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorFinish_hook = ExecutorFinish_hook; + stat_statements_parser_deinit(); +} + +#define CREATE_HOOK_WRAPPER(hookName, ...) \ + PG_TRY(); \ + { \ + EventSender::instance()->hookName(__VA_ARGS__); \ + } \ + PG_CATCH(); \ + { \ + ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ + PG_RE_THROW(); \ + } \ + PG_END_TRY(); \ + if (previous_##hookName##_hook) \ + (*previous_##hookName##_hook)(__VA_ARGS__); \ + else \ + standard_##hookName(__VA_ARGS__); + +void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags) +{ + CREATE_HOOK_WRAPPER(ExecutorStart, queryDesc, eflags); +} + +void ya_ExecutorFinish_hook(QueryDesc *queryDesc) +{ + CREATE_HOOK_WRAPPER(ExecutorFinish, queryDesc); +} \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h new file mode 100644 index 00000000000..815fcb7cd51 --- /dev/null +++ b/src/hook_wrappers.h @@ -0,0 +1,12 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +extern void hooks_init(); +extern void hooks_deinit(); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/src/stat_statements_parser/README.MD b/src/stat_statements_parser/README.MD new file mode 100644 index 00000000000..291e31a3099 --- /dev/null +++ b/src/stat_statements_parser/README.MD @@ -0,0 +1 @@ +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c new file mode 100644 index 00000000000..f14742337bd --- /dev/null +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -0,0 +1,771 @@ +#include "postgres.h" + +#include +#include + +#include "access/hash.h" +#include "executor/instrument.h" +#include "executor/execdesc.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "parser/analyze.h" +#include "parser/parsetree.h" +#include "parser/scanner.h" +#include "parser/gram.h" +#include "pgstat.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/spin.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + +#include "pg_stat_statements_ya_parser.h" + +static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; + +#define JUMBLE_SIZE 1024 /* query serialization buffer size */ + +/* + * Struct for tracking locations/lengths of constants during normalization + */ +typedef struct pgssLocationLen +{ + int location; /* start offset in query text */ + int length; /* length in bytes, or -1 to ignore */ +} pgssLocationLen; + +/* + * Working state for computing a query jumble and producing a normalized + * query string + */ +typedef struct pgssJumbleState +{ + /* Jumble of current query tree */ + unsigned char *jumble; + + /* Number of bytes used in jumble[] */ + Size jumble_len; + + /* Array of locations of constants that should be removed */ + pgssLocationLen *clocations; + + /* Allocated length of clocations array */ + int clocations_buf_size; + + /* Current number of valid entries in clocations array */ + int clocations_count; + + /* highest Param id we've seen, in order to start normalization correctly */ + int highest_extern_param_id; +} pgssJumbleState; + +static void AppendJumble(pgssJumbleState *jstate, + const unsigned char *item, Size size); +static void JumbleQuery(pgssJumbleState *jstate, Query *query); +static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); +static void JumbleExpr(pgssJumbleState *jstate, Node *node); +static void RecordConstLocation(pgssJumbleState *jstate, int location); + +static StringInfo gen_normplan(const char *execution_plan); + +static bool need_replace(int token); + +void pgss_post_parse_analyze(ParseState *pstate, Query *query); + +void stat_statements_parser_init() +{ + prev_post_parse_analyze_hook = post_parse_analyze_hook; + post_parse_analyze_hook = pgss_post_parse_analyze; +} + +void stat_statements_parser_deinit() +{ + post_parse_analyze_hook = prev_post_parse_analyze_hook; +} + +/* + * AppendJumble: Append a value that is substantive in a given query to + * the current jumble. + */ +static void +AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size) +{ + unsigned char *jumble = jstate->jumble; + Size jumble_len = jstate->jumble_len; + + /* + * Whenever the jumble buffer is full, we hash the current contents and + * reset the buffer to contain just that hash value, thus relying on the + * hash to summarize everything so far. + */ + while (size > 0) + { + Size part_size; + + if (jumble_len >= JUMBLE_SIZE) + { + uint32 start_hash = hash_any(jumble, JUMBLE_SIZE); + + memcpy(jumble, &start_hash, sizeof(start_hash)); + jumble_len = sizeof(start_hash); + } + part_size = Min(size, JUMBLE_SIZE - jumble_len); + memcpy(jumble + jumble_len, item, part_size); + jumble_len += part_size; + item += part_size; + size -= part_size; + } + jstate->jumble_len = jumble_len; +} + +/* + * Wrappers around AppendJumble to encapsulate details of serialization + * of individual local variable elements. + */ +#define APP_JUMB(item) \ + AppendJumble(jstate, (const unsigned char *)&(item), sizeof(item)) +#define APP_JUMB_STRING(str) \ + AppendJumble(jstate, (const unsigned char *)(str), strlen(str) + 1) + +/* + * JumbleQuery: Selectively serialize the query tree, appending significant + * data to the "query jumble" while ignoring nonsignificant data. + * + * Rule of thumb for what to include is that we should ignore anything not + * semantically significant (such as alias names) as well as anything that can + * be deduced from child nodes (else we'd just be double-hashing that piece + * of information). + */ +void JumbleQuery(pgssJumbleState *jstate, Query *query) +{ + Assert(IsA(query, Query)); + Assert(query->utilityStmt == NULL); + + APP_JUMB(query->commandType); + /* resultRelation is usually predictable from commandType */ + JumbleExpr(jstate, (Node *)query->cteList); + JumbleRangeTable(jstate, query->rtable); + JumbleExpr(jstate, (Node *)query->jointree); + JumbleExpr(jstate, (Node *)query->targetList); + JumbleExpr(jstate, (Node *)query->returningList); + JumbleExpr(jstate, (Node *)query->groupClause); + JumbleExpr(jstate, query->havingQual); + JumbleExpr(jstate, (Node *)query->windowClause); + JumbleExpr(jstate, (Node *)query->distinctClause); + JumbleExpr(jstate, (Node *)query->sortClause); + JumbleExpr(jstate, query->limitOffset); + JumbleExpr(jstate, query->limitCount); + /* we ignore rowMarks */ + JumbleExpr(jstate, query->setOperations); +} + +/* + * Jumble a range table + */ +static void +JumbleRangeTable(pgssJumbleState *jstate, List *rtable) +{ + ListCell *lc; + + foreach (lc, rtable) + { + RangeTblEntry *rte = (RangeTblEntry *)lfirst(lc); + + Assert(IsA(rte, RangeTblEntry)); + APP_JUMB(rte->rtekind); + switch (rte->rtekind) + { + case RTE_RELATION: + APP_JUMB(rte->relid); + break; + case RTE_SUBQUERY: + JumbleQuery(jstate, rte->subquery); + break; + case RTE_JOIN: + APP_JUMB(rte->jointype); + break; + case RTE_FUNCTION: + JumbleExpr(jstate, (Node *)rte->functions); + break; + case RTE_VALUES: + JumbleExpr(jstate, (Node *)rte->values_lists); + break; + case RTE_CTE: + + /* + * Depending on the CTE name here isn't ideal, but it's the + * only info we have to identify the referenced WITH item. + */ + APP_JUMB_STRING(rte->ctename); + APP_JUMB(rte->ctelevelsup); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); + break; + } + } +} + +/* + * Jumble an expression tree + * + * In general this function should handle all the same node types that + * expression_tree_walker() does, and therefore it's coded to be as parallel + * to that function as possible. However, since we are only invoked on + * queries immediately post-parse-analysis, we need not handle node types + * that only appear in planning. + * + * Note: the reason we don't simply use expression_tree_walker() is that the + * point of that function is to support tree walkers that don't care about + * most tree node types, but here we care about all types. We should complain + * about any unrecognized node type. + */ +static void +JumbleExpr(pgssJumbleState *jstate, Node *node) +{ + ListCell *temp; + + if (node == NULL) + return; + + /* Guard against stack overflow due to overly complex expressions */ + check_stack_depth(); + + /* + * We always emit the node's NodeTag, then any additional fields that are + * considered significant, and then we recurse to any child nodes. + */ + APP_JUMB(node->type); + + switch (nodeTag(node)) + { + case T_Var: + { + Var *var = (Var *)node; + + APP_JUMB(var->varno); + APP_JUMB(var->varattno); + APP_JUMB(var->varlevelsup); + } + break; + case T_Const: + { + Const *c = (Const *)node; + + /* We jumble only the constant's type, not its value */ + APP_JUMB(c->consttype); + /* Also, record its parse location for query normalization */ + RecordConstLocation(jstate, c->location); + } + break; + case T_Param: + { + Param *p = (Param *)node; + + APP_JUMB(p->paramkind); + APP_JUMB(p->paramid); + APP_JUMB(p->paramtype); + } + break; + case T_Aggref: + { + Aggref *expr = (Aggref *)node; + + APP_JUMB(expr->aggfnoid); + JumbleExpr(jstate, (Node *)expr->aggdirectargs); + JumbleExpr(jstate, (Node *)expr->args); + JumbleExpr(jstate, (Node *)expr->aggorder); + JumbleExpr(jstate, (Node *)expr->aggdistinct); + JumbleExpr(jstate, (Node *)expr->aggfilter); + } + break; + case T_WindowFunc: + { + WindowFunc *expr = (WindowFunc *)node; + + APP_JUMB(expr->winfnoid); + APP_JUMB(expr->winref); + JumbleExpr(jstate, (Node *)expr->args); + JumbleExpr(jstate, (Node *)expr->aggfilter); + } + break; + case T_ArrayRef: + { + ArrayRef *aref = (ArrayRef *)node; + + JumbleExpr(jstate, (Node *)aref->refupperindexpr); + JumbleExpr(jstate, (Node *)aref->reflowerindexpr); + JumbleExpr(jstate, (Node *)aref->refexpr); + JumbleExpr(jstate, (Node *)aref->refassgnexpr); + } + break; + case T_FuncExpr: + { + FuncExpr *expr = (FuncExpr *)node; + + APP_JUMB(expr->funcid); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_NamedArgExpr: + { + NamedArgExpr *nae = (NamedArgExpr *)node; + + APP_JUMB(nae->argnumber); + JumbleExpr(jstate, (Node *)nae->arg); + } + break; + case T_OpExpr: + case T_DistinctExpr: /* struct-equivalent to OpExpr */ + case T_NullIfExpr: /* struct-equivalent to OpExpr */ + { + OpExpr *expr = (OpExpr *)node; + + APP_JUMB(expr->opno); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *)node; + + APP_JUMB(expr->opno); + APP_JUMB(expr->useOr); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *)node; + + APP_JUMB(expr->boolop); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_SubLink: + { + SubLink *sublink = (SubLink *)node; + + APP_JUMB(sublink->subLinkType); + JumbleExpr(jstate, (Node *)sublink->testexpr); + JumbleQuery(jstate, (Query *)sublink->subselect); + } + break; + case T_FieldSelect: + { + FieldSelect *fs = (FieldSelect *)node; + + APP_JUMB(fs->fieldnum); + JumbleExpr(jstate, (Node *)fs->arg); + } + break; + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *)node; + + JumbleExpr(jstate, (Node *)fstore->arg); + JumbleExpr(jstate, (Node *)fstore->newvals); + } + break; + case T_RelabelType: + { + RelabelType *rt = (RelabelType *)node; + + APP_JUMB(rt->resulttype); + JumbleExpr(jstate, (Node *)rt->arg); + } + break; + case T_CoerceViaIO: + { + CoerceViaIO *cio = (CoerceViaIO *)node; + + APP_JUMB(cio->resulttype); + JumbleExpr(jstate, (Node *)cio->arg); + } + break; + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *)node; + + APP_JUMB(acexpr->resulttype); + JumbleExpr(jstate, (Node *)acexpr->arg); + } + break; + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *)node; + + APP_JUMB(crexpr->resulttype); + JumbleExpr(jstate, (Node *)crexpr->arg); + } + break; + case T_CollateExpr: + { + CollateExpr *ce = (CollateExpr *)node; + + APP_JUMB(ce->collOid); + JumbleExpr(jstate, (Node *)ce->arg); + } + break; + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *)node; + + JumbleExpr(jstate, (Node *)caseexpr->arg); + foreach (temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *)lfirst(temp); + + Assert(IsA(when, CaseWhen)); + JumbleExpr(jstate, (Node *)when->expr); + JumbleExpr(jstate, (Node *)when->result); + } + JumbleExpr(jstate, (Node *)caseexpr->defresult); + } + break; + case T_CaseTestExpr: + { + CaseTestExpr *ct = (CaseTestExpr *)node; + + APP_JUMB(ct->typeId); + } + break; + case T_ArrayExpr: + JumbleExpr(jstate, (Node *)((ArrayExpr *)node)->elements); + break; + case T_RowExpr: + JumbleExpr(jstate, (Node *)((RowExpr *)node)->args); + break; + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *)node; + + APP_JUMB(rcexpr->rctype); + JumbleExpr(jstate, (Node *)rcexpr->largs); + JumbleExpr(jstate, (Node *)rcexpr->rargs); + } + break; + case T_CoalesceExpr: + JumbleExpr(jstate, (Node *)((CoalesceExpr *)node)->args); + break; + case T_MinMaxExpr: + { + MinMaxExpr *mmexpr = (MinMaxExpr *)node; + + APP_JUMB(mmexpr->op); + JumbleExpr(jstate, (Node *)mmexpr->args); + } + break; + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *)node; + + APP_JUMB(xexpr->op); + JumbleExpr(jstate, (Node *)xexpr->named_args); + JumbleExpr(jstate, (Node *)xexpr->args); + } + break; + case T_NullTest: + { + NullTest *nt = (NullTest *)node; + + APP_JUMB(nt->nulltesttype); + JumbleExpr(jstate, (Node *)nt->arg); + } + break; + case T_BooleanTest: + { + BooleanTest *bt = (BooleanTest *)node; + + APP_JUMB(bt->booltesttype); + JumbleExpr(jstate, (Node *)bt->arg); + } + break; + case T_CoerceToDomain: + { + CoerceToDomain *cd = (CoerceToDomain *)node; + + APP_JUMB(cd->resulttype); + JumbleExpr(jstate, (Node *)cd->arg); + } + break; + case T_CoerceToDomainValue: + { + CoerceToDomainValue *cdv = (CoerceToDomainValue *)node; + + APP_JUMB(cdv->typeId); + } + break; + case T_SetToDefault: + { + SetToDefault *sd = (SetToDefault *)node; + + APP_JUMB(sd->typeId); + } + break; + case T_CurrentOfExpr: + { + CurrentOfExpr *ce = (CurrentOfExpr *)node; + + APP_JUMB(ce->cvarno); + if (ce->cursor_name) + APP_JUMB_STRING(ce->cursor_name); + APP_JUMB(ce->cursor_param); + } + break; + case T_TargetEntry: + { + TargetEntry *tle = (TargetEntry *)node; + + APP_JUMB(tle->resno); + APP_JUMB(tle->ressortgroupref); + JumbleExpr(jstate, (Node *)tle->expr); + } + break; + case T_RangeTblRef: + { + RangeTblRef *rtr = (RangeTblRef *)node; + + APP_JUMB(rtr->rtindex); + } + break; + case T_JoinExpr: + { + JoinExpr *join = (JoinExpr *)node; + + APP_JUMB(join->jointype); + APP_JUMB(join->isNatural); + APP_JUMB(join->rtindex); + JumbleExpr(jstate, join->larg); + JumbleExpr(jstate, join->rarg); + JumbleExpr(jstate, join->quals); + } + break; + case T_FromExpr: + { + FromExpr *from = (FromExpr *)node; + + JumbleExpr(jstate, (Node *)from->fromlist); + JumbleExpr(jstate, from->quals); + } + break; + case T_List: + foreach (temp, (List *)node) + { + JumbleExpr(jstate, (Node *)lfirst(temp)); + } + break; + case T_SortGroupClause: + { + SortGroupClause *sgc = (SortGroupClause *)node; + + APP_JUMB(sgc->tleSortGroupRef); + APP_JUMB(sgc->eqop); + APP_JUMB(sgc->sortop); + APP_JUMB(sgc->nulls_first); + } + break; + case T_WindowClause: + { + WindowClause *wc = (WindowClause *)node; + + APP_JUMB(wc->winref); + APP_JUMB(wc->frameOptions); + JumbleExpr(jstate, (Node *)wc->partitionClause); + JumbleExpr(jstate, (Node *)wc->orderClause); + JumbleExpr(jstate, wc->startOffset); + JumbleExpr(jstate, wc->endOffset); + } + break; + case T_CommonTableExpr: + { + CommonTableExpr *cte = (CommonTableExpr *)node; + + /* we store the string name because RTE_CTE RTEs need it */ + APP_JUMB_STRING(cte->ctename); + JumbleQuery(jstate, (Query *)cte->ctequery); + } + break; + case T_SetOperationStmt: + { + SetOperationStmt *setop = (SetOperationStmt *)node; + + APP_JUMB(setop->op); + APP_JUMB(setop->all); + JumbleExpr(jstate, setop->larg); + JumbleExpr(jstate, setop->rarg); + } + break; + case T_RangeTblFunction: + { + RangeTblFunction *rtfunc = (RangeTblFunction *)node; + + JumbleExpr(jstate, rtfunc->funcexpr); + } + break; + default: + /* Only a warning, since we can stumble along anyway */ + elog(WARNING, "unrecognized node type: %d", + (int)nodeTag(node)); + break; + } +} + +/* + * Record location of constant within query string of query tree + * that is currently being walked. + */ +static void +RecordConstLocation(pgssJumbleState *jstate, int location) +{ + /* -1 indicates unknown or undefined location */ + if (location >= 0) + { + /* enlarge array if needed */ + if (jstate->clocations_count >= jstate->clocations_buf_size) + { + jstate->clocations_buf_size *= 2; + jstate->clocations = (pgssLocationLen *) + repalloc(jstate->clocations, + jstate->clocations_buf_size * + sizeof(pgssLocationLen)); + } + jstate->clocations[jstate->clocations_count].location = location; + /* initialize lengths to -1 to simplify fill_in_constant_lengths */ + jstate->clocations[jstate->clocations_count].length = -1; + jstate->clocations_count++; + } +} + +/* check if token should be replaced by substitute varable */ +static bool +need_replace(int token) +{ + return (token == FCONST) || (token == ICONST) || (token == SCONST) || (token == BCONST) || (token == XCONST); +} + +/* + * gen_normplan - parse execution plan using flex and replace all CONST to + * substitute variables. + */ +static StringInfo +gen_normplan(const char *execution_plan) +{ + core_yyscan_t yyscanner; + core_yy_extra_type yyextra; + core_YYSTYPE yylval; + YYLTYPE yylloc; + int tok; + int bind_prefix = 1; + char *tmp_str; + YYLTYPE last_yylloc = 0; + int last_tok = 0; + StringInfo plan_out = makeStringInfo(); + ; + + yyscanner = scanner_init(execution_plan, + &yyextra, +#if PG_VERSION_NUM >= 120000 + &ScanKeywords, + ScanKeywordTokens +#else + ScanKeywords, + NumScanKeywords +#endif + ); + + for (;;) + { + /* get the next lexem */ + tok = core_yylex(&yylval, &yylloc, yyscanner); + + /* now we store end previsous lexem in yylloc - so could prcess it */ + if (need_replace(last_tok)) + { + /* substitute variable instead of CONST */ + int s_len = asprintf(&tmp_str, "$%i", bind_prefix++); + if (s_len > 0) + { + appendStringInfoString(plan_out, tmp_str); + free(tmp_str); + } + else + { + appendStringInfoString(plan_out, "??"); + } + } + else + { + /* do not change - just copy as-is */ + tmp_str = strndup((char *)execution_plan + last_yylloc, yylloc - last_yylloc); + appendStringInfoString(plan_out, tmp_str); + free(tmp_str); + } + /* check if further parsing not needed */ + if (tok == 0) + break; + last_tok = tok; + last_yylloc = yylloc; + } + + scanner_finish(yyscanner); + + return plan_out; +} + +uint64_t get_plan_id(QueryDesc *queryDesc) +{ + if (!queryDesc->sourceText) + return 0; + StringInfo normalized = gen_normplan(queryDesc->sourceText); + return hash_any((unsigned char *)normalized->data, normalized->len); +} + +/* + * Post-parse-analysis hook: mark query with a queryId + */ +void pgss_post_parse_analyze(ParseState *pstate, Query *query) +{ + pgssJumbleState jstate; + + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query); + + /* Assert we didn't do this already */ + Assert(query->queryId == 0); + + /* + * Utility statements get queryId zero. We do this even in cases where + * the statement contains an optimizable statement for which a queryId + * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, + * runtime control will first go through ProcessUtility and then the + * executor, and we don't want the executor hooks to do anything, since we + * are already measuring the statement's costs at the utility level. + */ + if (query->utilityStmt) + { + query->queryId = 0; + return; + } + + /* Set up workspace for query jumbling */ + jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble_len = 0; + jstate.clocations_buf_size = 32; + jstate.clocations = (pgssLocationLen *) + palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations_count = 0; + + /* Compute query ID and mark the Query node with it */ + JumbleQuery(&jstate, query); + query->queryId = hash_any(jstate.jumble, jstate.jumble_len); + + /* + * If we are unlucky enough to get a hash of zero, use 1 instead, to + * prevent confusion with the utility-statement case. + */ + if (query->queryId == 0) + query->queryId = 1; +} \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h new file mode 100644 index 00000000000..274f96aebaf --- /dev/null +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -0,0 +1,15 @@ +#pragma once + +#ifdef __cplusplus +extern "C" +{ +#endif + +extern void stat_statements_parser_init(void); +extern void stat_statements_parser_deinit(void); + +#ifdef __cplusplus +} +#endif + +uint64_t get_plan_id(QueryDesc *queryDesc); \ No newline at end of file diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c new file mode 100644 index 00000000000..69475ea5079 --- /dev/null +++ b/src/yagp_hooks_collector.c @@ -0,0 +1,22 @@ +#include "postgres.h" +#include "cdb/cdbvars.h" +#include "fmgr.h" + +#include "hook_wrappers.h" + +PG_MODULE_MAGIC; + +void _PG_init(void); +void _PG_fini(void); + +void _PG_init(void) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + hooks_init(); + } +} + +void _PG_fini(void) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + hooks_deinit(); + } +} diff --git a/yagp-hooks-collector.control b/yagp-hooks-collector.control new file mode 100644 index 00000000000..82c189a88fc --- /dev/null +++ b/yagp-hooks-collector.control @@ -0,0 +1,5 @@ +# yagp-hooks-collector extension +comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' +default_version = '1.0' +module_pathname = '$libdir/yagp-hooks-collector' +superuser = true From 208a37c0175dfcc9a7c03e262973b74b34f83efa Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 Mar 2023 17:07:23 +0300 Subject: [PATCH 037/128] [yagp_hooks_collector] Fix segfault in plan text generator Guard against NULL plan state when generating EXPLAIN output. --- src/EventSender.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index bb4765adeb1..b1815a22bf8 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -130,10 +130,13 @@ void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plantext(), queryDesc); - qi->set_plan_id(get_plan_id(queryDesc)); - qi->set_query_id(queryDesc->plannedstmt->queryId); + if (queryDesc->planstate) + { + set_plan_text(qi->mutable_plantext(), queryDesc); + qi->set_plan_id(get_plan_id(queryDesc)); + } } + qi->set_query_id(queryDesc->plannedstmt->queryId); } } // namespace From 9eaaaa5a1bc07f44bb2b152c9ba654601238e3cb Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 29 Mar 2023 16:10:20 +0300 Subject: [PATCH 038/128] [yagp_hooks_collector] Add executor instrumentation, /proc stats, and normalized texts Collect spill info (file count, bytes written). Generate normalized query and plan texts using a pg_stat_statements-derived parser. Collect buffer I/O counters, tuple counts, timing, and /proc/self CPU/memory/IO statistics. --- Makefile | 1 - protos/yagpcc_metrics.proto | 42 +-- protos/yagpcc_set_service.proto | 20 +- src/EventSender.cpp | 164 +++++++----- src/EventSender.h | 4 +- src/ProcStats.cpp | 119 +++++++++ src/ProcStats.h | 7 + src/SpillInfoWrapper.c | 21 ++ src/hook_wrappers.cpp | 22 +- .../pg_stat_statements_ya_parser.c | 248 +++++++++++++++++- .../pg_stat_statements_ya_parser.h | 3 +- 11 files changed, 519 insertions(+), 132 deletions(-) create mode 100644 src/ProcStats.cpp create mode 100644 src/ProcStats.h create mode 100644 src/SpillInfoWrapper.c diff --git a/Makefile b/Makefile index 15c5dabb70e..0a21cf136ff 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. -all: all check install installdirs installcheck installcheck-parallel uninstall clean distclean maintainer-clean dist distcheck world check-world install-world installcheck-world installcheck-resgroup installcheck-resgroup-v2: @if [ ! -f GNUmakefile ] ; then \ diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index b7e255484c7..f00f329a208 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -27,9 +27,12 @@ message QueryInfo { PlanGenerator generator = 1; uint64 query_id = 2; uint64 plan_id = 3; - string queryText = 4; - string planText = 5; - SessionInfo sessionInfo = 6; + string query_text = 4; + string plan_text = 5; + string temlate_query_text = 6; + string temlate_plan_text = 7; + string userName = 8; + string databaseName = 9; } enum PlanGenerator @@ -45,40 +48,17 @@ message GPMetrics { SpillInfo spill = 3; } -message QueryInfoHeader { - int32 pid = 1; - GpId gpIdentity = 2; - - int32 tmid = 3; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ - int32 ssid = 4; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ - int32 ccnt = 5; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ - int32 sliceid = 6; /* slice identificator, 0 means general info for the whole query */ +message QueryKey { + int32 tmid = 1; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ + int32 ssid = 2; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ + int32 ccnt = 3; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ } -message GpId { +message SegmentKey { int32 dbid = 1; /* the dbid of this database */ int32 segindex = 2; /* content indicator: -1 for entry database, * 0, ..., n-1 for segment database * * a primary and its mirror have the same segIndex */ - GpRole gp_role = 3; - GpRole gp_session_role = 4; -} - -enum GpRole -{ - GP_ROLE_UNSPECIFIED = 0; - GP_ROLE_UTILITY = 1; /* Operating as a simple database engine */ - GP_ROLE_DISPATCH = 2; /* Operating as the parallel query dispatcher */ - GP_ROLE_EXECUTE = 3; /* Operating as a parallel query executor */ - GP_ROLE_UNDEFINED = 4; /* Should never see this role in use */ -} - -message SessionInfo { - string sql = 1; - string userName = 2; - string databaseName = 3; - string resourceGroup = 4; - string applicationName = 5; } message SystemStat { diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 0bef72891ee..97c5691a6f5 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -27,19 +27,19 @@ enum MetricResponseStatusCode { } message SetQueryReq { - QueryStatus query_status = 1; + QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; - - QueryInfoHeader header = 3; - QueryInfo query_info = 4; - GPMetrics query_metrics = 5; - repeated MetricPlan plan_tree = 6; + QueryKey query_key = 3; + QueryInfo query_info = 4; + GPMetrics query_metrics = 5; + repeated MetricPlan plan_tree = 6; } message SetPlanNodeReq { - PlanNodeStatus node_status = 1; + PlanNodeStatus node_status = 1; google.protobuf.Timestamp datetime = 2; - QueryInfoHeader header = 3; - GPMetrics node_metrics = 4; - MetricPlan plan_node = 5; + QueryKey query_key = 3; + SegmentKey segment_key = 4; + GPMetrics node_metrics = 5; + MetricPlan plan_node = 6; } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b1815a22bf8..d8145b811a4 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,11 +1,13 @@ #include "EventSender.h" #include "GrpcConnector.h" +#include "ProcStats.h" #include "protos/yagpcc_set_service.pb.h" #include extern "C" { #include "postgres.h" +#include "access/hash.h" #include "utils/metrics_utils.h" #include "utils/elog.h" #include "executor/executor.h" @@ -18,10 +20,13 @@ extern "C" #include "tcop/utility.h" #include "pg_stat_statements_ya_parser.h" + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); } namespace { + std::string* get_user_name() { const char *username = GetConfigOption("session_authorization", false, false); @@ -36,26 +41,6 @@ std::string* get_db_name() return result; } -std::string* get_rg_name() -{ - auto userId = GetUserId(); - if (!OidIsValid(userId)) - return nullptr; - auto groupId = GetResGroupIdForRole(userId); - if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); - if (rgname == nullptr) - return nullptr; - pfree(rgname); - return new std::string(rgname); -} - -std::string* get_app_name() -{ - return application_name ? new std::string(application_name) : nullptr; -} - int get_cur_slice_id(QueryDesc *desc) { if (!desc->estate) @@ -75,33 +60,22 @@ google::protobuf::Timestamp current_ts() return current_ts; } -void set_header(yagpcc::QueryInfoHeader *header, QueryDesc *queryDesc) +void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { - header->set_pid(MyProcPid); - auto gpId = header->mutable_gpidentity(); - gpId->set_dbid(GpIdentity.dbid); - gpId->set_segindex(GpIdentity.segindex); - gpId->set_gp_role(static_cast(Gp_role)); - gpId->set_gp_session_role(static_cast(Gp_session_role)); - header->set_ssid(gp_session_id); - header->set_ccnt(gp_command_count); - header->set_sliceid(get_cur_slice_id(queryDesc)); + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); int32 tmid = 0; gpmon_gettmid(&tmid); - header->set_tmid(tmid); + key->set_tmid(tmid); } -void set_session_info(yagpcc::SessionInfo *si, QueryDesc *queryDesc) +void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { - if (queryDesc->sourceText) - *si->mutable_sql() = std::string(queryDesc->sourceText); - si->set_allocated_applicationname(get_app_name()); - si->set_allocated_databasename(get_db_name()); - si->set_allocated_resourcegroup(get_rg_name()); - si->set_allocated_username(get_user_name()); + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); } -ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { ExplainState es; ExplainInitState(&es); @@ -109,74 +83,130 @@ ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) es.verbose = true; es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); - ExplainPrintPlan(&es, queryDesc); + ExplainPrintPlan(&es, query_desc); ExplainEndOutput(&es); return es; } -void set_plan_text(std::string *plan_text, QueryDesc *queryDesc) +void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { - auto es = get_explain_state(queryDesc, true); + auto es = get_explain_state(query_desc, true); *plan_text = std::string(es.str->data, es.str->len); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) +void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - set_session_info(qi->mutable_sessioninfo(), queryDesc); - if (queryDesc->sourceText) - *qi->mutable_querytext() = queryDesc->sourceText; - if (queryDesc->plannedstmt) - { - qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - if (queryDesc->planstate) - { - set_plan_text(qi->mutable_plantext(), queryDesc); - qi->set_plan_id(get_plan_id(queryDesc)); - } + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + //TODO: free stringinfo? +} + +void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) +{ + *qi->mutable_query_text() = query_desc->sourceText; + char* norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_temlate_query_text() = std::string(norm_query); + pfree(norm_query); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) +{ + if (query_desc->sourceText) + set_query_text(qi, query_desc); + if (query_desc->plannedstmt) + { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); } - qi->set_query_id(queryDesc->plannedstmt->queryId); + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) +{ + auto instrument = query_desc->planstate->instrument; + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); } + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) +{ + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + if (query_desc->planstate->instrument) + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + fill_self_stats(metrics->mutable_systemstat()); +} + + } // namespace -void EventSender::ExecutorStart(QueryDesc *queryDesc, int /* eflags*/) +void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { - elog(DEBUG1, "Query %s start recording", queryDesc->sourceText); + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + + elog(DEBUG1, "Query %s start recording", query_desc->sourceText); yagpcc::SetQueryReq req; req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); *req.mutable_datetime() = current_ts(); - set_header(req.mutable_header(), queryDesc); - set_query_info(req.mutable_query_info(), queryDesc); + set_query_key(req.mutable_query_key(), query_desc); auto result = connector->set_metric_query(req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { elog(WARNING, "Query %s start reporting failed with an error %s", - queryDesc->sourceText, result.error_text().c_str()); + query_desc->sourceText, result.error_text().c_str()); } else { - elog(DEBUG1, "Query %s start successful", queryDesc->sourceText); + elog(DEBUG1, "Query %s start successful", query_desc->sourceText); } } -void EventSender::ExecutorFinish(QueryDesc *queryDesc) +void EventSender::ExecutorFinish(QueryDesc *query_desc) { - elog(DEBUG1, "Query %s finish recording", queryDesc->sourceText); + elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); yagpcc::SetQueryReq req; req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); *req.mutable_datetime() = current_ts(); - set_header(req.mutable_header(), queryDesc); - set_query_info(req.mutable_query_info(), queryDesc); + set_query_key(req.mutable_query_key(), query_desc); + set_query_info(req.mutable_query_info(), query_desc); + set_gp_metrics(req.mutable_query_metrics(), query_desc); auto result = connector->set_metric_query(req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { elog(WARNING, "Query %s finish reporting failed with an error %s", - queryDesc->sourceText, result.error_text().c_str()); + query_desc->sourceText, result.error_text().c_str()); } else { - elog(DEBUG1, "Query %s finish successful", queryDesc->sourceText); + elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); } } diff --git a/src/EventSender.h b/src/EventSender.h index 70868f6c757..bd02455ca7e 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -9,8 +9,8 @@ struct QueryDesc; class EventSender { public: - void ExecutorStart(QueryDesc *queryDesc, int eflags); - void ExecutorFinish(QueryDesc *queryDesc); + void ExecutorStart(QueryDesc *query_desc, int eflags); + void ExecutorFinish(QueryDesc *query_desc); static EventSender *instance(); private: diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp new file mode 100644 index 00000000000..34c5d05719e --- /dev/null +++ b/src/ProcStats.cpp @@ -0,0 +1,119 @@ +#include "ProcStats.h" +#include "yagpcc_metrics.pb.h" +#include +#include +#include + +extern "C" +{ +#include "postgres.h" +#include "utils/elog.h" +} + +namespace { +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name); + +void fill_io_stats(yagpcc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); +} + +void fill_cpu_stats(yagpcc::SystemStat *stats) +{ + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int STARTTIME_ID = 21; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); + + double uptime; + { + std::ifstream proc_stat("/proc/uptime"); + proc_stat >> uptime; + } + + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + double start_time = 0; + for (int i = 0; i <= RSS_ID; ++i) + { + switch (i) + { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps); + break; + case STARTTIME_ID: + uint64_t starttime; + proc_stat >> starttime; + start_time = static_cast(starttime) / tps; + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; + } + stats->set_runningtimeseconds(uptime - start_time); + } +} + +void fill_status_stats(yagpcc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) + { + if (key == "VmPeak:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") + elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); + } + else if (key == "VmSize:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") + elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); + } + } +} +} // namespace + +void fill_self_stats(yagpcc::SystemStat *stats) +{ + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); +} \ No newline at end of file diff --git a/src/ProcStats.h b/src/ProcStats.h new file mode 100644 index 00000000000..30a90a60519 --- /dev/null +++ b/src/ProcStats.h @@ -0,0 +1,7 @@ +#pragma once + +namespace yagpcc { +class SystemStat; +} + +void fill_self_stats(yagpcc::SystemStat *stats); \ No newline at end of file diff --git a/src/SpillInfoWrapper.c b/src/SpillInfoWrapper.c new file mode 100644 index 00000000000..c6ace0a693f --- /dev/null +++ b/src/SpillInfoWrapper.c @@ -0,0 +1,21 @@ +#include "postgres.h" +#include "utils/workfile_mgr.h" + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes) +{ + int count = 0; + int i = 0; + workfile_set *workfiles = workfile_mgr_cache_entries_get_copy(&count); + workfile_set *wf_iter = workfiles; + for (i = 0; i < count; ++i, ++wf_iter) + { + if (wf_iter->active && wf_iter->session_id == ssid && wf_iter->command_count == ccid) + { + *file_count += wf_iter->num_files; + *total_bytes += wf_iter->total_bytes; + } + } + pfree(workfiles); +} \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 9f3200c006f..1dabb59ab3f 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -19,8 +19,8 @@ extern "C" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; -static void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags); -static void ya_ExecutorFinish_hook(QueryDesc *queryDesc); +static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorFinish_hook(QueryDesc *query_desc); #define REPLACE_HOOK(hookName) \ previous_##hookName = hookName; \ @@ -56,12 +56,22 @@ void hooks_deinit() else \ standard_##hookName(__VA_ARGS__); -void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags) +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - CREATE_HOOK_WRAPPER(ExecutorStart, queryDesc, eflags); + CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); + PG_TRY(); + { + EventSender::instance()->ExecutorStart(query_desc, eflags); + } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + PG_RE_THROW(); + } + PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *queryDesc) +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - CREATE_HOOK_WRAPPER(ExecutorFinish, queryDesc); + CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index f14742337bd..ae79e7dc40a 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -1,3 +1,6 @@ +// NOTE: this file is just a bunch of code borrowed from pg_stat_statements for PG 9.4 +// and from our own inhouse implementation of pg_stat_statements for managed PG + #include "postgres.h" #include @@ -67,14 +70,15 @@ static void JumbleQuery(pgssJumbleState *jstate, Query *query); static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); static void JumbleExpr(pgssJumbleState *jstate, Node *node); static void RecordConstLocation(pgssJumbleState *jstate, int location); - -static StringInfo gen_normplan(const char *execution_plan); - +static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query); +static int comp_location(const void *a, const void *b); +StringInfo gen_normplan(const char *execution_plan); static bool need_replace(int token); - void pgss_post_parse_analyze(ParseState *pstate, Query *query); +static char *generate_normalized_query(pgssJumbleState *jstate, const char *query, + int *query_len_p, int encoding); -void stat_statements_parser_init() + void stat_statements_parser_init() { prev_post_parse_analyze_hook = post_parse_analyze_hook; post_parse_analyze_hook = pgss_post_parse_analyze; @@ -650,7 +654,7 @@ need_replace(int token) * gen_normplan - parse execution plan using flex and replace all CONST to * substitute variables. */ -static StringInfo +StringInfo gen_normplan(const char *execution_plan) { core_yyscan_t yyscanner; @@ -715,14 +719,6 @@ gen_normplan(const char *execution_plan) return plan_out; } -uint64_t get_plan_id(QueryDesc *queryDesc) -{ - if (!queryDesc->sourceText) - return 0; - StringInfo normalized = gen_normplan(queryDesc->sourceText); - return hash_any((unsigned char *)normalized->data, normalized->len); -} - /* * Post-parse-analysis hook: mark query with a queryId */ @@ -768,4 +764,228 @@ void pgss_post_parse_analyze(ParseState *pstate, Query *query) */ if (query->queryId == 0) query->queryId = 1; +} + +/* + * comp_location: comparator for qsorting pgssLocationLen structs by location + */ +static int +comp_location(const void *a, const void *b) +{ + int l = ((const pgssLocationLen *) a)->location; + int r = ((const pgssLocationLen *) b)->location; + + if (l < r) + return -1; + else if (l > r) + return +1; + else + return 0; +} + +/* + * Given a valid SQL string and an array of constant-location records, + * fill in the textual lengths of those constants. + * + * The constants may use any allowed constant syntax, such as float literals, + * bit-strings, single-quoted strings and dollar-quoted strings. This is + * accomplished by using the public API for the core scanner. + * + * It is the caller's job to ensure that the string is a valid SQL statement + * with constants at the indicated locations. Since in practice the string + * has already been parsed, and the locations that the caller provides will + * have originated from within the authoritative parser, this should not be + * a problem. + * + * Duplicate constant pointers are possible, and will have their lengths + * marked as '-1', so that they are later ignored. (Actually, we assume the + * lengths were initialized as -1 to start with, and don't change them here.) + * + * N.B. There is an assumption that a '-' character at a Const location begins + * a negative numeric constant. This precludes there ever being another + * reason for a constant to start with a '-'. + */ +static void +fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) +{ + pgssLocationLen *locs; + core_yyscan_t yyscanner; + core_yy_extra_type yyextra; + core_YYSTYPE yylval; + YYLTYPE yylloc; + int last_loc = -1; + int i; + + /* + * Sort the records by location so that we can process them in order while + * scanning the query text. + */ + if (jstate->clocations_count > 1) + qsort(jstate->clocations, jstate->clocations_count, + sizeof(pgssLocationLen), comp_location); + locs = jstate->clocations; + + /* initialize the flex scanner --- should match raw_parser() */ + yyscanner = scanner_init(query, + &yyextra, + ScanKeywords, + NumScanKeywords); + + /* Search for each constant, in sequence */ + for (i = 0; i < jstate->clocations_count; i++) + { + int loc = locs[i].location; + int tok; + + Assert(loc >= 0); + + if (loc <= last_loc) + continue; /* Duplicate constant, ignore */ + + /* Lex tokens until we find the desired constant */ + for (;;) + { + tok = core_yylex(&yylval, &yylloc, yyscanner); + + /* We should not hit end-of-string, but if we do, behave sanely */ + if (tok == 0) + break; /* out of inner for-loop */ + + /* + * We should find the token position exactly, but if we somehow + * run past it, work with that. + */ + if (yylloc >= loc) + { + if (query[loc] == '-') + { + /* + * It's a negative value - this is the one and only case + * where we replace more than a single token. + * + * Do not compensate for the core system's special-case + * adjustment of location to that of the leading '-' + * operator in the event of a negative constant. It is + * also useful for our purposes to start from the minus + * symbol. In this way, queries like "select * from foo + * where bar = 1" and "select * from foo where bar = -2" + * will have identical normalized query strings. + */ + tok = core_yylex(&yylval, &yylloc, yyscanner); + if (tok == 0) + break; /* out of inner for-loop */ + } + + /* + * We now rely on the assumption that flex has placed a zero + * byte after the text of the current token in scanbuf. + */ + locs[i].length = strlen(yyextra.scanbuf + loc); + break; /* out of inner for-loop */ + } + } + + /* If we hit end-of-string, give up, leaving remaining lengths -1 */ + if (tok == 0) + break; + + last_loc = loc; + } + + scanner_finish(yyscanner); +} + +/* + * Generate a normalized version of the query string that will be used to + * represent all similar queries. + * + * Note that the normalized representation may well vary depending on + * just which "equivalent" query is used to create the hashtable entry. + * We assume this is OK. + * + * *query_len_p contains the input string length, and is updated with + * the result string length (which cannot be longer) on exit. + * + * Returns a palloc'd string. + */ +static char * +generate_normalized_query(pgssJumbleState *jstate, const char *query, + int *query_len_p, int encoding) +{ + char *norm_query; + int query_len = *query_len_p; + int i, + len_to_wrt, /* Length (in bytes) to write */ + quer_loc = 0, /* Source query byte location */ + n_quer_loc = 0, /* Normalized query byte location */ + last_off = 0, /* Offset from start for previous tok */ + last_tok_len = 0; /* Length (in bytes) of that tok */ + + /* + * Get constants' lengths (core system only gives us locations). Note + * this also ensures the items are sorted by location. + */ + fill_in_constant_lengths(jstate, query); + + /* Allocate result buffer */ + norm_query = palloc(query_len + 1); + + for (i = 0; i < jstate->clocations_count; i++) + { + int off, /* Offset from start for cur tok */ + tok_len; /* Length (in bytes) of that tok */ + + off = jstate->clocations[i].location; + tok_len = jstate->clocations[i].length; + + if (tok_len < 0) + continue; /* ignore any duplicates */ + + /* Copy next chunk (what precedes the next constant) */ + len_to_wrt = off - last_off; + len_to_wrt -= last_tok_len; + + Assert(len_to_wrt >= 0); + memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); + n_quer_loc += len_to_wrt; + + /* And insert a '?' in place of the constant token */ + norm_query[n_quer_loc++] = '?'; + + quer_loc = off + tok_len; + last_off = off; + last_tok_len = tok_len; + } + + /* + * We've copied up until the last ignorable constant. Copy over the + * remaining bytes of the original query string. + */ + len_to_wrt = query_len - quer_loc; + + Assert(len_to_wrt >= 0); + memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); + n_quer_loc += len_to_wrt; + + Assert(n_quer_loc <= query_len); + norm_query[n_quer_loc] = '\0'; + + *query_len_p = n_quer_loc; + return norm_query; +} + +char *gen_normquery(const char *query) +{ + if (!query) { + return NULL; + } + pgssJumbleState jstate; + jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble_len = 0; + jstate.clocations_buf_size = 32; + jstate.clocations = (pgssLocationLen *) + palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations_count = 0; + int query_len = strlen(query); + return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index 274f96aebaf..aa9cd217e31 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -12,4 +12,5 @@ extern void stat_statements_parser_deinit(void); } #endif -uint64_t get_plan_id(QueryDesc *queryDesc); \ No newline at end of file +StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query); \ No newline at end of file From f1d5c16293e3f9f7caac35781616c98ecd9640c6 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 6 Apr 2023 13:24:25 +0300 Subject: [PATCH 039/128] [yagp_hooks_collector] Apply llvm code style --- Makefile | 1 - src/EventSender.cpp | 367 ++++++++++++++++++++---------------------- src/EventSender.h | 13 +- src/GrpcConnector.cpp | 68 ++++---- src/GrpcConnector.h | 13 +- src/ProcStats.cpp | 183 ++++++++++----------- src/hook_wrappers.cpp | 83 +++++----- 7 files changed, 338 insertions(+), 390 deletions(-) diff --git a/Makefile b/Makefile index 0a21cf136ff..91be52c4468 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. - # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. diff --git a/src/EventSender.cpp b/src/EventSender.cpp index d8145b811a4..b7c3cd70b85 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -4,8 +4,7 @@ #include "protos/yagpcc_set_service.pb.h" #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "access/hash.h" #include "utils/metrics_utils.h" @@ -21,202 +20,178 @@ extern "C" #include "tcop/utility.h" #include "pg_stat_statements_ya_parser.h" -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); -} - -namespace -{ - -std::string* get_user_name() -{ - const char *username = GetConfigOption("session_authorization", false, false); - return username ? new std::string(username) : nullptr; -} - -std::string* get_db_name() -{ - char *dbname = get_database_name(MyDatabaseId); - std::string* result = dbname ? new std::string(dbname) : nullptr; - pfree(dbname); - return result; -} - -int get_cur_slice_id(QueryDesc *desc) -{ - if (!desc->estate) - { - return 0; - } - return LocallyExecutingSliceIndex(desc->estate); -} - -google::protobuf::Timestamp current_ts() -{ - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; -} - -void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) -{ - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gpmon_gettmid(&tmid); - key->set_tmid(tmid); -} - -void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) -{ - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); -} - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) -{ - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; -} - -void set_plan_text(std::string *plan_text, QueryDesc *query_desc) -{ - auto es = get_explain_state(query_desc, true); - *plan_text = std::string(es.str->data, es.str->len); -} - -void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - //TODO: free stringinfo? -} - -void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - *qi->mutable_query_text() = query_desc->sourceText; - char* norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_temlate_query_text() = std::string(norm_query); - pfree(norm_query); -} - -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - if (query_desc->sourceText) - set_query_text(qi, query_desc); - if (query_desc->plannedstmt) - { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); -} - -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) -{ - auto instrument = query_desc->planstate->instrument; - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) -{ - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - if (query_desc->planstate->instrument) - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); - fill_self_stats(metrics->mutable_systemstat()); +void get_spill_info(int ssid, int ccid, int32_t *file_count, + int64_t *total_bytes); } +namespace { + +std::string *get_user_name() { + const char *username = GetConfigOption("session_authorization", false, false); + return username ? new std::string(username) : nullptr; +} + +std::string *get_db_name() { + char *dbname = get_database_name(MyDatabaseId); + std::string *result = dbname ? new std::string(dbname) : nullptr; + pfree(dbname); + return result; +} + +int get_cur_slice_id(QueryDesc *desc) { + if (!desc->estate) { + return 0; + } + return LocallyExecutingSliceIndex(desc->estate); +} + +google::protobuf::Timestamp current_ts() { + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gpmon_gettmid(&tmid); + key->set_tmid(tmid); +} + +void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); +} + +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; +} + +void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { + auto es = get_explain_state(query_desc, true); + *plan_text = std::string(es.str->data, es.str->len); +} + +void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + // TODO: free stringinfo? +} + +void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + *qi->mutable_query_text() = query_desc->sourceText; + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_temlate_query_text() = std::string(norm_query); + pfree(norm_query); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + if (query_desc->sourceText) { + set_query_text(qi, query_desc); + } + if (query_desc->plannedstmt) { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); + } + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, + QueryDesc *query_desc) { + auto instrument = query_desc->planstate->instrument; + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, + &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + if (query_desc->planstate->instrument) { + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + } + fill_self_stats(metrics->mutable_systemstat()); +} } // namespace -void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) -{ - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - - elog(DEBUG1, "Query %s start recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) - { - elog(WARNING, "Query %s start reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } - else - { - elog(DEBUG1, "Query %s start successful", query_desc->sourceText); - } -} - -void EventSender::ExecutorFinish(QueryDesc *query_desc) -{ - elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_query_info(req.mutable_query_info(), query_desc); - set_gp_metrics(req.mutable_query_metrics(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) - { - elog(WARNING, "Query %s finish reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } - else - { - elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); - } -} - -EventSender *EventSender::instance() -{ - static EventSender sender; - return &sender; -} - -EventSender::EventSender() -{ - connector = std::make_unique(); -} \ No newline at end of file +void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + + elog(DEBUG1, "Query %s start recording", query_desc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { + elog(WARNING, "Query %s start reporting failed with an error %s", + query_desc->sourceText, result.error_text().c_str()); + } else { + elog(DEBUG1, "Query %s start successful", query_desc->sourceText); + } +} + +void EventSender::ExecutorFinish(QueryDesc *query_desc) { + elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + set_query_info(req.mutable_query_info(), query_desc); + set_gp_metrics(req.mutable_query_metrics(), query_desc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { + elog(WARNING, "Query %s finish reporting failed with an error %s", + query_desc->sourceText, result.error_text().c_str()); + } else { + elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); + } +} + +EventSender *EventSender::instance() { + static EventSender sender; + return &sender; +} + +EventSender::EventSender() { connector = std::make_unique(); } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index bd02455ca7e..d69958db9b0 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -6,14 +6,13 @@ class GrpcConnector; struct QueryDesc; -class EventSender -{ +class EventSender { public: - void ExecutorStart(QueryDesc *query_desc, int eflags); - void ExecutorFinish(QueryDesc *query_desc); - static EventSender *instance(); + void ExecutorStart(QueryDesc *query_desc, int eflags); + void ExecutorFinish(QueryDesc *query_desc); + static EventSender *instance(); private: - EventSender(); - std::unique_ptr connector; + EventSender(); + std::unique_ptr connector; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 7329f392010..1a820404428 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -5,51 +5,43 @@ #include #include -class GrpcConnector::Impl -{ +class GrpcConnector::Impl { public: - Impl() - { - GOOGLE_PROTOBUF_VERIFY_VERSION; - this->stub = yagpcc::SetQueryInfo::NewStub(grpc::CreateChannel( - SOCKET_FILE, grpc::InsecureChannelCredentials())); + Impl() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + this->stub = yagpcc::SetQueryInfo::NewStub( + grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials())); + } + + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { + yagpcc::MetricResponse response; + grpc::ClientContext context; + auto deadline = + std::chrono::system_clock::now() + std::chrono::milliseconds(50); + context.set_deadline(deadline); + + grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); + + if (!status.ok()) { + response.set_error_text("Connection lost: " + status.error_message() + + "; " + status.error_details()); + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) - { - yagpcc::MetricResponse response; - grpc::ClientContext context; - auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(50); - context.set_deadline(deadline); - - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - - if (!status.ok()) - { - response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - } - - return response; - } + return response; + } private: - const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; - const std::string TCP_ADDRESS = "127.0.0.1:1432"; - std::unique_ptr stub; + const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; + const std::string TCP_ADDRESS = "127.0.0.1:1432"; + std::unique_ptr stub; }; -GrpcConnector::GrpcConnector() -{ - impl = new Impl(); -} +GrpcConnector::GrpcConnector() { impl = new Impl(); } -GrpcConnector::~GrpcConnector() -{ - delete impl; -} +GrpcConnector::~GrpcConnector() { delete impl; } -yagpcc::MetricResponse GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) -{ - return impl->set_metric_query(req); +yagpcc::MetricResponse +GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) { + return impl->set_metric_query(req); } \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index dc0f21706a3..810c0bd3e15 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -2,14 +2,13 @@ #include "yagpcc_set_service.pb.h" -class GrpcConnector -{ +class GrpcConnector { public: - GrpcConnector(); - ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + GrpcConnector(); + ~GrpcConnector(); + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); private: - class Impl; - Impl *impl; + class Impl; + Impl *impl; }; \ No newline at end of file diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 34c5d05719e..5c64f25ec09 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -4,116 +4,109 @@ #include #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "utils/elog.h" } namespace { -#define FILL_IO_STAT(stat_name) \ - uint64_t stat_name; \ - proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name); +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name); -void fill_io_stats(yagpcc::SystemStat *stats) -{ - std::ifstream proc_stat("/proc/self/io"); - std::string tmp; - FILL_IO_STAT(rchar); - FILL_IO_STAT(wchar); - FILL_IO_STAT(syscr); - FILL_IO_STAT(syscw); - FILL_IO_STAT(read_bytes); - FILL_IO_STAT(write_bytes); - FILL_IO_STAT(cancelled_write_bytes); +void fill_io_stats(yagpcc::SystemStat *stats) { + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(yagpcc::SystemStat *stats) -{ - static const int UTIME_ID = 13; - static const int STIME_ID = 14; - static const int STARTTIME_ID = 21; - static const int VSIZE_ID = 22; - static const int RSS_ID = 23; - static const double tps = sysconf(_SC_CLK_TCK); +void fill_cpu_stats(yagpcc::SystemStat *stats) { + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int STARTTIME_ID = 21; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); - double uptime; - { - std::ifstream proc_stat("/proc/uptime"); - proc_stat >> uptime; - } + double uptime; + { + std::ifstream proc_stat("/proc/uptime"); + proc_stat >> uptime; + } - std::ifstream proc_stat("/proc/self/stat"); - std::string trash; - double start_time = 0; - for (int i = 0; i <= RSS_ID; ++i) - { - switch (i) - { - case UTIME_ID: - double utime; - proc_stat >> utime; - stats->set_usertimeseconds(utime / tps); - break; - case STIME_ID: - double stime; - proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps); - break; - case STARTTIME_ID: - uint64_t starttime; - proc_stat >> starttime; - start_time = static_cast(starttime) / tps; - break; - case VSIZE_ID: - uint64_t vsize; - proc_stat >> vsize; - stats->set_vsize(vsize); - break; - case RSS_ID: - uint64_t rss; - proc_stat >> rss; - // NOTE: this is a double AFAIU, need to double-check - stats->set_rss(rss); - break; - default: - proc_stat >> trash; - } - stats->set_runningtimeseconds(uptime - start_time); + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + double start_time = 0; + for (int i = 0; i <= RSS_ID; ++i) { + switch (i) { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps); + break; + case STARTTIME_ID: + uint64_t starttime; + proc_stat >> starttime; + start_time = static_cast(starttime) / tps; + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; } + stats->set_runningtimeseconds(uptime - start_time); + } } -void fill_status_stats(yagpcc::SystemStat *stats) -{ - std::ifstream proc_stat("/proc/self/status"); - std::string key, measure; - while (proc_stat >> key) - { - if (key == "VmPeak:") - { - uint64_t value; - proc_stat >> value; - stats->set_vmpeakkb(value); - proc_stat >> measure; - if (measure != "kB") - elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); - } - else if (key == "VmSize:") - { - uint64_t value; - proc_stat >> value; - stats->set_vmsizekb(value); - if (measure != "kB") - elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); - } +void fill_status_stats(yagpcc::SystemStat *stats) { + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) { + if (key == "VmPeak:") { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") { + elog(FATAL, "Expected memory sizes in kB, but got in %s", + measure.c_str()); + } + } else if (key == "VmSize:") { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") { + elog(FATAL, "Expected memory sizes in kB, but got in %s", + measure.c_str()); + } } + } } } // namespace -void fill_self_stats(yagpcc::SystemStat *stats) -{ - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); +void fill_self_stats(yagpcc::SystemStat *stats) { + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 1dabb59ab3f..739cca80f01 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,8 +1,7 @@ #include "hook_wrappers.h" #include "EventSender.h" -extern "C" -{ +extern "C" { #include "postgres.h" #include "utils/metrics_utils.h" #include "utils/elog.h" @@ -22,56 +21,48 @@ static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorFinish_hook(QueryDesc *query_desc); -#define REPLACE_HOOK(hookName) \ - previous_##hookName = hookName; \ - hookName = ya_##hookName; +#define REPLACE_HOOK(hookName) \ + previous_##hookName = hookName; \ + hookName = ya_##hookName; -void hooks_init() -{ - REPLACE_HOOK(ExecutorStart_hook); - REPLACE_HOOK(ExecutorFinish_hook); - stat_statements_parser_init(); +void hooks_init() { + REPLACE_HOOK(ExecutorStart_hook); + REPLACE_HOOK(ExecutorFinish_hook); + stat_statements_parser_init(); } -void hooks_deinit() -{ - ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorFinish_hook = ExecutorFinish_hook; - stat_statements_parser_deinit(); +void hooks_deinit() { + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; + stat_statements_parser_deinit(); } -#define CREATE_HOOK_WRAPPER(hookName, ...) \ - PG_TRY(); \ - { \ - EventSender::instance()->hookName(__VA_ARGS__); \ - } \ - PG_CATCH(); \ - { \ - ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ - PG_RE_THROW(); \ - } \ - PG_END_TRY(); \ - if (previous_##hookName##_hook) \ - (*previous_##hookName##_hook)(__VA_ARGS__); \ - else \ - standard_##hookName(__VA_ARGS__); +#define CREATE_HOOK_WRAPPER(hookName, ...) \ + PG_TRY(); \ + { EventSender::instance()->hookName(__VA_ARGS__); } \ + PG_CATCH(); \ + { \ + ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ + PG_RE_THROW(); \ + } \ + PG_END_TRY(); \ + if (previous_##hookName##_hook) \ + (*previous_##hookName##_hook)(__VA_ARGS__); \ + else \ + standard_##hookName(__VA_ARGS__); -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) -{ - CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); - PG_TRY(); - { - EventSender::instance()->ExecutorStart(query_desc, eflags); - } - PG_CATCH(); - { - ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); - PG_RE_THROW(); - } - PG_END_TRY(); +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { + CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); + PG_TRY(); + { EventSender::instance()->ExecutorStart(query_desc, eflags); } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + PG_RE_THROW(); + } + PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) -{ - CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { + CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); } \ No newline at end of file From eb9c9a769a17944a36038d7f628b2b4f7c678253 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 10 Apr 2023 16:01:08 +0300 Subject: [PATCH 040/128] [yagp_hooks_collector] Switch to query_info_collect_hook and fix stability Use query_info_collect_hook for finer-grained lifecycle tracking. Fix two segfaults in early init paths. Skip hooks in UTILITY mode. General robustness improvements. --- protos/yagpcc_set_service.proto | 7 +- src/EventSender.cpp | 207 ++++++++++++------ src/EventSender.h | 13 +- src/GrpcConnector.cpp | 4 +- src/GrpcConnector.h | 2 +- src/hook_wrappers.cpp | 65 +++--- .../pg_stat_statements_ya_parser.c | 21 ++ 7 files changed, 206 insertions(+), 113 deletions(-) diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 97c5691a6f5..93c2f5a01d1 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -30,9 +30,10 @@ message SetQueryReq { QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; QueryKey query_key = 3; - QueryInfo query_info = 4; - GPMetrics query_metrics = 5; - repeated MetricPlan plan_tree = 6; + SegmentKey segment_key = 4; + QueryInfo query_info = 5; + GPMetrics query_metrics = 6; + repeated MetricPlan plan_tree = 7; } message SetPlanNodeReq { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b7c3cd70b85..5ab6bbd60df 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,29 +1,30 @@ -#include "EventSender.h" #include "GrpcConnector.h" #include "ProcStats.h" -#include "protos/yagpcc_set_service.pb.h" #include extern "C" { #include "postgres.h" + #include "access/hash.h" -#include "utils/metrics_utils.h" -#include "utils/elog.h" -#include "executor/executor.h" -#include "commands/explain.h" #include "commands/dbcommands.h" +#include "commands/explain.h" #include "commands/resgroupcmds.h" +#include "executor/executor.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" -#include "cdb/cdbvars.h" #include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" -#include "pg_stat_statements_ya_parser.h" void get_spill_info(int ssid, int ccid, int32_t *file_count, int64_t *total_bytes); } +#include "EventSender.h" + namespace { std::string *get_user_name() { @@ -102,90 +103,152 @@ void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { pfree(norm_query); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - if (query_desc->sourceText) { - set_query_text(qi, query_desc); +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, + bool with_text, bool with_plan) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + if (query_desc->sourceText && with_text) { + set_query_text(qi, query_desc); + } + if (query_desc->plannedstmt && with_plan) { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); + } + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); } - if (query_desc->plannedstmt) { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, - &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - if (query_desc->planstate->instrument) { + if (instrument) { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + bool need_spillinfo) { + if (need_spillinfo) { + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, + &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + } + if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); } +yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, + yagpcc::QueryStatus status) { + yagpcc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + set_segment_key(req.mutable_segment_key(), query_desc); + return req; +} + } // namespace -void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { +void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + switch (status) { + case METRICS_PLAN_NODE_INITIALIZE: + case METRICS_PLAN_NODE_EXECUTING: + case METRICS_PLAN_NODE_FINISHED: + // TODO + break; + case METRICS_QUERY_SUBMIT: + collect_query_submit(reinterpret_cast(arg)); + break; + case METRICS_QUERY_START: + // no-op: executor_after_start is enough + break; + case METRICS_QUERY_DONE: + collect_query_done(reinterpret_cast(arg), "done"); + break; + case METRICS_QUERY_ERROR: + collect_query_done(reinterpret_cast(arg), "error"); + break; + case METRICS_QUERY_CANCELING: + collect_query_done(reinterpret_cast(arg), "calcelling"); + break; + case METRICS_QUERY_CANCELED: + collect_query_done(reinterpret_cast(arg), "cancelled"); + break; + case METRICS_INNER_QUERY_DONE: + // TODO + break; + default: + elog(FATAL, "Unknown query status: %d", status); + } +} + +void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); + set_query_info(req.mutable_query_info(), query_desc, false, true); + send_query_info(&req, "started"); +} + +void EventSender::collect_query_submit(QueryDesc *query_desc) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; - elog(DEBUG1, "Query %s start recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query %s start reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } else { - elog(DEBUG1, "Query %s start successful", query_desc->sourceText); - } + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + set_query_info(req.mutable_query_info(), query_desc, true, false); + send_query_info(&req, "submit"); } -void EventSender::ExecutorFinish(QueryDesc *query_desc) { - elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_query_info(req.mutable_query_info(), query_desc); - set_gp_metrics(req.mutable_query_metrics(), query_desc); - auto result = connector->set_metric_query(req); +void EventSender::collect_query_done(QueryDesc *query_desc, + const std::string &status) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); + set_query_info(req.mutable_query_info(), query_desc, false, false); + // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to gather + // it here. It only makes sense when doing regular stat checks. + set_gp_metrics(req.mutable_query_metrics(), query_desc, + /*need_spillinfo*/ false); + send_query_info(&req, status); +} + +void EventSender::send_query_info(yagpcc::SetQueryReq *req, + const std::string &event) { + auto result = connector->set_metric_query(*req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query %s finish reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } else { - elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); + elog(WARNING, "Query {%d-%d-%d} %s reporting failed with an error %s", + req->query_key().tmid(), req->query_key().ssid(), + req->query_key().ccnt(), event.c_str(), result.error_text().c_str()); } } diff --git a/src/EventSender.h b/src/EventSender.h index d69958db9b0..9c574cba9a1 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,18 +1,25 @@ #pragma once #include +#include class GrpcConnector; - struct QueryDesc; +namespace yagpcc { +class SetQueryReq; +} class EventSender { public: - void ExecutorStart(QueryDesc *query_desc, int eflags); - void ExecutorFinish(QueryDesc *query_desc); + void executor_after_start(QueryDesc *query_desc, int eflags); + void query_metrics_collect(QueryMetricsStatus status, void *arg); static EventSender *instance(); private: + void collect_query_submit(QueryDesc *query_desc); + void collect_query_done(QueryDesc *query_desc, const std::string &status); + EventSender(); + void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 1a820404428..bca1acd9ce2 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -16,8 +16,10 @@ class GrpcConnector::Impl { yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { yagpcc::MetricResponse response; grpc::ClientContext context; + // TODO: find a more secure way to send messages than relying on a fixed + // timeout auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(50); + std::chrono::system_clock::now() + std::chrono::milliseconds(200); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index 810c0bd3e15..4fca6960a4e 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -1,6 +1,6 @@ #pragma once -#include "yagpcc_set_service.pb.h" +#include "protos/yagpcc_set_service.pb.h" class GrpcConnector { public: diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 739cca80f01..be39c953970 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,6 +1,3 @@ -#include "hook_wrappers.h" -#include "EventSender.h" - extern "C" { #include "postgres.h" #include "utils/metrics_utils.h" @@ -14,55 +11,57 @@ extern "C" { } #include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "hook_wrappers.h" +#include "EventSender.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; -static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; - -static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void ya_ExecutorFinish_hook(QueryDesc *query_desc); +static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; -#define REPLACE_HOOK(hookName) \ - previous_##hookName = hookName; \ - hookName = ya_##hookName; +static void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags); +static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); void hooks_init() { - REPLACE_HOOK(ExecutorStart_hook); - REPLACE_HOOK(ExecutorFinish_hook); + previous_ExecutorStart_hook = ExecutorStart_hook; + ExecutorStart_hook = ya_ExecutorAfterStart_hook; + previous_query_info_collect_hook = query_info_collect_hook; + query_info_collect_hook = ya_query_info_collect_hook; stat_statements_parser_init(); } void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorFinish_hook = previous_ExecutorFinish_hook; + query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); } -#define CREATE_HOOK_WRAPPER(hookName, ...) \ - PG_TRY(); \ - { EventSender::instance()->hookName(__VA_ARGS__); } \ - PG_CATCH(); \ - { \ - ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ - PG_RE_THROW(); \ - } \ - PG_END_TRY(); \ - if (previous_##hookName##_hook) \ - (*previous_##hookName##_hook)(__VA_ARGS__); \ - else \ - standard_##hookName(__VA_ARGS__); - -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); +void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { + if (previous_ExecutorStart_hook) { + (*previous_ExecutorStart_hook)(query_desc, eflags); + } else { + standard_ExecutorStart(query_desc, eflags); + } PG_TRY(); - { EventSender::instance()->ExecutorStart(query_desc, eflags); } + { EventSender::instance()->executor_after_start(query_desc, eflags); } PG_CATCH(); { - ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + ereport(WARNING, + (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); PG_RE_THROW(); } PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); +void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { + PG_TRY(); + { EventSender::instance()->query_metrics_collect(status, arg); } + PG_CATCH(); + { + ereport(WARNING, + (errmsg("EventSender failed in ya_query_info_collect_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); + if (previous_query_info_collect_hook) { + (*previous_query_info_collect_hook)(status, arg); + } } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index ae79e7dc40a..737e77745df 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -205,6 +205,13 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) APP_JUMB_STRING(rte->ctename); APP_JUMB(rte->ctelevelsup); break; + /* GPDB RTEs */ + case RTE_VOID: + break; + case RTE_TABLEFUNCTION: + JumbleQuery(jstate, rte->subquery); + JumbleExpr(jstate, (Node *)rte->functions); + break; default: elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); break; @@ -609,6 +616,20 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) JumbleExpr(jstate, rtfunc->funcexpr); } break; + /* GPDB nodes */ + case T_GroupingFunc: + { + GroupingFunc *grpnode = (GroupingFunc *)node; + + JumbleExpr(jstate, (Node *)grpnode->args); + } + break; + case T_Grouping: + case T_GroupId: + case T_Integer: + case T_Value: + // TODO: no idea what to do with those + break; default: /* Only a warning, since we can stumble along anyway */ elog(WARNING, "unrecognized node type: %d", From 2c5002174dff40ed814ad107babcf2b1665e49fd Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 1 May 2023 18:44:53 +0300 Subject: [PATCH 041/128] [yagp_hooks_collector] Add debian packaging and bionic GRPC compatibility --- debian/compat | 1 + debian/control | 11 +++++++++++ debian/postinst | 8 ++++++++ debian/rules | 10 ++++++++++ src/GrpcConnector.cpp | 4 ++-- 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/postinst create mode 100644 debian/rules diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000000..ec635144f60 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..600dd4d602e --- /dev/null +++ b/debian/control @@ -0,0 +1,11 @@ +Source: greenplum-6-yagpcc-hooks-collector-1 +Section: misc +Priority: optional +Maintainer: Maxim Smyatkin +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc +Standards-Version: 3.9.8 + +Package: greenplum-6-yagpcc-hooks-collector-1 +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) +Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 00000000000..27ddfc06a7d --- /dev/null +++ b/debian/postinst @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/opt/greenplum-db-6 + +chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} diff --git a/debian/rules b/debian/rules new file mode 100644 index 00000000000..6c2c7491067 --- /dev/null +++ b/debian/rules @@ -0,0 +1,10 @@ +#!/usr/bin/make -f +# You must remove unused comment lines for the released package. +export DH_VERBOSE = 1 + + +export GPHOME := /opt/greenplum-db-6 +export PATH := $(GPHOME)/bin:$(PATH) + +%: + dh $@ diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index bca1acd9ce2..5a24d576de1 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -1,8 +1,8 @@ #include "GrpcConnector.h" #include "yagpcc_set_service.grpc.pb.h" -#include -#include +#include +#include #include class GrpcConnector::Impl { From 0330cd6204f49d1929ee11db2d8ce0ea849eb455 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 4 May 2023 14:34:42 +0300 Subject: [PATCH 042/128] [yagp_hooks_collector] Add CDB metrics, query nesting, and configuration GUCs Add missing Greenplum node types to pg_stat_statements parser. Move stats reporting to ExecutorEnd hook. Improve GRPC failure handling. Track CDB-specific metrics and initial query nesting level. Add resource group collection. Add GUCs for controlling collection. Skip nested and utility statements by default. --- debian/control | 6 +- protos/yagpcc_metrics.proto | 1 + src/Config.cpp | 38 ++++++ src/Config.h | 12 ++ src/EventSender.cpp | 112 ++++++++++++++---- src/EventSender.h | 6 + src/GrpcConnector.cpp | 66 +++++++++-- src/hook_wrappers.cpp | 93 +++++++++++++-- .../pg_stat_statements_ya_parser.c | 29 ++++- 9 files changed, 318 insertions(+), 45 deletions(-) create mode 100644 src/Config.cpp create mode 100644 src/Config.h diff --git a/debian/control b/debian/control index 600dd4d602e..c740a8590ca 100644 --- a/debian/control +++ b/debian/control @@ -1,11 +1,11 @@ -Source: greenplum-6-yagpcc-hooks-collector-1 +Source: greenplum-6-yagpcc-hooks Section: misc Priority: optional Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc, libgrpc++1, libgrpc++-dev Standards-Version: 3.9.8 -Package: greenplum-6-yagpcc-hooks-collector-1 +Package: greenplum-6-yagpcc-hooks Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index f00f329a208..26e0a496460 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -33,6 +33,7 @@ message QueryInfo { string temlate_plan_text = 7; string userName = 8; string databaseName = 9; + string rsgname = 10; } enum PlanGenerator diff --git a/src/Config.cpp b/src/Config.cpp new file mode 100644 index 00000000000..d97e5d45984 --- /dev/null +++ b/src/Config.cpp @@ -0,0 +1,38 @@ +#include "Config.h" + +extern "C" { +#include "postgres.h" +#include "utils/builtins.h" +#include "utils/guc.h" +} + +static char *guc_uds_path = nullptr; +static bool guc_enable_analyze = true; +static bool guc_enable_cdbstats = true; +static bool guc_enable_collector = true; + +void Config::init() { + DefineCustomStringVariable( + "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, + true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable_analyze", "Collect analyze metrics in yagpcc", 0LL, + &guc_enable_analyze, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, + &guc_enable_cdbstats, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); +} + +std::string Config::uds_path() { return guc_uds_path; } +bool Config::enable_analyze() { return guc_enable_analyze; } +bool Config::enable_cdbstats() { return guc_enable_cdbstats; } +bool Config::enable_collector() { return guc_enable_collector; } diff --git a/src/Config.h b/src/Config.h new file mode 100644 index 00000000000..117481f219b --- /dev/null +++ b/src/Config.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +class Config { +public: + static void init(); + static std::string uds_path(); + static bool enable_analyze(); + static bool enable_cdbstats(); + static bool enable_collector(); +}; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 5ab6bbd60df..55858ed5183 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,3 +1,4 @@ +#include "Config.h" #include "GrpcConnector.h" #include "ProcStats.h" #include @@ -13,6 +14,7 @@ extern "C" { #include "utils/elog.h" #include "utils/metrics_utils.h" +#include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" @@ -25,6 +27,10 @@ void get_spill_info(int ssid, int ccid, int32_t *file_count, #include "EventSender.h" +#define need_collect() \ + (nesting_level == 0 && gp_command_count != 0 && \ + query_desc->sourceText != nullptr && Config::enable_collector()) + namespace { std::string *get_user_name() { @@ -39,6 +45,21 @@ std::string *get_db_name() { return result; } +std::string *get_rg_name() { + auto userId = GetUserId(); + if (!OidIsValid(userId)) + return nullptr; + auto groupId = GetResGroupIdForRole(userId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + auto result = new std::string(rgname); + pfree(rgname); + return result; +} + int get_cur_slice_id(QueryDesc *desc) { if (!desc->estate) { return 0; @@ -103,9 +124,10 @@ void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { pfree(norm_query); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, +void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, bool with_text, bool with_plan) { if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); if (query_desc->sourceText && with_text) { set_query_text(qi, query_desc); } @@ -115,6 +137,7 @@ void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, } qi->set_allocated_username(get_user_name()); qi->set_allocated_databasename(get_db_name()); + qi->set_allocated_rsgname(get_rg_name()); } } @@ -209,37 +232,79 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } } +void EventSender::executor_before_start(QueryDesc *query_desc, + int /* eflags*/) { + if (Gp_role == GP_ROLE_DISPATCH && need_collect() && + Config::enable_analyze()) { + instr_time starttime; + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + if (Config::enable_cdbstats()) { + query_desc->instrument_options |= INSTRUMENT_CDB; + + // TODO: there is a PR resolving some memory leak around auto-explain: + // https://github.com/greenplum-db/gpdb/pull/15164 + // Need to check if the memory leak applies here as well and fix it + Assert(query_desc->showstatctx == NULL); + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } + } +} + void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && + need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); + set_query_info(&req, query_desc, false, true); + send_query_info(&req, "started"); + } +} + +void EventSender::executor_end(QueryDesc *query_desc) { + if (!need_collect() || + (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } + if (query_desc->totaltime && Config::enable_analyze() && + Config::enable_cdbstats()) { + if (query_desc->estate->dispatcherState && + query_desc->estate->dispatcherState->primaryResults) { + cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, + DISPATCH_WAIT_NONE); + } + InstrEndLoop(query_desc->totaltime); + } auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); - set_query_info(req.mutable_query_info(), query_desc, false, true); - send_query_info(&req, "started"); + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); + set_query_info(&req, query_desc, false, false); + // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to + // gather it here. It only makes sense when doing regular stat checks. + set_gp_metrics(req.mutable_query_metrics(), query_desc, + /*need_spillinfo*/ false); + send_query_info(&req, "ended"); } void EventSender::collect_query_submit(QueryDesc *query_desc) { - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - set_query_info(req.mutable_query_info(), query_desc, true, false); - send_query_info(&req, "submit"); + if (need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + set_query_info(&req, query_desc, true, false); + send_query_info(&req, "submit"); + } } void EventSender::collect_query_done(QueryDesc *query_desc, const std::string &status) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - set_query_info(req.mutable_query_info(), query_desc, false, false); - // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to gather - // it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc, - /*need_spillinfo*/ false); - send_query_info(&req, status); + if (need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); + set_query_info(&req, query_desc, false, false); + send_query_info(&req, status); + } } void EventSender::send_query_info(yagpcc::SetQueryReq *req, @@ -257,4 +322,7 @@ EventSender *EventSender::instance() { return &sender; } -EventSender::EventSender() { connector = std::make_unique(); } \ No newline at end of file +EventSender::EventSender() { + Config::init(); + connector = std::make_unique(); +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 9c574cba9a1..9e2ef992f81 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -11,8 +11,12 @@ class SetQueryReq; class EventSender { public: + void executor_before_start(QueryDesc *query_desc, int eflags); void executor_after_start(QueryDesc *query_desc, int eflags); + void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); + void incr_depth() { nesting_level++; } + void decr_depth() { nesting_level--; } static EventSender *instance(); private: @@ -22,4 +26,6 @@ class EventSender { EventSender(); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; + + int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 5a24d576de1..276c9ceb8a8 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -1,42 +1,86 @@ #include "GrpcConnector.h" +#include "Config.h" #include "yagpcc_set_service.grpc.pb.h" -#include +#include +#include #include +#include +#include #include +#include + +extern "C" { +#include "postgres.h" +#include "cdb/cdbvars.h" +} class GrpcConnector::Impl { public: - Impl() { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { GOOGLE_PROTOBUF_VERIFY_VERSION; - this->stub = yagpcc::SetQueryInfo::NewStub( - grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials())); + channel = + grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); + stub = yagpcc::SetQueryInfo::NewStub(channel); + connected = true; + done = false; + reconnect_thread = std::thread(&Impl::reconnect, this); + } + + ~Impl() { + done = true; + cv.notify_one(); + reconnect_thread.join(); } yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { yagpcc::MetricResponse response; + if (!connected) { + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + response.set_error_text( + "Not tracing this query connection to agent has been lost"); + return response; + } grpc::ClientContext context; - // TODO: find a more secure way to send messages than relying on a fixed - // timeout + int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(200); + std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + connected = false; + cv.notify_one(); } return response; } private: - const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; - const std::string TCP_ADDRESS = "127.0.0.1:1432"; + const std::string SOCKET_FILE; std::unique_ptr stub; + std::shared_ptr channel; + std::atomic_bool connected; + std::thread reconnect_thread; + std::condition_variable cv; + std::mutex mtx; + bool done; + + void reconnect() { + while (!done) { + { + std::unique_lock lock(mtx); + cv.wait(lock); + } + while (!connected && !done) { + auto deadline = + std::chrono::system_clock::now() + std::chrono::milliseconds(100); + connected = channel->WaitForConnected(deadline); + } + } + } }; GrpcConnector::GrpcConnector() { impl = new Impl(); } diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index be39c953970..edad5798e44 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,28 +1,42 @@ extern "C" { #include "postgres.h" -#include "utils/metrics_utils.h" -#include "utils/elog.h" #include "executor/executor.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" -#include "cdb/cdbvars.h" #include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" #include "tcop/utility.h" } -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" -#include "hook_wrappers.h" +#include "Config.h" #include "EventSender.h" +#include "hook_wrappers.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; +static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; +static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; +static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; -static void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + long count); +static void ya_ExecutorFinish_hook(QueryDesc *query_desc); +static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); void hooks_init() { previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = ya_ExecutorAfterStart_hook; + ExecutorStart_hook = ya_ExecutorStart_hook; + previous_ExecutorRun_hook = ExecutorRun_hook; + ExecutorRun_hook = ya_ExecutorRun_hook; + previous_ExecutorFinish_hook = ExecutorFinish_hook; + ExecutorFinish_hook = ya_ExecutorFinish_hook; + previous_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = ya_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; query_info_collect_hook = ya_query_info_collect_hook; stat_statements_parser_init(); @@ -30,11 +44,21 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); } -void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { + PG_TRY(); + { EventSender::instance()->executor_before_start(query_desc, eflags); } + PG_CATCH(); + { + ereport(WARNING, + (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); if (previous_ExecutorStart_hook) { (*previous_ExecutorStart_hook)(query_desc, eflags); } else { @@ -51,6 +75,59 @@ void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { PG_END_TRY(); } +void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + long count) { + EventSender::instance()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorRun_hook) + previous_ExecutorRun_hook(query_desc, direction, count); + else + standard_ExecutorRun(query_desc, direction, count); + EventSender::instance()->decr_depth(); + } + PG_CATCH(); + { + EventSender::instance()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { + EventSender::instance()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorFinish_hook) + previous_ExecutorFinish_hook(query_desc); + else + standard_ExecutorFinish(query_desc); + EventSender::instance()->decr_depth(); + } + PG_CATCH(); + { + EventSender::instance()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +void ya_ExecutorEnd_hook(QueryDesc *query_desc) { + PG_TRY(); + { EventSender::instance()->executor_end(query_desc); } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); + if (previous_ExecutorEnd_hook) { + (*previous_ExecutorEnd_hook)(query_desc); + } else { + standard_ExecutorEnd(query_desc); + } +} + void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { PG_TRY(); { EventSender::instance()->query_metrics_collect(status, arg); } diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 737e77745df..a37ac0ef0bf 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -617,6 +617,13 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) } break; /* GPDB nodes */ + case T_GroupingClause: + { + GroupingClause *grpnode = (GroupingClause *)node; + + JumbleExpr(jstate, (Node *)grpnode->groupsets); + } + break; case T_GroupingFunc: { GroupingFunc *grpnode = (GroupingFunc *)node; @@ -628,7 +635,27 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) case T_GroupId: case T_Integer: case T_Value: - // TODO: no idea what to do with those + // TODO:seems like nothing to do with it + break; + /* GPDB-only additions, nothing to do */ + case T_PartitionBy: + case T_PartitionElem: + case T_PartitionRangeItem: + case T_PartitionBoundSpec: + case T_PartitionSpec: + case T_PartitionValuesSpec: + case T_AlterPartitionId: + case T_AlterPartitionCmd: + case T_InheritPartitionCmd: + case T_CreateFileSpaceStmt: + case T_FileSpaceEntry: + case T_DropFileSpaceStmt: + case T_TableValueExpr: + case T_DenyLoginInterval: + case T_DenyLoginPoint: + case T_AlterTypeStmt: + case T_SetDistributionCmd: + case T_ExpandStmtSpec: break; default: /* Only a warning, since we can stumble along anyway */ From f9f89ddc5496cb72d1e4424b78f66f6c3d636d09 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 7 Jun 2023 14:58:57 +0300 Subject: [PATCH 043/128] [yagp_hooks_collector] Diff system stats per-query and improve error safety Capture /proc stats at query start and compute diff at end rather than reporting lifetime totals. Suppress error rethrows from the collector to avoid breaking other extensions. Add missing hooks deinitialization. Modernize ereport style. --- src/EventSender.cpp | 24 +++++++++---- src/ProcStats.cpp | 36 +++++++------------ src/hook_wrappers.cpp | 10 ++---- .../pg_stat_statements_ya_parser.c | 6 ++-- 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 55858ed5183..b1f85cf9f1e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -2,6 +2,7 @@ #include "GrpcConnector.h" #include "ProcStats.h" #include +#include extern "C" { #include "postgres.h" @@ -168,6 +169,8 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, } } +decltype(std::chrono::high_resolution_clock::now()) query_start_time; + void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, bool need_spillinfo) { if (need_spillinfo) { @@ -182,6 +185,10 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); + std::chrono::duration elapsed_seconds = + std::chrono::high_resolution_clock::now() - query_start_time; + metrics->mutable_systemstat()->set_runningtimeseconds( + elapsed_seconds.count()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -228,14 +235,17 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; default: - elog(FATAL, "Unknown query status: %d", status); + ereport(FATAL, (errmsg("Unknown query status: %d", status))); } } void EventSender::executor_before_start(QueryDesc *query_desc, int /* eflags*/) { - if (Gp_role == GP_ROLE_DISPATCH && need_collect() && - Config::enable_analyze()) { + if (!need_collect()) { + return; + } + query_start_time = std::chrono::high_resolution_clock::now(); + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { instr_time starttime; query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -311,9 +321,11 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, const std::string &event) { auto result = connector->set_metric_query(*req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query {%d-%d-%d} %s reporting failed with an error %s", - req->query_key().tmid(), req->query_key().ssid(), - req->query_key().ccnt(), event.c_str(), result.error_text().c_str()); + ereport(WARNING, + (errmsg("Query {%d-%d-%d} %s reporting failed with an error %s", + req->query_key().tmid(), req->query_key().ssid(), + req->query_key().ccnt(), event.c_str(), + result.error_text().c_str()))); } } diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 5c64f25ec09..668173a0f7e 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -13,7 +13,7 @@ namespace { #define FILL_IO_STAT(stat_name) \ uint64_t stat_name; \ proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name); + stats->set_##stat_name(stat_name - stats->stat_name()); void fill_io_stats(yagpcc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/io"); @@ -30,36 +30,23 @@ void fill_io_stats(yagpcc::SystemStat *stats) { void fill_cpu_stats(yagpcc::SystemStat *stats) { static const int UTIME_ID = 13; static const int STIME_ID = 14; - static const int STARTTIME_ID = 21; static const int VSIZE_ID = 22; static const int RSS_ID = 23; static const double tps = sysconf(_SC_CLK_TCK); - double uptime; - { - std::ifstream proc_stat("/proc/uptime"); - proc_stat >> uptime; - } - std::ifstream proc_stat("/proc/self/stat"); std::string trash; - double start_time = 0; for (int i = 0; i <= RSS_ID; ++i) { switch (i) { case UTIME_ID: double utime; proc_stat >> utime; - stats->set_usertimeseconds(utime / tps); + stats->set_usertimeseconds(utime / tps - stats->usertimeseconds()); break; case STIME_ID: double stime; proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps); - break; - case STARTTIME_ID: - uint64_t starttime; - proc_stat >> starttime; - start_time = static_cast(starttime) / tps; + stats->set_kerneltimeseconds(stime / tps - stats->kerneltimeseconds()); break; case VSIZE_ID: uint64_t vsize; @@ -75,7 +62,6 @@ void fill_cpu_stats(yagpcc::SystemStat *stats) { default: proc_stat >> trash; } - stats->set_runningtimeseconds(uptime - start_time); } } @@ -89,16 +75,16 @@ void fill_status_stats(yagpcc::SystemStat *stats) { stats->set_vmpeakkb(value); proc_stat >> measure; if (measure != "kB") { - elog(FATAL, "Expected memory sizes in kB, but got in %s", - measure.c_str()); + ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", + measure.c_str()))); } } else if (key == "VmSize:") { uint64_t value; proc_stat >> value; stats->set_vmsizekb(value); if (measure != "kB") { - elog(FATAL, "Expected memory sizes in kB, but got in %s", - measure.c_str()); + ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", + measure.c_str()))); } } } @@ -106,7 +92,9 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } // namespace void fill_self_stats(yagpcc::SystemStat *stats) { - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); + static yagpcc::SystemStat prev_stats; + fill_io_stats(&prev_stats); + fill_cpu_stats(&prev_stats); + fill_status_stats(&prev_stats); + *stats = prev_stats; } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index edad5798e44..a904dc9bafd 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -44,6 +44,8 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorRun_hook = previous_ExecutorRun_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); @@ -56,7 +58,6 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); - PG_RE_THROW(); } PG_END_TRY(); if (previous_ExecutorStart_hook) { @@ -70,7 +71,6 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); - PG_RE_THROW(); } PG_END_TRY(); } @@ -116,10 +116,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { PG_TRY(); { EventSender::instance()->executor_end(query_desc); } PG_CATCH(); - { - ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); - PG_RE_THROW(); - } + { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } PG_END_TRY(); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); @@ -135,7 +132,6 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { { ereport(WARNING, (errmsg("EventSender failed in ya_query_info_collect_hook"))); - PG_RE_THROW(); } PG_END_TRY(); if (previous_query_info_collect_hook) { diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index a37ac0ef0bf..1c58d936093 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -213,7 +213,7 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) JumbleExpr(jstate, (Node *)rte->functions); break; default: - elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); + ereport(ERROR, (errmsg("unrecognized RTE kind: %d", (int)rte->rtekind))); break; } } @@ -659,8 +659,8 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) break; default: /* Only a warning, since we can stumble along anyway */ - elog(WARNING, "unrecognized node type: %d", - (int)nodeTag(node)); + ereport(WARNING, (errmsg("unrecognized node type: %d", + (int)nodeTag(node)))); break; } } From 0766e6253c7dbf4427e75adf7120f6e86353b30b Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 13 Jun 2023 16:51:40 +0300 Subject: [PATCH 044/128] [yagp_hooks_collector] Fix EventSender and GrpcConnector in forked processes Delay initialization of static singletons and GRPC connections to actual query handling time rather than _PG_init, since both are incompatible with fork(). --- debian/control | 4 ++-- src/EventSender.cpp | 10 ++-------- src/EventSender.h | 6 ++---- src/GrpcConnector.cpp | 33 ++++++++++++++++++++++----------- src/hook_wrappers.cpp | 33 +++++++++++++++++++++++---------- 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/debian/control b/debian/control index c740a8590ca..07176e94be5 100644 --- a/debian/control +++ b/debian/control @@ -2,10 +2,10 @@ Source: greenplum-6-yagpcc-hooks Section: misc Priority: optional Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc, libgrpc++1, libgrpc++-dev +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) Standards-Version: 3.9.8 Package: greenplum-6-yagpcc-hooks Architecture: any -Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) +Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b1f85cf9f1e..ec966e8686c 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -329,12 +329,6 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, } } -EventSender *EventSender::instance() { - static EventSender sender; - return &sender; -} +EventSender::EventSender() { connector = std::make_unique(); } -EventSender::EventSender() { - Config::init(); - connector = std::make_unique(); -} \ No newline at end of file +EventSender::~EventSender() { connector.release(); } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 9e2ef992f81..92e6937a690 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -17,15 +17,13 @@ class EventSender { void query_metrics_collect(QueryMetricsStatus status, void *arg); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } - static EventSender *instance(); + EventSender(); + ~EventSender(); private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - - EventSender(); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; - int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 276c9ceb8a8..966bfb4a780 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -10,14 +10,17 @@ #include #include -extern "C" { +extern "C" +{ #include "postgres.h" #include "cdb/cdbvars.h" } -class GrpcConnector::Impl { +class GrpcConnector::Impl +{ public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) + { GOOGLE_PROTOBUF_VERIFY_VERSION; channel = grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); @@ -27,15 +30,18 @@ class GrpcConnector::Impl { reconnect_thread = std::thread(&Impl::reconnect, this); } - ~Impl() { + ~Impl() + { done = true; cv.notify_one(); reconnect_thread.join(); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) + { yagpcc::MetricResponse response; - if (!connected) { + if (!connected) + { response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); response.set_error_text( "Not tracing this query connection to agent has been lost"); @@ -47,7 +53,8 @@ class GrpcConnector::Impl { std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { + if (!status.ok()) + { response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); @@ -68,13 +75,16 @@ class GrpcConnector::Impl { std::mutex mtx; bool done; - void reconnect() { - while (!done) { + void reconnect() + { + while (!done) + { { std::unique_lock lock(mtx); cv.wait(lock); } - while (!connected && !done) { + while (!connected && !done) + { auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(100); connected = channel->WaitForConnected(deadline); @@ -88,6 +98,7 @@ GrpcConnector::GrpcConnector() { impl = new Impl(); } GrpcConnector::~GrpcConnector() { delete impl; } yagpcc::MetricResponse -GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) { +GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) +{ return impl->set_metric_query(req); } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index a904dc9bafd..66ba6547ce2 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -28,7 +28,17 @@ static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static EventSender *sender = nullptr; + +static inline EventSender *get_sender() { + if (!sender) { + sender = new EventSender(); + } + return sender; +} + void hooks_init() { + Config::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; @@ -49,11 +59,14 @@ void hooks_deinit() { ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); + if (sender) { + delete sender; + } } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { PG_TRY(); - { EventSender::instance()->executor_before_start(query_desc, eflags); } + { get_sender()->executor_before_start(query_desc, eflags); } PG_CATCH(); { ereport(WARNING, @@ -66,7 +79,7 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { standard_ExecutorStart(query_desc, eflags); } PG_TRY(); - { EventSender::instance()->executor_after_start(query_desc, eflags); } + { get_sender()->executor_after_start(query_desc, eflags); } PG_CATCH(); { ereport(WARNING, @@ -77,36 +90,36 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, long count) { - EventSender::instance()->incr_depth(); + get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorRun_hook) previous_ExecutorRun_hook(query_desc, direction, count); else standard_ExecutorRun(query_desc, direction, count); - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); } PG_CATCH(); { - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); PG_RE_THROW(); } PG_END_TRY(); } void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - EventSender::instance()->incr_depth(); + get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorFinish_hook) previous_ExecutorFinish_hook(query_desc); else standard_ExecutorFinish(query_desc); - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); } PG_CATCH(); { - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); PG_RE_THROW(); } PG_END_TRY(); @@ -114,7 +127,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { void ya_ExecutorEnd_hook(QueryDesc *query_desc) { PG_TRY(); - { EventSender::instance()->executor_end(query_desc); } + { get_sender()->executor_end(query_desc); } PG_CATCH(); { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } PG_END_TRY(); @@ -127,7 +140,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { PG_TRY(); - { EventSender::instance()->query_metrics_collect(status, arg); } + { get_sender()->query_metrics_collect(status, arg); } PG_CATCH(); { ereport(WARNING, From 0f8a4303fb8be2c8f49c23cec664076cc5782838 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 16 Aug 2023 13:23:00 +0300 Subject: [PATCH 045/128] [yagp_hooks_collector] Fix memory leak in EXPLAIN ANALYZE code path --- protos/yagpcc_metrics.proto | 4 ++-- src/EventSender.cpp | 25 ++++++++----------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 26e0a496460..bc128a22f17 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -29,8 +29,8 @@ message QueryInfo { uint64 plan_id = 3; string query_text = 4; string plan_text = 5; - string temlate_query_text = 6; - string temlate_plan_text = 7; + string template_query_text = 6; + string template_plan_text = 7; string userName = 8; string databaseName = 9; string rsgname = 10; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index ec966e8686c..6d2ff4afd47 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -61,13 +61,6 @@ std::string *get_rg_name() { return result; } -int get_cur_slice_id(QueryDesc *desc) { - if (!desc->estate) { - return 0; - } - return LocallyExecutingSliceIndex(desc->estate); -} - google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -113,7 +106,7 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); set_plan_text(qi->mutable_plan_text(), query_desc); StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); // TODO: free stringinfo? } @@ -121,7 +114,7 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { *qi->mutable_query_text() = query_desc->sourceText; char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_temlate_query_text() = std::string(norm_query); + *qi->mutable_template_query_text() = std::string(norm_query); pfree(norm_query); } @@ -246,20 +239,18 @@ void EventSender::executor_before_start(QueryDesc *query_desc, } query_start_time = std::chrono::high_resolution_clock::now(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { - instr_time starttime; query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - // TODO: there is a PR resolving some memory leak around auto-explain: - // https://github.com/greenplum-db/gpdb/pull/15164 - // Need to check if the memory leak applies here as well and fix it - Assert(query_desc->showstatctx == NULL); - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + if (!query_desc->showstatctx) { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } } } } From 221311abc7d4284fe60ab0e6d22a0652e94f12ee Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 6 Sep 2023 16:10:04 +0300 Subject: [PATCH 046/128] [yagp_hooks_collector] Add motion network and workfile spill stats --- protos/yagpcc_metrics.proto | 8 ++++++++ src/EventSender.cpp | 41 ++++++++++++++++++++++++++++--------- src/EventSender.h | 2 +- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index bc128a22f17..2d20d3c46d9 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -84,6 +84,12 @@ message SystemStat { uint64 cancelled_write_bytes = 14; } +message NetworkStat { + uint32 total_bytes = 1; + uint32 tuple_bytes = 2; + uint32 chunks = 3; +} + message MetricInstrumentation { uint64 ntuples = 1; /* Total tuples produced */ uint64 nloops = 2; /* # of run cycles for this node */ @@ -103,6 +109,8 @@ message MetricInstrumentation { uint64 temp_blks_written = 16; double blk_read_time = 17; /* measured read/write time */ double blk_write_time = 18; + NetworkStat sent = 19; + NetworkStat received = 20; } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 6d2ff4afd47..2810e581313 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -4,6 +4,8 @@ #include #include +#define typeid __typeid +#define operator __operator extern "C" { #include "postgres.h" @@ -14,10 +16,12 @@ extern "C" { #include "executor/executor.h" #include "utils/elog.h" #include "utils/metrics_utils.h" +#include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/cdbinterconnect.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" @@ -25,6 +29,8 @@ extern "C" { void get_spill_info(int ssid, int ccid, int32_t *file_count, int64_t *total_bytes); } +#undef typeid +#undef operator #include "EventSender.h" @@ -160,6 +166,18 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, metrics->set_blk_write_time( INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); } + if (query_desc->estate && query_desc->estate->motionlayer_context) { + MotionLayerState *mlstate = + (MotionLayerState *)query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); + } } decltype(std::chrono::high_resolution_clock::now()) query_start_time; @@ -182,6 +200,8 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, std::chrono::high_resolution_clock::now() - query_start_time; metrics->mutable_systemstat()->set_runningtimeseconds( elapsed_seconds.count()); + metrics->mutable_spill()->set_filecount(WorkfileTotalFilesCreated()); + metrics->mutable_spill()->set_totalbytes(WorkfileTotalBytesWritten()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -238,6 +258,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } query_start_time = std::chrono::high_resolution_clock::now(); + WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -245,12 +266,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - if (!query_desc->showstatctx) { - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); - } + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -281,7 +300,6 @@ void EventSender::executor_end(QueryDesc *query_desc) { } auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); - set_query_info(&req, query_desc, false, false); // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to // gather it here. It only makes sense when doing regular stat checks. set_gp_metrics(req.mutable_query_metrics(), query_desc, @@ -303,7 +321,6 @@ void EventSender::collect_query_done(QueryDesc *query_desc, if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - set_query_info(&req, query_desc, false, false); send_query_info(&req, status); } } @@ -320,6 +337,10 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, } } -EventSender::EventSender() { connector = std::make_unique(); } +EventSender::EventSender() { + if (Config::enable_collector()) { + connector = new GrpcConnector(); + } +} -EventSender::~EventSender() { connector.release(); } \ No newline at end of file +EventSender::~EventSender() { delete connector; } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 92e6937a690..f53648bed36 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -24,6 +24,6 @@ class EventSender { void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); - std::unique_ptr connector; + GrpcConnector *connector; int nesting_level = 0; }; \ No newline at end of file From 6fa8c3d1653a930d437c3ba6897f7ea55f9157ae Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 6 Sep 2023 16:11:06 +0300 Subject: [PATCH 047/128] [yagp_hooks_collector] Clean up threading, signal handling, and logging Mute PG-destined signals in GRPC reconnection thread. Move debian config to CI. Redirect debug output to log file. Harden memory handling. Remove thread-unsafe logging and dead code. --- debian/compat | 1 - debian/control | 11 ------ debian/postinst | 8 ---- debian/rules | 10 ----- src/EventSender.cpp | 85 +++++++++++++++++++----------------------- src/EventSender.h | 1 - src/GrpcConnector.cpp | 85 ++++++++++++++++++++++++++++-------------- src/GrpcConnector.h | 3 +- src/SpillInfoWrapper.c | 21 ----------- 9 files changed, 97 insertions(+), 128 deletions(-) delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/postinst delete mode 100644 debian/rules delete mode 100644 src/SpillInfoWrapper.c diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f60..00000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index 07176e94be5..00000000000 --- a/debian/control +++ /dev/null @@ -1,11 +0,0 @@ -Source: greenplum-6-yagpcc-hooks -Section: misc -Priority: optional -Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) -Standards-Version: 3.9.8 - -Package: greenplum-6-yagpcc-hooks -Architecture: any -Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) -Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/debian/postinst b/debian/postinst deleted file mode 100644 index 27ddfc06a7d..00000000000 --- a/debian/postinst +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -GPADMIN=gpadmin -GPHOME=/opt/greenplum-db-6 - -chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} diff --git a/debian/rules b/debian/rules deleted file mode 100644 index 6c2c7491067..00000000000 --- a/debian/rules +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/make -f -# You must remove unused comment lines for the released package. -export DH_VERBOSE = 1 - - -export GPHOME := /opt/greenplum-db-6 -export PATH := $(GPHOME)/bin:$(PATH) - -%: - dh $@ diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 2810e581313..57fe6f13391 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,8 +1,8 @@ #include "Config.h" #include "GrpcConnector.h" #include "ProcStats.h" -#include #include +#include #define typeid __typeid #define operator __operator @@ -20,14 +20,11 @@ extern "C" { #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" -#include "cdb/cdbvars.h" #include "cdb/cdbinterconnect.h" +#include "cdb/cdbvars.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" - -void get_spill_info(int ssid, int ccid, int32_t *file_count, - int64_t *total_bytes); } #undef typeid #undef operator @@ -48,7 +45,6 @@ std::string *get_user_name() { std::string *get_db_name() { char *dbname = get_database_name(MyDatabaseId); std::string *result = dbname ? new std::string(dbname) : nullptr; - pfree(dbname); return result; } @@ -63,7 +59,6 @@ std::string *get_rg_name() { if (rgname == nullptr) return nullptr; auto result = new std::string(rgname); - pfree(rgname); return result; } @@ -114,14 +109,12 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - // TODO: free stringinfo? } void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { *qi->mutable_query_text() = query_desc->sourceText; char *norm_query = gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = std::string(norm_query); - pfree(norm_query); } void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, @@ -182,16 +175,7 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, decltype(std::chrono::high_resolution_clock::now()) query_start_time; -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, - bool need_spillinfo) { - if (need_spillinfo) { - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, - &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - } +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } @@ -254,6 +238,9 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { void EventSender::executor_before_start(QueryDesc *query_desc, int /* eflags*/) { + if (!connector) { + return; + } if (!need_collect()) { return; } @@ -275,71 +262,75 @@ void EventSender::executor_before_start(QueryDesc *query_desc, } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { + if (!connector) { + return; + } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); set_query_info(&req, query_desc, false, true); - send_query_info(&req, "started"); + connector->set_metric_query(req, "started"); } } void EventSender::executor_end(QueryDesc *query_desc) { + if (!connector) { + return; + } if (!need_collect() || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } - if (query_desc->totaltime && Config::enable_analyze() && - Config::enable_cdbstats()) { - if (query_desc->estate->dispatcherState && - query_desc->estate->dispatcherState->primaryResults) { - cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, - DISPATCH_WAIT_NONE); - } - InstrEndLoop(query_desc->totaltime); - } + /* TODO: when querying via CURSOR this call freezes. Need to investigate. + To reproduce - uncomment it and run installchecks. It will freeze around join test. + Needs investigation + + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + Config::enable_cdbstats() && query_desc->estate->dispatcherState && + query_desc->estate->dispatcherState->primaryResults) { + cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, + DISPATCH_WAIT_NONE); + }*/ auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to // gather it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc, - /*need_spillinfo*/ false); - send_query_info(&req, "ended"); + set_gp_metrics(req.mutable_query_metrics(), query_desc); + connector->set_metric_query(req, "ended"); } void EventSender::collect_query_submit(QueryDesc *query_desc) { + if (!connector) { + return; + } if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); set_query_info(&req, query_desc, true, false); - send_query_info(&req, "submit"); + connector->set_metric_query(req, "submit"); } } void EventSender::collect_query_done(QueryDesc *query_desc, const std::string &status) { + if (!connector) { + return; + } if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - send_query_info(&req, status); - } -} - -void EventSender::send_query_info(yagpcc::SetQueryReq *req, - const std::string &event) { - auto result = connector->set_metric_query(*req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - ereport(WARNING, - (errmsg("Query {%d-%d-%d} %s reporting failed with an error %s", - req->query_key().tmid(), req->query_key().ssid(), - req->query_key().ccnt(), event.c_str(), - result.error_text().c_str()))); + connector->set_metric_query(req, status); } } EventSender::EventSender() { if (Config::enable_collector()) { - connector = new GrpcConnector(); + try { + connector = new GrpcConnector(); + } catch (const std::exception &e) { + ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); + } } } diff --git a/src/EventSender.h b/src/EventSender.h index f53648bed36..ee0db2f0938 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,7 +23,6 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); GrpcConnector *connector; int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 966bfb4a780..73c1944fa04 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -7,45 +7,72 @@ #include #include #include +#include +#include #include #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "cdb/cdbvars.h" } -class GrpcConnector::Impl -{ +/* + * Set up the thread signal mask, we don't want to run our signal handlers + * in downloading and uploading threads. + */ +static void MaskThreadSignals() { + sigset_t sigs; + + if (pthread_equal(main_tid, pthread_self())) { + ereport(ERROR, (errmsg("thread_mask is called from main thread!"))); + return; + } + + sigemptyset(&sigs); + + /* make our thread to ignore these signals (which should allow that they be + * delivered to the main thread) */ + sigaddset(&sigs, SIGHUP); + sigaddset(&sigs, SIGINT); + sigaddset(&sigs, SIGTERM); + sigaddset(&sigs, SIGALRM); + sigaddset(&sigs, SIGUSR1); + sigaddset(&sigs, SIGUSR2); + + pthread_sigmask(SIG_BLOCK, &sigs, NULL); +} + +class GrpcConnector::Impl { public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) - { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { GOOGLE_PROTOBUF_VERIFY_VERSION; channel = grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); stub = yagpcc::SetQueryInfo::NewStub(channel); connected = true; + reconnected = false; done = false; reconnect_thread = std::thread(&Impl::reconnect, this); } - ~Impl() - { + ~Impl() { done = true; cv.notify_one(); reconnect_thread.join(); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) - { + yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event) { yagpcc::MetricResponse response; - if (!connected) - { + if (!connected) { response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); response.set_error_text( - "Not tracing this query connection to agent has been lost"); + "Not tracing this query because grpc connection has been lost"); return response; + } else if (reconnected) { + reconnected = false; + ereport(LOG, (errmsg("GRPC connection is restored"))); } grpc::ClientContext context; int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; @@ -53,12 +80,16 @@ class GrpcConnector::Impl std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) - { - response.set_error_text("Connection lost: " + status.error_message() + - "; " + status.error_details()); + if (!status.ok()) { + response.set_error_text("GRPC error: " + status.error_message() + "; " + + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str(), + response.error_text().c_str()))); connected = false; + reconnected = false; cv.notify_one(); } @@ -69,25 +100,23 @@ class GrpcConnector::Impl const std::string SOCKET_FILE; std::unique_ptr stub; std::shared_ptr channel; - std::atomic_bool connected; + std::atomic_bool connected, reconnected, done; std::thread reconnect_thread; std::condition_variable cv; std::mutex mtx; - bool done; - void reconnect() - { - while (!done) - { + void reconnect() { + MaskThreadSignals(); + while (!done) { { std::unique_lock lock(mtx); cv.wait(lock); } - while (!connected && !done) - { + while (!connected && !done) { auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(100); connected = channel->WaitForConnected(deadline); + reconnected = connected.load(); } } } @@ -98,7 +127,7 @@ GrpcConnector::GrpcConnector() { impl = new Impl(); } GrpcConnector::~GrpcConnector() { delete impl; } yagpcc::MetricResponse -GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) -{ - return impl->set_metric_query(req); +GrpcConnector::set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event) { + return impl->set_metric_query(req, event); } \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index 4fca6960a4e..6571c626dfd 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -6,7 +6,8 @@ class GrpcConnector { public: GrpcConnector(); ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event); private: class Impl; diff --git a/src/SpillInfoWrapper.c b/src/SpillInfoWrapper.c deleted file mode 100644 index c6ace0a693f..00000000000 --- a/src/SpillInfoWrapper.c +++ /dev/null @@ -1,21 +0,0 @@ -#include "postgres.h" -#include "utils/workfile_mgr.h" - -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); - -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes) -{ - int count = 0; - int i = 0; - workfile_set *workfiles = workfile_mgr_cache_entries_get_copy(&count); - workfile_set *wf_iter = workfiles; - for (i = 0; i < count; ++i, ++wf_iter) - { - if (wf_iter->active && wf_iter->session_id == ssid && wf_iter->command_count == ccid) - { - *file_count += wf_iter->num_files; - *total_bytes += wf_iter->total_bytes; - } - } - pfree(workfiles); -} \ No newline at end of file From c8dddf03c1703e6c9bea498150e6e1a1854df68c Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 21 Sep 2023 15:16:35 +0300 Subject: [PATCH 048/128] [yagp_hooks_collector] Add ignored_users_list GUC Add a comma-separated GUC to suppress metrics collection for specified roles. Parse using SplitIdentifierString and cache in an unordered_set. --- src/Config.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ src/Config.h | 1 + src/EventSender.cpp | 5 +++-- src/EventSender.h | 2 +- 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index d97e5d45984..c5c2c15f7e9 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,7 @@ #include "Config.h" +#include +#include +#include extern "C" { #include "postgres.h" @@ -10,6 +13,8 @@ static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; +static char *guc_ignored_users = nullptr; +static std::unique_ptr> ignored_users = nullptr; void Config::init() { DefineCustomStringVariable( @@ -30,9 +35,47 @@ void Config::init() { "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomStringVariable( + "yagpcc.ignored_users_list", + "Make yagpcc ignore queries issued by given users", 0LL, + &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); } std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } + +bool Config::filter_user(const std::string *username) { + if (!ignored_users) { + ignored_users.reset(new std::unordered_set()); + if (guc_ignored_users == nullptr || guc_ignored_users[0] == '0') { + return false; + } + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return false; + } + foreach (l, elemlist) { + ignored_users->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + return !username || ignored_users->find(*username) != ignored_users->end(); +} diff --git a/src/Config.h b/src/Config.h index 117481f219b..999d0300640 100644 --- a/src/Config.h +++ b/src/Config.h @@ -9,4 +9,5 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); + static bool filter_user(const std::string *username); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 57fe6f13391..9146078fd0e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -33,7 +33,8 @@ extern "C" { #define need_collect() \ (nesting_level == 0 && gp_command_count != 0 && \ - query_desc->sourceText != nullptr && Config::enable_collector()) + query_desc->sourceText != nullptr && Config::enable_collector() && \ + !Config::filter_user(get_user_name())) namespace { @@ -325,7 +326,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } EventSender::EventSender() { - if (Config::enable_collector()) { + if (Config::enable_collector() && !Config::filter_user(get_user_name())) { try { connector = new GrpcConnector(); } catch (const std::exception &e) { diff --git a/src/EventSender.h b/src/EventSender.h index ee0db2f0938..2af8b7ffa03 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,6 +23,6 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - GrpcConnector *connector; + GrpcConnector *connector = nullptr; int nesting_level = 0; }; \ No newline at end of file From ee9c8056d0cc7ca76c981420e3e2442972e66934 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 2 Oct 2023 12:54:32 +0300 Subject: [PATCH 049/128] [yagp_hooks_collector] Replace GRPC transport with protobuf-over-UDS Remove GRPC dependency. Serialize metrics as protobuf messages and deliver them over a Unix domain socket. Replace server-side message queue with incremental per-query message building. Add clang-format configuration. Use deprecated protobuf API for bionic compatibility. --- .clang-format | 2 + protos/yagpcc_set_service.proto | 23 ++---- src/EventSender.cpp | 115 ++++++++++++++++----------- src/EventSender.h | 10 ++- src/GrpcConnector.cpp | 133 -------------------------------- src/GrpcConnector.h | 15 ---- src/UDSConnector.cpp | 83 ++++++++++++++++++++ src/UDSConnector.h | 13 ++++ 8 files changed, 183 insertions(+), 211 deletions(-) create mode 100644 .clang-format delete mode 100644 src/GrpcConnector.cpp delete mode 100644 src/GrpcConnector.h create mode 100644 src/UDSConnector.cpp create mode 100644 src/UDSConnector.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..99130575c9a --- /dev/null +++ b/.clang-format @@ -0,0 +1,2 @@ +BasedOnStyle: LLVM +SortIncludes: false diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 93c2f5a01d1..e8fc7aaa99d 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -9,23 +9,6 @@ package yagpcc; option java_outer_classname = "SegmentYAGPCCAS"; option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; -service SetQueryInfo { - rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} - - rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} -} - -message MetricResponse { - MetricResponseStatusCode error_code = 1; - string error_text = 2; -} - -enum MetricResponseStatusCode { - METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; - METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; - METRIC_RESPONSE_STATUS_CODE_ERROR = 2; -} - message SetQueryReq { QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; @@ -34,6 +17,9 @@ message SetQueryReq { QueryInfo query_info = 5; GPMetrics query_metrics = 6; repeated MetricPlan plan_tree = 7; + google.protobuf.Timestamp submit_time = 8; + google.protobuf.Timestamp start_time = 9; + google.protobuf.Timestamp end_time = 10; } message SetPlanNodeReq { @@ -43,4 +29,7 @@ message SetPlanNodeReq { SegmentKey segment_key = 4; GPMetrics node_metrics = 5; MetricPlan plan_node = 6; + google.protobuf.Timestamp submit_time = 7; + google.protobuf.Timestamp start_time = 8; + google.protobuf.Timestamp end_time = 9; } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 9146078fd0e..834553a6187 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,6 @@ #include "Config.h" -#include "GrpcConnector.h" #include "ProcStats.h" +#include "UDSConnector.h" #include #include @@ -15,7 +15,6 @@ extern "C" { #include "commands/resgroupcmds.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/metrics_utils.h" #include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" @@ -102,33 +101,46 @@ void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { *plan_text = std::string(es.str->data, es.str->len); } -void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_template_plan_text() = std::string(norm_plan->data); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_template_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + // TODO: For now assume queryid equal to planid, which is wrong. The + // reason for doing so this bug + // https://github.com/greenplum-db/gpdb/pull/15385 (ORCA loses + // pg_stat_statements` queryid during planning phase). Need to fix it + // upstream, cherry-pick and bump gp + // qi->set_query_id(query_desc->plannedstmt->queryId); + qi->set_query_id(qi->plan_id()); + } } -void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - *qi->mutable_query_text() = query_desc->sourceText; - char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = std::string(norm_query); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = query_desc->sourceText; + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_template_query_text() = std::string(norm_query); + } } -void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, - bool with_text, bool with_plan) { +void clear_big_fields(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_query_text(); + } +} + +void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); - if (query_desc->sourceText && with_text) { - set_query_text(qi, query_desc); - } - if (query_desc->plannedstmt && with_plan) { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } qi->set_allocated_username(get_user_name()); qi->set_allocated_databasename(get_db_name()); qi->set_allocated_rsgname(get_rg_name()); @@ -245,6 +257,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!need_collect()) { return; } + if (query_msg->has_query_key()) { + connector->report_query(*query_msg, "previous query"); + query_msg->Clear(); + } query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -268,10 +284,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && need_collect()) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); - set_query_info(&req, query_desc, false, true); - connector->set_metric_query(req, "started"); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *query_msg->mutable_start_time() = current_ts(); + set_query_plan(query_msg, query_desc); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } } } @@ -284,21 +302,21 @@ void EventSender::executor_end(QueryDesc *query_desc) { return; } /* TODO: when querying via CURSOR this call freezes. Need to investigate. - To reproduce - uncomment it and run installchecks. It will freeze around join test. - Needs investigation - + To reproduce - uncomment it and run installchecks. It will freeze around + join test. Needs investigation + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && Config::enable_cdbstats() && query_desc->estate->dispatcherState && query_desc->estate->dispatcherState->primaryResults) { cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, DISPATCH_WAIT_NONE); }*/ - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); - // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to - // gather it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc); - connector->set_metric_query(req, "ended"); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + if (connector->report_query(*query_msg, "ended")) { + query_msg->Clear(); + } } void EventSender::collect_query_submit(QueryDesc *query_desc) { @@ -306,10 +324,14 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { return; } if (need_collect()) { - auto req = + *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - set_query_info(&req, query_desc, true, false); - connector->set_metric_query(req, "submit"); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg, query_desc); + set_query_text(query_msg, query_desc); + if (connector->report_query(*query_msg, "submit")) { + clear_big_fields(query_msg); + } } } @@ -319,20 +341,25 @@ void EventSender::collect_query_done(QueryDesc *query_desc, return; } if (need_collect()) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - connector->set_metric_query(req, status); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + if (connector->report_query(*query_msg, status)) { + clear_big_fields(query_msg); + } } } EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { + query_msg = new yagpcc::SetQueryReq(); try { - connector = new GrpcConnector(); + connector = new UDSConnector(); } catch (const std::exception &e) { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } } } -EventSender::~EventSender() { delete connector; } \ No newline at end of file +EventSender::~EventSender() { + delete query_msg; + delete connector; +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 2af8b7ffa03..161bf6ce037 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,9 +1,14 @@ #pragma once #include +#include #include -class GrpcConnector; +extern "C" { +#include "utils/metrics_utils.h" +} + +class UDSConnector; struct QueryDesc; namespace yagpcc { class SetQueryReq; @@ -23,6 +28,7 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - GrpcConnector *connector = nullptr; + UDSConnector *connector = nullptr; int nesting_level = 0; + yagpcc::SetQueryReq *query_msg; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp deleted file mode 100644 index 73c1944fa04..00000000000 --- a/src/GrpcConnector.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "GrpcConnector.h" -#include "Config.h" -#include "yagpcc_set_service.grpc.pb.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "postgres.h" -#include "cdb/cdbvars.h" -} - -/* - * Set up the thread signal mask, we don't want to run our signal handlers - * in downloading and uploading threads. - */ -static void MaskThreadSignals() { - sigset_t sigs; - - if (pthread_equal(main_tid, pthread_self())) { - ereport(ERROR, (errmsg("thread_mask is called from main thread!"))); - return; - } - - sigemptyset(&sigs); - - /* make our thread to ignore these signals (which should allow that they be - * delivered to the main thread) */ - sigaddset(&sigs, SIGHUP); - sigaddset(&sigs, SIGINT); - sigaddset(&sigs, SIGTERM); - sigaddset(&sigs, SIGALRM); - sigaddset(&sigs, SIGUSR1); - sigaddset(&sigs, SIGUSR2); - - pthread_sigmask(SIG_BLOCK, &sigs, NULL); -} - -class GrpcConnector::Impl { -public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - channel = - grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); - stub = yagpcc::SetQueryInfo::NewStub(channel); - connected = true; - reconnected = false; - done = false; - reconnect_thread = std::thread(&Impl::reconnect, this); - } - - ~Impl() { - done = true; - cv.notify_one(); - reconnect_thread.join(); - } - - yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event) { - yagpcc::MetricResponse response; - if (!connected) { - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - response.set_error_text( - "Not tracing this query because grpc connection has been lost"); - return response; - } else if (reconnected) { - reconnected = false; - ereport(LOG, (errmsg("GRPC connection is restored"))); - } - grpc::ClientContext context; - int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; - auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); - context.set_deadline(deadline); - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { - response.set_error_text("GRPC error: " + status.error_message() + "; " + - status.error_details()); - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str(), - response.error_text().c_str()))); - connected = false; - reconnected = false; - cv.notify_one(); - } - - return response; - } - -private: - const std::string SOCKET_FILE; - std::unique_ptr stub; - std::shared_ptr channel; - std::atomic_bool connected, reconnected, done; - std::thread reconnect_thread; - std::condition_variable cv; - std::mutex mtx; - - void reconnect() { - MaskThreadSignals(); - while (!done) { - { - std::unique_lock lock(mtx); - cv.wait(lock); - } - while (!connected && !done) { - auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(100); - connected = channel->WaitForConnected(deadline); - reconnected = connected.load(); - } - } - } -}; - -GrpcConnector::GrpcConnector() { impl = new Impl(); } - -GrpcConnector::~GrpcConnector() { delete impl; } - -yagpcc::MetricResponse -GrpcConnector::set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event) { - return impl->set_metric_query(req, event); -} \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h deleted file mode 100644 index 6571c626dfd..00000000000 --- a/src/GrpcConnector.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "protos/yagpcc_set_service.pb.h" - -class GrpcConnector { -public: - GrpcConnector(); - ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event); - -private: - class Impl; - Impl *impl; -}; \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp new file mode 100644 index 00000000000..339a5d4f374 --- /dev/null +++ b/src/UDSConnector.cpp @@ -0,0 +1,83 @@ +#include "UDSConnector.h" +#include "Config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "postgres.h" +#include "cdb/cdbvars.h" +} + +UDSConnector::UDSConnector() : uds_path("unix://" + Config::uds_path()) { + GOOGLE_PROTOBUF_VERIFY_VERSION; +} + +static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, + const std::string &event) { + ereport(LOG, + (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str(), strerror(errno)))); +} + +bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, + const std::string &event) { + sockaddr_un address; + address.sun_family = AF_UNIX; + strcpy(address.sun_path, uds_path.c_str()); + bool success = true; + auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd != -1) { + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) != -1) { + if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { + auto data_size = req.ByteSize(); + auto total_size = data_size + sizeof(uint32_t); + uint8_t *buf = (uint8_t *)palloc(total_size); + uint32_t *size_payload = (uint32_t *)buf; + *size_payload = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + int64_t sent = 0, sent_total = 0; + do { + sent = send(sockfd, buf + sent_total, total_size - sent_total, + MSG_DONTWAIT); + sent_total += sent; + } while ( + sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + if (sent < 0) { + log_tracing_failure(req, event); + success = false; + } + pfree(buf); + } else { + // log the error and go on + log_tracing_failure(req, event); + success = false; + } + } else { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %s", + strerror(errno)))); + success = false; + } + close(sockfd); + } else { + // log the error and go on + log_tracing_failure(req, event); + success = false; + } + return success; +} \ No newline at end of file diff --git a/src/UDSConnector.h b/src/UDSConnector.h new file mode 100644 index 00000000000..574653023e6 --- /dev/null +++ b/src/UDSConnector.h @@ -0,0 +1,13 @@ +#pragma once + +#include "protos/yagpcc_set_service.pb.h" +#include + +class UDSConnector { +public: + UDSConnector(); + bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); + +private: + const std::string uds_path; +}; \ No newline at end of file From 49af1b706bd0f3d20795681f370c8b7ff4d1119d Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 2 Nov 2023 14:38:24 +0300 Subject: [PATCH 050/128] [yagp_hooks_collector] Fix missing query statuses after protobuf migration --- src/EventSender.cpp | 47 ++++++++++++++++++++++++++++----------------- src/EventSender.h | 2 +- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 834553a6187..45d72b93e48 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -230,16 +230,10 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // no-op: executor_after_start is enough break; case METRICS_QUERY_DONE: - collect_query_done(reinterpret_cast(arg), "done"); - break; case METRICS_QUERY_ERROR: - collect_query_done(reinterpret_cast(arg), "error"); - break; case METRICS_QUERY_CANCELING: - collect_query_done(reinterpret_cast(arg), "calcelling"); - break; case METRICS_QUERY_CANCELED: - collect_query_done(reinterpret_cast(arg), "cancelled"); + collect_query_done(reinterpret_cast(arg), status); break; case METRICS_INNER_QUERY_DONE: // TODO @@ -320,10 +314,7 @@ void EventSender::executor_end(QueryDesc *query_desc) { } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (!connector) { - return; - } - if (need_collect()) { + if (connector && need_collect()) { *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); @@ -336,13 +327,33 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { } void EventSender::collect_query_done(QueryDesc *query_desc, - const std::string &status) { - if (!connector) { - return; - } - if (need_collect()) { - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - if (connector->report_query(*query_msg, status)) { + QueryMetricsStatus status) { + if (connector && need_collect()) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + query_msg->set_query_status(query_status); + if (connector->report_query(*query_msg, msg)) { clear_big_fields(query_msg); } } diff --git a/src/EventSender.h b/src/EventSender.h index 161bf6ce037..0e8985873b6 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -27,7 +27,7 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); - void collect_query_done(QueryDesc *query_desc, const std::string &status); + void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); UDSConnector *connector = nullptr; int nesting_level = 0; yagpcc::SetQueryReq *query_msg; From 6d42cd1de4e225b3c36b492035d2e9d2fad7b4e3 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 13 Nov 2023 15:38:31 +0300 Subject: [PATCH 051/128] [yagp_hooks_collector] Add stat_messages() runtime statistics view Add SQL functions stat_messages() and stat_messages_reset() exposing per-segment UDS transport counters: total_messages, send_failures, connection_failures, other_errors, max_message_size. --- sql/yagp-hooks-collector--1.0.sql | 2 - sql/yagp-hooks-collector--unpackaged--1.0.sql | 2 - sql/yagp_hooks_collector--1.0.sql | 55 +++++++++++ src/UDSConnector.cpp | 13 ++- src/UDSConnector.h | 3 - src/YagpStat.cpp | 91 +++++++++++++++++++ src/YagpStat.h | 21 +++++ src/hook_wrappers.cpp | 52 ++++++++++- src/hook_wrappers.h | 2 + src/yagp_hooks_collector.c | 13 ++- ...or.control => yagp_hooks_collector.control | 4 +- 11 files changed, 242 insertions(+), 16 deletions(-) delete mode 100644 sql/yagp-hooks-collector--1.0.sql delete mode 100644 sql/yagp-hooks-collector--unpackaged--1.0.sql create mode 100644 sql/yagp_hooks_collector--1.0.sql create mode 100644 src/YagpStat.cpp create mode 100644 src/YagpStat.h rename yagp-hooks-collector.control => yagp_hooks_collector.control (61%) diff --git a/sql/yagp-hooks-collector--1.0.sql b/sql/yagp-hooks-collector--1.0.sql deleted file mode 100644 index f9ab15fb400..00000000000 --- a/sql/yagp-hooks-collector--1.0.sql +++ /dev/null @@ -1,2 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use '''CREATE EXTENSION "yagp-hooks-collector"''' to load this file. \quit diff --git a/sql/yagp-hooks-collector--unpackaged--1.0.sql b/sql/yagp-hooks-collector--unpackaged--1.0.sql deleted file mode 100644 index 0441c97bd84..00000000000 --- a/sql/yagp-hooks-collector--unpackaged--1.0.sql +++ /dev/null @@ -1,2 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use '''CREATE EXTENSION "uuid-cb" FROM unpackaged''' to load this file. \quit diff --git a/sql/yagp_hooks_collector--1.0.sql b/sql/yagp_hooks_collector--1.0.sql new file mode 100644 index 00000000000..88bbe4e0dc7 --- /dev/null +++ b/sql/yagp_hooks_collector--1.0.sql @@ -0,0 +1,55 @@ +/* yagp_hooks_collector--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit + +CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagp_stat_messages_reset() +RETURNS void +AS +$$ + SELECT __yagp_stat_messages_reset_f_on_master(); + SELECT __yagp_stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagp_stat_messages AS + SELECT C.* + FROM __yagp_stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM __yagp_stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 339a5d4f374..b9088205250 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,5 +1,6 @@ #include "UDSConnector.h" #include "Config.h" +#include "YagpStat.h" #include #include @@ -15,9 +16,7 @@ extern "C" { #include "cdb/cdbvars.h" } -UDSConnector::UDSConnector() : uds_path("unix://" + Config::uds_path()) { - GOOGLE_PROTOBUF_VERIFY_VERSION; -} +UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { @@ -31,7 +30,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event) { sockaddr_un address; address.sun_family = AF_UNIX; - strcpy(address.sun_path, uds_path.c_str()); + strcpy(address.sun_path, Config::uds_path().c_str()); bool success = true; auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd != -1) { @@ -58,12 +57,16 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (sent < 0) { log_tracing_failure(req, event); success = false; + YagpStat::report_bad_send(total_size); + } else { + YagpStat::report_send(total_size); } pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); success = false; + YagpStat::report_bad_connection(); } } else { // That's a very important error that should never happen, so make it @@ -72,12 +75,14 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, (errmsg("Unable to create non-blocking socket connection %s", strerror(errno)))); success = false; + YagpStat::report_error(); } close(sockfd); } else { // log the error and go on log_tracing_failure(req, event); success = false; + YagpStat::report_error(); } return success; } \ No newline at end of file diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 574653023e6..42e0aa20968 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -7,7 +7,4 @@ class UDSConnector { public: UDSConnector(); bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); - -private: - const std::string uds_path; }; \ No newline at end of file diff --git a/src/YagpStat.cpp b/src/YagpStat.cpp new file mode 100644 index 00000000000..879cde85212 --- /dev/null +++ b/src/YagpStat.cpp @@ -0,0 +1,91 @@ +#include "YagpStat.h" + +#include + +extern "C" { +#include "postgres.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "storage/spin.h" +} + +namespace { +struct ProtectedData { + slock_t mutex; + YagpStat::Data data; +}; +shmem_startup_hook_type prev_shmem_startup_hook = NULL; +ProtectedData *data = nullptr; + +void yagp_shmem_startup() { + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + bool found; + data = reinterpret_cast( + ShmemInitStruct("yagp_stat_messages", sizeof(ProtectedData), &found)); + if (!found) { + SpinLockInit(&data->mutex); + data->data = YagpStat::Data(); + } + LWLockRelease(AddinShmemInitLock); +} + +class LockGuard { +public: + LockGuard(slock_t *mutex) : mutex_(mutex) { SpinLockAcquire(mutex_); } + ~LockGuard() { SpinLockRelease(mutex_); } + +private: + slock_t *mutex_; +}; +} // namespace + +void YagpStat::init() { + if (!process_shared_preload_libraries_in_progress) + return; + RequestAddinShmemSpace(sizeof(ProtectedData)); + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = yagp_shmem_startup; +} + +void YagpStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } + +void YagpStat::reset() { + LockGuard lg(&data->mutex); + data->data = YagpStat::Data(); +} + +void YagpStat::report_send(int32_t msg_size) { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +} + +void YagpStat::report_bad_connection() { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_connects++; +} + +void YagpStat::report_bad_send(int32_t msg_size) { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_sends++; + data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +} + +void YagpStat::report_error() { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_other++; +} + +YagpStat::Data YagpStat::get_stats() { + LockGuard lg(&data->mutex); + return data->data; +} + +bool YagpStat::loaded() { return data != nullptr; } diff --git a/src/YagpStat.h b/src/YagpStat.h new file mode 100644 index 00000000000..110b1fdcbb1 --- /dev/null +++ b/src/YagpStat.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +class YagpStat { +public: + struct Data { + int64_t total, failed_sends, failed_connects, failed_other; + int32_t max_message_size; + }; + + static void init(); + static void deinit(); + static void reset(); + static void report_send(int32_t msg_size); + static void report_bad_connection(); + static void report_bad_send(int32_t msg_size); + static void report_error(); + static Data get_stats(); + static bool loaded(); +}; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 66ba6547ce2..37f80385a6b 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,16 +1,17 @@ extern "C" { #include "postgres.h" +#include "funcapi.h" #include "executor/executor.h" #include "utils/elog.h" +#include "utils/builtins.h" #include "utils/metrics_utils.h" - #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" - #include "tcop/utility.h" } #include "Config.h" +#include "YagpStat.h" #include "EventSender.h" #include "hook_wrappers.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" @@ -39,6 +40,7 @@ static inline EventSender *get_sender() { void hooks_init() { Config::init(); + YagpStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; @@ -62,6 +64,7 @@ void hooks_deinit() { if (sender) { delete sender; } + YagpStat::deinit(); } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { @@ -150,4 +153,49 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } +} + +static void check_stats_loaded() { + if (!YagpStat::loaded()) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("yagp_hooks_collector must be loaded via " + "shared_preload_libraries"))); + } +} + +void yagp_functions_reset() { + check_stats_loaded(); + YagpStat::reset(); +} + +Datum yagp_functions_get(FunctionCallInfo fcinfo) { + const int ATTNUM = 6; + check_stats_loaded(); + auto stats = YagpStat::get_stats(); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM, false); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, + 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)3, "send_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)4, "connection_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)5, "other_errors", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)6, "max_message_size", INT4OID, + -1 /* typmod */, 0 /* attdim */); + tupdesc = BlessTupleDesc(tupdesc); + Datum values[ATTNUM]; + bool nulls[ATTNUM]; + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(GpIdentity.segindex); + values[1] = Int64GetDatum(stats.total); + values[2] = Int64GetDatum(stats.failed_sends); + values[3] = Int64GetDatum(stats.failed_connects); + values[4] = Int64GetDatum(stats.failed_other); + values[5] = Int32GetDatum(stats.max_message_size); + HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index 815fcb7cd51..c158f42cf1d 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -6,6 +6,8 @@ extern "C" { extern void hooks_init(); extern void hooks_deinit(); +extern void yagp_functions_reset(); +extern Datum yagp_functions_get(FunctionCallInfo fcinfo); #ifdef __cplusplus } diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 69475ea5079..2a9e7328e6d 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,6 +1,6 @@ #include "postgres.h" #include "cdb/cdbvars.h" -#include "fmgr.h" +#include "utils/builtins.h" #include "hook_wrappers.h" @@ -8,6 +8,8 @@ PG_MODULE_MAGIC; void _PG_init(void); void _PG_fini(void); +PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); +PG_FUNCTION_INFO_V1(yagp_stat_messages); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { @@ -20,3 +22,12 @@ void _PG_fini(void) { hooks_deinit(); } } + +Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { + yagp_functions_reset(); + PG_RETURN_VOID(); +} + +Datum yagp_stat_messages(PG_FUNCTION_ARGS) { + return yagp_functions_get(fcinfo); +} \ No newline at end of file diff --git a/yagp-hooks-collector.control b/yagp_hooks_collector.control similarity index 61% rename from yagp-hooks-collector.control rename to yagp_hooks_collector.control index 82c189a88fc..b5539dd6462 100644 --- a/yagp-hooks-collector.control +++ b/yagp_hooks_collector.control @@ -1,5 +1,5 @@ -# yagp-hooks-collector extension +# yagp_hooks_collector extension comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' default_version = '1.0' -module_pathname = '$libdir/yagp-hooks-collector' +module_pathname = '$libdir/yagp_hooks_collector' superuser = true From 6b41581460cdcace12a5fa8f75115323bd955153 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 15 Nov 2023 13:37:10 +0300 Subject: [PATCH 052/128] [yagp_hooks_collector] Fix message lifecycle ordering and memory leaks Move query message cleanup to the correct lifecycle point. Finalize fields before sending DONE event. Fix protobuf message memory leaks. --- src/EventSender.cpp | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 45d72b93e48..e3be58b194e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -39,12 +39,17 @@ namespace { std::string *get_user_name() { const char *username = GetConfigOption("session_authorization", false, false); + // username is not to be freed return username ? new std::string(username) : nullptr; } std::string *get_db_name() { char *dbname = get_database_name(MyDatabaseId); - std::string *result = dbname ? new std::string(dbname) : nullptr; + std::string *result = nullptr; + if (dbname) { + result = new std::string(dbname); + pfree(dbname); + } return result; } @@ -58,8 +63,7 @@ std::string *get_rg_name() { char *rgname = GetResGroupNameForId(groupId); if (rgname == nullptr) return nullptr; - auto result = new std::string(rgname); - return result; + return new std::string(rgname); } google::protobuf::Timestamp current_ts() { @@ -97,8 +101,12 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { } void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); *plan_text = std::string(es.str->data, es.str->len); + pfree(es.str->data); + MemoryContextSwitchTo(oldcxt); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -251,10 +259,6 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!need_collect()) { return; } - if (query_msg->has_query_key()) { - connector->report_query(*query_msg, "previous query"); - query_msg->Clear(); - } query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -263,11 +267,12 @@ void EventSender::executor_before_start(QueryDesc *query_desc, query_desc->instrument_options |= INSTRUMENT_TIMER; if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + if (!query_desc->showstatctx) { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } } } } @@ -309,12 +314,16 @@ void EventSender::executor_end(QueryDesc *query_desc) { *query_msg->mutable_end_time() = current_ts(); set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); if (connector->report_query(*query_msg, "ended")) { - query_msg->Clear(); + clear_big_fields(query_msg); } } void EventSender::collect_query_submit(QueryDesc *query_desc) { if (connector && need_collect()) { + if (query_msg && query_msg->has_query_key()) { + connector->report_query(*query_msg, "previous query"); + query_msg->Clear(); + } *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); @@ -354,7 +363,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } query_msg->set_query_status(query_status); if (connector->report_query(*query_msg, msg)) { - clear_big_fields(query_msg); + query_msg->Clear(); } } } From fad49ebf33da5ceb48d6de7b1b65e62e6fb1f449 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 26 Dec 2023 16:36:26 +0300 Subject: [PATCH 053/128] [yagp_hooks_collector] Improve query_id and resource group resolution Use core query_id from Query instead of a separate hash. Resolve resource group from the current session rather than the role default. --- src/EventSender.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index e3be58b194e..21c2e2117a3 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -54,10 +54,7 @@ std::string *get_db_name() { } std::string *get_rg_name() { - auto userId = GetUserId(); - if (!OidIsValid(userId)) - return nullptr; - auto groupId = GetResGroupIdForRole(userId); + auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); if (!OidIsValid(groupId)) return nullptr; char *rgname = GetResGroupNameForId(groupId); @@ -119,13 +116,7 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - // TODO: For now assume queryid equal to planid, which is wrong. The - // reason for doing so this bug - // https://github.com/greenplum-db/gpdb/pull/15385 (ORCA loses - // pg_stat_statements` queryid during planning phase). Need to fix it - // upstream, cherry-pick and bump gp - // qi->set_query_id(query_desc->plannedstmt->queryId); - qi->set_query_id(qi->plan_id()); + qi->set_query_id(query_desc->plannedstmt->queryId); } } From 63bfe5e3fdcea7ae2b4447bc229fb19f64d32bf1 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Fri, 17 May 2024 15:55:27 +0300 Subject: [PATCH 054/128] [yagp_hooks_collector] Add nested query tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track query nesting level using a per-query key (tmid, ssid, ccnt, nesting_level, query_desc_addr). Maintain a state machine per active query to correctly sequence submit→start→end→done across nesting boundaries. --- protos/yagpcc_metrics.proto | 10 ++- protos/yagpcc_set_service.proto | 32 ++++++-- src/Config.cpp | 7 ++ src/Config.h | 1 + src/EventSender.cpp | 138 ++++++++++++++++++++++++++------ src/EventSender.h | 26 +++++- src/hook_wrappers.cpp | 2 +- 7 files changed, 178 insertions(+), 38 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 2d20d3c46d9..68492732ece 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -36,6 +36,11 @@ message QueryInfo { string rsgname = 10; } +message AdditionalQueryInfo { + int64 nested_level = 1; + string error_message = 2; +} + enum PlanGenerator { PLAN_GENERATOR_UNSPECIFIED = 0; @@ -95,7 +100,7 @@ message MetricInstrumentation { uint64 nloops = 2; /* # of run cycles for this node */ uint64 tuplecount = 3; /* Tuples emitted so far this cycle */ double firsttuple = 4; /* Time for first tuple of this cycle */ - double startup = 5; /* Total startup time (in seconds) */ + double startup = 5; /* Total startup time (in seconds) (optimiser's cost estimation) */ double total = 6; /* Total total time (in seconds) */ uint64 shared_blks_hit = 7; /* shared blocks stats*/ uint64 shared_blks_read = 8; @@ -105,12 +110,13 @@ message MetricInstrumentation { uint64 local_blks_read = 12; uint64 local_blks_dirtied = 13; uint64 local_blks_written = 14; - uint64 temp_blks_read = 15; /* temporary tables read stat */ + uint64 temp_blks_read = 15; /* temporary tables read stat */ uint64 temp_blks_written = 16; double blk_read_time = 17; /* measured read/write time */ double blk_write_time = 18; NetworkStat sent = 19; NetworkStat received = 20; + double startup_time = 21; /* real query startup time (planning + queue time) */ } message SpillInfo { diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index e8fc7aaa99d..0b9e34df49d 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -9,17 +9,35 @@ package yagpcc; option java_outer_classname = "SegmentYAGPCCAS"; option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; +service SetQueryInfo { + rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} + + rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} +} + +message MetricResponse { + MetricResponseStatusCode error_code = 1; + string error_text = 2; +} + +enum MetricResponseStatusCode { + METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; + METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; + METRIC_RESPONSE_STATUS_CODE_ERROR = 2; +} + message SetQueryReq { - QueryStatus query_status = 1; - google.protobuf.Timestamp datetime = 2; - QueryKey query_key = 3; - SegmentKey segment_key = 4; - QueryInfo query_info = 5; - GPMetrics query_metrics = 6; - repeated MetricPlan plan_tree = 7; + QueryStatus query_status = 1; + google.protobuf.Timestamp datetime = 2; + QueryKey query_key = 3; + SegmentKey segment_key = 4; + QueryInfo query_info = 5; + GPMetrics query_metrics = 6; + repeated MetricPlan plan_tree = 7; google.protobuf.Timestamp submit_time = 8; google.protobuf.Timestamp start_time = 9; google.protobuf.Timestamp end_time = 10; + AdditionalQueryInfo add_info = 11; } message SetPlanNodeReq { diff --git a/src/Config.cpp b/src/Config.cpp index c5c2c15f7e9..1bbad9a6ea3 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -13,6 +13,7 @@ static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; +static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static std::unique_ptr> ignored_users = nullptr; @@ -36,6 +37,11 @@ void Config::init() { &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomBoolVariable( + "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, + &guc_report_nested_queries, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomStringVariable( "yagpcc.ignored_users_list", "Make yagpcc ignore queries issued by given users", 0LL, @@ -47,6 +53,7 @@ std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } +bool Config::report_nested_queries() { return guc_report_nested_queries; } bool Config::filter_user(const std::string *username) { if (!ignored_users) { diff --git a/src/Config.h b/src/Config.h index 999d0300640..15f425be67c 100644 --- a/src/Config.h +++ b/src/Config.h @@ -10,4 +10,5 @@ class Config { static bool enable_cdbstats(); static bool enable_collector(); static bool filter_user(const std::string *username); + static bool report_nested_queries(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 21c2e2117a3..116805d0646 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -10,6 +10,7 @@ extern "C" { #include "postgres.h" #include "access/hash.h" +#include "access/xact.h" #include "commands/dbcommands.h" #include "commands/explain.h" #include "commands/resgroupcmds.h" @@ -30,11 +31,6 @@ extern "C" { #include "EventSender.h" -#define need_collect() \ - (nesting_level == 0 && gp_command_count != 0 && \ - query_desc->sourceText != nullptr && Config::enable_collector() && \ - !Config::filter_user(get_user_name())) - namespace { std::string *get_user_name() { @@ -146,6 +142,11 @@ void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } } +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); +} + void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; @@ -210,6 +211,19 @@ yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, return req; } +inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +inline bool need_collect(QueryDesc *query_desc, int nesting_level) { + return (Config::report_nested_queries() || + is_top_level_query(query_desc, nesting_level)) && + gp_command_count != 0 && query_desc->sourceText != nullptr && + Config::enable_collector() && !Config::filter_user(get_user_name()); +} + } // namespace void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { @@ -223,7 +237,8 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - collect_query_submit(reinterpret_cast(arg)); + // don't collect anything here. We will fake this call in ExecutorStart as + // it really makes no difference. Just complicates things break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -232,10 +247,8 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELING: case METRICS_QUERY_CANCELED: - collect_query_done(reinterpret_cast(arg), status); - break; case METRICS_INNER_QUERY_DONE: - // TODO + collect_query_done(reinterpret_cast(arg), status); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -247,9 +260,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!connector) { return; } - if (!need_collect()) { + if (!need_collect(query_desc, nesting_level)) { return; } + collect_query_submit(query_desc); query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -273,8 +287,10 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { return; } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && - need_collect()) { - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + need_collect(query_desc, nesting_level)) { + auto *query = get_query_message(query_desc); + update_query_state(query_desc, query, QueryState::START); + auto query_msg = query->message; *query_msg->mutable_start_time() = current_ts(); set_query_plan(query_msg, query_desc); if (connector->report_query(*query_msg, "started")) { @@ -287,7 +303,7 @@ void EventSender::executor_end(QueryDesc *query_desc) { if (!connector) { return; } - if (!need_collect() || + if (!need_collect(query_desc, nesting_level) || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } @@ -301,7 +317,13 @@ void EventSender::executor_end(QueryDesc *query_desc) { cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, DISPATCH_WAIT_NONE); }*/ - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + auto *query = get_query_message(query_desc); + if (query->state == UNKNOWN && !Config::report_nested_queries()) { + // COMMIT/ROLLBACK of a nested query. Happens in top-level + return; + } + update_query_state(query_desc, query, QueryState::END); + auto query_msg = query->message; *query_msg->mutable_end_time() = current_ts(); set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); if (connector->report_query(*query_msg, "ended")) { @@ -310,15 +332,15 @@ void EventSender::executor_end(QueryDesc *query_desc) { } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (connector && need_collect()) { - if (query_msg && query_msg->has_query_key()) { - connector->report_query(*query_msg, "previous query"); - query_msg->Clear(); - } + if (connector && need_collect(query_desc, nesting_level)) { + auto *query = get_query_message(query_desc); + query->state = QueryState::SUBMIT; + auto query_msg = query->message; *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg, query_desc); + set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); set_query_text(query_msg, query_desc); if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); @@ -328,11 +350,12 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && need_collect()) { + if (connector && need_collect(query_desc, nesting_level)) { yagpcc::QueryStatus query_status; std::string msg; switch (status) { case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; msg = "done"; break; @@ -352,16 +375,26 @@ void EventSender::collect_query_done(QueryDesc *query_desc, ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", status))); } - query_msg->set_query_status(query_status); - if (connector->report_query(*query_msg, msg)) { - query_msg->Clear(); + auto *query = get_query_message(query_desc); + if (query->state != UNKNOWN || Config::report_nested_queries()) { + update_query_state(query_desc, query, QueryState::DONE, + query_status == + yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query->message; + query_msg->set_query_status(query_status); + connector->report_query(*query_msg, msg); + } else { + // otherwise it`s a nested query being committed/aborted at top level + // and we should ignore it } + query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}); + pfree(query_desc->gpmon_pkt); } } EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { - query_msg = new yagpcc::SetQueryReq(); try { connector = new UDSConnector(); } catch (const std::exception &e) { @@ -371,6 +404,59 @@ EventSender::EventSender() { } EventSender::~EventSender() { - delete query_msg; delete connector; -} \ No newline at end of file + for (auto iter = query_msgs.begin(); iter != query_msgs.end(); ++iter) { + delete iter->second.message; + } +} + +// That's basically a very simplistic state machine to fix or highlight any bugs +// coming from GP +void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, + QueryState new_state, bool success) { + if (query->state == UNKNOWN) { + collect_query_submit(query_desc); + } + switch (new_state) { + case QueryState::SUBMIT: + Assert(false); + break; + case QueryState::START: + if (query->state == QueryState::SUBMIT) { + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + } else { + Assert(false); + } + break; + case QueryState::END: + Assert(query->state == QueryState::START || IsAbortInProgress()); + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + break; + case QueryState::DONE: + Assert(query->state == QueryState::END || !success); + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + break; + default: + Assert(false); + } + query->state = new_state; +} + +EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { + if (query_desc->gpmon_pkt == nullptr || + query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}) == + query_msgs.end()) { + query_desc->gpmon_pkt = (gpmon_packet_t *)palloc0(sizeof(gpmon_packet_t)); + query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; + query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; + query_msgs.insert({{gp_command_count, nesting_level}, + QueryItem(UNKNOWN, new yagpcc::SetQueryReq())}); + } + return &query_msgs.at({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}); +} + +EventSender::QueryItem::QueryItem(EventSender::QueryState st, + yagpcc::SetQueryReq *msg) + : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 0e8985873b6..55b8daf9a91 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include extern "C" { @@ -26,9 +26,31 @@ class EventSender { ~EventSender(); private: + enum QueryState { UNKNOWN, SUBMIT, START, END, DONE }; + + struct QueryItem { + QueryState state = QueryState::UNKNOWN; + yagpcc::SetQueryReq *message = nullptr; + + QueryItem(QueryState st, yagpcc::SetQueryReq *msg); + }; + + struct pair_hash { + std::size_t operator()(const std::pair &p) const { + auto h1 = std::hash{}(p.first); + auto h2 = std::hash{}(p.second); + return h1 ^ h2; + } + }; + + void update_query_state(QueryDesc *query_desc, QueryItem *query, + QueryState new_state, bool success = true); + QueryItem *get_query_message(QueryDesc *query_desc); void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); + void cleanup_messages(); + UDSConnector *connector = nullptr; int nesting_level = 0; - yagpcc::SetQueryReq *query_msg; + std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 37f80385a6b..caf38a10f6e 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -56,9 +56,9 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; ExecutorRun_hook = previous_ExecutorRun_hook; ExecutorFinish_hook = previous_ExecutorFinish_hook; - ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); if (sender) { From e3cc4c73b5a4cebc6db18d5adf530a75b1590fc7 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 May 2024 15:25:35 +0300 Subject: [PATCH 055/128] [yagp_hooks_collector] Add configurable text field trimming Trim query text and plan text to max_text_size and max_plan_size limits. --- src/Config.cpp | 11 ++++++++++- src/Config.h | 1 + src/EventSender.cpp | 12 +++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 1bbad9a6ea3..c07a6948694 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,7 +1,8 @@ #include "Config.h" -#include +#include #include #include +#include extern "C" { #include "postgres.h" @@ -15,6 +16,7 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; +static int guc_max_text_size = 1024; // in KB static std::unique_ptr> ignored_users = nullptr; void Config::init() { @@ -47,6 +49,12 @@ void Config::init() { "Make yagpcc ignore queries issued by given users", 0LL, &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomIntVariable( + "yagpcc.max_text_size", + "Make yagpcc trim plan and query texts longer than configured size", NULL, + &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } @@ -54,6 +62,7 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } +size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { if (!ignored_users) { diff --git a/src/Config.h b/src/Config.h index 15f425be67c..f806bc0dbf5 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,4 +11,5 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); + static size_t max_text_size(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 116805d0646..4de5564533b 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -93,11 +93,15 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { return es; } +inline std::string char_to_trimmed_str(const char *str, size_t len) { + return std::string(str, std::min(len, Config::max_text_size())); +} + void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { MemoryContext oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); - *plan_text = std::string(es.str->data, es.str->len); + *plan_text = char_to_trimmed_str(es.str->data, es.str->len); pfree(es.str->data); MemoryContextSwitchTo(oldcxt); } @@ -119,9 +123,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = query_desc->sourceText; + *qi->mutable_query_text() = char_to_trimmed_str( + query_desc->sourceText, strlen(query_desc->sourceText)); char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = std::string(norm_query); + *qi->mutable_template_query_text() = + char_to_trimmed_str(norm_query, strlen(norm_query)); } } From ddf2f4e616e5078cd6478411969569ea94a2984c Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 May 2024 16:19:58 +0300 Subject: [PATCH 056/128] [yagp_hooks_collector] Add error message reporting for failed queries Capture elog error message at the done event for ERROR and CANCELED statuses. Properly send accumulated runtime metrics before teardown. Drop the intermediate CANCELLING event. --- src/EventSender.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 4de5564533b..8d202991986 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -153,6 +153,12 @@ void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { aqi->set_nested_level(nesting_level); } +void set_qi_error_message(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + auto error = elog_message(); + *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); +} + void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; @@ -249,9 +255,13 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_START: // no-op: executor_after_start is enough break; + case METRICS_QUERY_CANCELING: + // it appears we're unly interested in the actual CANCELED event. + // for now we will ignore CANCELING state unless otherwise requested from + // end users + break; case METRICS_QUERY_DONE: case METRICS_QUERY_ERROR: - case METRICS_QUERY_CANCELING: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: collect_query_done(reinterpret_cast(arg), status); @@ -370,6 +380,9 @@ void EventSender::collect_query_done(QueryDesc *query_desc, msg = "error"; break; case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code here + // just in case + Assert(false); query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; msg = "cancelling"; break; @@ -382,12 +395,21 @@ void EventSender::collect_query_done(QueryDesc *query_desc, status))); } auto *query = get_query_message(query_desc); + auto prev_state = query->state; if (query->state != UNKNOWN || Config::report_nested_queries()) { update_query_state(query_desc, query, QueryState::DONE, query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query->message; query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + } connector->report_query(*query_msg, msg); } else { // otherwise it`s a nested query being committed/aborted at top level @@ -435,7 +457,9 @@ void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, } break; case QueryState::END: - Assert(query->state == QueryState::START || IsAbortInProgress()); + // Example of below assert triggering: CURSOR closes before ever being + // executed Assert(query->state == QueryState::START || + // IsAbortInProgress()); query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: From 5592f44f3edc2dc36496782a0fe721fa29a5db42 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 3 Jun 2024 18:22:00 +0300 Subject: [PATCH 057/128] [yagp_hooks_collector] Change report_nested_queries to PGC_USERSET --- src/Config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.cpp b/src/Config.cpp index c07a6948694..42fa4b2fb12 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -41,7 +41,7 @@ void Config::init() { DefineCustomBoolVariable( "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, - &guc_report_nested_queries, true, PGC_SUSET, + &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomStringVariable( From 8cc491f6ada0bd676f7432a3ce2aae2024330760 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 13 Jun 2024 10:59:46 +0300 Subject: [PATCH 058/128] [yagp_hooks_collector] Diff per-query stats between submit and end Take an initial metrics snapshot at submit so incremental stats are computed as deltas. Required for correct per-query accounting with nested statements. --- src/EventSender.cpp | 21 +++++++++++---------- src/ProcStats.cpp | 8 +++----- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 8d202991986..60f21818d00 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,7 +1,6 @@ #include "Config.h" #include "ProcStats.h" #include "UDSConnector.h" -#include #include #define typeid __typeid @@ -198,19 +197,17 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, } } -decltype(std::chrono::high_resolution_clock::now()) query_start_time; - void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); - std::chrono::duration elapsed_seconds = - std::chrono::high_resolution_clock::now() - query_start_time; metrics->mutable_systemstat()->set_runningtimeseconds( - elapsed_seconds.count()); - metrics->mutable_spill()->set_filecount(WorkfileTotalFilesCreated()); - metrics->mutable_spill()->set_totalbytes(WorkfileTotalBytesWritten()); + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -280,8 +277,6 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } collect_query_submit(query_desc); - query_start_time = std::chrono::high_resolution_clock::now(); - WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -309,9 +304,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { auto query_msg = query->message; *query_msg->mutable_start_time() = current_ts(); set_query_plan(query_msg, query_desc); + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); if (connector->report_query(*query_msg, "started")) { clear_big_fields(query_msg); } + std::swap(stats, *query_msg->mutable_query_metrics()); } } @@ -361,6 +359,9 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); } } diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 668173a0f7e..a557a20cbb0 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -92,9 +92,7 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } // namespace void fill_self_stats(yagpcc::SystemStat *stats) { - static yagpcc::SystemStat prev_stats; - fill_io_stats(&prev_stats); - fill_cpu_stats(&prev_stats); - fill_status_stats(&prev_stats); - *stats = prev_stats; + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file From af843a5bf683505ddbda80ece9d8accf3dc3db41 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 7 Aug 2024 14:28:57 +0300 Subject: [PATCH 059/128] [yagp_hooks_collector] Fix try/catch block when calling C++ code from PG hooks --- src/hook_wrappers.cpp | 44 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index caf38a10f6e..93faaa0bf8f 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -38,6 +38,15 @@ static inline EventSender *get_sender() { return sender; } +template +R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { + try { + return (obj->*func)(args...); + } catch (const std::exception &e) { + ereport(FATAL, (errmsg("Unexpected exception in yagpcc %s", e.what()))); + } +} + void hooks_init() { Config::init(); YagpStat::init(); @@ -68,27 +77,15 @@ void hooks_deinit() { } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - PG_TRY(); - { get_sender()->executor_before_start(query_desc, eflags); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, + eflags); if (previous_ExecutorStart_hook) { (*previous_ExecutorStart_hook)(query_desc, eflags); } else { standard_ExecutorStart(query_desc, eflags); } - PG_TRY(); - { get_sender()->executor_after_start(query_desc, eflags); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, + eflags); } void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -129,11 +126,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { } void ya_ExecutorEnd_hook(QueryDesc *query_desc) { - PG_TRY(); - { get_sender()->executor_end(query_desc); } - PG_CATCH(); - { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_end, query_desc); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); } else { @@ -142,14 +135,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - PG_TRY(); - { get_sender()->query_metrics_collect(status, arg); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_query_info_collect_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg); if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } From 3def9fc9bca0ab2851ad81e47bca0dd9b3fc1b7f Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 12 Sep 2024 16:15:26 +0300 Subject: [PATCH 060/128] [yagp_hooks_collector] Improve nested query handling and add slice info Don't normalize trimmed plans. Clean up stale text fields between events. Report nested queries only from dispatcher. Add slice_id. Aggregate inherited_calls and inherited_time on segments. --- protos/yagpcc_metrics.proto | 3 + src/EventSender.cpp | 295 ++++++++++++++++++++++-------------- src/EventSender.h | 3 + 3 files changed, 191 insertions(+), 110 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 68492732ece..fc85386c6b0 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -39,6 +39,7 @@ message QueryInfo { message AdditionalQueryInfo { int64 nested_level = 1; string error_message = 2; + int64 slice_id = 3; } enum PlanGenerator @@ -117,6 +118,8 @@ message MetricInstrumentation { NetworkStat sent = 19; NetworkStat received = 20; double startup_time = 21; /* real query startup time (planning + queue time) */ + uint64 inherited_calls = 22; /* the number of executed sub-queries */ + double inherited_time = 23; /* total time spend on inherited execution */ } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 60f21818d00..7d2d5a1a2c2 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -58,6 +58,53 @@ std::string *get_rg_name() { return new std::string(rgname); } +/** + * Things get tricky with nested queries. + * a) A nested query on master is a real query optimized and executed from + * master. An example would be `select some_insert_function();`, where + * some_insert_function does something like `insert into tbl values (1)`. Master + * will create two statements. Outer select statement and inner insert statement + * with nesting level 1. + * For segments both statements are top-level statements with nesting level 0. + * b) A nested query on segment is something executed as sub-statement on + * segment. An example would be `select a from tbl where is_good_value(b);`. In + * this case master will issue one top-level statement, but segments will change + * contexts for UDF execution and execute is_good_value(b) once for each tuple + * as a nested query. Creating massive load on gpcc agent. + * + * Hence, here is a decision: + * 1) ignore all queries that are nested on segments + * 2) record (if enabled) all queries that are nested on master + * NODE: The truth is, we can't really ignore nested master queries, because + * segment sees those as top-level. + */ + +inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +inline bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return (Gp_session_role == GP_ROLE_DISPATCH && + Config::report_nested_queries()) || + is_top_level_query(query_desc, nesting_level); +} + +bool need_report_nested_query() { + return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; +} + +inline bool filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !Config::enable_collector() || Config::filter_user(get_user_name()); +} + +inline bool need_collect(QueryDesc *query_desc, int nesting_level) { + return !filter_query(query_desc) && + nesting_is_valid(query_desc, nesting_level); +} + google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -96,26 +143,24 @@ inline std::string char_to_trimmed_str(const char *str, size_t len) { return std::string(str, std::min(len, Config::max_text_size())); } -void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { - MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - *plan_text = char_to_trimmed_str(es.str->data, es.str->len); - pfree(es.str->data); - MemoryContextSwitchTo(oldcxt); -} - void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_template_plan_text() = std::string(norm_plan->data); + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + auto es = get_explain_state(query_desc, true); + MemoryContextSwitchTo(oldcxt); + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + StringInfo norm_plan = gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = + char_to_trimmed_str(norm_plan->data, norm_plan->len); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); + pfree(es.str->data); + pfree(norm_plan->data); } } @@ -134,7 +179,9 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); + qi->clear_template_plan_text(); qi->clear_query_text(); + qi->clear_template_query_text(); } } @@ -152,6 +199,11 @@ void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { aqi->set_nested_level(nesting_level); } +void set_qi_slice_id(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); +} + void set_qi_error_message(yagpcc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); auto error = elog_message(); @@ -159,7 +211,8 @@ void set_qi_error_message(yagpcc::SetQueryReq *req) { } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, - QueryDesc *query_desc) { + QueryDesc *query_desc, int nested_calls, + double nested_time) { auto instrument = query_desc->planstate->instrument; if (instrument) { metrics->set_ntuples(instrument->ntuples); @@ -195,11 +248,15 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, mlstate->stat_tuple_bytes_recvd); metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); } -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) { if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, + nested_calls, nested_time); } fill_self_stats(metrics->mutable_systemstat()); metrics->mutable_systemstat()->set_runningtimeseconds( @@ -220,17 +277,8 @@ yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, return req; } -inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; -} - -inline bool need_collect(QueryDesc *query_desc, int nesting_level) { - return (Config::report_nested_queries() || - is_top_level_query(query_desc, nesting_level)) && - gp_command_count != 0 && query_desc->sourceText != nullptr && - Config::enable_collector() && !Config::filter_user(get_user_name()); +double protots_to_double(const google::protobuf::Timestamp &ts) { + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } } // namespace @@ -273,6 +321,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!connector) { return; } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } if (!need_collect(query_desc, nesting_level)) { return; } @@ -297,51 +349,53 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { if (!connector) { return; } - if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && - need_collect(query_desc, nesting_level)) { - auto *query = get_query_message(query_desc); - update_query_state(query_desc, query, QueryState::START); - auto query_msg = query->message; - *query_msg->mutable_start_time() = current_ts(); - set_query_plan(query_msg, query_desc); - yagpcc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { - clear_big_fields(query_msg); + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (!filter_query(query_desc)) { + auto *query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_start_time() = current_ts(); + if (!nesting_is_valid(query_desc, nesting_level)) { + return; + } + update_query_state(query_desc, query, QueryState::START); + set_query_plan(query_msg, query_desc); + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } - std::swap(stats, *query_msg->mutable_query_metrics()); } } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector) { - return; - } - if (!need_collect(query_desc, nesting_level) || + if (!connector || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } - /* TODO: when querying via CURSOR this call freezes. Need to investigate. - To reproduce - uncomment it and run installchecks. It will freeze around - join test. Needs investigation - - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && - Config::enable_cdbstats() && query_desc->estate->dispatcherState && - query_desc->estate->dispatcherState->primaryResults) { - cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, - DISPATCH_WAIT_NONE); - }*/ - auto *query = get_query_message(query_desc); - if (query->state == UNKNOWN && !Config::report_nested_queries()) { - // COMMIT/ROLLBACK of a nested query. Happens in top-level - return; - } - update_query_state(query_desc, query, QueryState::END); - auto query_msg = query->message; - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); - if (connector->report_query(*query_msg, "ended")) { - clear_big_fields(query_msg); + if (!filter_query(query_desc)) { + auto *query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_end_time() = current_ts(); + if (nesting_is_valid(query_desc, nesting_level)) { + if (query->state == UNKNOWN && + // Yet another greenplum weirdness: thats actually a nested query + // which is being committed/rollbacked. Treat it accordingly. + !need_report_nested_query()) { + return; + } + update_query_state(query_desc, query, QueryState::END); + if (is_top_level_query(query_desc, nesting_level)) { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } else { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (connector->report_query(*query_msg, "ended")) { + clear_big_fields(query_msg); + } + } } } @@ -355,66 +409,70 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg, query_desc); set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); + set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); } // take initial metrics snapshot so that we can safely take diff afterwards // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); } } void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && need_collect(query_desc, nesting_level)) { - yagpcc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code here - // just in case - Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", - status))); - } + if (connector && !filter_query(query_desc)) { auto *query = get_query_message(query_desc); - auto prev_state = query->state; - if (query->state != UNKNOWN || Config::report_nested_queries()) { - update_query_state(query_desc, query, QueryState::DONE, - query_status == - yagpcc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query->message; - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); + if (query->state != UNKNOWN || need_report_nested_query()) { + if (nesting_is_valid(query_desc, nesting_level)) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, + (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + auto prev_state = query->state; + update_query_state(query_desc, query, QueryState::DONE, + query_status == + yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query->message; + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } + connector->report_query(*query_msg, msg); } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); - } - connector->report_query(*query_msg, msg); - } else { - // otherwise it`s a nested query being committed/aborted at top level - // and we should ignore it + update_nested_counters(query_desc); } query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}); @@ -488,6 +546,23 @@ EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { query_desc->gpmon_pkt->u.qexec.key.tmid}); } +void EventSender::update_nested_counters(QueryDesc *query_desc) { + if (!is_top_level_query(query_desc, nesting_level)) { + auto query_msg = get_query_message(query_desc); + nested_calls++; + double end_time = protots_to_double(query_msg->message->end_time()); + double start_time = protots_to_double(query_msg->message->start_time()); + if (end_time >= start_time) { + nested_timing += end_time - start_time; + } else { + ereport(WARNING, (errmsg("YAGPCC query start_time > end_time (%f > %f)", + start_time, end_time))); + ereport(DEBUG3, + (errmsg("YAGPCC nested query text %s", query_desc->sourceText))); + } + } +} + EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 55b8daf9a91..9470cbf1f98 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -49,8 +49,11 @@ class EventSender { void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); + void update_nested_counters(QueryDesc *query_desc); UDSConnector *connector = nullptr; int nesting_level = 0; + int64_t nested_calls = 0; + double nested_timing = 0; std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file From f4a3e7f8d581bef6e2c6eaaab9fa8442912eeff7 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 7 Nov 2024 13:09:44 +0300 Subject: [PATCH 061/128] [yagp_hooks_collector] Split EventSender into submodules Factor out ProtoUtils, ProcStats, and PgUtils from EventSender. --- src/EventSender.cpp | 275 +------------------------------------------- src/PgUtils.cpp | 94 +++++++++++++++ src/PgUtils.h | 16 +++ src/ProtoUtils.cpp | 185 +++++++++++++++++++++++++++++ src/ProtoUtils.h | 16 +++ 5 files changed, 315 insertions(+), 271 deletions(-) create mode 100644 src/PgUtils.cpp create mode 100644 src/PgUtils.h create mode 100644 src/ProtoUtils.cpp create mode 100644 src/ProtoUtils.h diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 7d2d5a1a2c2..cdb21ef7aa6 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,287 +1,21 @@ #include "Config.h" -#include "ProcStats.h" #include "UDSConnector.h" -#include -#define typeid __typeid -#define operator __operator extern "C" { #include "postgres.h" #include "access/hash.h" -#include "access/xact.h" -#include "commands/dbcommands.h" -#include "commands/explain.h" -#include "commands/resgroupcmds.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" -#include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" - -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" -#include "tcop/utility.h" } -#undef typeid -#undef operator #include "EventSender.h" - -namespace { - -std::string *get_user_name() { - const char *username = GetConfigOption("session_authorization", false, false); - // username is not to be freed - return username ? new std::string(username) : nullptr; -} - -std::string *get_db_name() { - char *dbname = get_database_name(MyDatabaseId); - std::string *result = nullptr; - if (dbname) { - result = new std::string(dbname); - pfree(dbname); - } - return result; -} - -std::string *get_rg_name() { - auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); - if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); - if (rgname == nullptr) - return nullptr; - return new std::string(rgname); -} - -/** - * Things get tricky with nested queries. - * a) A nested query on master is a real query optimized and executed from - * master. An example would be `select some_insert_function();`, where - * some_insert_function does something like `insert into tbl values (1)`. Master - * will create two statements. Outer select statement and inner insert statement - * with nesting level 1. - * For segments both statements are top-level statements with nesting level 0. - * b) A nested query on segment is something executed as sub-statement on - * segment. An example would be `select a from tbl where is_good_value(b);`. In - * this case master will issue one top-level statement, but segments will change - * contexts for UDF execution and execute is_good_value(b) once for each tuple - * as a nested query. Creating massive load on gpcc agent. - * - * Hence, here is a decision: - * 1) ignore all queries that are nested on segments - * 2) record (if enabled) all queries that are nested on master - * NODE: The truth is, we can't really ignore nested master queries, because - * segment sees those as top-level. - */ - -inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; -} - -inline bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return (Gp_session_role == GP_ROLE_DISPATCH && - Config::report_nested_queries()) || - is_top_level_query(query_desc, nesting_level); -} - -bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; -} - -inline bool filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !Config::enable_collector() || Config::filter_user(get_user_name()); -} - -inline bool need_collect(QueryDesc *query_desc, int nesting_level) { - return !filter_query(query_desc) && - nesting_is_valid(query_desc, nesting_level); -} - -google::protobuf::Timestamp current_ts() { - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; -} - -void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gpmon_gettmid(&tmid); - key->set_tmid(tmid); -} - -void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); -} - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; -} - -inline std::string char_to_trimmed_str(const char *str, size_t len) { - return std::string(str, std::min(len, Config::max_text_size())); -} - -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { - auto qi = req->mutable_query_info(); - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); - StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = - char_to_trimmed_str(norm_plan->data, norm_plan->len); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - qi->set_query_id(query_desc->plannedstmt->queryId); - pfree(es.str->data); - pfree(norm_plan->data); - } -} - -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { - auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = char_to_trimmed_str( - query_desc->sourceText, strlen(query_desc->sourceText)); - char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = - char_to_trimmed_str(norm_query, strlen(norm_query)); - } -} - -void clear_big_fields(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->clear_plan_text(); - qi->clear_template_plan_text(); - qi->clear_query_text(); - qi->clear_template_query_text(); - } -} - -void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); - qi->set_allocated_rsgname(get_rg_name()); - } -} - -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { - auto aqi = req->mutable_add_info(); - aqi->set_nested_level(nesting_level); -} - -void set_qi_slice_id(yagpcc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - aqi->set_slice_id(currentSliceId); -} - -void set_qi_error_message(yagpcc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - auto error = elog_message(); - *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); -} - -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, - QueryDesc *query_desc, int nested_calls, - double nested_time) { - auto instrument = query_desc->planstate->instrument; - if (instrument) { - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time( - INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); - } - if (query_desc->estate && query_desc->estate->motionlayer_context) { - MotionLayerState *mlstate = - (MotionLayerState *)query_desc->estate->motionlayer_context; - metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); - metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); - metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); - metrics->mutable_received()->set_total_bytes( - mlstate->stat_total_bytes_recvd); - metrics->mutable_received()->set_tuple_bytes( - mlstate->stat_tuple_bytes_recvd); - metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); - } - metrics->set_inherited_calls(nested_calls); - metrics->set_inherited_time(nested_time); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time) { - if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, - nested_calls, nested_time); - } - fill_self_stats(metrics->mutable_systemstat()); - metrics->mutable_systemstat()->set_runningtimeseconds( - time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); - metrics->mutable_spill()->set_filecount( - WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); - metrics->mutable_spill()->set_totalbytes( - WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); -} - -yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, - yagpcc::QueryStatus status) { - yagpcc::SetQueryReq req; - req.set_query_status(status); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_segment_key(req.mutable_segment_key(), query_desc); - return req; -} - -double protots_to_double(const google::protobuf::Timestamp &ts) { - return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; -} - -} // namespace +#include "PgUtils.h" +#include "ProtoUtils.h" void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { @@ -404,10 +138,9 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { auto *query = get_query_message(query_desc); query->state = QueryState::SUBMIT; auto query_msg = query->message; - *query_msg = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg, query_desc); + set_query_info(query_msg); set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp new file mode 100644 index 00000000000..528426e6c64 --- /dev/null +++ b/src/PgUtils.cpp @@ -0,0 +1,94 @@ +#include "PgUtils.h" +#include "Config.h" + +extern "C" { +#include "utils/guc.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" +#include "cdb/cdbvars.h" +} + +std::string *get_user_name() { + const char *username = GetConfigOption("session_authorization", false, false); + // username is not to be freed + return username ? new std::string(username) : nullptr; +} + +std::string *get_db_name() { + char *dbname = get_database_name(MyDatabaseId); + std::string *result = nullptr; + if (dbname) { + result = new std::string(dbname); + pfree(dbname); + } + return result; +} + +std::string *get_rg_name() { + auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + return new std::string(rgname); +} + +/** + * Things get tricky with nested queries. + * a) A nested query on master is a real query optimized and executed from + * master. An example would be `select some_insert_function();`, where + * some_insert_function does something like `insert into tbl values (1)`. Master + * will create two statements. Outer select statement and inner insert statement + * with nesting level 1. + * For segments both statements are top-level statements with nesting level 0. + * b) A nested query on segment is something executed as sub-statement on + * segment. An example would be `select a from tbl where is_good_value(b);`. In + * this case master will issue one top-level statement, but segments will change + * contexts for UDF execution and execute is_good_value(b) once for each tuple + * as a nested query. Creating massive load on gpcc agent. + * + * Hence, here is a decision: + * 1) ignore all queries that are nested on segments + * 2) record (if enabled) all queries that are nested on master + * NODE: The truth is, we can't really ignore nested master queries, because + * segment sees those as top-level. + */ + +bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return (Gp_session_role == GP_ROLE_DISPATCH && + Config::report_nested_queries()) || + is_top_level_query(query_desc, nesting_level); +} + +bool need_report_nested_query() { + return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; +} + +bool filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !Config::enable_collector() || Config::filter_user(get_user_name()); +} + +bool need_collect(QueryDesc *query_desc, int nesting_level) { + return !filter_query(query_desc) && + nesting_is_valid(query_desc, nesting_level); +} + +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; +} diff --git a/src/PgUtils.h b/src/PgUtils.h new file mode 100644 index 00000000000..85b1eb833cd --- /dev/null +++ b/src/PgUtils.h @@ -0,0 +1,16 @@ +extern "C" { +#include "postgres.h" +#include "commands/explain.h" +} + +#include + +std::string *get_user_name(); +std::string *get_db_name(); +std::string *get_rg_name(); +bool is_top_level_query(QueryDesc *query_desc, int nesting_level); +bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); +bool need_report_nested_query(); +bool filter_query(QueryDesc *query_desc); +bool need_collect(QueryDesc *query_desc, int nesting_level); +ExplainState get_explain_state(QueryDesc *query_desc, bool costs); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp new file mode 100644 index 00000000000..e1be25b8b1e --- /dev/null +++ b/src/ProtoUtils.cpp @@ -0,0 +1,185 @@ +#include "ProtoUtils.h" +#include "PgUtils.h" +#include "ProcStats.h" +#include "Config.h" + +#define typeid __typeid +#define operator __operator +extern "C" { +#include "postgres.h" +#include "access/hash.h" +#include "cdb/cdbinterconnect.h" +#include "cdb/cdbvars.h" +#include "gpmon/gpmon.h" +#include "utils/workfile_mgr.h" + +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +} +#undef typeid +#undef operator + +#include +#include + +google::protobuf::Timestamp current_ts() { + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_query_key(yagpcc::QueryKey *key) { + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gpmon_gettmid(&tmid); + key->set_tmid(tmid); +} + +void set_segment_key(yagpcc::SegmentKey *key) { + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); +} + +inline std::string char_to_trimmed_str(const char *str, size_t len) { + return std::string(str, std::min(len, Config::max_text_size())); +} + +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + auto es = get_explain_state(query_desc, true); + MemoryContextSwitchTo(oldcxt); + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + StringInfo norm_plan = gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = + char_to_trimmed_str(norm_plan->data, norm_plan->len); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + qi->set_query_id(query_desc->plannedstmt->queryId); + pfree(es.str->data); + pfree(norm_plan->data); + } +} + +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = char_to_trimmed_str( + query_desc->sourceText, strlen(query_desc->sourceText)); + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_template_query_text() = + char_to_trimmed_str(norm_query, strlen(norm_query)); + } +} + +void clear_big_fields(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_template_plan_text(); + qi->clear_query_text(); + qi->clear_template_query_text(); + } +} + +void set_query_info(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); + qi->set_allocated_rsgname(get_rg_name()); + } +} + +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); +} + +void set_qi_slice_id(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); +} + +void set_qi_error_message(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + auto error = elog_message(); + *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, + QueryDesc *query_desc, int nested_calls, + double nested_time) { + auto instrument = query_desc->planstate->instrument; + if (instrument) { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } + if (query_desc->estate && query_desc->estate->motionlayer_context) { + MotionLayerState *mlstate = + (MotionLayerState *)query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); + } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) { + if (query_desc->planstate && query_desc->planstate->instrument) { + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, + nested_calls, nested_time); + } + fill_self_stats(metrics->mutable_systemstat()); + metrics->mutable_systemstat()->set_runningtimeseconds( + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); +} + +yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { + yagpcc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key()); + set_segment_key(req.mutable_segment_key()); + return req; +} + +double protots_to_double(const google::protobuf::Timestamp &ts) { + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +} \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h new file mode 100644 index 00000000000..38aa75611b2 --- /dev/null +++ b/src/ProtoUtils.h @@ -0,0 +1,16 @@ +#include "protos/yagpcc_set_service.pb.h" + +struct QueryDesc; + +google::protobuf::Timestamp current_ts(); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void clear_big_fields(yagpcc::SetQueryReq *req); +void set_query_info(yagpcc::SetQueryReq *req); +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); +void set_qi_slice_id(yagpcc::SetQueryReq *req); +void set_qi_error_message(yagpcc::SetQueryReq *req); +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time); +yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); +double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file From d65fff52ba05ab946169f72b65798f7220a29864 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 7 Apr 2025 14:15:39 +0300 Subject: [PATCH 062/128] [yagp_hooks_collector] Ignore EXPLAIN VERBOSE errors for unsupported node types --- src/PgUtils.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 528426e6c64..5982ff77c1c 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -88,7 +88,24 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { es.verbose = true; es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); + PG_TRY(); + { ExplainPrintPlan(&es, query_desc); } + PG_CATCH(); + { + // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE + // implementation. We don't want any queries to fail due to those bugs, so + // we report the bug here for future investigatin and continue collecting + // metrics w/o reporting any plans + resetStringInfo(es.str); + appendStringInfo( + es.str, + "Unable to restore query plan due to PostgreSQL internal error. " + "See logs for more information"); + ereport(INFO, + (errmsg("YAGPCC failed to reconstruct explain text for query: %s", + query_desc->sourceText))); + } + PG_END_TRY(); ExplainEndOutput(&es); return es; } From 95a15c6f11b14a9a704b024dcec238c8dac64332 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Fri, 18 Apr 2025 14:58:52 +0300 Subject: [PATCH 063/128] [yagp_hooks_collector] Add per-slice interconnect statistics Hook into ic_teardown to collect UDP-IFC packet-level counters. Compile-time gated behind IC_TEARDOWN_HOOK. --- protos/yagpcc_metrics.proto | 56 +++++++++++++++++++++++++++++++++++++ src/EventSender.cpp | 53 ++++++++++++++++++++++++++++++++++- src/EventSender.h | 10 +++++++ src/ProtoUtils.cpp | 35 +++++++++++++++++++++++ src/ProtoUtils.h | 3 ++ src/hook_wrappers.cpp | 24 ++++++++++++++++ 6 files changed, 180 insertions(+), 1 deletion(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index fc85386c6b0..086f3e63379 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -42,6 +42,11 @@ message AdditionalQueryInfo { int64 slice_id = 3; } +message AdditionalQueryStat { + string error_message = 1; + repeated int64 slices = 2; +} + enum PlanGenerator { PLAN_GENERATOR_UNSPECIFIED = 0; @@ -96,6 +101,56 @@ message NetworkStat { uint32 chunks = 3; } +message InterconnectStat { + // Receive queue size sum when main thread is trying to get a packet + uint64 total_recv_queue_size = 1; + // Counting times when computing total_recv_queue_size + uint64 recv_queue_size_counting_time = 2; + + // The capacity sum when packets are tried to be sent + uint64 total_capacity = 3; + // Counting times used to compute total_capacity + uint64 capacity_counting_time = 4; + + // Total buffers available when sending packets + uint64 total_buffers = 5; + // Counting times when compute total_buffers + uint64 buffer_counting_time = 6; + + // The number of active connections + uint64 active_connections_num = 7; + + // The number of packet retransmits + int64 retransmits = 8; + + // The number of cached future packets + int64 startup_cached_pkt_num = 9; + + // The number of mismatched packets received + int64 mismatch_num = 10; + + // The number of crc errors + int64 crc_errors = 11; + + // The number of packets sent by sender + int64 snd_pkt_num = 12; + + // The number of packets received by receiver + int64 recv_pkt_num = 13; + + // Disordered packet number + int64 disordered_pkt_num = 14; + + // Duplicate packet number + int64 duplicated_pkt_num = 15; + + // The number of Acks received + int64 recv_ack_num = 16; + + // The number of status query messages sent + int64 status_query_msg_num = 17; +} + message MetricInstrumentation { uint64 ntuples = 1; /* Total tuples produced */ uint64 nloops = 2; /* # of run cycles for this node */ @@ -120,6 +175,7 @@ message MetricInstrumentation { double startup_time = 21; /* real query startup time (planning + queue time) */ uint64 inherited_calls = 22; /* the number of executed sub-queries */ double inherited_time = 23; /* total time spend on inherited execution */ + InterconnectStat interconnect = 24; } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index cdb21ef7aa6..2ba34d1e4cc 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,7 @@ #include "Config.h" #include "UDSConnector.h" +#define typeid __typeid extern "C" { #include "postgres.h" @@ -11,7 +12,9 @@ extern "C" { #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" } +#undef typeid #include "EventSender.h" #include "PgUtils.h" @@ -35,7 +38,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // no-op: executor_after_start is enough break; case METRICS_QUERY_CANCELING: - // it appears we're unly interested in the actual CANCELED event. + // it appears we're only interested in the actual CANCELED event. // for now we will ignore CANCELING state unless otherwise requested from // end users break; @@ -150,6 +153,12 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { // take initial metrics snapshot so that we can safely take diff afterwards // in END or DONE events. set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); +#ifdef IC_TEARDOWN_HOOK + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif } } @@ -203,6 +212,12 @@ void EventSender::collect_query_done(QueryDesc *query_desc, set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, nested_timing); } +#ifdef IC_TEARDOWN_HOOK + ic_metrics_collect(); + set_ic_stats( + query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif connector->report_query(*query_msg, msg); } update_nested_counters(query_desc); @@ -213,6 +228,39 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } } +void EventSender::ic_metrics_collect() { +#ifdef IC_TEARDOWN_HOOK + if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { + return; + } + if (!connector || gp_command_count == 0 || !Config::enable_collector() || + Config::filter_user(get_user_name())) { + return; + } + // we also would like to know nesting level here and filter queries BUT we + // don't have this kind of information from this callback. Will have to + // collect stats anyways and throw it away later, if necessary + auto metrics = UDPIFCGetICStats(); + ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; + ic_statistics.recvQueueSizeCountingTime += metrics.recvQueueSizeCountingTime; + ic_statistics.totalCapacity += metrics.totalCapacity; + ic_statistics.capacityCountingTime += metrics.capacityCountingTime; + ic_statistics.totalBuffers += metrics.totalBuffers; + ic_statistics.bufferCountingTime += metrics.bufferCountingTime; + ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; + ic_statistics.retransmits += metrics.retransmits; + ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; + ic_statistics.mismatchNum += metrics.mismatchNum; + ic_statistics.crcErrors += metrics.crcErrors; + ic_statistics.sndPktNum += metrics.sndPktNum; + ic_statistics.recvPktNum += metrics.recvPktNum; + ic_statistics.disorderedPktNum += metrics.disorderedPktNum; + ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; + ic_statistics.recvAckNum += metrics.recvAckNum; + ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; +#endif +} + EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { try { @@ -221,6 +269,9 @@ EventSender::EventSender() { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } } +#ifdef IC_TEARDOWN_HOOK + memset(&ic_statistics, 0, sizeof(ICStatistics)); +#endif } EventSender::~EventSender() { diff --git a/src/EventSender.h b/src/EventSender.h index 9470cbf1f98..99f7b24753d 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -4,9 +4,15 @@ #include #include +#define typeid __typeid extern "C" { #include "utils/metrics_utils.h" +#include "cdb/ml_ipc.h" +#ifdef IC_TEARDOWN_HOOK +#include "cdb/ic_udpifc.h" +#endif } +#undef typeid class UDSConnector; struct QueryDesc; @@ -20,6 +26,7 @@ class EventSender { void executor_after_start(QueryDesc *query_desc, int eflags); void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); + void ic_metrics_collect(); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } EventSender(); @@ -55,5 +62,8 @@ class EventSender { int nesting_level = 0; int64_t nested_calls = 0; double nested_timing = 0; +#ifdef IC_TEARDOWN_HOOK + ICStatistics ic_statistics; +#endif std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index e1be25b8b1e..c37cefb72d6 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -10,6 +10,10 @@ extern "C" { #include "access/hash.h" #include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" +#ifdef IC_TEARDOWN_HOOK +#include "cdb/ic_udpifc.h" +#endif #include "gpmon/gpmon.h" #include "utils/workfile_mgr.h" @@ -171,6 +175,37 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } +#define UPDATE_IC_STATS(proto_name, stat_name) \ + metrics->mutable_interconnect()->set_##proto_name( \ + ic_statistics->stat_name - \ + metrics->mutable_interconnect()->proto_name()); \ + Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ + metrics->mutable_interconnect()->proto_name() <= \ + ic_statistics->stat_name) + +void set_ic_stats(yagpcc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics) { +#ifdef IC_TEARDOWN_HOOK + UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); + UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); + UPDATE_IC_STATS(total_capacity, totalCapacity); + UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); + UPDATE_IC_STATS(total_buffers, totalBuffers); + UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); + UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); + UPDATE_IC_STATS(retransmits, retransmits); + UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); + UPDATE_IC_STATS(mismatch_num, mismatchNum); + UPDATE_IC_STATS(crc_errors, crcErrors); + UPDATE_IC_STATS(snd_pkt_num, sndPktNum); + UPDATE_IC_STATS(recv_pkt_num, recvPktNum); + UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); + UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); + UPDATE_IC_STATS(recv_ack_num, recvAckNum); + UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); +#endif +} + yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { yagpcc::SetQueryReq req; req.set_query_status(status); diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 38aa75611b2..4e4ed5e76a3 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,6 +1,7 @@ #include "protos/yagpcc_set_service.pb.h" struct QueryDesc; +struct ICStatistics; google::protobuf::Timestamp current_ts(); void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); @@ -12,5 +13,7 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req); void set_qi_error_message(yagpcc::SetQueryReq *req); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); +void set_ic_stats(yagpcc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 93faaa0bf8f..f1d403b82f1 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,3 +1,4 @@ +#define typeid __typeid extern "C" { #include "postgres.h" #include "funcapi.h" @@ -7,8 +8,10 @@ extern "C" { #include "utils/metrics_utils.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" #include "tcop/utility.h" } +#undef typeid #include "Config.h" #include "YagpStat.h" @@ -21,6 +24,9 @@ static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; +#ifdef IC_TEARDOWN_HOOK +static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; +#endif static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -28,6 +34,8 @@ static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static void ya_ic_teardown_hook(ChunkTransportState *transportStates, + bool hasErrors); static EventSender *sender = nullptr; @@ -60,6 +68,10 @@ void hooks_init() { ExecutorEnd_hook = ya_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; query_info_collect_hook = ya_query_info_collect_hook; +#ifdef IC_TEARDOWN_HOOK + previous_ic_teardown_hook = ic_teardown_hook; + ic_teardown_hook = ya_ic_teardown_hook; +#endif stat_statements_parser_init(); } @@ -69,6 +81,9 @@ void hooks_deinit() { ExecutorRun_hook = previous_ExecutorRun_hook; ExecutorFinish_hook = previous_ExecutorFinish_hook; query_info_collect_hook = previous_query_info_collect_hook; +#ifdef IC_TEARDOWN_HOOK + ic_teardown_hook = previous_ic_teardown_hook; +#endif stat_statements_parser_deinit(); if (sender) { delete sender; @@ -141,6 +156,15 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } } +void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { + cpp_call(get_sender(), &EventSender::ic_metrics_collect); +#ifdef IC_TEARDOWN_HOOK + if (previous_ic_teardown_hook) { + (*previous_ic_teardown_hook)(transportStates, hasErrors); + } +#endif +} + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), From 0f9290b4b9a4f6f31217f3750aa6a703884643f4 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 9 Jun 2025 16:59:13 +0300 Subject: [PATCH 064/128] [yagp_hooks_collector] Fix user filtering propagation timing --- src/Config.cpp | 51 ++++++++++++++------------------------------- src/Config.h | 1 + src/EventSender.cpp | 40 ++++++++++++++++++++++++++++++++++- src/EventSender.h | 1 + 4 files changed, 57 insertions(+), 36 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 42fa4b2fb12..19aa37d1b9d 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -6,7 +6,6 @@ extern "C" { #include "postgres.h" -#include "utils/builtins.h" #include "utils/guc.h" } @@ -17,7 +16,12 @@ static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1024; // in KB -static std::unique_ptr> ignored_users = nullptr; +std::unique_ptr> ignored_users_set = nullptr; +bool ignored_users_guc_dirty = false; + +static void assign_ignored_users_hook(const char *, void *) { + ignored_users_guc_dirty = true; +} void Config::init() { DefineCustomStringVariable( @@ -44,11 +48,12 @@ void Config::init() { &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - DefineCustomStringVariable( - "yagpcc.ignored_users_list", - "Make yagpcc ignore queries issued by given users", 0LL, - &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomStringVariable("yagpcc.ignored_users_list", + "Make yagpcc ignore queries issued by given users", + 0LL, &guc_ignored_users, + "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, + assign_ignored_users_hook, 0LL); DefineCustomIntVariable( "yagpcc.max_text_size", @@ -62,36 +67,12 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } +const char *Config::ignored_users() { return guc_ignored_users; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { - if (!ignored_users) { - ignored_users.reset(new std::unordered_set()); - if (guc_ignored_users == nullptr || guc_ignored_users[0] == '0') { - return false; - } - /* Need a modifiable copy of string */ - char *rawstring = pstrdup(guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { - /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return false; - } - foreach (l, elemlist) { - ignored_users->insert((char *)lfirst(l)); - } - pfree(rawstring); - list_free(elemlist); + if (!username || !ignored_users_set) { + return true; } - return !username || ignored_users->find(*username) != ignored_users->end(); + return ignored_users_set->find(*username) != ignored_users_set->end(); } diff --git a/src/Config.h b/src/Config.h index f806bc0dbf5..9dd33c68321 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,5 +11,6 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); + static const char *ignored_users(); static size_t max_text_size(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 2ba34d1e4cc..fed9b69911f 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,6 +8,7 @@ extern "C" { #include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" +#include "utils/builtins.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" @@ -20,6 +21,9 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" +extern std::unique_ptr> ignored_users_set; +extern bool ignored_users_guc_dirty; + void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -62,6 +66,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, nested_timing = 0; nested_calls = 0; } + if (ignored_users_guc_dirty) { + update_ignored_users(Config::ignored_users()); + ignored_users_guc_dirty = false; + } if (!need_collect(query_desc, nesting_level)) { return; } @@ -262,7 +270,7 @@ void EventSender::ic_metrics_collect() { } EventSender::EventSender() { - if (Config::enable_collector() && !Config::filter_user(get_user_name())) { + if (Config::enable_collector()) { try { connector = new UDSConnector(); } catch (const std::exception &e) { @@ -347,6 +355,36 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } +void EventSender::update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = + std::make_unique>(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + ignored_users_set = std::move(new_ignored_users_set); +} + EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 99f7b24753d..6919defbbb3 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -57,6 +57,7 @@ class EventSender { void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); + void update_ignored_users(const char *new_guc_ignored_users); UDSConnector *connector = nullptr; int nesting_level = 0; From 0e0fadd215c27b485aa8c3fbf138d86ec77d3a5a Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 16 Jun 2025 13:07:59 +0300 Subject: [PATCH 065/128] [yagp_hooks_collector] Miscellaneous fixes and refactoring Fix UB in strcpy. General code refactoring. --- src/Config.cpp | 44 +++++++++++++++++++++++++++++++++++++++++--- src/Config.h | 2 +- src/EventSender.cpp | 39 +-------------------------------------- src/EventSender.h | 1 - src/UDSConnector.cpp | 8 +++++++- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 19aa37d1b9d..5e0749f171d 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -6,6 +6,7 @@ extern "C" { #include "postgres.h" +#include "utils/builtins.h" #include "utils/guc.h" } @@ -16,8 +17,39 @@ static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1024; // in KB -std::unique_ptr> ignored_users_set = nullptr; -bool ignored_users_guc_dirty = false; +static std::unique_ptr> ignored_users_set = + nullptr; +static bool ignored_users_guc_dirty = false; + +static void update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = + std::make_unique>(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + ignored_users_set = std::move(new_ignored_users_set); +} static void assign_ignored_users_hook(const char *, void *) { ignored_users_guc_dirty = true; @@ -67,7 +99,6 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } -const char *Config::ignored_users() { return guc_ignored_users; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { @@ -76,3 +107,10 @@ bool Config::filter_user(const std::string *username) { } return ignored_users_set->find(*username) != ignored_users_set->end(); } + +void Config::sync() { + if (ignored_users_guc_dirty) { + update_ignored_users(guc_ignored_users); + ignored_users_guc_dirty = false; + } +} \ No newline at end of file diff --git a/src/Config.h b/src/Config.h index 9dd33c68321..3caa0c78339 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,6 +11,6 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); - static const char *ignored_users(); static size_t max_text_size(); + static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fed9b69911f..fc0f7e1aa07 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,7 +8,6 @@ extern "C" { #include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/builtins.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" @@ -21,9 +20,6 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" -extern std::unique_ptr> ignored_users_set; -extern bool ignored_users_guc_dirty; - void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -66,10 +62,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, nested_timing = 0; nested_calls = 0; } - if (ignored_users_guc_dirty) { - update_ignored_users(Config::ignored_users()); - ignored_users_guc_dirty = false; - } + Config::sync(); if (!need_collect(query_desc, nesting_level)) { return; } @@ -355,36 +348,6 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } -void EventSender::update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = - std::make_unique>(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { - /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - pfree(rawstring); - list_free(elemlist); - } - ignored_users_set = std::move(new_ignored_users_set); -} - EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 6919defbbb3..99f7b24753d 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -57,7 +57,6 @@ class EventSender { void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); - void update_ignored_users(const char *new_guc_ignored_users); UDSConnector *connector = nullptr; int nesting_level = 0; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b9088205250..8a5f754f3b4 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -30,7 +30,13 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event) { sockaddr_un address; address.sun_family = AF_UNIX; - strcpy(address.sun_path, Config::uds_path().c_str()); + std::string uds_path = Config::uds_path(); + if (uds_path.size() >= sizeof(address.sun_path)) { + ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); + YagpStat::report_error(); + return false; + } + strcpy(address.sun_path, uds_path.c_str()); bool success = true; auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd != -1) { From 0cb4930f98dad0741d8ba0047432d2247dba791a Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:41:03 +0300 Subject: [PATCH 066/128] [yagp_hooks_collector] Add conditional EXPLAIN ANALYZE collection When enable_analyze is true and execution time exceeds min_analyze_time, generate EXPLAIN (ANALYZE, BUFFERS, TIMING, VERBOSE) output and include it in the done event. --- protos/yagpcc_metrics.proto | 1 + src/Config.cpp | 22 +++++++++++++-- src/Config.h | 2 ++ src/EventSender.cpp | 51 ++++++++++++++++++++++++++++++++--- src/EventSender.h | 1 + src/PgUtils.cpp | 37 ++++++++++++++++++++++++++ src/PgUtils.h | 1 + src/ProtoUtils.cpp | 53 ++++++++++++++++++++++++++++++------- src/ProtoUtils.h | 4 ++- src/hook_wrappers.cpp | 24 +++++++++++++++++ 10 files changed, 180 insertions(+), 16 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 086f3e63379..91ac0c4941a 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -34,6 +34,7 @@ message QueryInfo { string userName = 8; string databaseName = 9; string rsgname = 10; + string analyze_text = 11; } message AdditionalQueryInfo { diff --git a/src/Config.cpp b/src/Config.cpp index 5e0749f171d..ac274a1e218 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,7 +16,10 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1024; // in KB +static int guc_max_text_size = 1024; // in KB +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = -1; // uninitialized state + static std::unique_ptr> ignored_users_set = nullptr; static bool ignored_users_guc_dirty = false; @@ -89,9 +92,22 @@ void Config::init() { DefineCustomIntVariable( "yagpcc.max_text_size", - "Make yagpcc trim plan and query texts longer than configured size", NULL, + "Make yagpcc trim query texts longer than configured size", NULL, &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "yagpcc.max_plan_size", + "Make yagpcc trim plan longer than configured size", NULL, + &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "yagpcc.min_analyze_time", + "Sets the minimum execution time above which plans will be logged.", + "Zero prints all plans. -1 turns this feature off.", + &guc_min_analyze_time, -1, -1, INT_MAX, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } @@ -100,6 +116,8 @@ bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } +size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } +int Config::min_analyze_time() { return guc_min_analyze_time; }; bool Config::filter_user(const std::string *username) { if (!username || !ignored_users_set) { diff --git a/src/Config.h b/src/Config.h index 3caa0c78339..dd081c41dd6 100644 --- a/src/Config.h +++ b/src/Config.h @@ -12,5 +12,7 @@ class Config { static bool filter_user(const std::string *username); static bool report_nested_queries(); static size_t max_text_size(); + static size_t max_plan_size(); + static int min_analyze_time(); static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fc0f7e1aa07..19787fe0db0 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -20,6 +20,10 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" +#define need_collect_analyze() \ + (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ + Config::enable_analyze()) + void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -53,8 +57,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } } -void EventSender::executor_before_start(QueryDesc *query_desc, - int /* eflags*/) { +void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { if (!connector) { return; } @@ -67,7 +70,8 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } collect_query_submit(query_desc); - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; @@ -97,6 +101,17 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } update_query_state(query_desc, query, QueryState::START); set_query_plan(query_msg, query_desc); + if (need_collect_analyze()) { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) { + MemoryContext oldcxt; + oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + query_desc->totaltime = InstrAlloc(1, INSTRUMENT_ALL); + MemoryContextSwitchTo(oldcxt); + } + } yagpcc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); if (connector->report_query(*query_msg, "started")) { @@ -262,6 +277,34 @@ void EventSender::ic_metrics_collect() { #endif } +void EventSender::analyze_stats_collect(QueryDesc *query_desc) { + if (!connector || Gp_role != GP_ROLE_DISPATCH) { + return; + } + if (!need_collect(query_desc, nesting_level)) { + return; + } + auto query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_end_time() = current_ts(); + // Yet another greenplum weirdness: thats actually a nested query + // which is being committed/rollbacked. Treat it accordingly. + if (query->state == UNKNOWN && !need_report_nested_query()) { + return; + } + if (!query_desc->totaltime || !need_collect_analyze()) { + return; + } + // Make sure stats accumulation is done. + // (Note: it's okay if several levels of hook all do this.) + InstrEndLoop(query_desc->totaltime); + + double ms = query_desc->totaltime->total * 1000.0; + if (ms >= Config::min_analyze_time()) { + set_analyze_plan_text_json(query_desc, query_msg); + } +} + EventSender::EventSender() { if (Config::enable_collector()) { try { @@ -350,4 +393,4 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) - : state(st), message(msg) {} \ No newline at end of file + : state(st), message(msg) {} diff --git a/src/EventSender.h b/src/EventSender.h index 99f7b24753d..4d09b429fc8 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -27,6 +27,7 @@ class EventSender { void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); void ic_metrics_collect(); + void analyze_stats_collect(QueryDesc *query_desc); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } EventSender(); diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 5982ff77c1c..ed3e69c6d44 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -109,3 +109,40 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { ExplainEndOutput(&es); return es; } + +ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze) { + ExplainState es; + ExplainInitState(&es); + es.analyze = analyze; + es.verbose = true; + es.buffers = es.analyze; + es.timing = es.analyze; + es.summary = es.analyze; + es.format = EXPLAIN_FORMAT_JSON; + ExplainBeginOutput(&es); + if (analyze) { + PG_TRY(); + { + ExplainPrintPlan(&es, query_desc); + ExplainPrintExecStatsEnd(&es, query_desc); + } + PG_CATCH(); + { + // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE + // implementation. We don't want any queries to fail due to those bugs, so + // we report the bug here for future investigatin and continue collecting + // metrics w/o reporting any plans + resetStringInfo(es.str); + appendStringInfo( + es.str, + "Unable to restore analyze plan due to PostgreSQL internal error. " + "See logs for more information"); + ereport(INFO, + (errmsg("YAGPCC failed to reconstruct analyze text for query: %s", + query_desc->sourceText))); + } + PG_END_TRY(); + } + ExplainEndOutput(&es); + return es; +} diff --git a/src/PgUtils.h b/src/PgUtils.h index 85b1eb833cd..81282a473a8 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -14,3 +14,4 @@ bool need_report_nested_query(); bool filter_query(QueryDesc *query_desc); bool need_collect(QueryDesc *query_desc, int nesting_level); ExplainState get_explain_state(QueryDesc *query_desc, bool costs); +ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index c37cefb72d6..6e9fa6bd5c5 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -8,6 +8,7 @@ extern "C" { #include "postgres.h" #include "access/hash.h" +#include "access/xact.h" #include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" @@ -47,8 +48,9 @@ void set_segment_key(yagpcc::SegmentKey *key) { key->set_segindex(GpIdentity.segindex); } -inline std::string char_to_trimmed_str(const char *str, size_t len) { - return std::string(str, std::min(len, Config::max_text_size())); +inline std::string char_to_trimmed_str(const char *str, size_t len, + size_t lim) { + return std::string(str, std::min(len, lim)); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -61,10 +63,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + *qi->mutable_plan_text() = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = - char_to_trimmed_str(norm_plan->data, norm_plan->len); + *qi->mutable_template_plan_text() = char_to_trimmed_str( + norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); pfree(es.str->data); @@ -76,10 +79,11 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = char_to_trimmed_str( - query_desc->sourceText, strlen(query_desc->sourceText)); + query_desc->sourceText, strlen(query_desc->sourceText), + Config::max_text_size()); char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = - char_to_trimmed_str(norm_query, strlen(norm_query)); + *qi->mutable_template_query_text() = char_to_trimmed_str( + norm_query, strlen(norm_query), Config::max_text_size()); } } @@ -90,6 +94,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { qi->clear_template_plan_text(); qi->clear_query_text(); qi->clear_template_query_text(); + qi->clear_analyze_text(); } } @@ -115,7 +120,8 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { void set_qi_error_message(yagpcc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); auto error = elog_message(); - *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); + *aqi->mutable_error_message() = + char_to_trimmed_str(error, strlen(error), Config::max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -217,4 +223,33 @@ yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +} + +void set_analyze_plan_text_json(QueryDesc *query_desc, + yagpcc::SetQueryReq *req) { + // Make sure it is a valid txn and it is not an utility + // statement for ExplainPrintPlan() later. + if (!IsTransactionState() || !query_desc->plannedstmt) { + return; + } + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + + ExplainState es = get_analyze_state_json( + query_desc, query_desc->instrument_options && Config::enable_analyze()); + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { + es.str->data[--es.str->len] = '\0'; + } + // Convert JSON array to JSON object. + if (es.str->len > 0) { + es.str->data[0] = '{'; + es.str->data[es.str->len - 1] = '}'; + } + auto trimmed_analyze = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + + pfree(es.str->data); + MemoryContextSwitchTo(oldcxt); } \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 4e4ed5e76a3..6fb880c2eb8 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -16,4 +16,6 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); -double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file +double protots_to_double(const google::protobuf::Timestamp &ts); +void set_analyze_plan_text_json(QueryDesc *query_desc, + yagpcc::SetQueryReq *message); \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index f1d403b82f1..79d3ec45881 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -3,6 +3,7 @@ extern "C" { #include "postgres.h" #include "funcapi.h" #include "executor/executor.h" +#include "executor/execUtils.h" #include "utils/elog.h" #include "utils/builtins.h" #include "utils/metrics_utils.h" @@ -24,6 +25,10 @@ static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; +#ifdef ANALYZE_STATS_COLLECT_HOOK +static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = + nullptr; +#endif #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif @@ -36,6 +41,9 @@ static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); static void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); +#ifdef ANALYZE_STATS_COLLECT_HOOK +static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); +#endif static EventSender *sender = nullptr; @@ -71,6 +79,10 @@ void hooks_init() { #ifdef IC_TEARDOWN_HOOK previous_ic_teardown_hook = ic_teardown_hook; ic_teardown_hook = ya_ic_teardown_hook; +#endif +#ifdef ANALYZE_STATS_COLLECT_HOOK + previous_analyze_stats_collect_hook = analyze_stats_collect_hook; + analyze_stats_collect_hook = ya_analyze_stats_collect_hook; #endif stat_statements_parser_init(); } @@ -83,6 +95,9 @@ void hooks_deinit() { query_info_collect_hook = previous_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK ic_teardown_hook = previous_ic_teardown_hook; +#endif +#ifdef ANALYZE_STATS_COLLECT_HOOK + analyze_stats_collect_hook = previous_analyze_stats_collect_hook; #endif stat_statements_parser_deinit(); if (sender) { @@ -165,6 +180,15 @@ void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { #endif } +#ifdef ANALYZE_STATS_COLLECT_HOOK +void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { + cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); + if (previous_analyze_stats_collect_hook) { + (*previous_analyze_stats_collect_hook)(query_desc); + } +} +#endif + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), From 4ee91cfc508b357b8da47146f862ba24ca345a92 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Fri, 27 Jun 2025 13:31:34 +0300 Subject: [PATCH 067/128] [yagp_hooks_collector] Fix memory leaks, add safe C++ wrappers, improve Makefile Fix memory leaks in C++ and PG contexts. Add safe C++ wrappers around PG functions. Improve error message logging. Enable parallel make. Fix variable expansion. --- Makefile | 2 + src/Config.cpp | 22 +-- src/Config.h | 2 +- src/EventSender.cpp | 20 +-- src/EventSender.h | 3 - src/PgUtils.cpp | 106 +++---------- src/PgUtils.h | 6 +- src/ProcStats.cpp | 8 +- src/ProtoUtils.cpp | 73 ++++----- src/ProtoUtils.h | 2 + src/UDSConnector.cpp | 6 +- src/UDSConnector.h | 1 - src/hook_wrappers.cpp | 6 +- src/memory/gpdbwrappers.cpp | 148 ++++++++++++++++++ src/memory/gpdbwrappers.h | 131 ++++++++++++++++ .../pg_stat_statements_ya_parser.h | 6 +- 16 files changed, 380 insertions(+), 162 deletions(-) create mode 100644 src/memory/gpdbwrappers.cpp create mode 100644 src/memory/gpdbwrappers.h diff --git a/Makefile b/Makefile index 91be52c4468..15c5dabb70e 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. + # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. +all: all check install installdirs installcheck installcheck-parallel uninstall clean distclean maintainer-clean dist distcheck world check-world install-world installcheck-world installcheck-resgroup installcheck-resgroup-v2: @if [ ! -f GNUmakefile ] ; then \ diff --git a/src/Config.cpp b/src/Config.cpp index ac274a1e218..a1289a48891 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,5 @@ #include "Config.h" +#include "memory/gpdbwrappers.h" #include #include #include @@ -6,7 +7,6 @@ extern "C" { #include "postgres.h" -#include "utils/builtins.h" #include "utils/guc.h" } @@ -29,15 +29,15 @@ static void update_ignored_users(const char *new_guc_ignored_users) { std::make_unique>(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = pstrdup(new_guc_ignored_users); + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), @@ -48,8 +48,8 @@ static void update_ignored_users(const char *new_guc_ignored_users) { foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - pfree(rawstring); - list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); } ignored_users_set = std::move(new_ignored_users_set); } @@ -119,11 +119,11 @@ size_t Config::max_text_size() { return guc_max_text_size * 1024; } size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } int Config::min_analyze_time() { return guc_min_analyze_time; }; -bool Config::filter_user(const std::string *username) { - if (!username || !ignored_users_set) { +bool Config::filter_user(std::string username) { + if (!ignored_users_set) { return true; } - return ignored_users_set->find(*username) != ignored_users_set->end(); + return ignored_users_set->find(username) != ignored_users_set->end(); } void Config::sync() { @@ -131,4 +131,4 @@ void Config::sync() { update_ignored_users(guc_ignored_users); ignored_users_guc_dirty = false; } -} \ No newline at end of file +} diff --git a/src/Config.h b/src/Config.h index dd081c41dd6..eff83f0960a 100644 --- a/src/Config.h +++ b/src/Config.h @@ -9,7 +9,7 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); - static bool filter_user(const std::string *username); + static bool filter_user(std::string username); static bool report_nested_queries(); static size_t max_text_size(); static size_t max_plan_size(); diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 19787fe0db0..8711c4cbd4f 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,15 +1,14 @@ #include "Config.h" #include "UDSConnector.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid extern "C" { #include "postgres.h" -#include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" -#include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" @@ -81,7 +80,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -106,10 +105,10 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { // Make sure the space is allocated in the per-query // context so it will go away at executor_end. if (query_desc->totaltime == NULL) { - MemoryContext oldcxt; - oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - query_desc->totaltime = InstrAlloc(1, INSTRUMENT_ALL); - MemoryContextSwitchTo(oldcxt); + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL); + gpdb::mem_ctx_switch_to(oldcxt); } } yagpcc::GPMetrics stats; @@ -240,7 +239,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}); - pfree(query_desc->gpmon_pkt); + gpdb::pfree(query_desc->gpmon_pkt); } } @@ -297,7 +296,7 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - InstrEndLoop(query_desc->totaltime); + gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { @@ -364,7 +363,8 @@ EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}) == query_msgs.end()) { - query_desc->gpmon_pkt = (gpmon_packet_t *)palloc0(sizeof(gpmon_packet_t)); + query_desc->gpmon_pkt = + (gpmon_packet_t *)gpdb::palloc0(sizeof(gpmon_packet_t)); query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; query_msgs.insert({{gp_command_count, nesting_level}, diff --git a/src/EventSender.h b/src/EventSender.h index 4d09b429fc8..f3dd1d2a528 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,13 +1,10 @@ #pragma once -#include #include -#include #define typeid __typeid extern "C" { #include "utils/metrics_utils.h" -#include "cdb/ml_ipc.h" #ifdef IC_TEARDOWN_HOOK #include "cdb/ic_udpifc.h" #endif diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index ed3e69c6d44..f36cd030a39 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -1,37 +1,41 @@ #include "PgUtils.h" #include "Config.h" +#include "memory/gpdbwrappers.h" extern "C" { -#include "utils/guc.h" -#include "commands/dbcommands.h" #include "commands/resgroupcmds.h" #include "cdb/cdbvars.h" } -std::string *get_user_name() { - const char *username = GetConfigOption("session_authorization", false, false); - // username is not to be freed - return username ? new std::string(username) : nullptr; +std::string get_user_name() { + // username is allocated on stack, we don't need to pfree it. + const char *username = + ya_gpdb::get_config_option("session_authorization", false, false); + return username ? std::string(username) : ""; } -std::string *get_db_name() { - char *dbname = get_database_name(MyDatabaseId); - std::string *result = nullptr; +std::string get_db_name() { + char *dbname = ya_gpdb::get_database_name(MyDatabaseId); if (dbname) { - result = new std::string(dbname); - pfree(dbname); + std::string result(dbname); + ya_gpdb::pfree(dbname); + return result; } - return result; + return ""; } -std::string *get_rg_name() { - auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); +std::string get_rg_name() { + auto groupId = ya_gpdb::get_rg_id_by_session_id(MySessionState->sessionId); if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); + return ""; + + char *rgname = ya_gpdb::get_rg_name_for_id(groupId); if (rgname == nullptr) - return nullptr; - return new std::string(rgname); + return ""; + + std::string result(rgname); + ya_gpdb::pfree(rgname); + return result; } /** @@ -80,69 +84,3 @@ bool need_collect(QueryDesc *query_desc, int nesting_level) { return !filter_query(query_desc) && nesting_is_valid(query_desc, nesting_level); } - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - PG_TRY(); - { ExplainPrintPlan(&es, query_desc); } - PG_CATCH(); - { - // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE - // implementation. We don't want any queries to fail due to those bugs, so - // we report the bug here for future investigatin and continue collecting - // metrics w/o reporting any plans - resetStringInfo(es.str); - appendStringInfo( - es.str, - "Unable to restore query plan due to PostgreSQL internal error. " - "See logs for more information"); - ereport(INFO, - (errmsg("YAGPCC failed to reconstruct explain text for query: %s", - query_desc->sourceText))); - } - PG_END_TRY(); - ExplainEndOutput(&es); - return es; -} - -ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze) { - ExplainState es; - ExplainInitState(&es); - es.analyze = analyze; - es.verbose = true; - es.buffers = es.analyze; - es.timing = es.analyze; - es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_JSON; - ExplainBeginOutput(&es); - if (analyze) { - PG_TRY(); - { - ExplainPrintPlan(&es, query_desc); - ExplainPrintExecStatsEnd(&es, query_desc); - } - PG_CATCH(); - { - // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE - // implementation. We don't want any queries to fail due to those bugs, so - // we report the bug here for future investigatin and continue collecting - // metrics w/o reporting any plans - resetStringInfo(es.str); - appendStringInfo( - es.str, - "Unable to restore analyze plan due to PostgreSQL internal error. " - "See logs for more information"); - ereport(INFO, - (errmsg("YAGPCC failed to reconstruct analyze text for query: %s", - query_desc->sourceText))); - } - PG_END_TRY(); - } - ExplainEndOutput(&es); - return es; -} diff --git a/src/PgUtils.h b/src/PgUtils.h index 81282a473a8..ceb07c2e8e5 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -5,9 +5,9 @@ extern "C" { #include -std::string *get_user_name(); -std::string *get_db_name(); -std::string *get_rg_name(); +std::string get_user_name(); +std::string get_db_name(); +std::string get_rg_name(); bool is_top_level_query(QueryDesc *query_desc, int nesting_level); bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); bool need_report_nested_query(); diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index a557a20cbb0..5c09fa0bce4 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -75,16 +75,16 @@ void fill_status_stats(yagpcc::SystemStat *stats) { stats->set_vmpeakkb(value); proc_stat >> measure; if (measure != "kB") { - ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", - measure.c_str()))); + throw std::runtime_error("Expected memory sizes in kB, but got in " + + measure); } } else if (key == "VmSize:") { uint64_t value; proc_stat >> value; stats->set_vmsizekb(value); if (measure != "kB") { - ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", - measure.c_str()))); + throw std::runtime_error("Expected memory sizes in kB, but got in " + + measure); } } } diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 6e9fa6bd5c5..6dc39278bcd 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -2,6 +2,7 @@ #include "PgUtils.h" #include "ProcStats.h" #include "Config.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid #define operator __operator @@ -15,10 +16,7 @@ extern "C" { #ifdef IC_TEARDOWN_HOOK #include "cdb/ic_udpifc.h" #endif -#include "gpmon/gpmon.h" #include "utils/workfile_mgr.h" - -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" } #undef typeid #undef operator @@ -60,18 +58,21 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = char_to_trimmed_str( - norm_plan->data, norm_plan->len, Config::max_plan_size()); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - qi->set_query_id(query_desc->plannedstmt->queryId); - pfree(es.str->data); - pfree(norm_plan->data); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); + if (es.str) { + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, + Config::max_plan_size()); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = char_to_trimmed_str( + norm_plan->data, norm_plan->len, Config::max_plan_size()); + qi->set_plan_id( + hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + qi->set_query_id(query_desc->plannedstmt->queryId); + gpdb::pfree(es.str->data); + gpdb::pfree(norm_plan->data); + } + gpdb::mem_ctx_switch_to(oldcxt); } } @@ -81,7 +82,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { *qi->mutable_query_text() = char_to_trimmed_str( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); - char *norm_query = gen_normquery(query_desc->sourceText); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = char_to_trimmed_str( norm_query, strlen(norm_query), Config::max_text_size()); } @@ -101,9 +102,9 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { void set_query_info(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); - qi->set_allocated_rsgname(get_rg_name()); + qi->set_username(get_user_name()); + qi->set_databasename(get_db_name()); + qi->set_rsgname(get_rg_name()); } } @@ -233,23 +234,23 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, return; } MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - - ExplainState es = get_analyze_state_json( + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state_json( query_desc, query_desc->instrument_options && Config::enable_analyze()); - // Remove last line break. - if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { - es.str->data[--es.str->len] = '\0'; - } - // Convert JSON array to JSON object. - if (es.str->len > 0) { - es.str->data[0] = '{'; - es.str->data[es.str->len - 1] = '}'; + gpdb::mem_ctx_switch_to(oldcxt); + if (es.str) { + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { + es.str->data[--es.str->len] = '\0'; + } + // Convert JSON array to JSON object. + if (es.str->len > 0) { + es.str->data[0] = '{'; + es.str->data[es.str->len - 1] = '}'; + } + auto trimmed_analyze = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + gpdb::pfree(es.str->data); } - auto trimmed_analyze = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - req->mutable_query_info()->set_analyze_text(trimmed_analyze); - - pfree(es.str->data); - MemoryContextSwitchTo(oldcxt); } \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 6fb880c2eb8..8287b3de7ea 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,3 +1,5 @@ +#pragma once + #include "protos/yagpcc_set_service.pb.h" struct QueryDesc; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 8a5f754f3b4..b5b70836db4 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,6 +1,7 @@ #include "UDSConnector.h" #include "Config.h" #include "YagpStat.h" +#include "memory/gpdbwrappers.h" #include #include @@ -13,7 +14,6 @@ extern "C" { #include "postgres.h" -#include "cdb/cdbvars.h" } UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } @@ -44,7 +44,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { auto data_size = req.ByteSize(); auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)palloc(total_size); + uint8_t *buf = (uint8_t *)gpdb::palloc(total_size); uint32_t *size_payload = (uint32_t *)buf; *size_payload = data_size; req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); @@ -67,7 +67,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, } else { YagpStat::report_send(total_size); } - pfree(buf); + gpdb::pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 42e0aa20968..67504fc8529 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -1,7 +1,6 @@ #pragma once #include "protos/yagpcc_set_service.pb.h" -#include class UDSConnector { public: diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 79d3ec45881..25a85f086d1 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -7,10 +7,10 @@ extern "C" { #include "utils/elog.h" #include "utils/builtins.h" #include "utils/metrics_utils.h" -#include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" #include "tcop/utility.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" } #undef typeid @@ -18,7 +18,7 @@ extern "C" { #include "YagpStat.h" #include "EventSender.h" #include "hook_wrappers.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "memory/gpdbwrappers.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; @@ -229,7 +229,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp new file mode 100644 index 00000000000..1fba702a9f5 --- /dev/null +++ b/src/memory/gpdbwrappers.cpp @@ -0,0 +1,148 @@ +#include "gpdbwrappers.h" + +extern "C" { +#include "postgres.h" +#include "utils/guc.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" +#include "utils/builtins.h" +#include "nodes/pg_list.h" +#include "commands/explain.h" +#include "executor/instrument.h" +#include "access/tupdesc.h" +#include "access/htup.h" +#include "utils/elog.h" +#include "cdb/cdbexplain.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +} + +void *gpdb::palloc(Size size) { return detail::wrap_throw(::palloc, size); } + +void *gpdb::palloc0(Size size) { return detail::wrap_throw(::palloc0, size); } + +char *gpdb::pstrdup(const char *str) { + return detail::wrap_throw(::pstrdup, str); +} + +char *gpdb::get_database_name(Oid dbid) noexcept { + return detail::wrap_noexcept(::get_database_name, dbid); +} + +bool gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept { + return detail::wrap_noexcept(SplitIdentifierString, rawstring, separator, + namelist); +} + +ExplainState gpdb::get_explain_state(QueryDesc *query_desc, + bool costs) noexcept { + return detail::wrap_noexcept([&]() { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; + }); +} + +ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept { + return detail::wrap_noexcept([&]() { + ExplainState es; + ExplainInitState(&es); + es.analyze = analyze; + es.verbose = true; + es.buffers = es.analyze; + es.timing = es.analyze; + es.summary = es.analyze; + es.format = EXPLAIN_FORMAT_JSON; + ExplainBeginOutput(&es); + if (analyze) { + ExplainPrintPlan(&es, query_desc); + ExplainPrintExecStatsEnd(&es, query_desc); + } + ExplainEndOutput(&es); + return es; + }); +} + +Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options) { + return detail::wrap_throw(InstrAlloc, n, instrument_options); +} + +HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull) { + if (!tupleDescriptor || !values || !isnull) + throw std::runtime_error( + "Invalid input parameters for heap tuple formation"); + + return detail::wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); +} + +void gpdb::pfree(void *pointer) noexcept { + // Note that ::pfree asserts that pointer != NULL. + if (!pointer) + return; + + detail::wrap_noexcept(::pfree, pointer); +} + +MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { + return MemoryContextSwitchTo(context); +} + +const char *gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept { + if (!name) + return nullptr; + + return detail::wrap_noexcept(GetConfigOption, name, missing_ok, + restrict_superuser); +} + +void gpdb::list_free(List *list) noexcept { + if (!list) + return; + + detail::wrap_noexcept(::list_free, list); +} + +CdbExplain_ShowStatCtx * +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime) { + if (!query_desc) + throw std::runtime_error("Invalid query descriptor"); + + return detail::wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, + starttime); +} + +void gpdb::instr_end_loop(Instrumentation *instr) { + if (!instr) + throw std::runtime_error("Invalid instrumentation pointer"); + + detail::wrap_throw(::InstrEndLoop, instr); +} + +char *gpdb::gen_normquery(const char *query) { + return detail::wrap_throw(::gen_normquery, query); +} + +StringInfo gpdb::gen_normplan(const char *exec_plan) { + if (!exec_plan) + throw std::runtime_error("Invalid execution plan string"); + + return detail::wrap_throw(::gen_normplan, exec_plan); +} + +char *gpdb::get_rg_name_for_id(Oid group_id) { + return detail::wrap_throw(GetResGroupNameForId, group_id); +} + +Oid gpdb::get_rg_id_by_session_id(int session_id) { + return detail::wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +} \ No newline at end of file diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h new file mode 100644 index 00000000000..437a5dd5d29 --- /dev/null +++ b/src/memory/gpdbwrappers.h @@ -0,0 +1,131 @@ +#pragma once + +extern "C" { +#include "postgres.h" +#include "nodes/pg_list.h" +#include "commands/explain.h" +#include "executor/instrument.h" +#include "access/htup.h" +#include "utils/elog.h" +#include "utils/memutils.h" +} + +#include +#include +#include +#include +#include + +namespace gpdb { +namespace detail { + +template +auto wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) { + + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult {}; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) { + result_holder.emplace(func(std::forward(args)...)); + } else { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) { + std::string err; + if (edata && edata->message) { + err = std::string(edata->message); + } else { + err = "Unknown error occurred"; + } + + if (edata) { + FreeErrorData(edata); + } + + if constexpr (Throws) { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) { + return RetType{}; + } else { + return; + } + } + + if constexpr (!std::is_void_v) { + return *std::move(result_holder); + } else { + return; + } +} + +template +auto wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) { + return detail::wrap(std::forward(func), + std::forward(args)...); +} + +template +auto wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) { + return detail::wrap(std::forward(func), + std::forward(args)...); +} +} // namespace detail + +// Functions that call palloc(). +// Make sure correct memory context is set. +void *palloc(Size size); +void *palloc0(Size size); +char *pstrdup(const char *str); +char *get_database_name(Oid dbid) noexcept; +bool split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept; +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; +ExplainState get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept; +Instrumentation *instr_alloc(size_t n, int instrument_options); +HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull); +CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime); +void instr_end_loop(Instrumentation *instr); +char *gen_normquery(const char *query); +StringInfo gen_normplan(const char *executionPlan); +char *get_rg_name_for_id(Oid group_id); + +// Palloc-free functions. +void pfree(void *pointer) noexcept; +MemoryContext mem_ctx_switch_to(MemoryContext context) noexcept; +const char *get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept; +void list_free(List *list) noexcept; +Oid get_rg_id_by_session_id(int session_id); + +} // namespace gpdb diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index aa9cd217e31..b08e8533992 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -8,9 +8,9 @@ extern "C" extern void stat_statements_parser_init(void); extern void stat_statements_parser_deinit(void); +StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query); + #ifdef __cplusplus } #endif - -StringInfo gen_normplan(const char *executionPlan); -char *gen_normquery(const char *query); \ No newline at end of file From 36cd8f592e4d80593fe913537144882654b41517 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 14 Jul 2025 16:14:49 +0300 Subject: [PATCH 068/128] [yagp_hooks_collector] Add utility statement tracking and metrics documentation Hook into ProcessUtility to emit submit and done events for DDL. Add metrics documentation (metric.md). Change namespace to avoid GPOS conflicts. Report incomplete queries at extension shutdown. Clean up stray files. --- metric.md | 125 +++++++++++ src/Config.cpp | 12 +- src/EventSender.cpp | 407 ++++++++++++++++++++---------------- src/EventSender.h | 94 +++++++-- src/PgUtils.cpp | 10 +- src/ProtoUtils.cpp | 22 +- src/UDSConnector.cpp | 4 +- src/hook_wrappers.cpp | 2 +- src/memory/gpdbwrappers.cpp | 163 +++++++++++---- src/memory/gpdbwrappers.h | 85 +------- 10 files changed, 572 insertions(+), 352 deletions(-) create mode 100644 metric.md diff --git a/metric.md b/metric.md new file mode 100644 index 00000000000..2d198391a67 --- /dev/null +++ b/metric.md @@ -0,0 +1,125 @@ +## YAGP Hooks Collector Metrics + +### States +A Postgres process goes through 4 executor functions to execute a query: +1) `ExecutorStart()` - resource allocation for the query. +2) `ExecutorRun()` - query execution. +3) `ExecutorFinish()` - cleanup. +4) `ExecutorEnd()` - cleanup. + +yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +``` +submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done +``` + +### Key Points +- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. +- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. +- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. +- Nested queries on _Dispatcher_ become top-level on _Execute_. +- Each process (_Dispatcher_/_Execute_) sends its own statistics. + +### Notations +- **S** = Submit event. +- **T** = Start event. +- **E** = End event. +- **D** = Done event. +- **DIFF** = current_value - submit_value (submit event). +- **ABS** = Absolute value, or where diff is not applicable, the value taken. +- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. + +### Statistics Table + +| Proto Field | Type | When | DIFF/ABS | Local* | Scope | Dispatcher | Execute | Units | Notes | +| :--------------------------- | :----- | :------ | :------- | ------ | :------ | :--------: | :-----: | :------ | :-------------------------------------------------- | +| **SystemStat** | | | | | | | | | | +| `runningTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | Wall clock time | +| `userTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat utime | +| `kernelTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat stime | +| `vsize` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat vsize | +| `rss` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat rss | +| `VmSizeKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmSize | +| `VmPeakKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmPeak | +| `rchar` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io rchar | +| `wchar` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io wchar | +| `syscr` | uint64 | E, D | DIFF | - | Node | + | + | count | /proc/pid/io syscr | +| `syscw` | uint64 | E, D | DIFF | - | Node | + | + | count | /proc/pid/io syscw | +| `read_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io read_bytes | +| `write_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io write_bytes | +| `cancelled_write_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io cancelled_write_bytes | +| **MetricInstrumentation** | | | | | | | | | | +| `ntuples` | uint64 | E, D | ABS | + | Node | + | + | tuples | Accumulated total tuples | +| `nloops` | uint64 | E, D | ABS | + | Node | + | + | count | Number of cycles | +| `tuplecount` | uint64 | E, D | ABS | + | Node | + | + | tuples | Accumulated tuples per cycle | +| `firsttuple` | double | E, D | ABS | + | Node | + | + | seconds | Time for first tuple of this cycle | +| `startup` | double | E, D | ABS | + | Node | + | + | seconds | Start time of current iteration | +| `total` | double | E, D | ABS | + | Node | + | + | seconds | Total time taken | +| `shared_blks_hit` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared buffer blocks found in cache | +| `shared_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared buffer blocks read from disk | +| `shared_blks_dirtied` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared blocks dirtied | +| `shared_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Dirty shared buffer blocks written to disk | +| `local_blks_hit` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local buffer hits | +| `local_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Disk blocks read | +| `local_blks_dirtied` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local blocks dirtied | +| `local_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local blocks written to disk | +| `temp_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks read | +| `temp_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks written | +| `blk_read_time` | double | E, D | ABS | + | Node | + | + | seconds | Time reading data blocks | +| `blk_write_time` | double | E, D | ABS | + | Node | + | + | seconds | Time writing data blocks | +| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (YAGPCC-specific) | +| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (YAGPCC-specific) | +| **NetworkStat (sent)** | | | | | | | | | | +| `sent.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes sent, including headers | +| `sent.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data sent | +| `sent.chunks` | uint32 | D | ABS | - | Node | + | + | count | Tuple-chunks sent | +| **NetworkStat (received)** | | | | | | | | | | +| `received.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data received | +| `received.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data received | +| `received.chunks` | uint32 | D | ABS | - | Node | + | + | count | Tuple-chunks received | +| **InterconnectStat** | | | | | | | | | | +| `total_recv_queue_size` | uint64 | D | DIFF | - | Node | + | + | bytes | Receive queue size sum | +| `recv_queue_size_counting_t` | uint64 | D | DIFF | - | Node | + | + | count | Counting times when computing total_recv_queue_size | +| `total_capacity` | uint64 | D | DIFF | - | Node | + | + | bytes | the capacity sum for sent packets | +| `capacity_counting_time` | uint64 | D | DIFF | - | Node | + | + | count | counting times used to compute total_capacity | +| `total_buffers` | uint64 | D | DIFF | - | Node | + | + | count | Available buffers | +| `buffer_counting_time` | uint64 | D | DIFF | - | Node | + | + | count | counting times when compute total_buffers | +| `active_connections_num` | uint64 | D | DIFF | - | Node | + | + | count | Active connections | +| `retransmits` | int64 | D | DIFF | - | Node | + | + | count | Packet retransmits | +| `startup_cached_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Startup cached packets | +| `mismatch_num` | int64 | D | DIFF | - | Node | + | + | count | Mismatched packets received | +| `crc_errors` | int64 | D | DIFF | - | Node | + | + | count | CRC errors | +| `snd_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Packets sent | +| `recv_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Packets received | +| `disordered_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Out-of-order packets | +| `duplicated_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Duplicate packets | +| `recv_ack_num` | int64 | D | DIFF | - | Node | + | + | count | ACKs received | +| `status_query_msg_num` | int64 | D | DIFF | - | Node | + | + | count | Status query messages sent | +| **SpillInfo** | | | | | | | | | | +| `fileCount` | int32 | E, D | DIFF | - | Node | + | + | count | Spill (temp) files created | +| `totalBytes` | int64 | E, D | DIFF | - | Node | + | + | bytes | Spill bytes written | +| **QueryInfo** | | | | | | | | | | +| `generator` | enum | T, E, D | ABS | - | Cluster | + | - | enum | Planner/Optimizer | +| `query_id` | uint64 | T, E, D | ABS | - | Cluster | + | - | id | Query ID | +| `plan_id` | uint64 | T, E, D | ABS | - | Cluster | + | - | id | Hash of normalized plan | +| `query_text` | string | S | ABS | - | Cluster | + | - | text | Query text | +| `plan_text` | string | T | ABS | - | Cluster | + | - | text | EXPLAIN text | +| `template_query_text` | string | S | ABS | - | Cluster | + | - | text | Normalized query text | +| `template_plan_text` | string | T | ABS | - | Cluster | + | - | text | Normalized plan text | +| `userName` | string | All | ABS | - | Cluster | + | - | text | Session user | +| `databaseName` | string | All | ABS | - | Cluster | + | - | text | Database name | +| `rsgname` | string | All | ABS | - | Cluster | + | - | text | Resource group name | +| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE JSON | +| **AdditionalQueryInfo** | | | | | | | | | | +| `nested_level` | int64 | All | ABS | - | Node | + | + | count | Current nesting level | +| `error_message` | string | D | ABS | - | Node | + | + | text | Error message | +| `slice_id` | int64 | All | ABS | - | Node | + | + | id | Slice ID | +| **QueryKey** | | | | | | | | | | +| `tmid` | int32 | All | ABS | - | Node | + | + | id | Time ID | +| `ssid` | int32 | All | ABS | - | Node | + | + | id | Session ID | +| `ccnt` | int32 | All | ABS | - | Node | + | + | count | Command counter | +| **SegmentKey** | | | | | | | | | | +| `dbid` | int32 | All | ABS | - | Node | + | + | id | Database ID | +| `segment_index` | int32 | All | ABS | - | Node | + | + | id | Segment index (-1=coordinator) | + +--- + diff --git a/src/Config.cpp b/src/Config.cpp index a1289a48891..aef09fc7d73 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -29,15 +29,15 @@ static void update_ignored_users(const char *new_guc_ignored_users) { std::make_unique>(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = gpdb::pstrdup(new_guc_ignored_users); + char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), @@ -48,8 +48,8 @@ static void update_ignored_users(const char *new_guc_ignored_users) { foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); } ignored_users_set = std::move(new_ignored_users_set); } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 8711c4cbd4f..133d409b574 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,6 +8,7 @@ extern "C" { #include "executor/executor.h" #include "utils/elog.h" +#include "utils/guc.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" @@ -27,6 +28,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; } + auto *query_desc = reinterpret_cast(arg); switch (status) { case METRICS_PLAN_NODE_INITIALIZE: case METRICS_PLAN_NODE_EXECUTING: @@ -34,8 +36,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - // don't collect anything here. We will fake this call in ExecutorStart as - // it really makes no difference. Just complicates things + collect_query_submit(query_desc); break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -49,7 +50,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: - collect_query_done(reinterpret_cast(arg), status); + collect_query_done(query_desc, status); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -60,15 +61,15 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { if (!connector) { return; } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; + if (filter_query(query_desc)) { + return; + } + if (!qdesc_submitted(query_desc)) { + collect_query_submit(query_desc); } - Config::sync(); if (!need_collect(query_desc, nesting_level)) { return; } - collect_query_submit(query_desc); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; @@ -80,167 +81,194 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + ya_gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!connector) { + if (!connector || !need_collect(query_desc, nesting_level)) { return; } - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { - if (!filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - auto query_msg = query->message; - *query_msg->mutable_start_time() = current_ts(); - if (!nesting_is_valid(query_desc, nesting_level)) { - return; - } - update_query_state(query_desc, query, QueryState::START); - set_query_plan(query_msg, query_desc); - if (need_collect_analyze()) { - // Set up to track total elapsed time during query run. - // Make sure the space is allocated in the per-query - // context so it will go away at executor_end. - if (query_desc->totaltime == NULL) { - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL); - gpdb::mem_ctx_switch_to(oldcxt); - } - } - yagpcc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { - clear_big_fields(query_msg); - } - std::swap(stats, *query_msg->mutable_query_metrics()); + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto &query = get_query(query_desc); + auto query_msg = query.message.get(); + *query_msg->mutable_start_time() = current_ts(); + update_query_state(query, QueryState::START); + set_query_plan(query_msg, query_desc); + if (need_collect_analyze()) { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) { + MemoryContext oldcxt = + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL); + ya_gpdb::mem_ctx_switch_to(oldcxt); } } + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector || - (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { + if (!connector || !need_collect(query_desc, nesting_level)) { return; } - if (!filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - auto query_msg = query->message; - *query_msg->mutable_end_time() = current_ts(); - if (nesting_is_valid(query_desc, nesting_level)) { - if (query->state == UNKNOWN && - // Yet another greenplum weirdness: thats actually a nested query - // which is being committed/rollbacked. Treat it accordingly. - !need_report_nested_query()) { - return; - } - update_query_state(query_desc, query, QueryState::END); - if (is_top_level_query(query_desc, nesting_level)) { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, - nested_calls, nested_timing); - } else { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); - } - if (connector->report_query(*query_msg, "ended")) { - clear_big_fields(query_msg); - } - } + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg->mutable_end_time() = current_ts(); + update_query_state(query, QueryState::END); + if (is_top_level_query(query_desc, nesting_level)) { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, + nested_timing); + } else { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (connector->report_query(*query_msg, "ended")) { + clear_big_fields(query_msg); } } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (connector && need_collect(query_desc, nesting_level)) { - auto *query = get_query_message(query_desc); - query->state = QueryState::SUBMIT; - auto query_msg = query->message; - *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg); - set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); - set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc); - if (connector->report_query(*query_msg, "submit")) { - clear_big_fields(query_msg); - } - // take initial metrics snapshot so that we can safely take diff afterwards - // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + if (!connector) { + return; + } + Config::sync(); + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) { + QueryKey::register_qkey(query_desc, nesting_level); + return; + } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } + if (!need_collect(query_desc, nesting_level)) { + return; + } + submit_query(query_desc); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg); + set_qi_nesting_level(query_msg, nesting_level); + set_qi_slice_id(query_msg); + set_query_text(query_msg, query_desc); + if (connector->report_query(*query_msg, "submit")) { + clear_big_fields(query_msg); + } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); #ifdef IC_TEARDOWN_HOOK - // same for interconnect statistics - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif +} + +void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, + (errmsg("Unexpected query status in query_done hook: %d", status))); } + auto prev_state = query.state; + update_query_state(query, QueryState::DONE, + query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query.message.get(); + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, + nested_timing); + } +#ifdef IC_TEARDOWN_HOOK + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif + connector->report_query(*query_msg, msg); } void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && !filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - if (query->state != UNKNOWN || need_report_nested_query()) { - if (nesting_is_valid(query_desc, nesting_level)) { - yagpcc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code - // here just in case - Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, - (errmsg("Unexpected query status in query_done hook: %d", - status))); - } - auto prev_state = query->state; - update_query_state(query_desc, query, QueryState::DONE, - query_status == - yagpcc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query->message; - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); - } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, - nested_calls, nested_timing); - } -#ifdef IC_TEARDOWN_HOOK - ic_metrics_collect(); - set_ic_stats( - query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); -#endif - connector->report_query(*query_msg, msg); - } - update_nested_counters(query_desc); + if (!connector || !need_collect(query_desc, nesting_level)) { + return; + } + + // Skip sending done message if query errored before submit. + if (!qdesc_submitted(query_desc)) { + if (status != METRICS_QUERY_ERROR) { + ereport(WARNING, (errmsg("YAGPCC trying to process DONE hook for " + "unsubmitted and unerrored query"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); } - query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}); - gpdb::pfree(query_desc->gpmon_pkt); + return; + } + + if (queries.empty()) { + ereport(WARNING, (errmsg("YAGPCC cannot find query to process DONE hook"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + return; } + auto &query = get_query(query_desc); + + bool report = need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); + if (report) + report_query_done(query_desc, query, status); + + if (need_report_nested_query()) + update_nested_counters(query_desc); + + queries.erase(QueryKey::from_qdesc(query_desc)); + pfree(query_desc->yagp_query_key); + query_desc->yagp_query_key = NULL; } void EventSender::ic_metrics_collect() { @@ -283,20 +311,15 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { if (!need_collect(query_desc, nesting_level)) { return; } - auto query = get_query_message(query_desc); - auto query_msg = query->message; + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); *query_msg->mutable_end_time() = current_ts(); - // Yet another greenplum weirdness: thats actually a nested query - // which is being committed/rollbacked. Treat it accordingly. - if (query->state == UNKNOWN && !need_report_nested_query()) { - return; - } if (!query_desc->totaltime || !need_collect_analyze()) { return; } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - gpdb::instr_end_loop(query_desc->totaltime); + ya_gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { @@ -318,26 +341,26 @@ EventSender::EventSender() { } EventSender::~EventSender() { - delete connector; - for (auto iter = query_msgs.begin(); iter != query_msgs.end(); ++iter) { - delete iter->second.message; + for (const auto &[qkey, _] : queries) { + ereport(LOG, + (errmsg("YAGPCC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } + delete connector; } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP -void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, - QueryState new_state, bool success) { - if (query->state == UNKNOWN) { - collect_query_submit(query_desc); - } +void EventSender::update_query_state(QueryItem &query, QueryState new_state, + bool success) { switch (new_state) { case QueryState::SUBMIT: Assert(false); break; case QueryState::START: - if (query->state == QueryState::SUBMIT) { - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + if (query.state == QueryState::SUBMIT) { + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); } else { Assert(false); } @@ -346,40 +369,52 @@ void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, // Example of below assert triggering: CURSOR closes before ever being // executed Assert(query->state == QueryState::START || // IsAbortInProgress()); - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: - Assert(query->state == QueryState::END || !success); - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + Assert(query.state == QueryState::END || !success); + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); break; default: Assert(false); } - query->state = new_state; + query.state = new_state; } -EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { - if (query_desc->gpmon_pkt == nullptr || - query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}) == - query_msgs.end()) { - query_desc->gpmon_pkt = - (gpmon_packet_t *)gpdb::palloc0(sizeof(gpmon_packet_t)); - query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; - query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; - query_msgs.insert({{gp_command_count, nesting_level}, - QueryItem(UNKNOWN, new yagpcc::SetQueryReq())}); - } - return &query_msgs.at({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}); +EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { + if (!qdesc_submitted(query_desc)) { + ereport(WARNING, + (errmsg("YAGPCC attempting to get query that was not submitted"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + throw std::runtime_error("Attempting to get query that was not submitted"); + } + return queries.find(QueryKey::from_qdesc(query_desc))->second; +} + +void EventSender::submit_query(QueryDesc *query_desc) { + if (query_desc->yagp_query_key) { + ereport(WARNING, + (errmsg("YAGPCC trying to submit already submitted query"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } + QueryKey::register_qkey(query_desc, nesting_level); + auto key = QueryKey::from_qdesc(query_desc); + auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); + if (!inserted) { + ereport(WARNING, (errmsg("YAGPCC duplicate query submit detected"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } } void EventSender::update_nested_counters(QueryDesc *query_desc) { if (!is_top_level_query(query_desc, nesting_level)) { - auto query_msg = get_query_message(query_desc); + auto &query = get_query(query_desc); nested_calls++; - double end_time = protots_to_double(query_msg->message->end_time()); - double start_time = protots_to_double(query_msg->message->start_time()); + double end_time = protots_to_double(query.message->end_time()); + double start_time = protots_to_double(query.message->start_time()); if (end_time >= start_time) { nested_timing += end_time - start_time; } else { @@ -391,6 +426,12 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } -EventSender::QueryItem::QueryItem(EventSender::QueryState st, - yagpcc::SetQueryReq *msg) - : state(st), message(msg) {} +bool EventSender::qdesc_submitted(QueryDesc *query_desc) { + if (query_desc->yagp_query_key == NULL) { + return false; + } + return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); +} + +EventSender::QueryItem::QueryItem(QueryState st) + : message(std::make_unique()), state(st) {} diff --git a/src/EventSender.h b/src/EventSender.h index f3dd1d2a528..4071d580ff9 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #define typeid __typeid extern "C" { @@ -11,12 +13,75 @@ extern "C" { } #undef typeid +#include "memory/gpdbwrappers.h" + class UDSConnector; struct QueryDesc; namespace yagpcc { class SetQueryReq; } +#include + +struct QueryKey { + int tmid; + int ssid; + int ccnt; + int nesting_level; + uintptr_t query_desc_addr; + + bool operator==(const QueryKey &other) const { + return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == + std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, + other.query_desc_addr); + } + + static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { + query_desc->yagp_query_key = + (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); + int32 tmid; + gpmon_gettmid(&tmid); + query_desc->yagp_query_key->tmid = tmid; + query_desc->yagp_query_key->ssid = gp_session_id; + query_desc->yagp_query_key->ccnt = gp_command_count; + query_desc->yagp_query_key->nesting_level = nesting_level; + query_desc->yagp_query_key->query_desc_addr = (uintptr_t)query_desc; + } + + static QueryKey from_qdesc(QueryDesc *query_desc) { + return { + .tmid = query_desc->yagp_query_key->tmid, + .ssid = query_desc->yagp_query_key->ssid, + .ccnt = query_desc->yagp_query_key->ccnt, + .nesting_level = query_desc->yagp_query_key->nesting_level, + .query_desc_addr = query_desc->yagp_query_key->query_desc_addr, + }; + } +}; + +// https://www.boost.org/doc/libs/1_35_0/doc/html/boost/hash_combine_id241013.html +template inline void hash_combine(std::size_t &seed, const T &v) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +namespace std { +template <> struct hash { + size_t operator()(const QueryKey &k) const noexcept { + size_t seed = hash{}(k.tmid); + hash_combine(seed, k.ssid); + hash_combine(seed, k.ccnt); + hash_combine(seed, k.nesting_level); + uintptr_t addr = k.query_desc_addr; + if constexpr (SIZE_MAX < UINTPTR_MAX) { + addr %= SIZE_MAX; + } + hash_combine(seed, addr); + return seed; + } +}; +} // namespace std + class EventSender { public: void executor_before_start(QueryDesc *query_desc, int eflags); @@ -31,30 +96,25 @@ class EventSender { ~EventSender(); private: - enum QueryState { UNKNOWN, SUBMIT, START, END, DONE }; + enum QueryState { SUBMIT, START, END, DONE }; struct QueryItem { - QueryState state = QueryState::UNKNOWN; - yagpcc::SetQueryReq *message = nullptr; + std::unique_ptr message; + QueryState state; - QueryItem(QueryState st, yagpcc::SetQueryReq *msg); - }; - - struct pair_hash { - std::size_t operator()(const std::pair &p) const { - auto h1 = std::hash{}(p.first); - auto h2 = std::hash{}(p.second); - return h1 ^ h2; - } + explicit QueryItem(QueryState st); }; - void update_query_state(QueryDesc *query_desc, QueryItem *query, - QueryState new_state, bool success = true); - QueryItem *get_query_message(QueryDesc *query_desc); + void update_query_state(QueryItem &query, QueryState new_state, + bool success = true); + QueryItem &get_query(QueryDesc *query_desc); + void submit_query(QueryDesc *query_desc); void collect_query_submit(QueryDesc *query_desc); + void report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); - void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); + bool qdesc_submitted(QueryDesc *query_desc); UDSConnector *connector = nullptr; int nesting_level = 0; @@ -63,5 +123,5 @@ class EventSender { #ifdef IC_TEARDOWN_HOOK ICStatistics ic_statistics; #endif - std::unordered_map, QueryItem, pair_hash> query_msgs; + std::unordered_map queries; }; \ No newline at end of file diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index f36cd030a39..929f0cf2681 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -60,14 +60,14 @@ std::string get_rg_name() { */ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; + if (query_desc->yagp_query_key == NULL) { + return nesting_level == 0; + } + return query_desc->yagp_query_key->nesting_level == 0; } bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return (Gp_session_role == GP_ROLE_DISPATCH && - Config::report_nested_queries()) || + return need_report_nested_query() || is_top_level_query(query_desc, nesting_level); } diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 6dc39278bcd..4655433c806 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -58,21 +58,21 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_explain_state(query_desc, true); + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); *qi->mutable_template_plan_text() = char_to_trimmed_str( norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); - gpdb::pfree(es.str->data); - gpdb::pfree(norm_plan->data); + ya_gpdb::pfree(es.str->data); + ya_gpdb::pfree(norm_plan->data); } - gpdb::mem_ctx_switch_to(oldcxt); + ya_gpdb::mem_ctx_switch_to(oldcxt); } } @@ -82,7 +82,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { *qi->mutable_query_text() = char_to_trimmed_str( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); - char *norm_query = gpdb::gen_normquery(query_desc->sourceText); + char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = char_to_trimmed_str( norm_query, strlen(norm_query), Config::max_text_size()); } @@ -234,10 +234,10 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, return; } MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_analyze_state_json( + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = ya_gpdb::get_analyze_state_json( query_desc, query_desc->instrument_options && Config::enable_analyze()); - gpdb::mem_ctx_switch_to(oldcxt); + ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { @@ -251,6 +251,6 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, auto trimmed_analyze = char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); - gpdb::pfree(es.str->data); + ya_gpdb::pfree(es.str->data); } } \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b5b70836db4..f8c4586126d 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -44,7 +44,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { auto data_size = req.ByteSize(); auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)gpdb::palloc(total_size); + uint8_t *buf = (uint8_t *)ya_gpdb::palloc(total_size); uint32_t *size_payload = (uint32_t *)buf; *size_payload = data_size; req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); @@ -67,7 +67,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, } else { YagpStat::report_send(total_size); } - gpdb::pfree(buf); + ya_gpdb::pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 25a85f086d1..d76b7c64e10 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -229,7 +229,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 1fba702a9f5..9d579a91a30 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -16,27 +16,104 @@ extern "C" { #include "stat_statements_parser/pg_stat_statements_ya_parser.h" } -void *gpdb::palloc(Size size) { return detail::wrap_throw(::palloc, size); } +namespace { -void *gpdb::palloc0(Size size) { return detail::wrap_throw(::palloc0, size); } +template +auto wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) { -char *gpdb::pstrdup(const char *str) { - return detail::wrap_throw(::pstrdup, str); + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult {}; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) { + result_holder.emplace(func(std::forward(args)...)); + } else { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) { + std::string err; + if (edata && edata->message) { + err = std::string(edata->message); + } else { + err = "Unknown error occurred"; + } + + if (edata) { + FreeErrorData(edata); + } + + if constexpr (Throws) { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) { + return RetType{}; + } else { + return; + } + } + + if constexpr (!std::is_void_v) { + return *std::move(result_holder); + } else { + return; + } +} + +template +auto wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) { + return wrap(std::forward(func), std::forward(args)...); } -char *gpdb::get_database_name(Oid dbid) noexcept { - return detail::wrap_noexcept(::get_database_name, dbid); +template +auto wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) { + return wrap(std::forward(func), std::forward(args)...); +} +} // namespace + +void *ya_gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } + +void *ya_gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } + +char *ya_gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } + +char *ya_gpdb::get_database_name(Oid dbid) noexcept { + return wrap_noexcept(::get_database_name, dbid); } -bool gpdb::split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept { - return detail::wrap_noexcept(SplitIdentifierString, rawstring, separator, - namelist); +bool ya_gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept { + return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState gpdb::get_explain_state(QueryDesc *query_desc, - bool costs) noexcept { - return detail::wrap_noexcept([&]() { +ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, + bool costs) noexcept { + return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); es.costs = costs; @@ -49,9 +126,9 @@ ExplainState gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept { - return detail::wrap_noexcept([&]() { +ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept { + return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); es.analyze = analyze; @@ -70,79 +147,77 @@ ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, }); } -Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options) { - return detail::wrap_throw(InstrAlloc, n, instrument_options); +Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options) { + return wrap_throw(InstrAlloc, n, instrument_options); } -HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull) { +HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull) { if (!tupleDescriptor || !values || !isnull) throw std::runtime_error( "Invalid input parameters for heap tuple formation"); - return detail::wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); + return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void gpdb::pfree(void *pointer) noexcept { +void ya_gpdb::pfree(void *pointer) noexcept { // Note that ::pfree asserts that pointer != NULL. if (!pointer) return; - detail::wrap_noexcept(::pfree, pointer); + wrap_noexcept(::pfree, pointer); } -MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { +MemoryContext ya_gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { return MemoryContextSwitchTo(context); } -const char *gpdb::get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept { +const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept { if (!name) return nullptr; - return detail::wrap_noexcept(GetConfigOption, name, missing_ok, - restrict_superuser); + return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void gpdb::list_free(List *list) noexcept { +void ya_gpdb::list_free(List *list) noexcept { if (!list) return; - detail::wrap_noexcept(::list_free, list); + wrap_noexcept(::list_free, list); } CdbExplain_ShowStatCtx * -gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime) { +ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime) { if (!query_desc) throw std::runtime_error("Invalid query descriptor"); - return detail::wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, - starttime); + return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void gpdb::instr_end_loop(Instrumentation *instr) { +void ya_gpdb::instr_end_loop(Instrumentation *instr) { if (!instr) throw std::runtime_error("Invalid instrumentation pointer"); - detail::wrap_throw(::InstrEndLoop, instr); + wrap_throw(::InstrEndLoop, instr); } -char *gpdb::gen_normquery(const char *query) { - return detail::wrap_throw(::gen_normquery, query); +char *ya_gpdb::gen_normquery(const char *query) { + return wrap_throw(::gen_normquery, query); } -StringInfo gpdb::gen_normplan(const char *exec_plan) { +StringInfo ya_gpdb::gen_normplan(const char *exec_plan) { if (!exec_plan) throw std::runtime_error("Invalid execution plan string"); - return detail::wrap_throw(::gen_normplan, exec_plan); + return wrap_throw(::gen_normplan, exec_plan); } -char *gpdb::get_rg_name_for_id(Oid group_id) { - return detail::wrap_throw(GetResGroupNameForId, group_id); +char *ya_gpdb::get_rg_name_for_id(Oid group_id) { + return wrap_throw(GetResGroupNameForId, group_id); } -Oid gpdb::get_rg_id_by_session_id(int session_id) { - return detail::wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { + return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 437a5dd5d29..ad7ae96c362 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -16,88 +16,7 @@ extern "C" { #include #include -namespace gpdb { -namespace detail { - -template -auto wrap(Func &&func, Args &&...args) noexcept(!Throws) - -> decltype(func(std::forward(args)...)) { - - using RetType = decltype(func(std::forward(args)...)); - - // Empty struct for void return type. - struct VoidResult {}; - using ResultHolder = std::conditional_t, VoidResult, - std::optional>; - - bool success; - ErrorData *edata; - ResultHolder result_holder; - - PG_TRY(); - { - if constexpr (!std::is_void_v) { - result_holder.emplace(func(std::forward(args)...)); - } else { - func(std::forward(args)...); - } - edata = NULL; - success = true; - } - PG_CATCH(); - { - MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - MemoryContextSwitchTo(oldctx); - FlushErrorState(); - success = false; - } - PG_END_TRY(); - - if (!success) { - std::string err; - if (edata && edata->message) { - err = std::string(edata->message); - } else { - err = "Unknown error occurred"; - } - - if (edata) { - FreeErrorData(edata); - } - - if constexpr (Throws) { - throw std::runtime_error(err); - } - - if constexpr (!std::is_void_v) { - return RetType{}; - } else { - return; - } - } - - if constexpr (!std::is_void_v) { - return *std::move(result_holder); - } else { - return; - } -} - -template -auto wrap_throw(Func &&func, Args &&...args) - -> decltype(func(std::forward(args)...)) { - return detail::wrap(std::forward(func), - std::forward(args)...); -} - -template -auto wrap_noexcept(Func &&func, Args &&...args) noexcept - -> decltype(func(std::forward(args)...)) { - return detail::wrap(std::forward(func), - std::forward(args)...); -} -} // namespace detail +namespace ya_gpdb { // Functions that call palloc(). // Make sure correct memory context is set. @@ -128,4 +47,4 @@ const char *get_config_option(const char *name, bool missing_ok, void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace gpdb +} // namespace ya_gpdb From 82d11bca7b6a42d2e4d8c0fba1b474207e837a5d Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:26:16 +0300 Subject: [PATCH 069/128] [yagp_hooks_collector] Add regression tests, ANALYZE text output, and UTF-8 trimming Add PG-style regression tests. Enable sending EXPLAIN ANALYZE as text. Add utility statement hook coverage. Implement UTF-8 safe trimming: discard partial multi-byte characters at cut boundaries. Clean up stray gmon.out. --- expected/yagp_cursors.out | 165 +++++++++++ expected/yagp_dist.out | 177 ++++++++++++ expected/yagp_select.out | 138 +++++++++ expected/yagp_utf8_trim.out | 66 +++++ expected/yagp_utility.out | 272 ++++++++++++++++++ metric.md | 49 ++-- sql/yagp_cursors.sql | 83 ++++++ sql/yagp_dist.sql | 86 ++++++ sql/yagp_select.sql | 67 +++++ sql/yagp_utf8_trim.sql | 43 +++ sql/yagp_utility.sql | 133 +++++++++ src/Config.cpp | 36 ++- src/Config.h | 5 + src/EventSender.cpp | 191 +++++++----- src/EventSender.h | 18 +- src/PgUtils.cpp | 5 - src/PgUtils.h | 3 - src/ProtoUtils.cpp | 69 +++-- src/ProtoUtils.h | 5 +- src/UDSConnector.cpp | 3 +- src/UDSConnector.h | 4 +- src/hook_wrappers.cpp | 60 +++- src/hook_wrappers.h | 3 + src/log/LogOps.cpp | 131 +++++++++ src/log/LogOps.h | 19 ++ src/log/LogSchema.cpp | 135 +++++++++ src/log/LogSchema.h | 166 +++++++++++ src/memory/gpdbwrappers.cpp | 13 +- src/memory/gpdbwrappers.h | 8 +- src/yagp_hooks_collector.c | 14 +- yagp_hooks_collector--1.0--1.1.sql | 113 ++++++++ ...--1.0.sql => yagp_hooks_collector--1.0.sql | 2 +- yagp_hooks_collector--1.1.sql | 95 ++++++ yagp_hooks_collector.control | 2 +- 34 files changed, 2224 insertions(+), 155 deletions(-) create mode 100644 expected/yagp_cursors.out create mode 100644 expected/yagp_dist.out create mode 100644 expected/yagp_select.out create mode 100644 expected/yagp_utf8_trim.out create mode 100644 expected/yagp_utility.out create mode 100644 sql/yagp_cursors.sql create mode 100644 sql/yagp_dist.sql create mode 100644 sql/yagp_select.sql create mode 100644 sql/yagp_utf8_trim.sql create mode 100644 sql/yagp_utility.sql create mode 100644 src/log/LogOps.cpp create mode 100644 src/log/LogOps.h create mode 100644 src/log/LogSchema.cpp create mode 100644 src/log/LogSchema.h create mode 100644 yagp_hooks_collector--1.0--1.1.sql rename sql/yagp_hooks_collector--1.0.sql => yagp_hooks_collector--1.0.sql (99%) create mode 100644 yagp_hooks_collector--1.1.sql diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out new file mode 100644 index 00000000000..9587c00b550 --- /dev/null +++ b/expected/yagp_cursors.out @@ -0,0 +1,165 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +-- DECLARE +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- DECLARE WITH HOLD +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(14 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- ROLLBACK +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(12 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- FETCH +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; + ?column? +---------- + 2 +(1 row) + +FETCH 1 IN cursor_stats_6; + ?column? +---------- + 3 +(1 row) + +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(18 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out new file mode 100644 index 00000000000..ebaf839601d --- /dev/null +++ b/expected/yagp_dist.out @@ -0,0 +1,177 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; +-- Hash distributed table +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; +SET yagpcc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; + id +---- + 1 +(1 row) + +RESET optimizer_enable_direct_dispatch; +RESET yagpcc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +SET yagpcc.logging_mode to 'TBL'; +-- Scan all segments. +select * from test_hash_dist; + id +---- + 1 +(1 row) + +DROP TABLE test_hash_dist; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------+--------------------- + -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist; | QUERY_STATUS_START + -1 | select * from test_hash_dist; | QUERY_STATUS_END + -1 | select * from test_hash_dist; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); + count +------- + 3 +(1 row) + +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------------------+--------------------- + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; + count +------- + 100 +(1 row) + +RESET yagpcc.logging_mode; +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + query_text | query_status +-----------------------------------------+--------------------- + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE +(12 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_select.out b/expected/yagp_select.out new file mode 100644 index 00000000000..4c4a0218150 --- /dev/null +++ b/expected/yagp_select.out @@ -0,0 +1,138 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; +-- Basic SELECT tests +SET yagpcc.logging_mode to 'TBL'; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +SELECT COUNT(*) FROM generate_series(1,10); + count +------- + 10 +(1 row) + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Transaction test +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- CTE test +SET yagpcc.logging_mode to 'TBL'; +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + column1 +--------- + 1 + 2 +(2 rows) + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-----------------------------+--------------------- + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE + | SELECT * FROM t; | +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Prepared statement test +SET yagpcc.logging_mode to 'TBL'; +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; + ?column? +---------- + 1 +(1 row) + +DEALLOCATE test_stmt; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------+--------------------- + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_utf8_trim.out b/expected/yagp_utf8_trim.out new file mode 100644 index 00000000000..194ee6b3609 --- /dev/null +++ b/expected/yagp_utf8_trim.out @@ -0,0 +1,66 @@ +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM yagpcc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; +SET yagpcc.enable TO TRUE; +-- Test 1: 1 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; + ?column? +------------ + HelloWorld +(1 row) + +RESET yagpcc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 2: 2 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; + ?column? +------------- + РУССКИЙЯЗЫК +(1 row) + +RESET yagpcc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 3: 4 byte chars +SET yagpcc.max_text_size to 21; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; + ?column? +---------- + 😀 +(1 row) + +RESET yagpcc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET yagpcc.max_text_size; +RESET yagpcc.logging_mode; +RESET yagpcc.enable; +DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out new file mode 100644 index 00000000000..03c17713575 --- /dev/null +++ b/expected/yagp_utility.out @@ -0,0 +1,272 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE test_table (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE + -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE test_table; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Partitioning +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +NOTICE: CREATE TABLE will create partition "pt_test_1_prt_1" for table "pt_test" +NOTICE: CREATE TABLE will create partition "pt_test_1_prt_2" for table "pt_test" +DROP TABLE pt_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Views and Functions +SET yagpcc.logging_mode to 'TBL'; +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE + -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT + -1 | DROP VIEW test_view; | QUERY_STATUS_DONE + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Transaction Operations +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; +BEGIN; +SAVEPOINT sp2; +ABORT; +BEGIN; +ROLLBACK; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(18 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- DML Operations +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE dml_test (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE + -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(6 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- COPY Operations +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE copy_test (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +COPY (SELECT 1) TO STDOUT; +1 +DROP TABLE copy_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Prepared Statements and error during execute +SET yagpcc.logging_mode to 'TBL'; +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +ERROR: division by zero +DEALLOCATE test_prep; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR + -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT + -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- GUC Settings +SET yagpcc.logging_mode to 'TBL'; +SET yagpcc.report_nested_queries TO FALSE; +RESET yagpcc.report_nested_queries; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(6 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/metric.md b/metric.md index 2d198391a67..5df56877edb 100644 --- a/metric.md +++ b/metric.md @@ -1,32 +1,33 @@ ## YAGP Hooks Collector Metrics -### States -A Postgres process goes through 4 executor functions to execute a query: -1) `ExecutorStart()` - resource allocation for the query. -2) `ExecutorRun()` - query execution. -3) `ExecutorFinish()` - cleanup. -4) `ExecutorEnd()` - cleanup. +### States +A Postgres process goes through 4 executor functions to execute a query: +1) `ExecutorStart()` - resource allocation for the query. +2) `ExecutorRun()` - query execution. +3) `ExecutorFinish()` - cleanup. +4) `ExecutorEnd()` - cleanup. -yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: ``` submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done ``` -### Key Points -- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. -- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. -- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. -- Nested queries on _Dispatcher_ become top-level on _Execute_. -- Each process (_Dispatcher_/_Execute_) sends its own statistics. +### Key Points +- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. +- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. +- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. +- Nested queries on _Dispatcher_ become top-level on _Execute_. +- Each process (_Dispatcher_/_Execute_) sends its own statistics -### Notations -- **S** = Submit event. -- **T** = Start event. -- **E** = End event. -- **D** = Done event. -- **DIFF** = current_value - submit_value (submit event). -- **ABS** = Absolute value, or where diff is not applicable, the value taken. -- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. +### Notations +- **S** = Submit event. +- **T** = Start event. +- **E** = End event. +- **D** = Done event. +- **DIFF** = current_value - submit_value (submit event). +- **ABS** = Absolute value, or where diff is not applicable, the value taken. +- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. +- **Node** - PG process, either a `Query Dispatcher` (on master) or an `Execute` (on segment). ### Statistics Table @@ -36,7 +37,7 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `runningTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | Wall clock time | | `userTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat utime | | `kernelTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat stime | -| `vsize` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat vsize | +| `vsize` | uint64 | E, D | ABS | - | Node | + | + | bytes | /proc/pid/stat vsize | | `rss` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat rss | | `VmSizeKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmSize | | `VmPeakKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmPeak | @@ -108,13 +109,13 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `userName` | string | All | ABS | - | Cluster | + | - | text | Session user | | `databaseName` | string | All | ABS | - | Cluster | + | - | text | Database name | | `rsgname` | string | All | ABS | - | Cluster | + | - | text | Resource group name | -| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE JSON | +| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE | | **AdditionalQueryInfo** | | | | | | | | | | | `nested_level` | int64 | All | ABS | - | Node | + | + | count | Current nesting level | | `error_message` | string | D | ABS | - | Node | + | + | text | Error message | | `slice_id` | int64 | All | ABS | - | Node | + | + | id | Slice ID | | **QueryKey** | | | | | | | | | | -| `tmid` | int32 | All | ABS | - | Node | + | + | id | Time ID | +| `tmid` | int32 | All | ABS | - | Node | + | + | id | Transaction start time | | `ssid` | int32 | All | ABS | - | Node | + | + | id | Session ID | | `ccnt` | int32 | All | ABS | - | Node | + | + | count | Command counter | | **SegmentKey** | | | | | | | | | | diff --git a/sql/yagp_cursors.sql b/sql/yagp_cursors.sql new file mode 100644 index 00000000000..5d5bde58110 --- /dev/null +++ b/sql/yagp_cursors.sql @@ -0,0 +1,83 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; + +-- DECLARE +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- DECLARE WITH HOLD +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- ROLLBACK +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- FETCH +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; +FETCH 1 IN cursor_stats_6; +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_dist.sql b/sql/yagp_dist.sql new file mode 100644 index 00000000000..b837ef05335 --- /dev/null +++ b/sql/yagp_dist.sql @@ -0,0 +1,86 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; + +-- Hash distributed table + +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; + +SET yagpcc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; +RESET optimizer_enable_direct_dispatch; + +RESET yagpcc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +SET yagpcc.logging_mode to 'TBL'; + +-- Scan all segments. +select * from test_hash_dist; + +DROP TABLE test_hash_dist; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; + +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); + +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; +RESET yagpcc.logging_mode; + +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_select.sql b/sql/yagp_select.sql new file mode 100644 index 00000000000..4038c6b7b63 --- /dev/null +++ b/sql/yagp_select.sql @@ -0,0 +1,67 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; + +-- Basic SELECT tests +SET yagpcc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Transaction test +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +SELECT 1; +COMMIT; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- CTE test +SET yagpcc.logging_mode to 'TBL'; + +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Prepared statement test +SET yagpcc.logging_mode to 'TBL'; + +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; +DEALLOCATE test_stmt; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_utf8_trim.sql b/sql/yagp_utf8_trim.sql new file mode 100644 index 00000000000..c0fdcce24a5 --- /dev/null +++ b/sql/yagp_utf8_trim.sql @@ -0,0 +1,43 @@ +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM yagpcc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; + +SET yagpcc.enable TO TRUE; + +-- Test 1: 1 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; +RESET yagpcc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + +-- Test 2: 2 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; +RESET yagpcc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + +-- Test 3: 4 byte chars +SET yagpcc.max_text_size to 21; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; +RESET yagpcc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET yagpcc.max_text_size; +RESET yagpcc.logging_mode; +RESET yagpcc.enable; + +DROP EXTENSION yagp_hooks_collector; diff --git a/sql/yagp_utility.sql b/sql/yagp_utility.sql new file mode 100644 index 00000000000..b4cca6f5421 --- /dev/null +++ b/sql/yagp_utility.sql @@ -0,0 +1,133 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; + +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE test_table (a int, b text); +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Partitioning +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Views and Functions +SET yagpcc.logging_mode to 'TBL'; + +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Transaction Operations +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; + +BEGIN; +SAVEPOINT sp2; +ABORT; + +BEGIN; +ROLLBACK; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- DML Operations +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE dml_test (a int, b text); +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- COPY Operations +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE copy_test (a int); +COPY (SELECT 1) TO STDOUT; +DROP TABLE copy_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Prepared Statements and error during execute +SET yagpcc.logging_mode to 'TBL'; + +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +DEALLOCATE test_prep; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- GUC Settings +SET yagpcc.logging_mode to 'TBL'; + +SET yagpcc.report_nested_queries TO FALSE; +RESET yagpcc.report_nested_queries; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/src/Config.cpp b/src/Config.cpp index aef09fc7d73..dbd7e25b483 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,9 +16,16 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1024; // in KB -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = -1; // uninitialized state +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms +static int guc_logging_mode = LOG_MODE_UDS; +static bool guc_enable_utility = false; + +static const struct config_enum_entry logging_mode_options[] = { + {"uds", LOG_MODE_UDS, false /* hidden */}, + {"tbl", LOG_MODE_TBL, false}, + {NULL, 0, false}}; static std::unique_ptr> ignored_users_set = nullptr; @@ -92,9 +99,9 @@ void Config::init() { DefineCustomIntVariable( "yagpcc.max_text_size", - "Make yagpcc trim query texts longer than configured size", NULL, - &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + "Make yagpcc trim query texts longer than configured size in bytes", NULL, + &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); DefineCustomIntVariable( "yagpcc.max_plan_size", @@ -106,18 +113,31 @@ void Config::init() { "yagpcc.min_analyze_time", "Sets the minimum execution time above which plans will be logged.", "Zero prints all plans. -1 turns this feature off.", - &guc_min_analyze_time, -1, -1, INT_MAX, PGC_USERSET, + &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); + + DefineCustomEnumVariable( + "yagpcc.logging_mode", "Logging mode: UDS or PG Table", NULL, + &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, + NULL); + + DefineCustomBoolVariable( + "yagpcc.enable_utility", "Collect utility statement stats", NULL, + &guc_enable_utility, false, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } +bool Config::enable_utility() { return guc_enable_utility; } bool Config::report_nested_queries() { return guc_report_nested_queries; } -size_t Config::max_text_size() { return guc_max_text_size * 1024; } +size_t Config::max_text_size() { return guc_max_text_size; } size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } int Config::min_analyze_time() { return guc_min_analyze_time; }; +int Config::logging_mode() { return guc_logging_mode; } bool Config::filter_user(std::string username) { if (!ignored_users_set) { diff --git a/src/Config.h b/src/Config.h index eff83f0960a..7501c727a44 100644 --- a/src/Config.h +++ b/src/Config.h @@ -2,6 +2,9 @@ #include +#define LOG_MODE_UDS 0 +#define LOG_MODE_TBL 1 + class Config { public: static void init(); @@ -9,10 +12,12 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); + static bool enable_utility(); static bool filter_user(std::string username); static bool report_nested_queries(); static size_t max_text_size(); static size_t max_plan_size(); static int min_analyze_time(); + static int logging_mode(); static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 133d409b574..fee435a6dcc 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,7 @@ #include "Config.h" #include "UDSConnector.h" #include "memory/gpdbwrappers.h" +#include "log/LogOps.h" #define typeid __typeid extern "C" { @@ -24,10 +25,82 @@ extern "C" { (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ Config::enable_analyze()) -void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { +static bool enable_utility = Config::enable_utility(); + +bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, + bool utility) { + if (!proto_verified) { + return false; + } if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { - return; + return false; + } + + switch (state) { + case QueryState::SUBMIT: + // Cache enable_utility at SUBMIT to ensure consistent behavior at DONE. + // Without caching, a query that sets enable_utility to false from true + // would be accepted at SUBMIT (guc is true) but rejected at DONE (guc + // is false), causing a leak. + enable_utility = Config::enable_utility(); + if (utility && enable_utility == false) { + return false; + } + // Sync config in case current query changes it. + Config::sync(); + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) { + QueryKey::register_qkey(query_desc, nesting_level); + return false; + } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } + break; + case QueryState::START: + if (!qdesc_submitted(query_desc)) { + collect_query_submit(query_desc, false /* utility */); + } + break; + case QueryState::DONE: + if (utility && enable_utility == false) { + return false; + } + default: + break; + } + + if (filter_query(query_desc)) { + return false; + } + if (!nesting_is_valid(query_desc, nesting_level)) { + return false; + } + + return true; +} + +bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, + const std::string &event, bool utility) { + bool clear_big_fields = false; + switch (Config::logging_mode()) { + case LOG_MODE_UDS: + clear_big_fields = UDSConnector::report_query(req, event); + break; + case LOG_MODE_TBL: + ya_gpdb::insert_log(req, utility); + clear_big_fields = false; + break; + default: + Assert(false); } + return clear_big_fields; +} + +void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata) { auto *query_desc = reinterpret_cast(arg); switch (status) { case METRICS_PLAN_NODE_INITIALIZE: @@ -36,7 +109,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - collect_query_submit(query_desc); + collect_query_submit(query_desc, utility); break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -50,7 +123,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: - collect_query_done(query_desc, status); + collect_query_done(query_desc, utility, status, edata); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -58,18 +131,10 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { - if (!connector) { - return; - } - if (filter_query(query_desc)) { - return; - } - if (!qdesc_submitted(query_desc)) { - collect_query_submit(query_desc); - } - if (!need_collect(query_desc, nesting_level)) { + if (!verify_query(query_desc, QueryState::START, false /* utility*/)) { return; } + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; @@ -88,16 +153,14 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!connector || !need_collect(query_desc, nesting_level)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if (!verify_query(query_desc, QueryState::START, false /* utility */)) { return; } + auto &query = get_query(query_desc); auto query_msg = query.message.get(); *query_msg->mutable_start_time() = current_ts(); - update_query_state(query, QueryState::START); + update_query_state(query, QueryState::START, false /* utility */); set_query_plan(query_msg, query_desc); if (need_collect_analyze()) { // Set up to track total elapsed time during query run. @@ -112,52 +175,37 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } yagpcc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { + if (log_query_req(*query_msg, "started", false /* utility */)) { clear_big_fields(query_msg); } std::swap(stats, *query_msg->mutable_query_metrics()); } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector || !need_collect(query_desc, nesting_level)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if (!verify_query(query_desc, QueryState::END, false /* utility */)) { return; } + auto &query = get_query(query_desc); auto *query_msg = query.message.get(); *query_msg->mutable_end_time() = current_ts(); - update_query_state(query, QueryState::END); + update_query_state(query, QueryState::END, false /* utility */); if (is_top_level_query(query_desc, nesting_level)) { set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, nested_timing); } else { set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); } - if (connector->report_query(*query_msg, "ended")) { + if (log_query_req(*query_msg, "ended", false /* utility */)) { clear_big_fields(query_msg); } } -void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (!connector) { - return; - } - Config::sync(); - // Register qkey for a nested query we won't report, - // so we can detect nesting_level > 0 and skip reporting at end/done. - if (!need_report_nested_query() && nesting_level > 0) { - QueryKey::register_qkey(query_desc, nesting_level); - return; - } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; - } - if (!need_collect(query_desc, nesting_level)) { +void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { + if (!verify_query(query_desc, QueryState::SUBMIT, utility)) { return; } + submit_query(query_desc); auto &query = get_query(query_desc); auto *query_msg = query.message.get(); @@ -167,7 +215,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { set_qi_nesting_level(query_msg, nesting_level); set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); - if (connector->report_query(*query_msg, "submit")) { + if (log_query_req(*query_msg, "submit", utility)) { clear_big_fields(query_msg); } // take initial metrics snapshot so that we can safely take diff afterwards @@ -182,7 +230,8 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { } void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status) { + QueryMetricsStatus status, bool utility, + ErrorData *edata) { yagpcc::QueryStatus query_status; std::string msg; switch (status) { @@ -211,12 +260,20 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, (errmsg("Unexpected query status in query_done hook: %d", status))); } auto prev_state = query.state; - update_query_state(query, QueryState::DONE, + update_query_state(query, QueryState::DONE, utility, query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query.message.get(); query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); + bool error_flushed = elog_message() == NULL; + if (error_flushed && edata->message == NULL) { + ereport(WARNING, (errmsg("YAGPCC missing error message"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } else { + set_qi_error_message(query_msg, + error_flushed ? edata->message : elog_message()); + } } if (prev_state == START) { // We've missed ExecutorEnd call due to query cancel or error. It's @@ -230,12 +287,13 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), &ic_statistics); #endif - connector->report_query(*query_msg, msg); + (void)log_query_req(*query_msg, msg, utility); } -void EventSender::collect_query_done(QueryDesc *query_desc, - QueryMetricsStatus status) { - if (!connector || !need_collect(query_desc, nesting_level)) { +void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, + ErrorData *edata) { + if (!verify_query(query_desc, QueryState::DONE, utility)) { return; } @@ -258,10 +316,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } auto &query = get_query(query_desc); - bool report = need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); - if (report) - report_query_done(query_desc, query, status); + report_query_done(query_desc, query, status, utility, edata); if (need_report_nested_query()) update_nested_counters(query_desc); @@ -276,7 +331,7 @@ void EventSender::ic_metrics_collect() { if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { return; } - if (!connector || gp_command_count == 0 || !Config::enable_collector() || + if (!proto_verified || gp_command_count == 0 || !Config::enable_collector() || Config::filter_user(get_user_name())) { return; } @@ -305,15 +360,12 @@ void EventSender::ic_metrics_collect() { } void EventSender::analyze_stats_collect(QueryDesc *query_desc) { - if (!connector || Gp_role != GP_ROLE_DISPATCH) { + if (!verify_query(query_desc, QueryState::END, false /* utility */)) { return; } - if (!need_collect(query_desc, nesting_level)) { + if (Gp_role != GP_ROLE_DISPATCH) { return; } - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg->mutable_end_time() = current_ts(); if (!query_desc->totaltime || !need_collect_analyze()) { return; } @@ -323,14 +375,17 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { - set_analyze_plan_text_json(query_desc, query_msg); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + set_analyze_plan_text(query_desc, query_msg); } } EventSender::EventSender() { if (Config::enable_collector()) { try { - connector = new UDSConnector(); + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; } catch (const std::exception &e) { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } @@ -342,18 +397,16 @@ EventSender::EventSender() { EventSender::~EventSender() { for (const auto &[qkey, _] : queries) { - ereport(LOG, - (errmsg("YAGPCC query with missing done event: " - "tmid=%d ssid=%d ccnt=%d nlvl=%d", - qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); + ereport(LOG, (errmsg("YAGPCC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } - delete connector; } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP void EventSender::update_query_state(QueryItem &query, QueryState new_state, - bool success) { + bool utility, bool success) { switch (new_state) { case QueryState::SUBMIT: Assert(false); @@ -372,7 +425,7 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: - Assert(query.state == QueryState::END || !success); + Assert(query.state == QueryState::END || !success || utility); query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); break; default: diff --git a/src/EventSender.h b/src/EventSender.h index 4071d580ff9..4afdf1e14a4 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -87,7 +87,8 @@ class EventSender { void executor_before_start(QueryDesc *query_desc, int eflags); void executor_after_start(QueryDesc *query_desc, int eflags); void executor_end(QueryDesc *query_desc); - void query_metrics_collect(QueryMetricsStatus status, void *arg); + void query_metrics_collect(QueryMetricsStatus status, void *arg, bool utility, + ErrorData *edata = NULL); void ic_metrics_collect(); void analyze_stats_collect(QueryDesc *query_desc); void incr_depth() { nesting_level++; } @@ -105,18 +106,23 @@ class EventSender { explicit QueryItem(QueryState st); }; - void update_query_state(QueryItem &query, QueryState new_state, + static bool log_query_req(const yagpcc::SetQueryReq &req, + const std::string &event, bool utility); + bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); + void update_query_state(QueryItem &query, QueryState new_state, bool utility, bool success = true); QueryItem &get_query(QueryDesc *query_desc); void submit_query(QueryDesc *query_desc); - void collect_query_submit(QueryDesc *query_desc); + void collect_query_submit(QueryDesc *query_desc, bool utility); void report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status); - void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); + QueryMetricsStatus status, bool utility, + ErrorData *edata = NULL); + void collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata = NULL); void update_nested_counters(QueryDesc *query_desc); bool qdesc_submitted(QueryDesc *query_desc); - UDSConnector *connector = nullptr; + bool proto_verified = false; int nesting_level = 0; int64_t nested_calls = 0; double nested_timing = 0; diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 929f0cf2681..fc58112bfaa 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -79,8 +79,3 @@ bool filter_query(QueryDesc *query_desc) { return gp_command_count == 0 || query_desc->sourceText == nullptr || !Config::enable_collector() || Config::filter_user(get_user_name()); } - -bool need_collect(QueryDesc *query_desc, int nesting_level) { - return !filter_query(query_desc) && - nesting_is_valid(query_desc, nesting_level); -} diff --git a/src/PgUtils.h b/src/PgUtils.h index ceb07c2e8e5..02f084c597a 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -12,6 +12,3 @@ bool is_top_level_query(QueryDesc *query_desc, int nesting_level); bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); bool need_report_nested_query(); bool filter_query(QueryDesc *query_desc); -bool need_collect(QueryDesc *query_desc, int nesting_level); -ExplainState get_explain_state(QueryDesc *query_desc, bool costs); -ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 4655433c806..f28714da6ec 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -24,6 +24,18 @@ extern "C" { #include #include +namespace { +constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); +constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); +constexpr uint8_t UTF8_MAX_SYMBOL_BYTES = 4; + +// Returns true if byte is the starting byte of utf8 +// character, false if byte is the continuation (10xxxxxx). +inline bool utf8_start_byte(uint8_t byte) { + return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; +} +} // namespace + google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -46,9 +58,26 @@ void set_segment_key(yagpcc::SegmentKey *key) { key->set_segindex(GpIdentity.segindex); } -inline std::string char_to_trimmed_str(const char *str, size_t len, - size_t lim) { - return std::string(str, std::min(len, lim)); +std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { + if (unlikely(str == nullptr)) { + return std::string(); + } + if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) { + return std::string(str, std::min(len, lim)); + } + + // Handle trimming of utf8 correctly, do not cut multi-byte characters. + size_t cut_pos = lim; + size_t visited_bytes = 1; + while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) { + if (utf8_start_byte(static_cast(str[cut_pos]))) { + break; + } + ++visited_bytes; + --cut_pos; + } + + return std::string(str, cut_pos); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -61,10 +90,10 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, - Config::max_plan_size()); + *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, + Config::max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = char_to_trimmed_str( + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); @@ -79,11 +108,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = char_to_trimmed_str( + *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = char_to_trimmed_str( + *qi->mutable_template_query_text() = trim_str_shrink_utf8( norm_query, strlen(norm_query), Config::max_text_size()); } } @@ -103,7 +132,8 @@ void set_query_info(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); - qi->set_databasename(get_db_name()); + if (IsTransactionState()) + qi->set_databasename(get_db_name()); qi->set_rsgname(get_rg_name()); } } @@ -118,11 +148,10 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req) { +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg) { auto aqi = req->mutable_add_info(); - auto error = elog_message(); *aqi->mutable_error_message() = - char_to_trimmed_str(error, strlen(error), Config::max_text_size()); + trim_str_shrink_utf8(err_msg, strlen(err_msg), Config::max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -226,8 +255,7 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text_json(QueryDesc *query_desc, - yagpcc::SetQueryReq *req) { +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. if (!IsTransactionState() || !query_desc->plannedstmt) { @@ -235,7 +263,7 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, } MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_analyze_state_json( + ExplainState es = ya_gpdb::get_analyze_state( query_desc, query_desc->instrument_options && Config::enable_analyze()); ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { @@ -243,14 +271,9 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { es.str->data[--es.str->len] = '\0'; } - // Convert JSON array to JSON object. - if (es.str->len > 0) { - es.str->data[0] = '{'; - es.str->data[es.str->len - 1] = '}'; - } - auto trimmed_analyze = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, + Config::max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); ya_gpdb::pfree(es.str->data); } -} \ No newline at end of file +} diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 8287b3de7ea..725a634f765 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -12,12 +12,11 @@ void clear_big_fields(yagpcc::SetQueryReq *req); void set_query_info(yagpcc::SetQueryReq *req); void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req); +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text_json(QueryDesc *query_desc, - yagpcc::SetQueryReq *message); \ No newline at end of file +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message); \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index f8c4586126d..b6af303218d 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -2,6 +2,7 @@ #include "Config.h" #include "YagpStat.h" #include "memory/gpdbwrappers.h" +#include "log/LogOps.h" #include #include @@ -16,8 +17,6 @@ extern "C" { #include "postgres.h" } -UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } - static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { ereport(LOG, diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 67504fc8529..f0dfcb77a3f 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -4,6 +4,6 @@ class UDSConnector { public: - UDSConnector(); - bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); + bool static report_query(const yagpcc::SetQueryReq &req, + const std::string &event); }; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index d76b7c64e10..07ac511d546 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -32,6 +32,7 @@ static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif +static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -44,6 +45,10 @@ static void ya_ic_teardown_hook(ChunkTransportState *transportStates, #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif +static void ya_process_utility_hook(Node *parsetree, const char *queryString, + ProcessUtilityContext context, + ParamListInfo params, DestReceiver *dest, + char *completionTag); static EventSender *sender = nullptr; @@ -85,6 +90,8 @@ void hooks_init() { analyze_stats_collect_hook = ya_analyze_stats_collect_hook; #endif stat_statements_parser_init(); + previous_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = ya_process_utility_hook; } void hooks_deinit() { @@ -104,6 +111,7 @@ void hooks_deinit() { delete sender; } YagpStat::deinit(); + ProcessUtility_hook = previous_ProcessUtility_hook; } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { @@ -165,7 +173,8 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg); + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, + arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } @@ -189,6 +198,55 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif +static void ya_process_utility_hook(Node *parsetree, const char *queryString, + ProcessUtilityContext context, + ParamListInfo params, DestReceiver *dest, + char *completionTag) { + /* Project utility data on QueryDesc to use existing logic */ + QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); + query_desc->sourceText = queryString; + + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_SUBMIT, (void *)query_desc, true /* utility */, + (ErrorData *)NULL); + + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ProcessUtility_hook) { + (*previous_ProcessUtility_hook)(parsetree, queryString, context, params, + dest, completionTag); + } else { + standard_ProcessUtility(parsetree, queryString, context, params, dest, + completionTag); + } + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_DONE, + (void *)query_desc, true /* utility */, (ErrorData *)NULL); + + pfree(query_desc); + } + PG_CATCH(); + { + ErrorData *edata; + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(oldctx); + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_ERROR, + (void *)query_desc, true /* utility */, edata); + + pfree(query_desc); + ReThrowError(edata); + } + PG_END_TRY(); +} + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index c158f42cf1d..cfabf39485e 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -9,6 +9,9 @@ extern void hooks_deinit(); extern void yagp_functions_reset(); extern Datum yagp_functions_get(FunctionCallInfo fcinfo); +extern void init_log(); +extern void truncate_log(); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp new file mode 100644 index 00000000000..0868dd9fc1c --- /dev/null +++ b/src/log/LogOps.cpp @@ -0,0 +1,131 @@ +#include "protos/yagpcc_set_service.pb.h" + +#include "LogOps.h" +#include "LogSchema.h" + +extern "C" { +#include "postgres.h" + +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/xact.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/namespace.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_type.h" +#include "cdb/cdbvars.h" +#include "commands/tablecmds.h" +#include "funcapi.h" +#include "fmgr.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" +} + +void init_log() { + Oid namespaceId; + Oid relationId; + ObjectAddress tableAddr; + ObjectAddress schemaAddr; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + + /* Create table */ + relationId = heap_create_with_catalog( + log_relname.data() /* relname */, namespaceId /* namespace */, + 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, + InvalidOid /* reloftypeid */, GetUserId() /* owner */, + DescribeTuple() /* rel tuple */, NIL, InvalidOid /* relam */, + RELKIND_RELATION, RELPERSISTENCE_PERMANENT, RELSTORAGE_HEAP, false, false, + true, 0, ONCOMMIT_NOOP, NULL /* GP Policy */, (Datum)0, + false /* use_user_acl */, true, true, false /* valid_opts */, + false /* is_part_child */, false /* is part parent */, NULL); + + /* Make the table visible */ + CommandCounterIncrement(); + + /* Record dependency of the table on the schema */ + if (OidIsValid(relationId) && OidIsValid(namespaceId)) { + ObjectAddressSet(tableAddr, RelationRelationId, relationId); + ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); + + /* Table can be dropped only via DROP EXTENSION */ + recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); + } else { + ereport(NOTICE, (errmsg("YAGPCC failed to create log table or schema"))); + } + + /* Make changes visible */ + CommandCounterIncrement(); +} + +void insert_log(const yagpcc::SetQueryReq &req, bool utility) { + Oid namespaceId; + Oid relationId; + Relation rel; + HeapTuple tuple; + + /* Return if xact is not valid (needed for catalog lookups). */ + if (!IsTransactionState()) { + return; + } + + /* Return if extension was not loaded */ + namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); + if (!OidIsValid(namespaceId)) { + return; + } + + /* Return if the table was not created yet */ + relationId = get_relname_relid(log_relname.data(), namespaceId); + if (!OidIsValid(relationId)) { + return; + } + + bool nulls[natts_yagp_log]; + Datum values[natts_yagp_log]; + + memset(nulls, true, sizeof(nulls)); + memset(values, 0, sizeof(values)); + + extract_query_req(req, "", values, nulls); + nulls[attnum_yagp_log_utility] = false; + values[attnum_yagp_log_utility] = BoolGetDatum(utility); + + rel = heap_open(relationId, RowExclusiveLock); + + /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls + * back or aborts */ + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + frozen_heap_insert(rel, tuple); + + heap_freetuple(tuple); + /* Keep lock on rel until end of xact */ + heap_close(rel, NoLock); + + /* Make changes visible */ + CommandCounterIncrement(); +} + +void truncate_log() { + Oid namespaceId; + Oid relationId; + Relation relation; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + relationId = get_relname_relid(log_relname.data(), namespaceId); + + relation = heap_open(relationId, AccessExclusiveLock); + + /* Truncate the main table */ + heap_truncate_one_rel(relation); + + /* Keep lock on rel until end of xact */ + heap_close(relation, NoLock); + + /* Make changes visible */ + CommandCounterIncrement(); +} \ No newline at end of file diff --git a/src/log/LogOps.h b/src/log/LogOps.h new file mode 100644 index 00000000000..bad03d09a8f --- /dev/null +++ b/src/log/LogOps.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +extern "C" { +#include "postgres.h" +#include "fmgr.h" +} + +extern "C" { +/* CREATE TABLE yagpcc.__log (...); */ +void init_log(); + +/* TRUNCATE yagpcc.__log */ +void truncate_log(); +} + +/* INSERT INTO yagpcc.__log VALUES (...) */ +void insert_log(const yagpcc::SetQueryReq &req, bool utility); diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp new file mode 100644 index 00000000000..335a3103cfd --- /dev/null +++ b/src/log/LogSchema.cpp @@ -0,0 +1,135 @@ +#include "google/protobuf/reflection.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/timestamp.pb.h" + +#include "LogSchema.h" + +const std::unordered_map &proto_name_to_col_idx() { + static const auto name_col_idx = [] { + std::unordered_map map; + map.reserve(log_tbl_desc.size()); + + for (size_t idx = 0; idx < natts_yagp_log; ++idx) { + map.emplace(log_tbl_desc[idx].proto_field_name, idx); + } + + return map; + }(); + return name_col_idx; +} + +TupleDesc DescribeTuple() { + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log, false); + + for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { + TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), + log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, + 0 /* attdim */); + } + + return tupdesc; +} + +Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts) { + TimestampTz pgtimestamp = + (TimestampTz)ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); + pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; + return TimestampTzGetDatum(pgtimestamp); +} + +Datum field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg) { + using namespace google::protobuf; + + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return Int32GetDatum(reflection->GetInt32(msg, field)); + case FieldDescriptor::CPPTYPE_INT64: + return Int64GetDatum(reflection->GetInt64(msg, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return Int64GetDatum(reflection->GetUInt32(msg, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return Int64GetDatum( + static_cast(reflection->GetUInt64(msg, field))); + case FieldDescriptor::CPPTYPE_DOUBLE: + return Float8GetDatum(reflection->GetDouble(msg, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return Float4GetDatum(reflection->GetFloat(msg, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return BoolGetDatum(reflection->GetBool(msg, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return CStringGetTextDatum(reflection->GetEnum(msg, field)->name().data()); + case FieldDescriptor::CPPTYPE_STRING: + return CStringGetTextDatum(reflection->GetString(msg, field).c_str()); + default: + return (Datum)0; + } +} + +void process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls) { + + auto proto_idx_map = proto_name_to_col_idx(); + auto it = proto_idx_map.find(field_name); + + if (it == proto_idx_map.end()) { + ereport(NOTICE, + (errmsg("YAGPCC protobuf field %s is not registered in log table", + field_name.c_str()))); + return; + } + + int idx = it->second; + + if (!reflection->HasField(msg, field)) { + nulls[idx] = true; + return; + } + + if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() == "google.protobuf.Timestamp") { + const auto &ts = static_cast( + reflection->GetMessage(msg, field)); + values[idx] = protots_to_timestamptz(ts); + } else { + values[idx] = field_to_datum(field, reflection, msg); + } + nulls[idx] = false; + + return; +} + +void extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls) { + using namespace google::protobuf; + + const Descriptor *descriptor = msg.GetDescriptor(); + const Reflection *reflection = msg.GetReflection(); + + for (int i = 0; i < descriptor->field_count(); ++i) { + const FieldDescriptor *field = descriptor->field(i); + + // For now, we do not log any repeated fields plus they need special + // treatment. + if (field->is_repeated()) { + continue; + } + + std::string curr_pref = prefix.empty() ? "" : prefix + "."; + std::string field_name = curr_pref + field->name().data(); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() != "google.protobuf.Timestamp") { + + if (reflection->HasField(msg, field)) { + const Message &nested = reflection->GetMessage(msg, field); + extract_query_req(nested, field_name, values, nulls); + } + } else { + process_field(field, reflection, msg, field_name, values, nulls); + } + } +} diff --git a/src/log/LogSchema.h b/src/log/LogSchema.h new file mode 100644 index 00000000000..f713c1e9b0e --- /dev/null +++ b/src/log/LogSchema.h @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include +#include + +extern "C" { +#include "postgres.h" +#include "access/htup_details.h" +#include "access/tupdesc.h" +#include "catalog/pg_type.h" +#include "utils/timestamp.h" +#include "utils/builtins.h" +} + +namespace google { +namespace protobuf { +class FieldDescriptor; +class Message; +class Reflection; +class Timestamp; +} // namespace protobuf +} // namespace google + +inline constexpr std::string_view schema_name = "yagpcc"; +inline constexpr std::string_view log_relname = "__log"; + +struct LogDesc { + std::string_view pg_att_name; + std::string_view proto_field_name; + Oid type_oid; +}; + +/* + * Definition of the log table structure. + * + * System stats collected as %lu (unsigned) may + * overflow INT8OID (signed), but this is acceptable. + */ +/* clang-format off */ +inline constexpr std::array log_tbl_desc = { + /* 8-byte aligned types first - Query Info */ + LogDesc{"query_id", "query_info.query_id", INT8OID}, + LogDesc{"plan_id", "query_info.plan_id", INT8OID}, + LogDesc{"nested_level", "add_info.nested_level", INT8OID}, + LogDesc{"slice_id", "add_info.slice_id", INT8OID}, + /* 8-byte aligned types - System Stats */ + LogDesc{"systemstat_vsize", "query_metrics.systemStat.vsize", INT8OID}, + LogDesc{"systemstat_rss", "query_metrics.systemStat.rss", INT8OID}, + LogDesc{"systemstat_vmsizekb", "query_metrics.systemStat.VmSizeKb", INT8OID}, + LogDesc{"systemstat_vmpeakkb", "query_metrics.systemStat.VmPeakKb", INT8OID}, + LogDesc{"systemstat_rchar", "query_metrics.systemStat.rchar", INT8OID}, + LogDesc{"systemstat_wchar", "query_metrics.systemStat.wchar", INT8OID}, + LogDesc{"systemstat_syscr", "query_metrics.systemStat.syscr", INT8OID}, + LogDesc{"systemstat_syscw", "query_metrics.systemStat.syscw", INT8OID}, + LogDesc{"systemstat_read_bytes", "query_metrics.systemStat.read_bytes", INT8OID}, + LogDesc{"systemstat_write_bytes", "query_metrics.systemStat.write_bytes", INT8OID}, + LogDesc{"systemstat_cancelled_write_bytes", "query_metrics.systemStat.cancelled_write_bytes", INT8OID}, + /* 8-byte aligned types - Metric Instrumentation */ + LogDesc{"instrumentation_ntuples", "query_metrics.instrumentation.ntuples", INT8OID}, + LogDesc{"instrumentation_nloops", "query_metrics.instrumentation.nloops", INT8OID}, + LogDesc{"instrumentation_tuplecount", "query_metrics.instrumentation.tuplecount", INT8OID}, + LogDesc{"instrumentation_shared_blks_hit", "query_metrics.instrumentation.shared_blks_hit", INT8OID}, + LogDesc{"instrumentation_shared_blks_read", "query_metrics.instrumentation.shared_blks_read", INT8OID}, + LogDesc{"instrumentation_shared_blks_dirtied", "query_metrics.instrumentation.shared_blks_dirtied", INT8OID}, + LogDesc{"instrumentation_shared_blks_written", "query_metrics.instrumentation.shared_blks_written", INT8OID}, + LogDesc{"instrumentation_local_blks_hit", "query_metrics.instrumentation.local_blks_hit", INT8OID}, + LogDesc{"instrumentation_local_blks_read", "query_metrics.instrumentation.local_blks_read", INT8OID}, + LogDesc{"instrumentation_local_blks_dirtied", "query_metrics.instrumentation.local_blks_dirtied", INT8OID}, + LogDesc{"instrumentation_local_blks_written", "query_metrics.instrumentation.local_blks_written", INT8OID}, + LogDesc{"instrumentation_temp_blks_read", "query_metrics.instrumentation.temp_blks_read", INT8OID}, + LogDesc{"instrumentation_temp_blks_written", "query_metrics.instrumentation.temp_blks_written", INT8OID}, + LogDesc{"instrumentation_inherited_calls", "query_metrics.instrumentation.inherited_calls", INT8OID}, + /* 8-byte aligned types - Network Stats */ + LogDesc{"instrumentation_sent_total_bytes", "query_metrics.instrumentation.sent.total_bytes", INT8OID}, + LogDesc{"instrumentation_sent_tuple_bytes", "query_metrics.instrumentation.sent.tuple_bytes", INT8OID}, + LogDesc{"instrumentation_sent_chunks", "query_metrics.instrumentation.sent.chunks", INT8OID}, + LogDesc{"instrumentation_received_total_bytes", "query_metrics.instrumentation.received.total_bytes", INT8OID}, + LogDesc{"instrumentation_received_tuple_bytes", "query_metrics.instrumentation.received.tuple_bytes", INT8OID}, + LogDesc{"instrumentation_received_chunks", "query_metrics.instrumentation.received.chunks", INT8OID}, + /* 8-byte aligned types - Interconnect Stats and spilled bytes */ + LogDesc{"interconnect_total_recv_queue_size", "query_metrics.instrumentation.interconnect.total_recv_queue_size", INT8OID}, + LogDesc{"interconnect_recv_queue_size_counting_time", "query_metrics.instrumentation.interconnect.recv_queue_size_counting_time", INT8OID}, + LogDesc{"interconnect_total_capacity", "query_metrics.instrumentation.interconnect.total_capacity", INT8OID}, + LogDesc{"interconnect_capacity_counting_time", "query_metrics.instrumentation.interconnect.capacity_counting_time", INT8OID}, + LogDesc{"interconnect_total_buffers", "query_metrics.instrumentation.interconnect.total_buffers", INT8OID}, + LogDesc{"interconnect_buffer_counting_time", "query_metrics.instrumentation.interconnect.buffer_counting_time", INT8OID}, + LogDesc{"interconnect_active_connections_num", "query_metrics.instrumentation.interconnect.active_connections_num", INT8OID}, + LogDesc{"interconnect_retransmits", "query_metrics.instrumentation.interconnect.retransmits", INT8OID}, + LogDesc{"interconnect_startup_cached_pkt_num", "query_metrics.instrumentation.interconnect.startup_cached_pkt_num", INT8OID}, + LogDesc{"interconnect_mismatch_num", "query_metrics.instrumentation.interconnect.mismatch_num", INT8OID}, + LogDesc{"interconnect_crc_errors", "query_metrics.instrumentation.interconnect.crc_errors", INT8OID}, + LogDesc{"interconnect_snd_pkt_num", "query_metrics.instrumentation.interconnect.snd_pkt_num", INT8OID}, + LogDesc{"interconnect_recv_pkt_num", "query_metrics.instrumentation.interconnect.recv_pkt_num", INT8OID}, + LogDesc{"interconnect_disordered_pkt_num", "query_metrics.instrumentation.interconnect.disordered_pkt_num", INT8OID}, + LogDesc{"interconnect_duplicated_pkt_num", "query_metrics.instrumentation.interconnect.duplicated_pkt_num", INT8OID}, + LogDesc{"interconnect_recv_ack_num", "query_metrics.instrumentation.interconnect.recv_ack_num", INT8OID}, + LogDesc{"interconnect_status_query_msg_num", "query_metrics.instrumentation.interconnect.status_query_msg_num", INT8OID}, + LogDesc{"spill_totalbytes", "query_metrics.spill.totalBytes", INT8OID}, + /* 8-byte aligned types - Float and Timestamp */ + LogDesc{"systemstat_runningtimeseconds", "query_metrics.systemStat.runningTimeSeconds", FLOAT8OID}, + LogDesc{"systemstat_usertimeseconds", "query_metrics.systemStat.userTimeSeconds", FLOAT8OID}, + LogDesc{"systemstat_kerneltimeseconds", "query_metrics.systemStat.kernelTimeSeconds", FLOAT8OID}, + LogDesc{"instrumentation_firsttuple", "query_metrics.instrumentation.firsttuple", FLOAT8OID}, + LogDesc{"instrumentation_startup", "query_metrics.instrumentation.startup", FLOAT8OID}, + LogDesc{"instrumentation_total", "query_metrics.instrumentation.total", FLOAT8OID}, + LogDesc{"instrumentation_blk_read_time", "query_metrics.instrumentation.blk_read_time", FLOAT8OID}, + LogDesc{"instrumentation_blk_write_time", "query_metrics.instrumentation.blk_write_time", FLOAT8OID}, + LogDesc{"instrumentation_startup_time", "query_metrics.instrumentation.startup_time", FLOAT8OID}, + LogDesc{"instrumentation_inherited_time", "query_metrics.instrumentation.inherited_time", FLOAT8OID}, + LogDesc{"datetime", "datetime", TIMESTAMPTZOID}, + LogDesc{"submit_time", "submit_time", TIMESTAMPTZOID}, + LogDesc{"start_time", "start_time", TIMESTAMPTZOID}, + LogDesc{"end_time", "end_time", TIMESTAMPTZOID}, + /* 4-byte aligned types - Query Key */ + LogDesc{"tmid", "query_key.tmid", INT4OID}, + LogDesc{"ssid", "query_key.ssid", INT4OID}, + LogDesc{"ccnt", "query_key.ccnt", INT4OID}, + /* 4-byte aligned types - Segment Key */ + LogDesc{"dbid", "segment_key.dbid", INT4OID}, + LogDesc{"segid", "segment_key.segindex", INT4OID}, + LogDesc{"spill_filecount", "query_metrics.spill.fileCount", INT4OID}, + /* Variable-length types - Query Info */ + LogDesc{"generator", "query_info.generator", TEXTOID}, + LogDesc{"query_text", "query_info.query_text", TEXTOID}, + LogDesc{"plan_text", "query_info.plan_text", TEXTOID}, + LogDesc{"template_query_text", "query_info.template_query_text", TEXTOID}, + LogDesc{"template_plan_text", "query_info.template_plan_text", TEXTOID}, + LogDesc{"user_name", "query_info.userName", TEXTOID}, + LogDesc{"database_name", "query_info.databaseName", TEXTOID}, + LogDesc{"rsgname", "query_info.rsgname", TEXTOID}, + LogDesc{"analyze_text", "query_info.analyze_text", TEXTOID}, + LogDesc{"error_message", "add_info.error_message", TEXTOID}, + LogDesc{"query_status", "query_status", TEXTOID}, + /* Extra field */ + LogDesc{"utility", "", BOOLOID}, +}; +/* clang-format on */ + +inline constexpr size_t natts_yagp_log = log_tbl_desc.size(); +inline constexpr size_t attnum_yagp_log_utility = natts_yagp_log - 1; + +const std::unordered_map &proto_name_to_col_idx(); + +TupleDesc DescribeTuple(); + +Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts); + +Datum field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg); + +/* Process a single proto field and store in values/nulls arrays */ +void process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls); + +/* + * Extracts values from msg into values/nulls arrays. Caller must + * pre-init nulls[] to true (this function does net set nulls + * to true for nested messages if parent message is missing). + */ +void extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls); diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 9d579a91a30..0824a3a6808 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -1,4 +1,5 @@ #include "gpdbwrappers.h" +#include "log/LogOps.h" extern "C" { #include "postgres.h" @@ -126,8 +127,8 @@ ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept { +ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, + bool analyze) noexcept { return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); @@ -136,7 +137,7 @@ ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, es.buffers = es.analyze; es.timing = es.analyze; es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_JSON; + es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); if (analyze) { ExplainPrintPlan(&es, query_desc); @@ -220,4 +221,8 @@ char *ya_gpdb::get_rg_name_for_id(Oid group_id) { Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); -} \ No newline at end of file +} + +void ya_gpdb::insert_log(const yagpcc::SetQueryReq &req, bool utility) { + return wrap_throw(::insert_log, req, utility); +} diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index ad7ae96c362..8f5f146cc67 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -16,6 +16,10 @@ extern "C" { #include #include +namespace yagpcc { +class SetQueryReq; +} // namespace yagpcc + namespace ya_gpdb { // Functions that call palloc(). @@ -27,8 +31,7 @@ char *get_database_name(Oid dbid) noexcept; bool split_identifier_string(char *rawstring, char separator, List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; -ExplainState get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept; +ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; Instrumentation *instr_alloc(size_t n, int instrument_options); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull); @@ -38,6 +41,7 @@ void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query); StringInfo gen_normplan(const char *executionPlan); char *get_rg_name_for_id(Oid group_id); +void insert_log(const yagpcc::SetQueryReq &req, bool utility); // Palloc-free functions. void pfree(void *pointer) noexcept; diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 2a9e7328e6d..9db73638b24 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -10,6 +10,8 @@ void _PG_init(void); void _PG_fini(void); PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); PG_FUNCTION_INFO_V1(yagp_stat_messages); +PG_FUNCTION_INFO_V1(yagp_init_log); +PG_FUNCTION_INFO_V1(yagp_truncate_log); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { @@ -30,4 +32,14 @@ Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { Datum yagp_stat_messages(PG_FUNCTION_ARGS) { return yagp_functions_get(fcinfo); -} \ No newline at end of file +} + +Datum yagp_init_log(PG_FUNCTION_ARGS) { + init_log(); + PG_RETURN_VOID(); +} + +Datum yagp_truncate_log(PG_FUNCTION_ARGS) { + truncate_log(); + PG_RETURN_VOID(); +} diff --git a/yagp_hooks_collector--1.0--1.1.sql b/yagp_hooks_collector--1.0--1.1.sql new file mode 100644 index 00000000000..959d4f235d1 --- /dev/null +++ b/yagp_hooks_collector--1.0--1.1.sql @@ -0,0 +1,113 @@ +/* yagp_hooks_collector--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION yagp_hooks_collector UPDATE TO '1.1'" to load this file. \quit + +CREATE SCHEMA yagpcc; + +-- Unlink existing objects from extension. +ALTER EXTENSION yagp_hooks_collector DROP VIEW yagp_stat_messages; +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION yagp_stat_messages_reset(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_segments(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_master(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); + +-- Now drop the objects. +DROP VIEW yagp_stat_messages; +DROP FUNCTION yagp_stat_messages_reset(); +DROP FUNCTION __yagp_stat_messages_f_on_segments(); +DROP FUNCTION __yagp_stat_messages_f_on_master(); +DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); +DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); + +-- Recreate functions and view in new schema. +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.stat_messages_reset() +RETURNS void +AS +$$ + SELECT yagpcc.__stat_messages_reset_f_on_master(); + SELECT yagpcc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagpcc.stat_messages AS + SELECT C.* + FROM yagpcc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM yagpcc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +-- Create new objects. +CREATE FUNCTION yagpcc.__init_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__init_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside yagpcc schema. +SELECT yagpcc.__init_log_on_master(); +SELECT yagpcc.__init_log_on_segments(); + +CREATE VIEW yagpcc.log AS + SELECT * FROM yagpcc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('yagpcc.__log') -- segments + ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION yagpcc.__truncate_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__truncate_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.truncate_log() +RETURNS void AS $$ +BEGIN + PERFORM yagpcc.__truncate_log_on_master(); + PERFORM yagpcc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/sql/yagp_hooks_collector--1.0.sql b/yagp_hooks_collector--1.0.sql similarity index 99% rename from sql/yagp_hooks_collector--1.0.sql rename to yagp_hooks_collector--1.0.sql index 88bbe4e0dc7..7ab4e1b2fb7 100644 --- a/sql/yagp_hooks_collector--1.0.sql +++ b/yagp_hooks_collector--1.0.sql @@ -15,7 +15,7 @@ LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagp_stat_messages_reset() RETURNS void -AS +AS $$ SELECT __yagp_stat_messages_reset_f_on_master(); SELECT __yagp_stat_messages_reset_f_on_segments(); diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql new file mode 100644 index 00000000000..657720a88f2 --- /dev/null +++ b/yagp_hooks_collector--1.1.sql @@ -0,0 +1,95 @@ +/* yagp_hooks_collector--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit + +CREATE SCHEMA yagpcc; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.stat_messages_reset() +RETURNS void +AS +$$ + SELECT yagpcc.__stat_messages_reset_f_on_master(); + SELECT yagpcc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagpcc.stat_messages AS + SELECT C.* + FROM yagpcc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM yagpcc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION yagpcc.__init_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__init_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside yagpcc schema. +SELECT yagpcc.__init_log_on_master(); +SELECT yagpcc.__init_log_on_segments(); + +CREATE VIEW yagpcc.log AS + SELECT * FROM yagpcc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('yagpcc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION yagpcc.__truncate_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__truncate_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.truncate_log() +RETURNS void AS $$ +BEGIN + PERFORM yagpcc.__truncate_log_on_master(); + PERFORM yagpcc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/yagp_hooks_collector.control b/yagp_hooks_collector.control index b5539dd6462..cb5906a1302 100644 --- a/yagp_hooks_collector.control +++ b/yagp_hooks_collector.control @@ -1,5 +1,5 @@ # yagp_hooks_collector extension comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' -default_version = '1.0' +default_version = '1.1' module_pathname = '$libdir/yagp_hooks_collector' superuser = true From 16b42c858c3a9fa2104862dbd010441e7d26e11f Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 10:17:05 +0300 Subject: [PATCH 070/128] [yagp_hooks_collector] Port backend infrastructure and adapt for Cloudberry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port GpscQueryKey to QueryDesc and workfile spill counters to workfile_mgr.c from gpdb. Update for Cloudberry API changes: ExplainInitState→NewExplainState, gpmon_gettmid→gp_gettmid, Gp_session_role→Gp_role, signature changes in standard_ExecutorRun, standard_ProcessUtility, InstrAlloc, CreateTemplateTupleDesc. Change test functions to SRF. Remove redundant jumbling copies. --- expected/yagp_cursors.out | 12 +- expected/yagp_dist.out | 12 +- expected/yagp_select.out | 12 +- expected/yagp_utility.out | 52 +- src/EventSender.cpp | 2 +- src/EventSender.h | 4 +- src/PgUtils.cpp | 2 +- src/ProtoUtils.cpp | 12 +- src/UDSConnector.cpp | 10 +- src/backend/tcop/pquery.c | 3 + .../utils/workfile_manager/workfile_mgr.c | 24 + src/hook_wrappers.cpp | 34 +- src/include/executor/execdesc.h | 11 + src/include/utils/workfile_mgr.h | 4 + src/log/LogOps.cpp | 12 +- src/log/LogSchema.cpp | 2 +- src/memory/gpdbwrappers.cpp | 48 +- src/memory/gpdbwrappers.h | 2 +- .../pg_stat_statements_ya_parser.c | 760 +----------------- src/yagp_hooks_collector.c | 34 +- yagp_hooks_collector--1.0--1.1.sql | 16 +- yagp_hooks_collector--1.0.sql | 6 +- yagp_hooks_collector--1.1.sql | 16 +- 23 files changed, 217 insertions(+), 873 deletions(-) diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index 9587c00b550..d251ddd3e1c 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -40,8 +40,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- DECLARE WITH HOLD SET yagpcc.logging_mode to 'TBL'; @@ -74,8 +73,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- ROLLBACK SET yagpcc.logging_mode to 'TBL'; @@ -105,8 +103,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- FETCH SET yagpcc.logging_mode to 'TBL'; @@ -155,8 +152,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out index ebaf839601d..5fd5ea5fb3e 100644 --- a/expected/yagp_dist.out +++ b/expected/yagp_dist.out @@ -46,8 +46,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) SET yagpcc.logging_mode to 'TBL'; -- Scan all segments. @@ -83,8 +82,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Replicated table CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ @@ -128,8 +126,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Partially distributed table (2 numsegments) SET allow_system_table_mods = ON; @@ -167,8 +164,7 @@ SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_statu SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_select.out b/expected/yagp_select.out index 4c4a0218150..b6e18dc862f 100644 --- a/expected/yagp_select.out +++ b/expected/yagp_select.out @@ -46,8 +46,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Transaction test SET yagpcc.logging_mode to 'TBL'; @@ -72,8 +71,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- CTE test SET yagpcc.logging_mode to 'TBL'; @@ -102,8 +100,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Prepared statement test SET yagpcc.logging_mode to 'TBL'; @@ -128,8 +125,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 03c17713575..057f7d7a556 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -17,7 +17,7 @@ SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.logging_mode to 'TBL'; CREATE TABLE test_table (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE INDEX test_idx ON test_table(a); ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; @@ -41,8 +41,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Partitioning SET yagpcc.logging_mode to 'TBL'; @@ -50,34 +49,16 @@ CREATE TABLE pt_test (a int, b int) DISTRIBUTED BY (a) PARTITION BY RANGE (a) (START (0) END (100) EVERY (50)); -NOTICE: CREATE TABLE will create partition "pt_test_1_prt_1" for table "pt_test" -NOTICE: CREATE TABLE will create partition "pt_test_1_prt_2" for table "pt_test" DROP TABLE pt_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| @@ -85,13 +66,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT -(10 rows) +(6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Views and Functions SET yagpcc.logging_mode to 'TBL'; @@ -118,8 +98,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Transaction Operations SET yagpcc.logging_mode to 'TBL'; @@ -159,13 +138,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- DML Operations SET yagpcc.logging_mode to 'TBL'; CREATE TABLE dml_test (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO dml_test VALUES (1, 'test'); UPDATE dml_test SET b = 'updated' WHERE a = 1; @@ -186,13 +164,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- COPY Operations SET yagpcc.logging_mode to 'TBL'; CREATE TABLE copy_test (a int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. COPY (SELECT 1) TO STDOUT; 1 @@ -214,8 +191,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Prepared Statements and error during execute SET yagpcc.logging_mode to 'TBL'; @@ -240,8 +216,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- GUC Settings SET yagpcc.logging_mode to 'TBL'; @@ -262,8 +237,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fee435a6dcc..d638d275548 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -169,7 +169,7 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { if (query_desc->totaltime == NULL) { MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL); + query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL, false); ya_gpdb::mem_ctx_switch_to(oldcxt); } } diff --git a/src/EventSender.h b/src/EventSender.h index 4afdf1e14a4..6e195eeacdf 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,6 +23,8 @@ class SetQueryReq; #include +extern void gp_gettmid(int32 *); + struct QueryKey { int tmid; int ssid; @@ -40,7 +42,7 @@ struct QueryKey { query_desc->yagp_query_key = (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); int32 tmid; - gpmon_gettmid(&tmid); + gp_gettmid(&tmid); query_desc->yagp_query_key->tmid = tmid; query_desc->yagp_query_key->ssid = gp_session_id; query_desc->yagp_query_key->ccnt = gp_command_count; diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index fc58112bfaa..96f46429643 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -72,7 +72,7 @@ bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { } bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; + return Config::report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; } bool filter_query(QueryDesc *query_desc) { diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index f28714da6ec..aa8632477f5 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -24,6 +24,8 @@ extern "C" { #include #include +extern void gp_gettmid(int32 *); + namespace { constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); @@ -49,7 +51,7 @@ void set_query_key(yagpcc::QueryKey *key) { key->set_ccnt(gp_command_count); key->set_ssid(gp_session_id); int32 tmid = 0; - gpmon_gettmid(&tmid); + gp_gettmid(&tmid); key->set_tmid(tmid); } @@ -81,7 +83,7 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER @@ -106,7 +108,7 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), @@ -118,7 +120,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } void clear_big_fields(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { + if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); qi->clear_template_plan_text(); @@ -129,7 +131,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { } void set_query_info(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { + if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); if (IsTransactionState()) diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b6af303218d..a7eaed539f7 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -19,10 +19,9 @@ extern "C" { static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { - ereport(LOG, - (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str(), strerror(errno)))); + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str()))); } bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, @@ -77,8 +76,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, // That's a very important error that should never happen, so make it // visible to an end-user and admins. ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %s", - strerror(errno)))); + (errmsg("Unable to create non-blocking socket connection %m"))); success = false; YagpStat::report_error(); } diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 532690f1d51..7c1dbc480bc 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -127,6 +127,9 @@ CreateQueryDesc(PlannedStmt *plannedstmt, if (Gp_role != GP_ROLE_EXECUTE) increment_command_count(); + /* null this field until set by YAGP Hooks collector */ + qd->yagp_query_key = NULL; + return qd; } diff --git a/src/backend/utils/workfile_manager/workfile_mgr.c b/src/backend/utils/workfile_manager/workfile_mgr.c index e5b311cf9ba..21b4463e5f1 100644 --- a/src/backend/utils/workfile_manager/workfile_mgr.c +++ b/src/backend/utils/workfile_manager/workfile_mgr.c @@ -192,6 +192,9 @@ static void unpin_workset(workfile_set *work_set); static bool proc_exit_hook_registered = false; +static uint64 total_bytes_written = 0; +static uint64 total_files_created = 0; + Datum gp_workfile_mgr_cache_entries(PG_FUNCTION_ARGS); Datum gp_workfile_mgr_used_diskspace(PG_FUNCTION_ARGS); @@ -371,6 +374,7 @@ RegisterFileWithSet(File file, workfile_set *work_set) localCtl.entries[file].work_set = work_set; work_set->num_files++; work_set->perquery->num_files++; + total_files_created++; /* Enforce the limit on number of files */ if (gp_workfile_limit_files_per_query > 0 && @@ -447,6 +451,7 @@ UpdateWorkFileSize(File file, uint64 newsize) (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("workfile per segment size limit exceeded"))); } + total_bytes_written += diff; } /* @@ -986,3 +991,22 @@ workfile_is_active(workfile_set *workfile) { return workfile ? workfile->active : false; } + +uint64 +WorkfileTotalBytesWritten(void) +{ + return total_bytes_written; +} + +uint64 +WorkfileTotalFilesCreated(void) +{ + return total_files_created; +} + +void +WorkfileResetBackendStats(void) +{ + total_bytes_written = 0; + total_files_created = 0; +} diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 07ac511d546..56c1da9f4f6 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -36,7 +36,7 @@ static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - long count); + uint64 count, bool execute_once); static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); @@ -45,10 +45,12 @@ static void ya_ic_teardown_hook(ChunkTransportState *transportStates, #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void ya_process_utility_hook(Node *parsetree, const char *queryString, +static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, - ParamListInfo params, DestReceiver *dest, - char *completionTag); + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc); static EventSender *sender = nullptr; @@ -127,14 +129,14 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { } void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - long count) { + uint64 count, bool execute_once) { get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorRun_hook) - previous_ExecutorRun_hook(query_desc, direction, count); + previous_ExecutorRun_hook(query_desc, direction, count, execute_once); else - standard_ExecutorRun(query_desc, direction, count); + standard_ExecutorRun(query_desc, direction, count, execute_once); get_sender()->decr_depth(); } PG_CATCH(); @@ -198,10 +200,12 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif -static void ya_process_utility_hook(Node *parsetree, const char *queryString, +static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, - ParamListInfo params, DestReceiver *dest, - char *completionTag) { + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) { /* Project utility data on QueryDesc to use existing logic */ QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); query_desc->sourceText = queryString; @@ -214,11 +218,11 @@ static void ya_process_utility_hook(Node *parsetree, const char *queryString, PG_TRY(); { if (previous_ProcessUtility_hook) { - (*previous_ProcessUtility_hook)(parsetree, queryString, context, params, - dest, completionTag); + (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); } else { - standard_ProcessUtility(parsetree, queryString, context, params, dest, - completionTag); + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, + queryEnv, dest, qc); } get_sender()->decr_depth(); @@ -264,7 +268,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { const int ATTNUM = 6; check_stats_loaded(); auto stats = YagpStat::get_stats(); - TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM, false); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, 0 /* attdim */); TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index e3ecf31b664..e469945a4c5 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -22,6 +22,14 @@ struct CdbExplain_ShowStatCtx; /* private, in "cdb/cdbexplain.c" */ +typedef struct YagpQueryKey +{ + int tmid; /* transaction time */ + int ssid; /* session id */ + int ccnt; /* command count */ + int nesting_level; + uintptr_t query_desc_addr; +} YagpQueryKey; /* * SerializedParams is used to serialize external query parameters @@ -330,6 +338,9 @@ typedef struct QueryDesc /* This is always set NULL by the core system, but plugins can change it */ struct Instrumentation *totaltime; /* total time spent in ExecutorRun */ + + /* YAGP Hooks collector */ + YagpQueryKey *yagp_query_key; } QueryDesc; /* in pquery.c */ diff --git a/src/include/utils/workfile_mgr.h b/src/include/utils/workfile_mgr.h index dfbd17bca57..48c83620610 100644 --- a/src/include/utils/workfile_mgr.h +++ b/src/include/utils/workfile_mgr.h @@ -74,4 +74,8 @@ extern workfile_set *workfile_mgr_cache_entries_get_copy(int* num_actives); extern uint64 WorkfileSegspace_GetSize(void); extern bool workfile_is_active(workfile_set *workfile); +extern uint64 WorkfileTotalBytesWritten(void); +extern uint64 WorkfileTotalFilesCreated(void); +extern void WorkfileResetBackendStats(void); + #endif /* __WORKFILE_MGR_H__ */ diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index 0868dd9fc1c..cec9e33693a 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -37,12 +37,12 @@ void init_log() { relationId = heap_create_with_catalog( log_relname.data() /* relname */, namespaceId /* namespace */, 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, - InvalidOid /* reloftypeid */, GetUserId() /* owner */, - DescribeTuple() /* rel tuple */, NIL, InvalidOid /* relam */, - RELKIND_RELATION, RELPERSISTENCE_PERMANENT, RELSTORAGE_HEAP, false, false, - true, 0, ONCOMMIT_NOOP, NULL /* GP Policy */, (Datum)0, - false /* use_user_acl */, true, true, false /* valid_opts */, - false /* is_part_child */, false /* is part parent */, NULL); + InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, + DescribeTuple() /* rel tuple */, NIL, RELKIND_RELATION, + RELPERSISTENCE_PERMANENT, false, false, ONCOMMIT_NOOP, + NULL /* GP Policy */, (Datum)0, false /* use_user_acl */, true, true, + InvalidOid /* relrewrite */, NULL /* typaddress */, + false /* valid_opts */); /* Make the table visible */ CommandCounterIncrement(); diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp index 335a3103cfd..2fadcc46599 100644 --- a/src/log/LogSchema.cpp +++ b/src/log/LogSchema.cpp @@ -19,7 +19,7 @@ const std::unordered_map &proto_name_to_col_idx() { } TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log, false); + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log); for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 0824a3a6808..763e32e539c 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -7,6 +7,7 @@ extern "C" { #include "commands/dbcommands.h" #include "commands/resgroupcmds.h" #include "utils/builtins.h" +#include "utils/varlena.h" #include "nodes/pg_list.h" #include "commands/explain.h" #include "executor/instrument.h" @@ -115,41 +116,40 @@ bool ya_gpdb::split_identifier_string(char *rawstring, char separator, ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept { return wrap_noexcept([&]() { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; + ExplainState *es = NewExplainState(); + es->costs = costs; + es->verbose = true; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + ExplainPrintPlan(es, query_desc); + ExplainEndOutput(es); + return *es; }); } ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept { return wrap_noexcept([&]() { - ExplainState es; - ExplainInitState(&es); - es.analyze = analyze; - es.verbose = true; - es.buffers = es.analyze; - es.timing = es.analyze; - es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); + ExplainState *es = NewExplainState(); + es->analyze = analyze; + es->verbose = true; + es->buffers = es->analyze; + es->timing = es->analyze; + es->summary = es->analyze; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); if (analyze) { - ExplainPrintPlan(&es, query_desc); - ExplainPrintExecStatsEnd(&es, query_desc); + ExplainPrintPlan(es, query_desc); + ExplainPrintExecStatsEnd(es, query_desc); } - ExplainEndOutput(&es); - return es; + ExplainEndOutput(es); + return *es; }); } -Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options) { - return wrap_throw(InstrAlloc, n, instrument_options); +Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options, + bool async_mode) { + return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 8f5f146cc67..920fc1ae6e7 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -32,7 +32,7 @@ bool split_identifier_string(char *rawstring, char separator, List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; -Instrumentation *instr_alloc(size_t n, int instrument_options); +Instrumentation *instr_alloc(size_t n, int instrument_options, bool async_mode); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull); CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 1c58d936093..c19805ce506 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -6,689 +6,48 @@ #include #include -#include "access/hash.h" -#include "executor/instrument.h" -#include "executor/execdesc.h" -#include "funcapi.h" +#include "common/hashfn.h" +#include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "miscadmin.h" -#include "parser/analyze.h" -#include "parser/parsetree.h" #include "parser/scanner.h" -#include "parser/gram.h" -#include "pgstat.h" -#include "storage/fd.h" -#include "storage/ipc.h" -#include "storage/spin.h" -#include "tcop/utility.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "utils/queryjumble.h" #include "pg_stat_statements_ya_parser.h" -static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; - -#define JUMBLE_SIZE 1024 /* query serialization buffer size */ - -/* - * Struct for tracking locations/lengths of constants during normalization - */ -typedef struct pgssLocationLen -{ - int location; /* start offset in query text */ - int length; /* length in bytes, or -1 to ignore */ -} pgssLocationLen; - -/* - * Working state for computing a query jumble and producing a normalized - * query string - */ -typedef struct pgssJumbleState -{ - /* Jumble of current query tree */ - unsigned char *jumble; - - /* Number of bytes used in jumble[] */ - Size jumble_len; - - /* Array of locations of constants that should be removed */ - pgssLocationLen *clocations; - - /* Allocated length of clocations array */ - int clocations_buf_size; - - /* Current number of valid entries in clocations array */ - int clocations_count; - - /* highest Param id we've seen, in order to start normalization correctly */ - int highest_extern_param_id; -} pgssJumbleState; +#ifndef ICONST +#define ICONST 276 +#endif +#ifndef FCONST +#define FCONST 277 +#endif +#ifndef SCONST +#define SCONST 278 +#endif +#ifndef BCONST +#define BCONST 279 +#endif +#ifndef XCONST +#define XCONST 280 +#endif -static void AppendJumble(pgssJumbleState *jstate, - const unsigned char *item, Size size); -static void JumbleQuery(pgssJumbleState *jstate, Query *query); -static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); -static void JumbleExpr(pgssJumbleState *jstate, Node *node); -static void RecordConstLocation(pgssJumbleState *jstate, int location); -static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query); +static void fill_in_constant_lengths(JumbleState *jstate, const char *query); static int comp_location(const void *a, const void *b); StringInfo gen_normplan(const char *execution_plan); static bool need_replace(int token); -void pgss_post_parse_analyze(ParseState *pstate, Query *query); -static char *generate_normalized_query(pgssJumbleState *jstate, const char *query, +static char *generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding); - void stat_statements_parser_init() -{ - prev_post_parse_analyze_hook = post_parse_analyze_hook; - post_parse_analyze_hook = pgss_post_parse_analyze; -} - -void stat_statements_parser_deinit() +void stat_statements_parser_init(void) { - post_parse_analyze_hook = prev_post_parse_analyze_hook; -} - -/* - * AppendJumble: Append a value that is substantive in a given query to - * the current jumble. - */ -static void -AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size) -{ - unsigned char *jumble = jstate->jumble; - Size jumble_len = jstate->jumble_len; - - /* - * Whenever the jumble buffer is full, we hash the current contents and - * reset the buffer to contain just that hash value, thus relying on the - * hash to summarize everything so far. - */ - while (size > 0) - { - Size part_size; - - if (jumble_len >= JUMBLE_SIZE) - { - uint32 start_hash = hash_any(jumble, JUMBLE_SIZE); - - memcpy(jumble, &start_hash, sizeof(start_hash)); - jumble_len = sizeof(start_hash); - } - part_size = Min(size, JUMBLE_SIZE - jumble_len); - memcpy(jumble + jumble_len, item, part_size); - jumble_len += part_size; - item += part_size; - size -= part_size; - } - jstate->jumble_len = jumble_len; + EnableQueryId(); } -/* - * Wrappers around AppendJumble to encapsulate details of serialization - * of individual local variable elements. - */ -#define APP_JUMB(item) \ - AppendJumble(jstate, (const unsigned char *)&(item), sizeof(item)) -#define APP_JUMB_STRING(str) \ - AppendJumble(jstate, (const unsigned char *)(str), strlen(str) + 1) - -/* - * JumbleQuery: Selectively serialize the query tree, appending significant - * data to the "query jumble" while ignoring nonsignificant data. - * - * Rule of thumb for what to include is that we should ignore anything not - * semantically significant (such as alias names) as well as anything that can - * be deduced from child nodes (else we'd just be double-hashing that piece - * of information). - */ -void JumbleQuery(pgssJumbleState *jstate, Query *query) +void stat_statements_parser_deinit(void) { - Assert(IsA(query, Query)); - Assert(query->utilityStmt == NULL); - - APP_JUMB(query->commandType); - /* resultRelation is usually predictable from commandType */ - JumbleExpr(jstate, (Node *)query->cteList); - JumbleRangeTable(jstate, query->rtable); - JumbleExpr(jstate, (Node *)query->jointree); - JumbleExpr(jstate, (Node *)query->targetList); - JumbleExpr(jstate, (Node *)query->returningList); - JumbleExpr(jstate, (Node *)query->groupClause); - JumbleExpr(jstate, query->havingQual); - JumbleExpr(jstate, (Node *)query->windowClause); - JumbleExpr(jstate, (Node *)query->distinctClause); - JumbleExpr(jstate, (Node *)query->sortClause); - JumbleExpr(jstate, query->limitOffset); - JumbleExpr(jstate, query->limitCount); - /* we ignore rowMarks */ - JumbleExpr(jstate, query->setOperations); -} - -/* - * Jumble a range table - */ -static void -JumbleRangeTable(pgssJumbleState *jstate, List *rtable) -{ - ListCell *lc; - - foreach (lc, rtable) - { - RangeTblEntry *rte = (RangeTblEntry *)lfirst(lc); - - Assert(IsA(rte, RangeTblEntry)); - APP_JUMB(rte->rtekind); - switch (rte->rtekind) - { - case RTE_RELATION: - APP_JUMB(rte->relid); - break; - case RTE_SUBQUERY: - JumbleQuery(jstate, rte->subquery); - break; - case RTE_JOIN: - APP_JUMB(rte->jointype); - break; - case RTE_FUNCTION: - JumbleExpr(jstate, (Node *)rte->functions); - break; - case RTE_VALUES: - JumbleExpr(jstate, (Node *)rte->values_lists); - break; - case RTE_CTE: - - /* - * Depending on the CTE name here isn't ideal, but it's the - * only info we have to identify the referenced WITH item. - */ - APP_JUMB_STRING(rte->ctename); - APP_JUMB(rte->ctelevelsup); - break; - /* GPDB RTEs */ - case RTE_VOID: - break; - case RTE_TABLEFUNCTION: - JumbleQuery(jstate, rte->subquery); - JumbleExpr(jstate, (Node *)rte->functions); - break; - default: - ereport(ERROR, (errmsg("unrecognized RTE kind: %d", (int)rte->rtekind))); - break; - } - } -} - -/* - * Jumble an expression tree - * - * In general this function should handle all the same node types that - * expression_tree_walker() does, and therefore it's coded to be as parallel - * to that function as possible. However, since we are only invoked on - * queries immediately post-parse-analysis, we need not handle node types - * that only appear in planning. - * - * Note: the reason we don't simply use expression_tree_walker() is that the - * point of that function is to support tree walkers that don't care about - * most tree node types, but here we care about all types. We should complain - * about any unrecognized node type. - */ -static void -JumbleExpr(pgssJumbleState *jstate, Node *node) -{ - ListCell *temp; - - if (node == NULL) - return; - - /* Guard against stack overflow due to overly complex expressions */ - check_stack_depth(); - - /* - * We always emit the node's NodeTag, then any additional fields that are - * considered significant, and then we recurse to any child nodes. - */ - APP_JUMB(node->type); - - switch (nodeTag(node)) - { - case T_Var: - { - Var *var = (Var *)node; - - APP_JUMB(var->varno); - APP_JUMB(var->varattno); - APP_JUMB(var->varlevelsup); - } - break; - case T_Const: - { - Const *c = (Const *)node; - - /* We jumble only the constant's type, not its value */ - APP_JUMB(c->consttype); - /* Also, record its parse location for query normalization */ - RecordConstLocation(jstate, c->location); - } - break; - case T_Param: - { - Param *p = (Param *)node; - - APP_JUMB(p->paramkind); - APP_JUMB(p->paramid); - APP_JUMB(p->paramtype); - } - break; - case T_Aggref: - { - Aggref *expr = (Aggref *)node; - - APP_JUMB(expr->aggfnoid); - JumbleExpr(jstate, (Node *)expr->aggdirectargs); - JumbleExpr(jstate, (Node *)expr->args); - JumbleExpr(jstate, (Node *)expr->aggorder); - JumbleExpr(jstate, (Node *)expr->aggdistinct); - JumbleExpr(jstate, (Node *)expr->aggfilter); - } - break; - case T_WindowFunc: - { - WindowFunc *expr = (WindowFunc *)node; - - APP_JUMB(expr->winfnoid); - APP_JUMB(expr->winref); - JumbleExpr(jstate, (Node *)expr->args); - JumbleExpr(jstate, (Node *)expr->aggfilter); - } - break; - case T_ArrayRef: - { - ArrayRef *aref = (ArrayRef *)node; - - JumbleExpr(jstate, (Node *)aref->refupperindexpr); - JumbleExpr(jstate, (Node *)aref->reflowerindexpr); - JumbleExpr(jstate, (Node *)aref->refexpr); - JumbleExpr(jstate, (Node *)aref->refassgnexpr); - } - break; - case T_FuncExpr: - { - FuncExpr *expr = (FuncExpr *)node; - - APP_JUMB(expr->funcid); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_NamedArgExpr: - { - NamedArgExpr *nae = (NamedArgExpr *)node; - - APP_JUMB(nae->argnumber); - JumbleExpr(jstate, (Node *)nae->arg); - } - break; - case T_OpExpr: - case T_DistinctExpr: /* struct-equivalent to OpExpr */ - case T_NullIfExpr: /* struct-equivalent to OpExpr */ - { - OpExpr *expr = (OpExpr *)node; - - APP_JUMB(expr->opno); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_ScalarArrayOpExpr: - { - ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *)node; - - APP_JUMB(expr->opno); - APP_JUMB(expr->useOr); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_BoolExpr: - { - BoolExpr *expr = (BoolExpr *)node; - - APP_JUMB(expr->boolop); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_SubLink: - { - SubLink *sublink = (SubLink *)node; - - APP_JUMB(sublink->subLinkType); - JumbleExpr(jstate, (Node *)sublink->testexpr); - JumbleQuery(jstate, (Query *)sublink->subselect); - } - break; - case T_FieldSelect: - { - FieldSelect *fs = (FieldSelect *)node; - - APP_JUMB(fs->fieldnum); - JumbleExpr(jstate, (Node *)fs->arg); - } - break; - case T_FieldStore: - { - FieldStore *fstore = (FieldStore *)node; - - JumbleExpr(jstate, (Node *)fstore->arg); - JumbleExpr(jstate, (Node *)fstore->newvals); - } - break; - case T_RelabelType: - { - RelabelType *rt = (RelabelType *)node; - - APP_JUMB(rt->resulttype); - JumbleExpr(jstate, (Node *)rt->arg); - } - break; - case T_CoerceViaIO: - { - CoerceViaIO *cio = (CoerceViaIO *)node; - - APP_JUMB(cio->resulttype); - JumbleExpr(jstate, (Node *)cio->arg); - } - break; - case T_ArrayCoerceExpr: - { - ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *)node; - - APP_JUMB(acexpr->resulttype); - JumbleExpr(jstate, (Node *)acexpr->arg); - } - break; - case T_ConvertRowtypeExpr: - { - ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *)node; - - APP_JUMB(crexpr->resulttype); - JumbleExpr(jstate, (Node *)crexpr->arg); - } - break; - case T_CollateExpr: - { - CollateExpr *ce = (CollateExpr *)node; - - APP_JUMB(ce->collOid); - JumbleExpr(jstate, (Node *)ce->arg); - } - break; - case T_CaseExpr: - { - CaseExpr *caseexpr = (CaseExpr *)node; - - JumbleExpr(jstate, (Node *)caseexpr->arg); - foreach (temp, caseexpr->args) - { - CaseWhen *when = (CaseWhen *)lfirst(temp); - - Assert(IsA(when, CaseWhen)); - JumbleExpr(jstate, (Node *)when->expr); - JumbleExpr(jstate, (Node *)when->result); - } - JumbleExpr(jstate, (Node *)caseexpr->defresult); - } - break; - case T_CaseTestExpr: - { - CaseTestExpr *ct = (CaseTestExpr *)node; - - APP_JUMB(ct->typeId); - } - break; - case T_ArrayExpr: - JumbleExpr(jstate, (Node *)((ArrayExpr *)node)->elements); - break; - case T_RowExpr: - JumbleExpr(jstate, (Node *)((RowExpr *)node)->args); - break; - case T_RowCompareExpr: - { - RowCompareExpr *rcexpr = (RowCompareExpr *)node; - - APP_JUMB(rcexpr->rctype); - JumbleExpr(jstate, (Node *)rcexpr->largs); - JumbleExpr(jstate, (Node *)rcexpr->rargs); - } - break; - case T_CoalesceExpr: - JumbleExpr(jstate, (Node *)((CoalesceExpr *)node)->args); - break; - case T_MinMaxExpr: - { - MinMaxExpr *mmexpr = (MinMaxExpr *)node; - - APP_JUMB(mmexpr->op); - JumbleExpr(jstate, (Node *)mmexpr->args); - } - break; - case T_XmlExpr: - { - XmlExpr *xexpr = (XmlExpr *)node; - - APP_JUMB(xexpr->op); - JumbleExpr(jstate, (Node *)xexpr->named_args); - JumbleExpr(jstate, (Node *)xexpr->args); - } - break; - case T_NullTest: - { - NullTest *nt = (NullTest *)node; - - APP_JUMB(nt->nulltesttype); - JumbleExpr(jstate, (Node *)nt->arg); - } - break; - case T_BooleanTest: - { - BooleanTest *bt = (BooleanTest *)node; - - APP_JUMB(bt->booltesttype); - JumbleExpr(jstate, (Node *)bt->arg); - } - break; - case T_CoerceToDomain: - { - CoerceToDomain *cd = (CoerceToDomain *)node; - - APP_JUMB(cd->resulttype); - JumbleExpr(jstate, (Node *)cd->arg); - } - break; - case T_CoerceToDomainValue: - { - CoerceToDomainValue *cdv = (CoerceToDomainValue *)node; - - APP_JUMB(cdv->typeId); - } - break; - case T_SetToDefault: - { - SetToDefault *sd = (SetToDefault *)node; - - APP_JUMB(sd->typeId); - } - break; - case T_CurrentOfExpr: - { - CurrentOfExpr *ce = (CurrentOfExpr *)node; - - APP_JUMB(ce->cvarno); - if (ce->cursor_name) - APP_JUMB_STRING(ce->cursor_name); - APP_JUMB(ce->cursor_param); - } - break; - case T_TargetEntry: - { - TargetEntry *tle = (TargetEntry *)node; - - APP_JUMB(tle->resno); - APP_JUMB(tle->ressortgroupref); - JumbleExpr(jstate, (Node *)tle->expr); - } - break; - case T_RangeTblRef: - { - RangeTblRef *rtr = (RangeTblRef *)node; - - APP_JUMB(rtr->rtindex); - } - break; - case T_JoinExpr: - { - JoinExpr *join = (JoinExpr *)node; - - APP_JUMB(join->jointype); - APP_JUMB(join->isNatural); - APP_JUMB(join->rtindex); - JumbleExpr(jstate, join->larg); - JumbleExpr(jstate, join->rarg); - JumbleExpr(jstate, join->quals); - } - break; - case T_FromExpr: - { - FromExpr *from = (FromExpr *)node; - - JumbleExpr(jstate, (Node *)from->fromlist); - JumbleExpr(jstate, from->quals); - } - break; - case T_List: - foreach (temp, (List *)node) - { - JumbleExpr(jstate, (Node *)lfirst(temp)); - } - break; - case T_SortGroupClause: - { - SortGroupClause *sgc = (SortGroupClause *)node; - - APP_JUMB(sgc->tleSortGroupRef); - APP_JUMB(sgc->eqop); - APP_JUMB(sgc->sortop); - APP_JUMB(sgc->nulls_first); - } - break; - case T_WindowClause: - { - WindowClause *wc = (WindowClause *)node; - - APP_JUMB(wc->winref); - APP_JUMB(wc->frameOptions); - JumbleExpr(jstate, (Node *)wc->partitionClause); - JumbleExpr(jstate, (Node *)wc->orderClause); - JumbleExpr(jstate, wc->startOffset); - JumbleExpr(jstate, wc->endOffset); - } - break; - case T_CommonTableExpr: - { - CommonTableExpr *cte = (CommonTableExpr *)node; - - /* we store the string name because RTE_CTE RTEs need it */ - APP_JUMB_STRING(cte->ctename); - JumbleQuery(jstate, (Query *)cte->ctequery); - } - break; - case T_SetOperationStmt: - { - SetOperationStmt *setop = (SetOperationStmt *)node; - - APP_JUMB(setop->op); - APP_JUMB(setop->all); - JumbleExpr(jstate, setop->larg); - JumbleExpr(jstate, setop->rarg); - } - break; - case T_RangeTblFunction: - { - RangeTblFunction *rtfunc = (RangeTblFunction *)node; - - JumbleExpr(jstate, rtfunc->funcexpr); - } - break; - /* GPDB nodes */ - case T_GroupingClause: - { - GroupingClause *grpnode = (GroupingClause *)node; - - JumbleExpr(jstate, (Node *)grpnode->groupsets); - } - break; - case T_GroupingFunc: - { - GroupingFunc *grpnode = (GroupingFunc *)node; - - JumbleExpr(jstate, (Node *)grpnode->args); - } - break; - case T_Grouping: - case T_GroupId: - case T_Integer: - case T_Value: - // TODO:seems like nothing to do with it - break; - /* GPDB-only additions, nothing to do */ - case T_PartitionBy: - case T_PartitionElem: - case T_PartitionRangeItem: - case T_PartitionBoundSpec: - case T_PartitionSpec: - case T_PartitionValuesSpec: - case T_AlterPartitionId: - case T_AlterPartitionCmd: - case T_InheritPartitionCmd: - case T_CreateFileSpaceStmt: - case T_FileSpaceEntry: - case T_DropFileSpaceStmt: - case T_TableValueExpr: - case T_DenyLoginInterval: - case T_DenyLoginPoint: - case T_AlterTypeStmt: - case T_SetDistributionCmd: - case T_ExpandStmtSpec: - break; - default: - /* Only a warning, since we can stumble along anyway */ - ereport(WARNING, (errmsg("unrecognized node type: %d", - (int)nodeTag(node)))); - break; - } -} - -/* - * Record location of constant within query string of query tree - * that is currently being walked. - */ -static void -RecordConstLocation(pgssJumbleState *jstate, int location) -{ - /* -1 indicates unknown or undefined location */ - if (location >= 0) - { - /* enlarge array if needed */ - if (jstate->clocations_count >= jstate->clocations_buf_size) - { - jstate->clocations_buf_size *= 2; - jstate->clocations = (pgssLocationLen *) - repalloc(jstate->clocations, - jstate->clocations_buf_size * - sizeof(pgssLocationLen)); - } - jstate->clocations[jstate->clocations_count].location = location; - /* initialize lengths to -1 to simplify fill_in_constant_lengths */ - jstate->clocations[jstate->clocations_count].length = -1; - jstate->clocations_count++; - } + /* NO-OP */ } /* check if token should be replaced by substitute varable */ @@ -768,60 +127,13 @@ gen_normplan(const char *execution_plan) } /* - * Post-parse-analysis hook: mark query with a queryId - */ -void pgss_post_parse_analyze(ParseState *pstate, Query *query) -{ - pgssJumbleState jstate; - - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query); - - /* Assert we didn't do this already */ - Assert(query->queryId == 0); - - /* - * Utility statements get queryId zero. We do this even in cases where - * the statement contains an optimizable statement for which a queryId - * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, - * runtime control will first go through ProcessUtility and then the - * executor, and we don't want the executor hooks to do anything, since we - * are already measuring the statement's costs at the utility level. - */ - if (query->utilityStmt) - { - query->queryId = 0; - return; - } - - /* Set up workspace for query jumbling */ - jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); - jstate.jumble_len = 0; - jstate.clocations_buf_size = 32; - jstate.clocations = (pgssLocationLen *) - palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); - jstate.clocations_count = 0; - - /* Compute query ID and mark the Query node with it */ - JumbleQuery(&jstate, query); - query->queryId = hash_any(jstate.jumble, jstate.jumble_len); - - /* - * If we are unlucky enough to get a hash of zero, use 1 instead, to - * prevent confusion with the utility-statement case. - */ - if (query->queryId == 0) - query->queryId = 1; -} - -/* - * comp_location: comparator for qsorting pgssLocationLen structs by location + * comp_location: comparator for qsorting LocationLen structs by location */ static int comp_location(const void *a, const void *b) { - int l = ((const pgssLocationLen *) a)->location; - int r = ((const pgssLocationLen *) b)->location; + int l = ((const LocationLen *) a)->location; + int r = ((const LocationLen *) b)->location; if (l < r) return -1; @@ -854,9 +166,9 @@ comp_location(const void *a, const void *b) * reason for a constant to start with a '-'. */ static void -fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) +fill_in_constant_lengths(JumbleState *jstate, const char *query) { - pgssLocationLen *locs; + LocationLen *locs; core_yyscan_t yyscanner; core_yy_extra_type yyextra; core_YYSTYPE yylval; @@ -870,14 +182,14 @@ fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) */ if (jstate->clocations_count > 1) qsort(jstate->clocations, jstate->clocations_count, - sizeof(pgssLocationLen), comp_location); + sizeof(LocationLen), comp_location); locs = jstate->clocations; /* initialize the flex scanner --- should match raw_parser() */ yyscanner = scanner_init(query, &yyextra, - ScanKeywords, - NumScanKeywords); + &ScanKeywords, + ScanKeywordTokens); /* Search for each constant, in sequence */ for (i = 0; i < jstate->clocations_count; i++) @@ -957,7 +269,7 @@ fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) * Returns a palloc'd string. */ static char * -generate_normalized_query(pgssJumbleState *jstate, const char *query, +generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding) { char *norm_query; @@ -1027,12 +339,12 @@ char *gen_normquery(const char *query) if (!query) { return NULL; } - pgssJumbleState jstate; + JumbleState jstate; jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); jstate.jumble_len = 0; jstate.clocations_buf_size = 32; - jstate.clocations = (pgssLocationLen *) - palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations = (LocationLen *) + palloc(jstate.clocations_buf_size * sizeof(LocationLen)); jstate.clocations_count = 0; int query_len = strlen(query); return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 9db73638b24..27fd0e04b26 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,5 +1,6 @@ #include "postgres.h" #include "cdb/cdbvars.h" +#include "funcapi.h" #include "utils/builtins.h" #include "hook_wrappers.h" @@ -26,8 +27,15 @@ void _PG_fini(void) { } Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { - yagp_functions_reset(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + yagp_functions_reset(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } Datum yagp_stat_messages(PG_FUNCTION_ARGS) { @@ -35,11 +43,25 @@ Datum yagp_stat_messages(PG_FUNCTION_ARGS) { } Datum yagp_init_log(PG_FUNCTION_ARGS) { - init_log(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + init_log(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } Datum yagp_truncate_log(PG_FUNCTION_ARGS) { - truncate_log(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + truncate_log(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } diff --git a/yagp_hooks_collector--1.0--1.1.sql b/yagp_hooks_collector--1.0--1.1.sql index 959d4f235d1..8684ca73915 100644 --- a/yagp_hooks_collector--1.0--1.1.sql +++ b/yagp_hooks_collector--1.0--1.1.sql @@ -23,17 +23,17 @@ DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); -- Recreate functions and view in new schema. CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT yagpcc.__stat_messages_reset_f_on_master(); @@ -75,12 +75,12 @@ ORDER BY segid; -- Create new objects. CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; @@ -95,17 +95,17 @@ CREATE VIEW yagpcc.log AS ORDER BY tmid, ssid, ccnt; CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.truncate_log() -RETURNS void AS $$ +RETURNS SETOF void AS $$ BEGIN PERFORM yagpcc.__truncate_log_on_master(); PERFORM yagpcc.__truncate_log_on_segments(); diff --git a/yagp_hooks_collector--1.0.sql b/yagp_hooks_collector--1.0.sql index 7ab4e1b2fb7..270cab92382 100644 --- a/yagp_hooks_collector--1.0.sql +++ b/yagp_hooks_collector--1.0.sql @@ -4,17 +4,17 @@ \echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagp_stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT __yagp_stat_messages_reset_f_on_master(); diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql index 657720a88f2..e0e94b51493 100644 --- a/yagp_hooks_collector--1.1.sql +++ b/yagp_hooks_collector--1.1.sql @@ -6,17 +6,17 @@ CREATE SCHEMA yagpcc; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT yagpcc.__stat_messages_reset_f_on_master(); @@ -57,12 +57,12 @@ CREATE VIEW yagpcc.stat_messages AS ORDER BY segid; CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; @@ -77,17 +77,17 @@ CREATE VIEW yagpcc.log AS ORDER BY tmid, ssid, ccnt; CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.truncate_log() -RETURNS void AS $$ +RETURNS SETOF void AS $$ BEGIN PERFORM yagpcc.__truncate_log_on_master(); PERFORM yagpcc.__truncate_log_on_segments(); From 3563d78432aa71d2df145f0b06e7b75cf7b2e3c1 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 11:13:00 +0300 Subject: [PATCH 071/128] [yagp_hooks_collector] Add --with-yagp-hooks-collector configure option and CI Add configure.ac option with protobuf dependency. Add CI test configuration. Change env script from greenplum_path.sh to cloudberry-env.sh. --- .github/workflows/build-cloudberry.yml | 32 ++++++++- configure | 28 ++++++++ configure.ac | 7 ++ .../scripts/configure-cloudberry.sh | 4 +- expected/yagp_cursors.out | 10 +-- expected/yagp_dist.out | 2 + expected/yagp_select.out | 2 + expected/yagp_utf8_trim.out | 2 + expected/yagp_utility.out | 72 ++++++++++--------- gpcontrib/Makefile | 3 + gpcontrib/yagp_hooks_collector/Makefile | 41 +++++++++++ sql/yagp_cursors.sql | 2 + sql/yagp_dist.sql | 2 + sql/yagp_select.sql | 2 + sql/yagp_utf8_trim.sql | 2 + sql/yagp_utility.sql | 2 + src/Makefile.global.in | 1 + 17 files changed, 173 insertions(+), 41 deletions(-) create mode 100644 gpcontrib/yagp_hooks_collector/Makefile diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index adb57fb85ec..8484331998f 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -271,6 +271,10 @@ jobs: }, "enable_core_check":false }, + {"test":"gpcontrib-yagp-hooks-collector", + "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"], + "extension":"yagp_hooks_collector" + }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] }, @@ -535,10 +539,11 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1403,6 +1408,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: /usr/local/cloudberry-db shell: bash {0} run: | set -o pipefail @@ -1432,6 +1438,30 @@ jobs: PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" fi + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + yagp_hooks_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating yagp_hooks_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" fi diff --git a/configure b/configure index e91414fb52c..a3b0bef7b6f 100755 --- a/configure +++ b/configure @@ -722,6 +722,7 @@ with_apr_config with_libcurl with_rt with_zstd +with_yagp_hooks_collector with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -943,6 +944,7 @@ with_zlib with_lz4 with_libbz2 with_zstd +with_yagp_hooks_collector with_rt with_libcurl with_apr_config @@ -11251,6 +11253,32 @@ $as_echo "yes" >&6; } fi fi +# +# yagp_hooks_collector +# + + + +# Check whether --with-yagp-hooks-collector was given. +if test "${with_yagp_hooks_collector+set}" = set; then : + withval=$with_yagp_hooks_collector; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-yagp-hooks-collector option" "$LINENO" 5 + ;; + esac + +else + with_yagp_hooks_collector=no + +fi + # # Realtime library # diff --git a/configure.ac b/configure.ac index 9a07159cecf..7e19e927868 100644 --- a/configure.ac +++ b/configure.ac @@ -1368,6 +1368,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# yagp_hooks_collector +# +PGAC_ARG_BOOL(with, yagp_hooks_collector, no, + [build with YAGP hooks collector extension]) +AC_SUBST(with_yagp_hooks_collector) + if test "$with_zstd" = yes; then dnl zstd_errors.h was renamed from error_public.h in v1.4.0 PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0]) diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 2d7ad04aed8..d30a0b794f0 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -53,6 +53,7 @@ # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) +# CONFIGURE_EXTRA_OPTS - Args to pass to configure command # ENABLE_DEBUG - Enable debug build options (true/false, defaults to # false) # @@ -177,7 +178,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-uuid=e2fs \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib \ + ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 log_section_end "Configure" # Capture version information diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index d251ddd3e1c..46e124df5e8 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -25,7 +26,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT @@ -54,7 +55,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -86,7 +87,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -129,7 +130,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT @@ -159,3 +160,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out index 5fd5ea5fb3e..3b1e3504923 100644 --- a/expected/yagp_dist.out +++ b/expected/yagp_dist.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -171,3 +172,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_select.out b/expected/yagp_select.out index b6e18dc862f..af08f2d1def 100644 --- a/expected/yagp_select.out +++ b/expected/yagp_select.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -132,3 +133,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_utf8_trim.out b/expected/yagp_utf8_trim.out index 194ee6b3609..9de126dd882 100644 --- a/expected/yagp_utf8_trim.out +++ b/expected/yagp_utf8_trim.out @@ -7,6 +7,7 @@ RETURNS TEXT AS $$ ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; -- Test 1: 1 byte chars SET yagpcc.max_text_size to 19; @@ -63,4 +64,5 @@ DROP FUNCTION get_marked_query(TEXT); RESET yagpcc.max_text_size; RESET yagpcc.logging_mode; RESET yagpcc.enable; +RESET yagpcc.ignored_users_list; DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 057f7d7a556..0a77859d8d4 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -26,7 +27,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT @@ -83,7 +84,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT @@ -113,26 +114,26 @@ BEGIN; ROLLBACK; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------+--------------------- - -1 | | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+-----------------------------------+--------------------- + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -153,7 +154,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT @@ -176,16 +177,16 @@ COPY (SELECT 1) TO STDOUT; DROP TABLE copy_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+-----------------------------------+--------------------- + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -203,7 +204,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT @@ -226,7 +227,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT @@ -244,3 +245,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 8d95a14f876..8b98dc9142c 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,6 +35,9 @@ else diskquota endif +ifeq "$(with_yagp_hooks_collector)" "yes" + recurse_targets += yagp_hooks_collector +endif ifeq "$(with_zstd)" "yes" recurse_targets += zstd endif diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile new file mode 100644 index 00000000000..be46eb7149c --- /dev/null +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -0,0 +1,41 @@ +MODULE_big = yagp_hooks_collector +EXTENSION = yagp_hooks_collector +DATA = $(wildcard *--*.sql) +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility + +PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service +PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) + +C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) +CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) +OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) + +override CXXFLAGS = -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ + -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ + -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ + -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD + +PG_CXXFLAGS += -Isrc -Iinclude +SHLIB_LINK += -lprotobuf -lpthread -lstdc++ +EXTRA_CLEAN = src/protos + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = gpcontrib/yagp_hooks_collector +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto + @mkdir -p src/protos + sed -i 's/optional //g' $^ + sed -i 's|cloud/mdb/yagpcc/api/proto/common/|protos/|g' $^ + protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ + mv src/protos/$*.pb.cc src/protos/$*.pb.cpp + +$(CPP_OBJS): src/protos/yagpcc_metrics.pb.h src/protos/yagpcc_plan.pb.h src/protos/yagpcc_set_service.pb.h +src/protos/yagpcc_set_service.pb.o: src/protos/yagpcc_metrics.pb.h diff --git a/sql/yagp_cursors.sql b/sql/yagp_cursors.sql index 5d5bde58110..f56351e0d43 100644 --- a/sql/yagp_cursors.sql +++ b/sql/yagp_cursors.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -81,3 +82,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_dist.sql b/sql/yagp_dist.sql index b837ef05335..d5519d0cd96 100644 --- a/sql/yagp_dist.sql +++ b/sql/yagp_dist.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -84,3 +85,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_select.sql b/sql/yagp_select.sql index 4038c6b7b63..90e972ae4c1 100644 --- a/sql/yagp_select.sql +++ b/sql/yagp_select.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -65,3 +66,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_utf8_trim.sql b/sql/yagp_utf8_trim.sql index c0fdcce24a5..c3053e4af0c 100644 --- a/sql/yagp_utf8_trim.sql +++ b/sql/yagp_utf8_trim.sql @@ -9,6 +9,7 @@ RETURNS TEXT AS $$ LIMIT 1 $$ LANGUAGE sql VOLATILE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; -- Test 1: 1 byte chars @@ -39,5 +40,6 @@ DROP FUNCTION get_marked_query(TEXT); RESET yagpcc.max_text_size; RESET yagpcc.logging_mode; RESET yagpcc.enable; +RESET yagpcc.ignored_users_list; DROP EXTENSION yagp_hooks_collector; diff --git a/sql/yagp_utility.sql b/sql/yagp_utility.sql index b4cca6f5421..cf9c1d253d0 100644 --- a/sql/yagp_utility.sql +++ b/sql/yagp_utility.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -131,3 +132,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 77b58e7aa76..d520faeaf85 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -271,6 +271,7 @@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ EVENT_LIBS = @EVENT_LIBS@ +with_yagp_hooks_collector = @with_yagp_hooks_collector@ ########################################################################## # From 0984ee47297c6fb57c9c918e080a03507b75d095 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 17:05:08 +0300 Subject: [PATCH 072/128] [yagp_hooks_collector] Add consistent GUC filtering and submit/done hook callsites Cache GUC values at SUBMIT so filtering criteria remain consistent across the full query lifecycle. Add query_info_collect_hook calls in ExecCreateTableAs, refresh_matview_datafill, and PortalCleanup. Correct tokens from gram.y. --- expected/yagp_cursors.out | 8 +- expected/yagp_guc_cache.out | 57 ++++++++++++ expected/yagp_utility.out | 72 +++++++-------- gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_guc_cache.sql | 43 +++++++++ src/Config.cpp | 90 +++++++++---------- src/Config.h | 49 +++++++--- src/EventSender.cpp | 68 ++++++++------ src/EventSender.h | 10 ++- src/PgUtils.cpp | 14 --- src/PgUtils.h | 3 - src/ProtoUtils.cpp | 28 +++--- src/ProtoUtils.h | 13 ++- src/UDSConnector.cpp | 5 +- src/UDSConnector.h | 6 +- src/backend/commands/createas.c | 8 +- src/backend/commands/matview.c | 5 ++ src/backend/commands/portalcmds.c | 5 ++ src/hook_wrappers.cpp | 2 +- src/log/LogOps.cpp | 6 +- .../pg_stat_statements_ya_parser.c | 14 +-- 21 files changed, 325 insertions(+), 183 deletions(-) create mode 100644 expected/yagp_guc_cache.out create mode 100644 sql/yagp_guc_cache.sql diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index 46e124df5e8..df12e3e1b66 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -26,7 +26,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT @@ -36,6 +35,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -55,7 +55,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -69,6 +68,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (14 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -87,7 +87,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -99,6 +98,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | ROLLBACK; | QUERY_STATUS_SUBMIT -1 | ROLLBACK; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (12 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -130,7 +130,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT @@ -148,6 +147,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; diff --git a/expected/yagp_guc_cache.out b/expected/yagp_guc_cache.out new file mode 100644 index 00000000000..3085cfa42e1 --- /dev/null +++ b/expected/yagp_guc_cache.out @@ -0,0 +1,57 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +SELECT yagpcc.truncate_log(); +-- end_ignore +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM yagpcc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.logging_mode TO 'TBL'; +-- SET below disables utility logging and DONE must still be logged. +SET yagpcc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_DONE +(2 rows) + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('yagpcc.ignored_users_list', current_user, false); + set_config +------------ + gpadmin +(1 row) + +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_START + QUERY_STATUS_END + QUERY_STATUS_DONE +(4 rows) + +DROP FUNCTION print_last_query(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +RESET yagpcc.logging_mode; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 0a77859d8d4..7df1d2816eb 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -27,7 +27,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT @@ -37,6 +36,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT -1 | DROP TABLE test_table; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -55,7 +55,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| @@ -67,6 +66,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -84,7 +84,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT @@ -94,6 +93,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -114,26 +114,26 @@ BEGIN; ROLLBACK; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+----------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -154,12 +154,12 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -177,16 +177,16 @@ COPY (SELECT 1) TO STDOUT; DROP TABLE copy_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -204,7 +204,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT @@ -212,6 +211,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -227,12 +227,12 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index be46eb7149c..79f5401c8d1 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_guc_cache.sql b/sql/yagp_guc_cache.sql new file mode 100644 index 00000000000..9e6de69d61e --- /dev/null +++ b/sql/yagp_guc_cache.sql @@ -0,0 +1,43 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +SELECT yagpcc.truncate_log(); +-- end_ignore + +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM yagpcc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; + +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.logging_mode TO 'TBL'; + +-- SET below disables utility logging and DONE must still be logged. +SET yagpcc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('yagpcc.ignored_users_list', current_user, false); +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); + +DROP FUNCTION print_last_query(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +RESET yagpcc.logging_mode; diff --git a/src/Config.cpp b/src/Config.cpp index dbd7e25b483..4fb58677018 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -27,45 +27,13 @@ static const struct config_enum_entry logging_mode_options[] = { {"tbl", LOG_MODE_TBL, false}, {NULL, 0, false}}; -static std::unique_ptr> ignored_users_set = - nullptr; static bool ignored_users_guc_dirty = false; -static void update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = - std::make_unique>(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { - /* syntax error in list */ - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); - } - ignored_users_set = std::move(new_ignored_users_set); -} - static void assign_ignored_users_hook(const char *, void *) { ignored_users_guc_dirty = true; } -void Config::init() { +void Config::init_gucs() { DefineCustomStringVariable( "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, @@ -128,22 +96,40 @@ void Config::init() { GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } -std::string Config::uds_path() { return guc_uds_path; } -bool Config::enable_analyze() { return guc_enable_analyze; } -bool Config::enable_cdbstats() { return guc_enable_cdbstats; } -bool Config::enable_collector() { return guc_enable_collector; } -bool Config::enable_utility() { return guc_enable_utility; } -bool Config::report_nested_queries() { return guc_report_nested_queries; } -size_t Config::max_text_size() { return guc_max_text_size; } -size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } -int Config::min_analyze_time() { return guc_min_analyze_time; }; -int Config::logging_mode() { return guc_logging_mode; } - -bool Config::filter_user(std::string username) { - if (!ignored_users_set) { +void Config::update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = std::make_unique(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + /* syntax error in list */ + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); + } + ignored_users_ = std::move(new_ignored_users_set); +} + +bool Config::filter_user(const std::string &username) const { + if (!ignored_users_) { return true; } - return ignored_users_set->find(username) != ignored_users_set->end(); + return ignored_users_->find(username) != ignored_users_->end(); } void Config::sync() { @@ -151,4 +137,14 @@ void Config::sync() { update_ignored_users(guc_ignored_users); ignored_users_guc_dirty = false; } + uds_path_ = guc_uds_path; + enable_analyze_ = guc_enable_analyze; + enable_cdbstats_ = guc_enable_cdbstats; + enable_collector_ = guc_enable_collector; + enable_utility_ = guc_enable_utility; + report_nested_queries_ = guc_report_nested_queries; + max_text_size_ = static_cast(guc_max_text_size); + max_plan_size_ = static_cast(guc_max_plan_size); + min_analyze_time_ = guc_min_analyze_time; + logging_mode_ = guc_logging_mode; } diff --git a/src/Config.h b/src/Config.h index 7501c727a44..b4a393b0383 100644 --- a/src/Config.h +++ b/src/Config.h @@ -1,23 +1,44 @@ #pragma once +#include #include +#include #define LOG_MODE_UDS 0 #define LOG_MODE_TBL 1 +using IgnoredUsers = std::unordered_set; + class Config { public: - static void init(); - static std::string uds_path(); - static bool enable_analyze(); - static bool enable_cdbstats(); - static bool enable_collector(); - static bool enable_utility(); - static bool filter_user(std::string username); - static bool report_nested_queries(); - static size_t max_text_size(); - static size_t max_plan_size(); - static int min_analyze_time(); - static int logging_mode(); - static void sync(); -}; \ No newline at end of file + static void init_gucs(); + + void sync(); + + const std::string &uds_path() const { return uds_path_; } + bool enable_analyze() const { return enable_analyze_; } + bool enable_cdbstats() const { return enable_cdbstats_; } + bool enable_collector() const { return enable_collector_; } + bool enable_utility() const { return enable_utility_; } + bool report_nested_queries() const { return report_nested_queries_; } + size_t max_text_size() const { return max_text_size_; } + size_t max_plan_size() const { return max_plan_size_ * 1024; } + int min_analyze_time() const { return min_analyze_time_; } + int logging_mode() const { return logging_mode_; } + bool filter_user(const std::string &username) const; + +private: + void update_ignored_users(const char *new_guc_ignored_users); + + std::unique_ptr ignored_users_; + std::string uds_path_; + bool enable_analyze_; + bool enable_cdbstats_; + bool enable_collector_; + bool enable_utility_; + bool report_nested_queries_; + size_t max_text_size_; + size_t max_plan_size_; + int min_analyze_time_; + int logging_mode_; +}; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index d638d275548..853a0c43fb9 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,4 +1,3 @@ -#include "Config.h" #include "UDSConnector.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" @@ -22,10 +21,8 @@ extern "C" { #include "ProtoUtils.h" #define need_collect_analyze() \ - (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ - Config::enable_analyze()) - -static bool enable_utility = Config::enable_utility(); + (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ + config.enable_analyze()) bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool utility) { @@ -38,16 +35,16 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, switch (state) { case QueryState::SUBMIT: - // Cache enable_utility at SUBMIT to ensure consistent behavior at DONE. - // Without caching, a query that sets enable_utility to false from true - // would be accepted at SUBMIT (guc is true) but rejected at DONE (guc - // is false), causing a leak. - enable_utility = Config::enable_utility(); - if (utility && enable_utility == false) { + // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent + // states. Without caching, a query that unsets/sets filtering GUCs would + // see different filter criteria at DONE, because at SUBMIT the query was + // not executed yet, causing DONE to be skipped/added. + config.sync(); + + if (utility && !config.enable_utility()) { return false; } - // Sync config in case current query changes it. - Config::sync(); + // Register qkey for a nested query we won't report, // so we can detect nesting_level > 0 and skip reporting at end/done. if (!need_report_nested_query() && nesting_level > 0) { @@ -65,7 +62,7 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, } break; case QueryState::DONE: - if (utility && enable_utility == false) { + if (utility && !config.enable_utility()) { return false; } default: @@ -85,9 +82,9 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, bool utility) { bool clear_big_fields = false; - switch (Config::logging_mode()) { + switch (config.logging_mode()) { case LOG_MODE_UDS: - clear_big_fields = UDSConnector::report_query(req, event); + clear_big_fields = UDSConnector::report_query(req, event, config); break; case LOG_MODE_TBL: ya_gpdb::insert_log(req, utility); @@ -135,12 +132,12 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { return; } - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; - if (Config::enable_cdbstats()) { + if (config.enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; if (!query_desc->showstatctx) { instr_time starttime; @@ -161,7 +158,7 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { auto query_msg = query.message.get(); *query_msg->mutable_start_time() = current_ts(); update_query_state(query, QueryState::START, false /* utility */); - set_query_plan(query_msg, query_desc); + set_query_plan(query_msg, query_desc, config); if (need_collect_analyze()) { // Set up to track total elapsed time during query run. // Make sure the space is allocated in the per-query @@ -214,7 +211,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { set_query_info(query_msg); set_qi_nesting_level(query_msg, nesting_level); set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc); + set_query_text(query_msg, query_desc, config); if (log_query_req(*query_msg, "submit", utility)) { clear_big_fields(query_msg); } @@ -271,8 +268,8 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, ereport(DEBUG3, (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); } else { - set_qi_error_message(query_msg, - error_flushed ? edata->message : elog_message()); + set_qi_error_message( + query_msg, error_flushed ? edata->message : elog_message(), config); } } if (prev_state == START) { @@ -331,8 +328,8 @@ void EventSender::ic_metrics_collect() { if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { return; } - if (!proto_verified || gp_command_count == 0 || !Config::enable_collector() || - Config::filter_user(get_user_name())) { + if (!proto_verified || gp_command_count == 0 || !config.enable_collector() || + config.filter_user(get_user_name())) { return; } // we also would like to know nesting level here and filter queries BUT we @@ -374,15 +371,18 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { ya_gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; - if (ms >= Config::min_analyze_time()) { + if (ms >= config.min_analyze_time()) { auto &query = get_query(query_desc); auto *query_msg = query.message.get(); - set_analyze_plan_text(query_desc, query_msg); + set_analyze_plan_text(query_desc, query_msg, config); } } EventSender::EventSender() { - if (Config::enable_collector()) { + // Perform initial sync to get default GUC values + config.sync(); + + if (config.enable_collector()) { try { GOOGLE_PROTOBUF_VERIFY_VERSION; proto_verified = true; @@ -486,5 +486,19 @@ bool EventSender::qdesc_submitted(QueryDesc *query_desc) { return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); } +bool EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); +} + +bool EventSender::need_report_nested_query() { + return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; +} + +bool EventSender::filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !config.enable_collector() || config.filter_user(get_user_name()); +} + EventSender::QueryItem::QueryItem(QueryState st) : message(std::make_unique()), state(st) {} diff --git a/src/EventSender.h b/src/EventSender.h index 6e195eeacdf..e9acb04422b 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -14,6 +14,7 @@ extern "C" { #undef typeid #include "memory/gpdbwrappers.h" +#include "Config.h" class UDSConnector; struct QueryDesc; @@ -108,8 +109,8 @@ class EventSender { explicit QueryItem(QueryState st); }; - static bool log_query_req(const yagpcc::SetQueryReq &req, - const std::string &event, bool utility); + bool log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, + bool utility); bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); void update_query_state(QueryItem &query, QueryState new_state, bool utility, bool success = true); @@ -123,6 +124,9 @@ class EventSender { QueryMetricsStatus status, ErrorData *edata = NULL); void update_nested_counters(QueryDesc *query_desc); bool qdesc_submitted(QueryDesc *query_desc); + bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); + bool need_report_nested_query(); + bool filter_query(QueryDesc *query_desc); bool proto_verified = false; int nesting_level = 0; @@ -132,4 +136,6 @@ class EventSender { ICStatistics ic_statistics; #endif std::unordered_map queries; + + Config config; }; \ No newline at end of file diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 96f46429643..7e53abdabbf 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -65,17 +65,3 @@ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { } return query_desc->yagp_query_key->nesting_level == 0; } - -bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); -} - -bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; -} - -bool filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !Config::enable_collector() || Config::filter_user(get_user_name()); -} diff --git a/src/PgUtils.h b/src/PgUtils.h index 02f084c597a..e9715ce10f4 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -9,6 +9,3 @@ std::string get_user_name(); std::string get_db_name(); std::string get_rg_name(); bool is_top_level_query(QueryDesc *query_desc, int nesting_level); -bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); -bool need_report_nested_query(); -bool filter_query(QueryDesc *query_desc); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index aa8632477f5..8ebbe19e289 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -82,7 +82,8 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { return std::string(str, cut_pos); } -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER @@ -93,10 +94,10 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, - Config::max_plan_size()); + config.max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, Config::max_plan_size()); + norm_plan->data, norm_plan->len, config.max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); @@ -107,15 +108,16 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } } -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), - Config::max_text_size()); + config.max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), Config::max_text_size()); + norm_query, strlen(norm_query), config.max_text_size()); } } @@ -150,10 +152,11 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg) { +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, + const Config &config) { auto aqi = req->mutable_add_info(); *aqi->mutable_error_message() = - trim_str_shrink_utf8(err_msg, strlen(err_msg), Config::max_text_size()); + trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -257,7 +260,8 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, + const Config &config) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. if (!IsTransactionState() || !query_desc->plannedstmt) { @@ -266,15 +270,15 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); ExplainState es = ya_gpdb::get_analyze_state( - query_desc, query_desc->instrument_options && Config::enable_analyze()); + query_desc, query_desc->instrument_options && config.enable_analyze()); ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { es.str->data[--es.str->len] = '\0'; } - auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, - Config::max_plan_size()); + auto trimmed_analyze = + trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); ya_gpdb::pfree(es.str->data); } diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 725a634f765..37b7e4a8a29 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -4,19 +4,24 @@ struct QueryDesc; struct ICStatistics; +class Config; google::protobuf::Timestamp current_ts(); -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config); void clear_big_fields(yagpcc::SetQueryReq *req); void set_query_info(yagpcc::SetQueryReq *req); void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg); +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, + const Config &config); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message); \ No newline at end of file +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message, + const Config &config); diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index a7eaed539f7..74fd57a3ac0 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -25,10 +25,11 @@ static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, } bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, - const std::string &event) { + const std::string &event, + const Config &config) { sockaddr_un address; address.sun_family = AF_UNIX; - std::string uds_path = Config::uds_path(); + const std::string &uds_path = config.uds_path(); if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); YagpStat::report_error(); diff --git a/src/UDSConnector.h b/src/UDSConnector.h index f0dfcb77a3f..9483407159d 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -2,8 +2,10 @@ #include "protos/yagpcc_set_service.pb.h" +class Config; + class UDSConnector { public: bool static report_query(const yagpcc::SetQueryReq &req, - const std::string &event); -}; \ No newline at end of file + const std::string &event, const Config &config); +}; diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 6822032fe0d..a3d2f155fd8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -478,10 +478,6 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, dest, params, queryEnv, 0); } - /* GPDB hook for collecting query info */ - if (query_info_collect_hook) - (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); - if (into->skipData) { /* @@ -495,6 +491,10 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, } else { + /* GPDB hook for collecting query info */ + if (query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); + check_and_unassign_from_resgroup(queryDesc->plannedstmt); queryDesc->plannedstmt->query_mem = ResourceManagerGetQueryMemoryLimit(queryDesc->plannedstmt); diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 1555ea9d334..dc8efd4d892 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -63,6 +63,7 @@ #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/metrics_utils.h" #include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -842,6 +843,10 @@ refresh_matview_datafill(DestReceiver *dest, Query *query, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); + /* GPDB hook for collecting query info */ + if (query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); + RestoreOidAssignments(saved_dispatch_oids); /* call ExecutorStart to prepare the plan for execution */ diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 4817c14f07d..553830e8599 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -35,6 +35,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" +#include "utils/metrics_utils.h" #include "utils/snapmgr.h" #include "cdb/cdbendpoint.h" @@ -373,6 +374,10 @@ PortalCleanup(Portal portal) FreeQueryDesc(queryDesc); CurrentResourceOwner = saveResourceOwner; + } else { + /* GPDB hook for collecting query info */ + if (queryDesc->yagp_query_key && query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); } } diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 56c1da9f4f6..8cf74641c29 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -71,7 +71,7 @@ R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { } void hooks_init() { - Config::init(); + Config::init_gucs(); YagpStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index cec9e33693a..56bdf1dca62 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -38,9 +38,9 @@ void init_log() { log_relname.data() /* relname */, namespaceId /* namespace */, 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, - DescribeTuple() /* rel tuple */, NIL, RELKIND_RELATION, - RELPERSISTENCE_PERMANENT, false, false, ONCOMMIT_NOOP, - NULL /* GP Policy */, (Datum)0, false /* use_user_acl */, true, true, + DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, RELKIND_RELATION, + RELPERSISTENCE_PERMANENT, false /* shared_relation */, false /* mapped_relation */, ONCOMMIT_NOOP, + NULL /* GP Policy */, (Datum)0 /* reloptions */, false /* use_user_acl */, true /* allow_system_table_mods */, true /* is_internal */, InvalidOid /* relrewrite */, NULL /* typaddress */, false /* valid_opts */); diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index c19805ce506..54c8b2cf59f 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -17,20 +17,20 @@ #include "pg_stat_statements_ya_parser.h" -#ifndef ICONST -#define ICONST 276 -#endif #ifndef FCONST -#define FCONST 277 +#define FCONST 260 #endif #ifndef SCONST -#define SCONST 278 +#define SCONST 261 #endif #ifndef BCONST -#define BCONST 279 +#define BCONST 263 #endif #ifndef XCONST -#define XCONST 280 +#define XCONST 264 +#endif +#ifndef ICONST +#define ICONST 266 #endif static void fill_in_constant_lengths(JumbleState *jstate, const char *query); From d7cc4fc9a0c0ac94c6ab499435b48ddbd1207eae Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 20 Jan 2026 17:03:53 +0300 Subject: [PATCH 073/128] [yagp_hooks_collector] Add UDS round-trip test and fix send() accounting Add regression test for UDS transport. Fix send() return value: do not add -1 to total_bytes_sent on error. General refactoring. --- expected/yagp_uds.out | 42 +++++++++ gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_uds.sql | 31 +++++++ src/Config.cpp | 10 +- src/Config.h | 8 +- src/UDSConnector.cpp | 117 +++++++++++++----------- src/hook_wrappers.cpp | 96 ++++++++++++++++++- src/hook_wrappers.h | 4 + src/yagp_hooks_collector.c | 64 ++++++++++++- yagp_hooks_collector--1.1.sql | 15 +++ 10 files changed, 318 insertions(+), 71 deletions(-) create mode 100644 expected/yagp_uds.out create mode 100644 sql/yagp_uds.sql diff --git a/expected/yagp_uds.out b/expected/yagp_uds.out new file mode 100644 index 00000000000..d04929ffb4a --- /dev/null +++ b/expected/yagp_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +-- end_ignore +\set UDS_PATH '/tmp/yagpcc_test.sock' +-- Configure extension to send via UDS +SET yagpcc.uds_path TO :'UDS_PATH'; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.logging_mode TO 'UDS'; +-- Start receiver +SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT yagpcc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT yagpcc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.uds_path; +RESET yagpcc.ignored_users_list; +RESET yagpcc.enable; +RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index 79f5401c8d1..eb6541b7687 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_uds.sql b/sql/yagp_uds.sql new file mode 100644 index 00000000000..3eef697a4e7 --- /dev/null +++ b/sql/yagp_uds.sql @@ -0,0 +1,31 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +-- end_ignore + +\set UDS_PATH '/tmp/yagpcc_test.sock' + +-- Configure extension to send via UDS +SET yagpcc.uds_path TO :'UDS_PATH'; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.logging_mode TO 'UDS'; + +-- Start receiver +SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); + +-- Send +SELECT 1; + +-- Receive +SELECT yagpcc.__test_uds_receive() > 0 as received; + +-- Stop receiver +SELECT yagpcc.__test_uds_stop_server(); + +-- Cleanup +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.uds_path; +RESET yagpcc.ignored_users_list; +RESET yagpcc.enable; +RESET yagpcc.logging_mode; diff --git a/src/Config.cpp b/src/Config.cpp index 4fb58677018..2c2032ebb03 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,9 +16,9 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1 << 20; // in bytes (1MB) -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = 10000; // in ms +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms static int guc_logging_mode = LOG_MODE_UDS; static bool guc_enable_utility = false; @@ -143,8 +143,8 @@ void Config::sync() { enable_collector_ = guc_enable_collector; enable_utility_ = guc_enable_utility; report_nested_queries_ = guc_report_nested_queries; - max_text_size_ = static_cast(guc_max_text_size); - max_plan_size_ = static_cast(guc_max_plan_size); + max_text_size_ = guc_max_text_size; + max_plan_size_ = guc_max_plan_size; min_analyze_time_ = guc_min_analyze_time; logging_mode_ = guc_logging_mode; } diff --git a/src/Config.h b/src/Config.h index b4a393b0383..aa6b5bdc0ba 100644 --- a/src/Config.h +++ b/src/Config.h @@ -21,8 +21,8 @@ class Config { bool enable_collector() const { return enable_collector_; } bool enable_utility() const { return enable_utility_; } bool report_nested_queries() const { return report_nested_queries_; } - size_t max_text_size() const { return max_text_size_; } - size_t max_plan_size() const { return max_plan_size_ * 1024; } + int max_text_size() const { return max_text_size_; } + int max_plan_size() const { return max_plan_size_ * 1024; } int min_analyze_time() const { return min_analyze_time_; } int logging_mode() const { return logging_mode_; } bool filter_user(const std::string &username) const; @@ -37,8 +37,8 @@ class Config { bool enable_collector_; bool enable_utility_; bool report_nested_queries_; - size_t max_text_size_; - size_t max_plan_size_; + int max_text_size_; + int max_plan_size_; int min_analyze_time_; int logging_mode_; }; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 74fd57a3ac0..ea118fca783 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -27,66 +27,77 @@ static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event, const Config &config) { - sockaddr_un address; + sockaddr_un address{}; address.sun_family = AF_UNIX; - const std::string &uds_path = config.uds_path(); + const auto &uds_path = config.uds_path(); + if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); YagpStat::report_error(); return false; } strcpy(address.sun_path, uds_path.c_str()); - bool success = true; - auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd != -1) { - if (fcntl(sockfd, F_SETFL, O_NONBLOCK) != -1) { - if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { - auto data_size = req.ByteSize(); - auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)ya_gpdb::palloc(total_size); - uint32_t *size_payload = (uint32_t *)buf; - *size_payload = data_size; - req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); - int64_t sent = 0, sent_total = 0; - do { - sent = send(sockfd, buf + sent_total, total_size - sent_total, - MSG_DONTWAIT); - sent_total += sent; - } while ( - sent > 0 && size_t(sent_total) != total_size && - // the line below is a small throttling hack: - // if a message does not fit a single packet, we take a nap - // before sending the next one. - // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); - if (sent < 0) { - log_tracing_failure(req, event); - success = false; - YagpStat::report_bad_send(total_size); - } else { - YagpStat::report_send(total_size); - } - ya_gpdb::pfree(buf); - } else { - // log the error and go on - log_tracing_failure(req, event); - success = false; - YagpStat::report_bad_connection(); - } - } else { - // That's a very important error that should never happen, so make it - // visible to an end-user and admins. - ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %m"))); - success = false; - YagpStat::report_error(); - } - close(sockfd); - } else { - // log the error and go on + + const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) { log_tracing_failure(req, event); - success = false; YagpStat::report_error(); + return false; } - return success; -} \ No newline at end of file + + // Close socket automatically on error path. + struct SockGuard { + int fd; + ~SockGuard() { close(fd); } + } sock_guard{sockfd}; + + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %m"))); + YagpStat::report_error(); + return false; + } + + if (connect(sockfd, reinterpret_cast(&address), + sizeof(address)) == -1) { + log_tracing_failure(req, event); + YagpStat::report_bad_connection(); + return false; + } + + const auto data_size = req.ByteSize(); + const auto total_size = data_size + sizeof(uint32_t); + auto *buf = static_cast(ya_gpdb::palloc(total_size)); + // Free buf automatically on error path. + struct BufGuard { + void *p; + ~BufGuard() { ya_gpdb::pfree(p); } + } buf_guard{buf}; + + *reinterpret_cast(buf) = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + + int64_t sent = 0, sent_total = 0; + do { + sent = + send(sockfd, buf + sent_total, total_size - sent_total, MSG_DONTWAIT); + if (sent > 0) + sent_total += sent; + } while (sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + + if (sent < 0) { + log_tracing_failure(req, event); + YagpStat::report_bad_send(total_size); + return false; + } + + YagpStat::report_send(total_size); + return true; +} diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 8cf74641c29..602a2470805 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -11,6 +11,12 @@ extern "C" { #include "cdb/ml_ipc.h" #include "tcop/utility.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" + +#include +#include +#include +#include +#include } #undef typeid @@ -52,6 +58,13 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); +#define TEST_MAX_CONNECTIONS 4 +#define TEST_RCV_BUF_SIZE 8192 +#define TEST_POLL_TIMEOUT_MS 200 + +static int test_server_fd = -1; +static char *test_sock_path = NULL; + static EventSender *sender = nullptr; static inline EventSender *get_sender() { @@ -226,8 +239,9 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, } get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_DONE, - (void *)query_desc, true /* utility */, (ErrorData *)NULL); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_DONE, (void *)query_desc, true /* utility */, + (ErrorData *)NULL); pfree(query_desc); } @@ -242,8 +256,9 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, MemoryContextSwitchTo(oldctx); get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_ERROR, - (void *)query_desc, true /* utility */, edata); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_ERROR, (void *)query_desc, true /* utility */, + edata); pfree(query_desc); ReThrowError(edata); @@ -294,4 +309,77 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); +} + +void test_uds_stop_server() { + if (test_server_fd >= 0) { + close(test_server_fd); + test_server_fd = -1; + } + if (test_sock_path) { + unlink(test_sock_path); + pfree(test_sock_path); + test_sock_path = NULL; + } +} + +void test_uds_start_server(const char *path) { + struct sockaddr_un addr = {.sun_family = AF_UNIX}; + + if (strlen(path) >= sizeof(addr.sun_path)) + ereport(ERROR, (errmsg("path too long"))); + + test_uds_stop_server(); + + strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); + test_sock_path = MemoryContextStrdup(TopMemoryContext, path); + unlink(path); + + if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || + bind(test_server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0 || + listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) { + test_uds_stop_server(); + ereport(ERROR, (errmsg("socket setup failed: %m"))); + } +} + +int64 test_uds_receive(int timeout_ms) { + char buf[TEST_RCV_BUF_SIZE]; + int rc; + struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; + int64 total = 0; + + if (test_server_fd < 0) + ereport(ERROR, (errmsg("server not started"))); + + for (;;) { + CHECK_FOR_INTERRUPTS(); + rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); + if (rc > 0) + break; + if (rc < 0 && errno != EINTR) + ereport(ERROR, (errmsg("poll: %m"))); + timeout_ms -= TEST_POLL_TIMEOUT_MS; + if (timeout_ms <= 0) + return total; + } + + if (pfd.revents & POLLIN) { + int client = accept(test_server_fd, NULL, NULL); + ssize_t n; + + if (client < 0) + ereport(ERROR, (errmsg("accept: %m"))); + + while ((n = recv(client, buf, sizeof(buf), 0)) != 0) { + if (n > 0) + total += n; + else if (errno != EINTR) + break; + } + + close(client); + } + + return total; } \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index cfabf39485e..236c6eb9d79 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -12,6 +12,10 @@ extern Datum yagp_functions_get(FunctionCallInfo fcinfo); extern void init_log(); extern void truncate_log(); +extern void test_uds_start_server(const char *path); +extern int64_t test_uds_receive(int timeout_ms); +extern void test_uds_stop_server(); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 27fd0e04b26..f7863a38921 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -14,16 +14,18 @@ PG_FUNCTION_INFO_V1(yagp_stat_messages); PG_FUNCTION_INFO_V1(yagp_init_log); PG_FUNCTION_INFO_V1(yagp_truncate_log); +PG_FUNCTION_INFO_V1(yagp_test_uds_start_server); +PG_FUNCTION_INFO_V1(yagp_test_uds_receive); +PG_FUNCTION_INFO_V1(yagp_test_uds_stop_server); + void _PG_init(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_init(); - } } void _PG_fini(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_deinit(); - } } Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { @@ -65,3 +67,57 @@ Datum yagp_truncate_log(PG_FUNCTION_ARGS) { funcctx = SRF_PERCALL_SETUP(); SRF_RETURN_DONE(funcctx); } + +Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + test_uds_start_server(path); + pfree(path); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); +} + +Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + int64 *result; + + if (SRF_IS_FIRSTCALL()) { + MemoryContext oldcontext; + + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + result = (int64 *)palloc(sizeof(int64)); + funcctx->user_fctx = result; + funcctx->max_calls = 1; + MemoryContextSwitchTo(oldcontext); + + int timeout_ms = PG_GETARG_INT32(0); + *result = test_uds_receive(timeout_ms); + } + + funcctx = SRF_PERCALL_SETUP(); + + if (funcctx->call_cntr < funcctx->max_calls) { + result = (int64 *)funcctx->user_fctx; + SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); + } + + SRF_RETURN_DONE(funcctx); +} + +Datum yagp_test_uds_stop_server(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + test_uds_stop_server(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); +} diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql index e0e94b51493..83bfb553638 100644 --- a/yagp_hooks_collector--1.1.sql +++ b/yagp_hooks_collector--1.1.sql @@ -93,3 +93,18 @@ BEGIN PERFORM yagpcc.__truncate_log_on_segments(); END; $$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION yagpcc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'yagp_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'yagp_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'yagp_test_uds_stop_server' +LANGUAGE C EXECUTE ON MASTER; From 2a83ca01f1ccf24189503c8c9983b30617f3d4e8 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 21 Jan 2026 12:53:47 +0000 Subject: [PATCH 074/128] [yagp_hooks_collector] Fix locale-dependent normalization crash Make gen_normquery() and gen_normplan() noexcept. Wide-character conversion can fail for locales that cannot handle the input charset. --- expected/yagp_locale.out | 23 ++++++++++++++++++++ gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_locale.sql | 29 +++++++++++++++++++++++++ src/ProtoUtils.cpp | 19 ++++++++++------ src/memory/gpdbwrappers.cpp | 11 ++++------ src/memory/gpdbwrappers.h | 4 ++-- 6 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 expected/yagp_locale.out create mode 100644 sql/yagp_locale.sql diff --git a/expected/yagp_locale.out b/expected/yagp_locale.out new file mode 100644 index 00000000000..6689b6a4ed3 --- /dev/null +++ b/expected/yagp_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS yagp_test_locale; +-- end_ignore +CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c yagp_test_locale +CREATE EXTENSION yagp_hooks_collector; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.enable TO TRUE; +CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +DROP TABLE yagp_hi_안녕세계; +DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index eb6541b7687..d145ae46dbe 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds yagp_locale PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_locale.sql b/sql/yagp_locale.sql new file mode 100644 index 00000000000..65d867d1680 --- /dev/null +++ b/sql/yagp_locale.sql @@ -0,0 +1,29 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. + +-- start_ignore +DROP DATABASE IF EXISTS yagp_test_locale; +-- end_ignore + +CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c yagp_test_locale + +CREATE EXTENSION yagp_hooks_collector; + +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.enable TO TRUE; + +CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; + +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +DROP TABLE yagp_hi_안녕세계; +DROP EXTENSION yagp_hooks_collector; diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 8ebbe19e289..f9119ca4b14 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -96,13 +96,15 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, config.max_plan_size()); - qi->set_plan_id( - hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + if (norm_plan) { + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( + norm_plan->data, norm_plan->len, config.max_plan_size()); + qi->set_plan_id( + hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + ya_gpdb::pfree(norm_plan->data); + } qi->set_query_id(query_desc->plannedstmt->queryId); ya_gpdb::pfree(es.str->data); - ya_gpdb::pfree(norm_plan->data); } ya_gpdb::mem_ctx_switch_to(oldcxt); } @@ -116,8 +118,11 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, query_desc->sourceText, strlen(query_desc->sourceText), config.max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), config.max_text_size()); + if (norm_query) { + *qi->mutable_template_query_text() = trim_str_shrink_utf8( + norm_query, strlen(norm_query), config.max_text_size()); + ya_gpdb::pfree(norm_query); + } } } diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 763e32e539c..8cc483a39de 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -204,15 +204,12 @@ void ya_gpdb::instr_end_loop(Instrumentation *instr) { wrap_throw(::InstrEndLoop, instr); } -char *ya_gpdb::gen_normquery(const char *query) { - return wrap_throw(::gen_normquery, query); +char *ya_gpdb::gen_normquery(const char *query) noexcept { + return wrap_noexcept(::gen_normquery, query); } -StringInfo ya_gpdb::gen_normplan(const char *exec_plan) { - if (!exec_plan) - throw std::runtime_error("Invalid execution plan string"); - - return wrap_throw(::gen_normplan, exec_plan); +StringInfo ya_gpdb::gen_normplan(const char *exec_plan) noexcept { + return wrap_noexcept(::gen_normplan, exec_plan); } char *ya_gpdb::get_rg_name_for_id(Oid group_id) { diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 920fc1ae6e7..e080ef5cdd4 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -38,8 +38,8 @@ HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime); void instr_end_loop(Instrumentation *instr); -char *gen_normquery(const char *query); -StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query) noexcept; +StringInfo gen_normplan(const char *executionPlan) noexcept; char *get_rg_name_for_id(Oid group_id); void insert_log(const yagpcc::SetQueryReq &req, bool utility); From 643a68142c2c8b85dd009f34f8bccc573b1083b7 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 21 Jan 2026 13:52:56 +0000 Subject: [PATCH 075/128] [yagp_hooks_collector] Add Apache license headers and enable -Werror --- gpcontrib/yagp_hooks_collector/Makefile | 2 +- pom.xml | 6 ++++ src/Config.cpp | 27 +++++++++++++++ src/Config.h | 27 +++++++++++++++ src/EventSender.cpp | 27 +++++++++++++++ src/EventSender.h | 27 +++++++++++++++ src/PgUtils.cpp | 27 +++++++++++++++ src/PgUtils.h | 27 +++++++++++++++ src/ProcStats.cpp | 27 +++++++++++++++ src/ProcStats.h | 27 +++++++++++++++ src/ProtoUtils.cpp | 27 +++++++++++++++ src/ProtoUtils.h | 27 +++++++++++++++ src/UDSConnector.cpp | 29 +++++++++++++++- src/UDSConnector.h | 27 +++++++++++++++ src/YagpStat.cpp | 27 +++++++++++++++ src/YagpStat.h | 27 +++++++++++++++ src/hook_wrappers.cpp | 33 +++++++++++++++++-- src/hook_wrappers.h | 27 +++++++++++++++ src/log/LogOps.cpp | 27 +++++++++++++++ src/log/LogOps.h | 27 +++++++++++++++ src/log/LogSchema.cpp | 27 +++++++++++++++ src/log/LogSchema.h | 27 +++++++++++++++ src/memory/gpdbwrappers.cpp | 27 +++++++++++++++ src/memory/gpdbwrappers.h | 27 +++++++++++++++ src/stat_statements_parser/README.md | 1 + .../pg_stat_statements_ya_parser.c | 27 +++++++++++++++ .../pg_stat_statements_ya_parser.h | 27 +++++++++++++++ src/yagp_hooks_collector.c | 27 +++++++++++++++ 28 files changed, 688 insertions(+), 4 deletions(-) create mode 100644 src/stat_statements_parser/README.md diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index d145ae46dbe..49825c55f35 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -10,7 +10,7 @@ C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) -override CXXFLAGS = -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ +override CXXFLAGS = -Werror -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD diff --git a/pom.xml b/pom.xml index 5d27326e4b6..2af9aa19a35 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,12 @@ code or new licensing patterns. gpcontrib/gp_exttable_fdw/gp_exttable_fdw.control gpcontrib/diskquota/** + gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control + gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto + gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto + gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto + gpcontrib/yagp_hooks_collector/.clang-format + gpcontrib/yagp_hooks_collector/Makefile getversion .git-blame-ignore-revs diff --git a/src/Config.cpp b/src/Config.cpp index 2c2032ebb03..62c16e91d1f 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * Config.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/Config.cpp + * + *------------------------------------------------------------------------- + */ + #include "Config.h" #include "memory/gpdbwrappers.h" #include diff --git a/src/Config.h b/src/Config.h index aa6b5bdc0ba..01ae5ea328e 100644 --- a/src/Config.h +++ b/src/Config.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * Config.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/Config.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 853a0c43fb9..f1cc0cc6ea1 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * EventSender.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/EventSender.cpp + * + *------------------------------------------------------------------------- + */ + #include "UDSConnector.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" diff --git a/src/EventSender.h b/src/EventSender.h index e9acb04422b..ef7dcb0bf8c 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * EventSender.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/EventSender.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 7e53abdabbf..ed4bf4d7e64 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PgUtils.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/PgUtils.cpp + * + *------------------------------------------------------------------------- + */ + #include "PgUtils.h" #include "Config.h" #include "memory/gpdbwrappers.h" diff --git a/src/PgUtils.h b/src/PgUtils.h index e9715ce10f4..5113fadbff2 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PgUtils.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/PgUtils.h + * + *------------------------------------------------------------------------- + */ + extern "C" { #include "postgres.h" #include "commands/explain.h" diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 5c09fa0bce4..72a12e8ca00 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProcStats.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProcStats.cpp + * + *------------------------------------------------------------------------- + */ + #include "ProcStats.h" #include "yagpcc_metrics.pb.h" #include diff --git a/src/ProcStats.h b/src/ProcStats.h index 30a90a60519..7629edd0aea 100644 --- a/src/ProcStats.h +++ b/src/ProcStats.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProcStats.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProcStats.h + * + *------------------------------------------------------------------------- + */ + #pragma once namespace yagpcc { diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index f9119ca4b14..b449ae20900 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProtoUtils.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp + * + *------------------------------------------------------------------------- + */ + #include "ProtoUtils.h" #include "PgUtils.h" #include "ProcStats.h" diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 37b7e4a8a29..c954545494f 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProtoUtils.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProtoUtils.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include "protos/yagpcc_set_service.pb.h" diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index ea118fca783..d13a82a5ca9 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * UDSConnector.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp + * + *------------------------------------------------------------------------- + */ + #include "UDSConnector.h" #include "Config.h" #include "YagpStat.h" @@ -67,7 +94,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, return false; } - const auto data_size = req.ByteSize(); + const auto data_size = req.ByteSizeLong(); const auto total_size = data_size + sizeof(uint32_t); auto *buf = static_cast(ya_gpdb::palloc(total_size)); // Free buf automatically on error path. diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 9483407159d..be5ab1ef413 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * UDSConnector.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/UDSConnector.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include "protos/yagpcc_set_service.pb.h" diff --git a/src/YagpStat.cpp b/src/YagpStat.cpp index 879cde85212..3a760b6ea97 100644 --- a/src/YagpStat.cpp +++ b/src/YagpStat.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * YagpStat.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/YagpStat.cpp + * + *------------------------------------------------------------------------- + */ + #include "YagpStat.h" #include diff --git a/src/YagpStat.h b/src/YagpStat.h index 110b1fdcbb1..57fc90cd4d1 100644 --- a/src/YagpStat.h +++ b/src/YagpStat.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * YagpStat.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/YagpStat.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 602a2470805..cb4970d60d9 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * hook_wrappers.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp + * + *------------------------------------------------------------------------- + */ + #define typeid __typeid extern "C" { #include "postgres.h" @@ -46,8 +73,10 @@ static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +#ifdef IC_TEARDOWN_HOOK static void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); +#endif #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif @@ -195,14 +224,14 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } } +#ifdef IC_TEARDOWN_HOOK void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { cpp_call(get_sender(), &EventSender::ic_metrics_collect); -#ifdef IC_TEARDOWN_HOOK if (previous_ic_teardown_hook) { (*previous_ic_teardown_hook)(transportStates, hasErrors); } -#endif } +#endif #ifdef ANALYZE_STATS_COLLECT_HOOK void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index 236c6eb9d79..443406a5259 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * hook_wrappers.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/hook_wrappers.h + * + *------------------------------------------------------------------------- + */ + #pragma once #ifdef __cplusplus diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index 56bdf1dca62..e8c927ece84 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogOps.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp + * + *------------------------------------------------------------------------- + */ + #include "protos/yagpcc_set_service.pb.h" #include "LogOps.h" diff --git a/src/log/LogOps.h b/src/log/LogOps.h index bad03d09a8f..1fc30c21030 100644 --- a/src/log/LogOps.h +++ b/src/log/LogOps.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogOps.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogOps.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp index 2fadcc46599..a391b1a2209 100644 --- a/src/log/LogSchema.cpp +++ b/src/log/LogSchema.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogSchema.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp + * + *------------------------------------------------------------------------- + */ + #include "google/protobuf/reflection.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/timestamp.pb.h" diff --git a/src/log/LogSchema.h b/src/log/LogSchema.h index f713c1e9b0e..f78acec7ce9 100644 --- a/src/log/LogSchema.h +++ b/src/log/LogSchema.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogSchema.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogSchema.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 8cc483a39de..22083e8bdaf 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gpdbwrappers.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp + * + *------------------------------------------------------------------------- + */ + #include "gpdbwrappers.h" #include "log/LogOps.h" diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index e080ef5cdd4..fe9b3ba0487 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gpdbwrappers.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h + * + *------------------------------------------------------------------------- + */ + #pragma once extern "C" { diff --git a/src/stat_statements_parser/README.md b/src/stat_statements_parser/README.md new file mode 100644 index 00000000000..291e31a3099 --- /dev/null +++ b/src/stat_statements_parser/README.md @@ -0,0 +1 @@ +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 54c8b2cf59f..7404208055f 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_stat_statements_ya_parser.c + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * + *------------------------------------------------------------------------- + */ + // NOTE: this file is just a bunch of code borrowed from pg_stat_statements for PG 9.4 // and from our own inhouse implementation of pg_stat_statements for managed PG diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index b08e8533992..96c6a776dba 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_stat_statements_ya_parser.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * + *------------------------------------------------------------------------- + */ + #pragma once #ifdef __cplusplus diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index f7863a38921..271bceee178 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * yagp_hooks_collector.c + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c + * + *------------------------------------------------------------------------- + */ + #include "postgres.h" #include "cdb/cdbvars.h" #include "funcapi.h" From 5bc75ae8e8d47dd50dd33634b4337cab01baf998 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:41:58 +0300 Subject: [PATCH 076/128] [yagp_hooks_collector] Fix null ErrorData dereference on segments Guard against NULL ErrorData in set_qi_error_message(). For some query types ErrorData can be NULL despite an error occurring. --- src/EventSender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index f1cc0cc6ea1..6993814ffbf 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -290,7 +290,7 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { bool error_flushed = elog_message() == NULL; - if (error_flushed && edata->message == NULL) { + if (error_flushed && (edata == NULL || edata->message == NULL)) { ereport(WARNING, (errmsg("YAGPCC missing error message"))); ereport(DEBUG3, (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); From aa20a4df16b0c92d8957299d7bcaa8c3e1d4f38b Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 18 Mar 2026 15:17:52 +0000 Subject: [PATCH 077/128] [yagp_hooks_collector] Move to gpcontrib directory --- .../yagp_hooks_collector/.clang-format | 0 gpcontrib/yagp_hooks_collector/README.md | 28 +++++++++++++++++++ .../expected}/yagp_cursors.out | 0 .../expected}/yagp_dist.out | 0 .../expected}/yagp_guc_cache.out | 0 .../expected}/yagp_locale.out | 0 .../expected}/yagp_select.out | 0 .../expected}/yagp_uds.out | 0 .../expected}/yagp_utf8_trim.out | 0 .../expected}/yagp_utility.out | 0 .../yagp_hooks_collector/metric.md | 0 .../protos}/yagpcc_metrics.proto | 0 .../protos}/yagpcc_plan.proto | 0 .../protos}/yagpcc_set_service.proto | 0 .../sql}/yagp_cursors.sql | 0 .../yagp_hooks_collector/sql}/yagp_dist.sql | 0 .../sql}/yagp_guc_cache.sql | 0 .../yagp_hooks_collector/sql}/yagp_locale.sql | 0 .../yagp_hooks_collector/sql}/yagp_select.sql | 0 .../yagp_hooks_collector/sql}/yagp_uds.sql | 0 .../sql}/yagp_utf8_trim.sql | 0 .../sql}/yagp_utility.sql | 0 .../yagp_hooks_collector/src}/Config.cpp | 0 .../yagp_hooks_collector/src}/Config.h | 0 .../yagp_hooks_collector/src}/EventSender.cpp | 0 .../yagp_hooks_collector/src}/EventSender.h | 0 .../yagp_hooks_collector/src}/PgUtils.cpp | 0 .../yagp_hooks_collector/src}/PgUtils.h | 0 .../yagp_hooks_collector/src}/ProcStats.cpp | 0 .../yagp_hooks_collector/src}/ProcStats.h | 0 .../yagp_hooks_collector/src}/ProtoUtils.cpp | 0 .../yagp_hooks_collector/src}/ProtoUtils.h | 0 .../src}/UDSConnector.cpp | 0 .../yagp_hooks_collector/src}/UDSConnector.h | 0 .../yagp_hooks_collector/src}/YagpStat.cpp | 0 .../yagp_hooks_collector/src}/YagpStat.h | 0 .../src}/hook_wrappers.cpp | 0 .../yagp_hooks_collector/src}/hook_wrappers.h | 0 .../yagp_hooks_collector/src}/log/LogOps.cpp | 0 .../yagp_hooks_collector/src}/log/LogOps.h | 0 .../src}/log/LogSchema.cpp | 0 .../yagp_hooks_collector/src}/log/LogSchema.h | 0 .../src}/memory/gpdbwrappers.cpp | 0 .../src}/memory/gpdbwrappers.h | 0 .../src}/stat_statements_parser/README.md | 0 .../pg_stat_statements_ya_parser.c | 0 .../pg_stat_statements_ya_parser.h | 0 .../src}/yagp_hooks_collector.c | 0 .../yagp_hooks_collector--1.0--1.1.sql | 0 .../yagp_hooks_collector--1.0.sql | 0 .../yagp_hooks_collector--1.1.sql | 0 .../yagp_hooks_collector.control | 0 src/stat_statements_parser/README.MD | 1 - 53 files changed, 28 insertions(+), 1 deletion(-) rename .clang-format => gpcontrib/yagp_hooks_collector/.clang-format (100%) create mode 100644 gpcontrib/yagp_hooks_collector/README.md rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_cursors.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_dist.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_guc_cache.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_locale.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_select.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_uds.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_utf8_trim.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_utility.out (100%) rename metric.md => gpcontrib/yagp_hooks_collector/metric.md (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_metrics.proto (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_plan.proto (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_set_service.proto (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_cursors.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_dist.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_guc_cache.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_locale.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_select.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_uds.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_utf8_trim.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_utility.sql (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/Config.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/Config.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/EventSender.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/EventSender.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/PgUtils.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/PgUtils.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProcStats.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProcStats.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProtoUtils.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProtoUtils.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/UDSConnector.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/UDSConnector.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/YagpStat.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/YagpStat.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/hook_wrappers.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/hook_wrappers.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogOps.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogOps.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogSchema.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogSchema.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/memory/gpdbwrappers.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/memory/gpdbwrappers.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/stat_statements_parser/README.md (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/stat_statements_parser/pg_stat_statements_ya_parser.c (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/stat_statements_parser/pg_stat_statements_ya_parser.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/yagp_hooks_collector.c (100%) rename yagp_hooks_collector--1.0--1.1.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql (100%) rename yagp_hooks_collector--1.0.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql (100%) rename yagp_hooks_collector--1.1.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql (100%) rename yagp_hooks_collector.control => gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control (100%) delete mode 100644 src/stat_statements_parser/README.MD diff --git a/.clang-format b/gpcontrib/yagp_hooks_collector/.clang-format similarity index 100% rename from .clang-format rename to gpcontrib/yagp_hooks_collector/.clang-format diff --git a/gpcontrib/yagp_hooks_collector/README.md b/gpcontrib/yagp_hooks_collector/README.md new file mode 100644 index 00000000000..9f465a190cb --- /dev/null +++ b/gpcontrib/yagp_hooks_collector/README.md @@ -0,0 +1,28 @@ +## YAGP Hooks Collector + +An extension for collecting greenplum query execution metrics and reporting them to an external agent. + +### Collected Statistics + +#### 1. Query Lifecycle +- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. +- **GUC:** `yagpcc.enable`. + +#### 2. `EXPLAIN` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. +- **GUC:** `yagpcc.enable`. + +#### 3. `EXPLAIN ANALYZE` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. +- **GUCs:** `yagpcc.enable`, `yagpcc.min_analyze_time`, `yagpcc.enable_cdbstats`(ANALYZE), `yagpcc.enable_analyze`(BUFFERS, TIMING, VERBOSE). + +#### 4. Other Metrics +- **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics. +- **GUC:** `yagpcc.enable`. + +### General Configuration +- **Nested Queries:** When `yagpcc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. +- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `yagpcc.uds_path`. +- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `yagpcc.ignored_users_list`. +- **Trimming plans:** Query texts and execution plans are trimmed based on `yagpcc.max_text_size` and `yagpcc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. +- **Analyze collection:** Analyze is sent if execution time exceeds `yagpcc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `yagpcc.enable_analyze` is true. diff --git a/expected/yagp_cursors.out b/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out similarity index 100% rename from expected/yagp_cursors.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out diff --git a/expected/yagp_dist.out b/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out similarity index 100% rename from expected/yagp_dist.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_dist.out diff --git a/expected/yagp_guc_cache.out b/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out similarity index 100% rename from expected/yagp_guc_cache.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out diff --git a/expected/yagp_locale.out b/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out similarity index 100% rename from expected/yagp_locale.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_locale.out diff --git a/expected/yagp_select.out b/gpcontrib/yagp_hooks_collector/expected/yagp_select.out similarity index 100% rename from expected/yagp_select.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_select.out diff --git a/expected/yagp_uds.out b/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out similarity index 100% rename from expected/yagp_uds.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_uds.out diff --git a/expected/yagp_utf8_trim.out b/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out similarity index 100% rename from expected/yagp_utf8_trim.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out diff --git a/expected/yagp_utility.out b/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out similarity index 100% rename from expected/yagp_utility.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_utility.out diff --git a/metric.md b/gpcontrib/yagp_hooks_collector/metric.md similarity index 100% rename from metric.md rename to gpcontrib/yagp_hooks_collector/metric.md diff --git a/protos/yagpcc_metrics.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto similarity index 100% rename from protos/yagpcc_metrics.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto diff --git a/protos/yagpcc_plan.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto similarity index 100% rename from protos/yagpcc_plan.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto diff --git a/protos/yagpcc_set_service.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto similarity index 100% rename from protos/yagpcc_set_service.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto diff --git a/sql/yagp_cursors.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql similarity index 100% rename from sql/yagp_cursors.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql diff --git a/sql/yagp_dist.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql similarity index 100% rename from sql/yagp_dist.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql diff --git a/sql/yagp_guc_cache.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql similarity index 100% rename from sql/yagp_guc_cache.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql diff --git a/sql/yagp_locale.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql similarity index 100% rename from sql/yagp_locale.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql diff --git a/sql/yagp_select.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql similarity index 100% rename from sql/yagp_select.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_select.sql diff --git a/sql/yagp_uds.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql similarity index 100% rename from sql/yagp_uds.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql diff --git a/sql/yagp_utf8_trim.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql similarity index 100% rename from sql/yagp_utf8_trim.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql diff --git a/sql/yagp_utility.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql similarity index 100% rename from sql/yagp_utility.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql diff --git a/src/Config.cpp b/gpcontrib/yagp_hooks_collector/src/Config.cpp similarity index 100% rename from src/Config.cpp rename to gpcontrib/yagp_hooks_collector/src/Config.cpp diff --git a/src/Config.h b/gpcontrib/yagp_hooks_collector/src/Config.h similarity index 100% rename from src/Config.h rename to gpcontrib/yagp_hooks_collector/src/Config.h diff --git a/src/EventSender.cpp b/gpcontrib/yagp_hooks_collector/src/EventSender.cpp similarity index 100% rename from src/EventSender.cpp rename to gpcontrib/yagp_hooks_collector/src/EventSender.cpp diff --git a/src/EventSender.h b/gpcontrib/yagp_hooks_collector/src/EventSender.h similarity index 100% rename from src/EventSender.h rename to gpcontrib/yagp_hooks_collector/src/EventSender.h diff --git a/src/PgUtils.cpp b/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp similarity index 100% rename from src/PgUtils.cpp rename to gpcontrib/yagp_hooks_collector/src/PgUtils.cpp diff --git a/src/PgUtils.h b/gpcontrib/yagp_hooks_collector/src/PgUtils.h similarity index 100% rename from src/PgUtils.h rename to gpcontrib/yagp_hooks_collector/src/PgUtils.h diff --git a/src/ProcStats.cpp b/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp similarity index 100% rename from src/ProcStats.cpp rename to gpcontrib/yagp_hooks_collector/src/ProcStats.cpp diff --git a/src/ProcStats.h b/gpcontrib/yagp_hooks_collector/src/ProcStats.h similarity index 100% rename from src/ProcStats.h rename to gpcontrib/yagp_hooks_collector/src/ProcStats.h diff --git a/src/ProtoUtils.cpp b/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp similarity index 100% rename from src/ProtoUtils.cpp rename to gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp diff --git a/src/ProtoUtils.h b/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h similarity index 100% rename from src/ProtoUtils.h rename to gpcontrib/yagp_hooks_collector/src/ProtoUtils.h diff --git a/src/UDSConnector.cpp b/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp similarity index 100% rename from src/UDSConnector.cpp rename to gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp diff --git a/src/UDSConnector.h b/gpcontrib/yagp_hooks_collector/src/UDSConnector.h similarity index 100% rename from src/UDSConnector.h rename to gpcontrib/yagp_hooks_collector/src/UDSConnector.h diff --git a/src/YagpStat.cpp b/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp similarity index 100% rename from src/YagpStat.cpp rename to gpcontrib/yagp_hooks_collector/src/YagpStat.cpp diff --git a/src/YagpStat.h b/gpcontrib/yagp_hooks_collector/src/YagpStat.h similarity index 100% rename from src/YagpStat.h rename to gpcontrib/yagp_hooks_collector/src/YagpStat.h diff --git a/src/hook_wrappers.cpp b/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp similarity index 100% rename from src/hook_wrappers.cpp rename to gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp diff --git a/src/hook_wrappers.h b/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h similarity index 100% rename from src/hook_wrappers.h rename to gpcontrib/yagp_hooks_collector/src/hook_wrappers.h diff --git a/src/log/LogOps.cpp b/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp similarity index 100% rename from src/log/LogOps.cpp rename to gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp diff --git a/src/log/LogOps.h b/gpcontrib/yagp_hooks_collector/src/log/LogOps.h similarity index 100% rename from src/log/LogOps.h rename to gpcontrib/yagp_hooks_collector/src/log/LogOps.h diff --git a/src/log/LogSchema.cpp b/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp similarity index 100% rename from src/log/LogSchema.cpp rename to gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp diff --git a/src/log/LogSchema.h b/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h similarity index 100% rename from src/log/LogSchema.h rename to gpcontrib/yagp_hooks_collector/src/log/LogSchema.h diff --git a/src/memory/gpdbwrappers.cpp b/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp similarity index 100% rename from src/memory/gpdbwrappers.cpp rename to gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp diff --git a/src/memory/gpdbwrappers.h b/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h similarity index 100% rename from src/memory/gpdbwrappers.h rename to gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h diff --git a/src/stat_statements_parser/README.md b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md similarity index 100% rename from src/stat_statements_parser/README.md rename to gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c similarity index 100% rename from src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h similarity index 100% rename from src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h diff --git a/src/yagp_hooks_collector.c b/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c similarity index 100% rename from src/yagp_hooks_collector.c rename to gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c diff --git a/yagp_hooks_collector--1.0--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql similarity index 100% rename from yagp_hooks_collector--1.0--1.1.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql diff --git a/yagp_hooks_collector--1.0.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql similarity index 100% rename from yagp_hooks_collector--1.0.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql diff --git a/yagp_hooks_collector--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql similarity index 100% rename from yagp_hooks_collector--1.1.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql diff --git a/yagp_hooks_collector.control b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control similarity index 100% rename from yagp_hooks_collector.control rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control diff --git a/src/stat_statements_parser/README.MD b/src/stat_statements_parser/README.MD deleted file mode 100644 index 291e31a3099..00000000000 --- a/src/stat_statements_parser/README.MD +++ /dev/null @@ -1 +0,0 @@ -This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. From 8d4750a41c0e3a894c34bb0b4b2538d87fa62f3f Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Wed, 25 Mar 2026 18:37:39 +0300 Subject: [PATCH 078/128] [gp_stats_collector] Rename yagp_hooks_collector to gp_stats_collector Rename extension, shared library, SQL objects, GUC prefix, test files, and all internal identifiers. Restore accidentally deleted files. Clean up stray gmon.out. --- .github/workflows/build-cloudberry-rocky8.yml | 32 ++- .github/workflows/build-cloudberry.yml | 16 +- .github/workflows/build-deb-cloudberry.yml | 32 ++- .gitignore | 5 - Makefile | 2 + configure | 16 +- configure.ac | 19 +- gpcontrib/Makefile | 4 +- .../.clang-format | 0 gpcontrib/gp_stats_collector/.gitignore | 5 + .../Makefile | 16 +- gpcontrib/gp_stats_collector/README.md | 47 ++++ .../expected/gpsc_cursors.out} | 72 ++--- .../expected/gpsc_dist.out} | 56 ++-- .../expected/gpsc_guc_cache.out} | 32 +-- .../expected/gpsc_locale.out | 23 ++ .../expected/gpsc_select.out} | 56 ++-- .../gp_stats_collector/expected/gpsc_uds.out | 42 +++ .../expected/gpsc_utf8_trim.out} | 36 +-- .../expected/gpsc_utility.out} | 172 ++++++------ .../gp_stats_collector--1.0--1.1.sql | 113 ++++++++ .../gp_stats_collector--1.0.sql | 55 ++++ .../gp_stats_collector--1.1.sql | 110 ++++++++ .../gp_stats_collector.control | 5 + .../metric.md | 27 +- .../protos/gpsc_metrics.proto} | 4 +- .../protos/gpsc_plan.proto} | 4 +- .../protos/gpsc_set_service.proto} | 8 +- .../results/gpsc_cursors.out | 163 ++++++++++++ .../gp_stats_collector/results/gpsc_dist.out | 175 ++++++++++++ .../results/gpsc_guc_cache.out | 61 +++++ .../results/gpsc_locale.out | 23 ++ .../results/gpsc_select.out | 136 ++++++++++ .../gp_stats_collector/results/gpsc_uds.out | 42 +++ .../results/gpsc_utf8_trim.out | 68 +++++ .../results/gpsc_utility.out | 248 ++++++++++++++++++ .../gp_stats_collector/sql/gpsc_cursors.sql | 85 ++++++ .../sql/gpsc_dist.sql} | 58 ++-- .../sql/gpsc_guc_cache.sql} | 32 +-- .../gp_stats_collector/sql/gpsc_locale.sql | 29 ++ .../gp_stats_collector/sql/gpsc_select.sql | 69 +++++ gpcontrib/gp_stats_collector/sql/gpsc_uds.sql | 31 +++ .../sql/gpsc_utf8_trim.sql} | 36 +-- .../gp_stats_collector/sql/gpsc_utility.sql | 135 ++++++++++ .../src/Config.cpp | 46 ++-- .../src/Config.h | 2 +- .../src/EventSender.cpp | 78 +++--- .../src/EventSender.h | 32 +-- .../src/GpscStat.cpp} | 36 +-- .../src/GpscStat.h} | 6 +- .../src/PgUtils.cpp | 20 +- .../src/PgUtils.h | 2 +- .../src/ProcStats.cpp | 12 +- .../src/ProcStats.h | 6 +- .../src/ProtoUtils.cpp | 60 ++--- .../src/ProtoUtils.h | 26 +- .../src/UDSConnector.cpp | 24 +- .../src/UDSConnector.h | 6 +- .../src/gp_stats_collector.c} | 36 +-- .../src/hook_wrappers.cpp | 72 ++--- .../src/hook_wrappers.h | 6 +- .../src/log/LogOps.cpp | 16 +- .../src/log/LogOps.h | 10 +- .../src/log/LogSchema.cpp | 10 +- .../src/log/LogSchema.h | 8 +- .../src/memory/gpdbwrappers.cpp | 42 +-- .../src/memory/gpdbwrappers.h | 12 +- .../src/stat_statements_parser/README.md | 20 ++ .../pg_stat_statements_ya_parser.c | 2 +- .../pg_stat_statements_ya_parser.h | 2 +- gpcontrib/yagp_hooks_collector/README.md | 28 -- .../expected/yagp_locale.out | 23 -- .../expected/yagp_uds.out | 42 --- .../yagp_hooks_collector/sql/yagp_cursors.sql | 85 ------ .../yagp_hooks_collector/sql/yagp_locale.sql | 29 -- .../yagp_hooks_collector/sql/yagp_select.sql | 69 ----- .../yagp_hooks_collector/sql/yagp_uds.sql | 31 --- .../yagp_hooks_collector/sql/yagp_utility.sql | 135 ---------- .../src/stat_statements_parser/README.md | 1 - .../yagp_hooks_collector--1.0--1.1.sql | 113 -------- .../yagp_hooks_collector--1.0.sql | 55 ---- .../yagp_hooks_collector--1.1.sql | 110 -------- .../yagp_hooks_collector.control | 5 - pom.xml | 16 +- src/Makefile.global.in | 2 +- src/backend/commands/portalcmds.c | 2 +- src/backend/tcop/pquery.c | 4 +- src/include/executor/execdesc.h | 8 +- 88 files changed, 2396 insertions(+), 1354 deletions(-) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/.clang-format (100%) create mode 100644 gpcontrib/gp_stats_collector/.gitignore rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/Makefile (66%) create mode 100644 gpcontrib/gp_stats_collector/README.md rename gpcontrib/{yagp_hooks_collector/expected/yagp_cursors.out => gp_stats_collector/expected/gpsc_cursors.out} (73%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_dist.out => gp_stats_collector/expected/gpsc_dist.out} (81%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_guc_cache.out => gp_stats_collector/expected/gpsc_guc_cache.out} (64%) create mode 100644 gpcontrib/gp_stats_collector/expected/gpsc_locale.out rename gpcontrib/{yagp_hooks_collector/expected/yagp_select.out => gp_stats_collector/expected/gpsc_select.out} (67%) create mode 100644 gpcontrib/gp_stats_collector/expected/gpsc_uds.out rename gpcontrib/{yagp_hooks_collector/expected/yagp_utf8_trim.out => gp_stats_collector/expected/gpsc_utf8_trim.out} (65%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_utility.out => gp_stats_collector/expected/gpsc_utility.out} (57%) create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector.control rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/metric.md (94%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_metrics.proto => gp_stats_collector/protos/gpsc_metrics.proto} (97%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_plan.proto => gp_stats_collector/protos/gpsc_plan.proto} (98%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_set_service.proto => gp_stats_collector/protos/gpsc_set_service.proto} (86%) create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_cursors.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_dist.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_locale.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_select.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_uds.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utility.out create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql rename gpcontrib/{yagp_hooks_collector/sql/yagp_dist.sql => gp_stats_collector/sql/gpsc_dist.sql} (53%) rename gpcontrib/{yagp_hooks_collector/sql/yagp_guc_cache.sql => gp_stats_collector/sql/gpsc_guc_cache.sql} (58%) create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_locale.sql create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_select.sql create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_uds.sql rename gpcontrib/{yagp_hooks_collector/sql/yagp_utf8_trim.sql => gp_stats_collector/sql/gpsc_utf8_trim.sql} (58%) create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_utility.sql rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/Config.cpp (79%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/Config.h (97%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/EventSender.cpp (86%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/EventSender.h (84%) rename gpcontrib/{yagp_hooks_collector/src/YagpStat.cpp => gp_stats_collector/src/GpscStat.cpp} (78%) rename gpcontrib/{yagp_hooks_collector/src/YagpStat.h => gp_stats_collector/src/GpscStat.h} (94%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/PgUtils.cpp (83%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/PgUtils.h (96%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProcStats.cpp (92%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProcStats.h (89%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProtoUtils.cpp (85%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProtoUtils.h (65%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/UDSConnector.cpp (87%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/UDSConnector.h (88%) rename gpcontrib/{yagp_hooks_collector/src/yagp_hooks_collector.c => gp_stats_collector/src/gp_stats_collector.c} (79%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/hook_wrappers.cpp (84%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/hook_wrappers.h (89%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogOps.cpp (91%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogOps.h (83%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogSchema.cpp (94%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogSchema.h (98%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/memory/gpdbwrappers.cpp (81%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/memory/gpdbwrappers.h (92%) create mode 100644 gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/stat_statements_parser/pg_stat_statements_ya_parser.c (99%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/stat_statements_parser/pg_stat_statements_ya_parser.h (93%) delete mode 100644 gpcontrib/yagp_hooks_collector/README.md delete mode 100644 gpcontrib/yagp_hooks_collector/expected/yagp_locale.out delete mode 100644 gpcontrib/yagp_hooks_collector/expected/yagp_uds.out delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_select.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql delete mode 100644 gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index e0936c725c8..39175753a99 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -320,6 +320,10 @@ jobs: "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, {"test":"ic-fixme", "make_configs":["src/test/regress:installcheck-fixme"], "enable_core_check":false @@ -540,10 +544,11 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1400,6 +1405,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: /usr/local/cloudberry-db shell: bash {0} run: | set -o pipefail @@ -1423,6 +1429,30 @@ jobs: # 2. Follow the same pattern as optimizer # 3. Update matrix entries to include the new setting + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + # Set PostgreSQL options if defined PG_OPTS="" if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 8484331998f..cbd4fd753dc 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -271,9 +271,9 @@ jobs: }, "enable_core_check":false }, - {"test":"gpcontrib-yagp-hooks-collector", - "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"], - "extension":"yagp_hooks_collector" + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] @@ -539,7 +539,7 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -1441,17 +1441,17 @@ jobs: # Create extension if required if [[ "${{ matrix.extension != '' }}" == "true" ]]; then case "${{ matrix.extension }}" in - yagp_hooks_collector) + gp_stats_collector) if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ - gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ gpstop -ra && \ - echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ SHOW shared_preload_libraries; \ TABLE pg_extension;' | \ psql postgres" then - echo "Error creating yagp_hooks_collector extension" + echo "Error creating gp_stats_collector extension" exit 1 fi ;; diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 85d917b8ff0..bf85a107b31 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -252,6 +252,10 @@ jobs: "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, {"test":"ic-cbdb-parallel", "make_configs":["src/test/regress:installcheck-cbdb-parallel"] } @@ -448,13 +452,14 @@ jobs: shell: bash env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail export BUILD_DESTINATION=${SRC_DIR}/debian/build chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1341,6 +1346,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: ${{ github.workspace }}/debian/build shell: bash {0} run: | set -o pipefail @@ -1365,6 +1371,30 @@ jobs: # 3. Update matrix entries to include the new setting + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + # Set PostgreSQL options if defined PG_OPTS="" if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then diff --git a/.gitignore b/.gitignore index 29b40ee096c..7f5110d5c8e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,8 +74,3 @@ lib*.pc /tmp_install/ /.cache/ /install/ -*.o -*.so -src/protos/ -.vscode -compile_commands.json diff --git a/Makefile b/Makefile index 15c5dabb70e..e9ab3fbf2d4 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,14 @@ # to build Postgres with a different make, we have this make file # that, as a service, will look for a GNU make and invoke it, or show # an error message if none could be found. + # If the user were using GNU make now, this file would not get used # because GNU make uses a make file named "GNUmakefile" in preference # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. + # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. all: diff --git a/configure b/configure index a3b0bef7b6f..5127039e72b 100755 --- a/configure +++ b/configure @@ -722,7 +722,7 @@ with_apr_config with_libcurl with_rt with_zstd -with_yagp_hooks_collector +with_gp_stats_collector with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -944,7 +944,7 @@ with_zlib with_lz4 with_libbz2 with_zstd -with_yagp_hooks_collector +with_gp_stats_collector with_rt with_libcurl with_apr_config @@ -11254,14 +11254,14 @@ fi fi # -# yagp_hooks_collector +# gp_stats_collector # -# Check whether --with-yagp-hooks-collector was given. -if test "${with_yagp_hooks_collector+set}" = set; then : - withval=$with_yagp_hooks_collector; +# Check whether --with-gp-stats-collector was given. +if test "${with_gp_stats_collector+set}" = set; then : + withval=$with_gp_stats_collector; case $withval in yes) : @@ -11270,12 +11270,12 @@ if test "${with_yagp_hooks_collector+set}" = set; then : : ;; *) - as_fn_error $? "no argument expected for --with-yagp-hooks-collector option" "$LINENO" 5 + as_fn_error $? "no argument expected for --with-gp-stats-collector option" "$LINENO" 5 ;; esac else - with_yagp_hooks_collector=no + with_gp_stats_collector=no fi diff --git a/configure.ac b/configure.ac index 7e19e927868..9664aa86e57 100644 --- a/configure.ac +++ b/configure.ac @@ -1369,11 +1369,22 @@ AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) # -# yagp_hooks_collector +# gp_stats_collector # -PGAC_ARG_BOOL(with, yagp_hooks_collector, no, - [build with YAGP hooks collector extension]) -AC_SUBST(with_yagp_hooks_collector) +PGAC_ARG_BOOL(with, gp_stats_collector, no, + [build with stats collector extension]) +AC_SUBST(with_gp_stats_collector) + +if test "$with_gp_stats_collector" = yes; then + PKG_CHECK_MODULES([PROTOBUF], [protobuf >= 3.0.0], + [], + [AC_MSG_ERROR([protobuf >= 3.0.0 is required for gp_stats_collector])] + ) + AC_PATH_PROG([PROTOC], [protoc], [no]) + if test "$PROTOC" = no; then + AC_MSG_ERROR([protoc is required for gp_stats_collector but was not found in PATH]) + fi +fi if test "$with_zstd" = yes; then dnl zstd_errors.h was renamed from error_public.h in v1.4.0 diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 8b98dc9142c..956cb470477 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,8 +35,8 @@ else diskquota endif -ifeq "$(with_yagp_hooks_collector)" "yes" - recurse_targets += yagp_hooks_collector +ifeq "$(with_gp_stats_collector)" "yes" + recurse_targets += gp_stats_collector endif ifeq "$(with_zstd)" "yes" recurse_targets += zstd diff --git a/gpcontrib/yagp_hooks_collector/.clang-format b/gpcontrib/gp_stats_collector/.clang-format similarity index 100% rename from gpcontrib/yagp_hooks_collector/.clang-format rename to gpcontrib/gp_stats_collector/.clang-format diff --git a/gpcontrib/gp_stats_collector/.gitignore b/gpcontrib/gp_stats_collector/.gitignore new file mode 100644 index 00000000000..e8dfe855dad --- /dev/null +++ b/gpcontrib/gp_stats_collector/.gitignore @@ -0,0 +1,5 @@ +*.o +*.so +src/protos/ +.vscode +compile_commands.json diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile similarity index 66% rename from gpcontrib/yagp_hooks_collector/Makefile rename to gpcontrib/gp_stats_collector/Makefile index 49825c55f35..c8f7b3c30fe 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -1,9 +1,9 @@ -MODULE_big = yagp_hooks_collector -EXTENSION = yagp_hooks_collector +MODULE_big = gp_stats_collector +EXTENSION = gp_stats_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds yagp_locale +REGRESS = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_uds gpsc_locale -PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service +PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) @@ -24,7 +24,7 @@ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = gpcontrib/yagp_hooks_collector +subdir = gpcontrib/gp_stats_collector top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk @@ -32,10 +32,8 @@ endif src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto @mkdir -p src/protos - sed -i 's/optional //g' $^ - sed -i 's|cloud/mdb/yagpcc/api/proto/common/|protos/|g' $^ protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ mv src/protos/$*.pb.cc src/protos/$*.pb.cpp -$(CPP_OBJS): src/protos/yagpcc_metrics.pb.h src/protos/yagpcc_plan.pb.h src/protos/yagpcc_set_service.pb.h -src/protos/yagpcc_set_service.pb.o: src/protos/yagpcc_metrics.pb.h +$(CPP_OBJS): src/protos/gpsc_metrics.pb.h src/protos/gpsc_plan.pb.h src/protos/gpsc_set_service.pb.h +src/protos/gpsc_set_service.pb.o: src/protos/gpsc_metrics.pb.h diff --git a/gpcontrib/gp_stats_collector/README.md b/gpcontrib/gp_stats_collector/README.md new file mode 100644 index 00000000000..8c2d5c6868e --- /dev/null +++ b/gpcontrib/gp_stats_collector/README.md @@ -0,0 +1,47 @@ + + +## GP Stats Collector + +An extension for collecting query execution metrics and reporting them to an external agent. + +### Collected Statistics + +#### 1. Query Lifecycle +- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. +- **GUC:** `gpsc.enable`. + +#### 2. `EXPLAIN` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. +- **GUC:** `gpsc.enable`. + +#### 3. `EXPLAIN ANALYZE` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. +- **GUCs:** `gpsc.enable`, `gpsc.min_analyze_time`, `gpsc.enable_cdbstats`(ANALYZE), `gpsc.enable_analyze`(BUFFERS, TIMING, VERBOSE). + +#### 4. Other Metrics +- **What:** Captures Instrument, System, Network, Interconnect, Spill metrics. +- **GUC:** `gpsc.enable`. + +### General Configuration +- **Nested Queries:** When `gpsc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. +- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `gpsc.uds_path`. +- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`. +- **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. +- **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true. diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out b/gpcontrib/gp_stats_collector/expected/gpsc_cursors.out similarity index 73% rename from gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out rename to gpcontrib/gp_stats_collector/expected/gpsc_cursors.out index df12e3e1b66..282d9ac49e1 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_cursors.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,18 +12,18 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; -- DECLARE -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_0 CURSOR FOR SELECT 0; CLOSE cursor_stats_0; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -34,25 +34,25 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- DECLARE WITH HOLD -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; CLOSE cursor_stats_1; DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; CLOSE cursor_stats_2; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -67,24 +67,24 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (14 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- ROLLBACK -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_3 CURSOR FOR SELECT 1; CLOSE cursor_stats_3; DECLARE cursor_stats_4 CURSOR FOR SELECT 1; ROLLBACK; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -97,17 +97,17 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE -1 | ROLLBACK; | QUERY_STATUS_SUBMIT -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (12 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- FETCH -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; @@ -126,8 +126,8 @@ FETCH 1 IN cursor_stats_6; CLOSE cursor_stats_5; CLOSE cursor_stats_6; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -146,18 +146,18 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (18 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out b/gpcontrib/gp_stats_collector/expected/gpsc_dist.out similarity index 81% rename from gpcontrib/yagp_hooks_collector/expected/yagp_dist.out rename to gpcontrib/gp_stats_collector/expected/gpsc_dist.out index 3b1e3504923..92e8678767b 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_dist.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,14 +12,14 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Hash distributed table CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); INSERT INTO test_hash_dist SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SET optimizer_enable_direct_dispatch TO TRUE; -- Direct dispatch is used here, only one segment is scanned. select * from test_hash_dist where id = 1; @@ -29,9 +29,9 @@ select * from test_hash_dist where id = 1; (1 row) RESET optimizer_enable_direct_dispatch; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Should see 8 rows. -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT @@ -44,12 +44,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag 1 | | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; -- Scan all segments. select * from test_hash_dist; id @@ -58,8 +58,8 @@ select * from test_hash_dist; (1 row) DROP TABLE test_hash_dist; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------+--------------------- -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT @@ -80,7 +80,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | | QUERY_STATUS_DONE (16 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) @@ -93,7 +93,7 @@ END; $$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; INSERT INTO test_replicated SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_replicated, force_segments(); count ------- @@ -102,8 +102,8 @@ SELECT COUNT(*) FROM test_replicated, force_segments(); DROP TABLE test_replicated; DROP FUNCTION force_segments(); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------------------+--------------------- -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT @@ -124,7 +124,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | | QUERY_STATUS_DONE (16 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) @@ -134,18 +134,18 @@ SET allow_system_table_mods = ON; CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_partial_dist; count ------- 100 (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; DROP TABLE test_partial_dist; RESET allow_system_table_mods; -- Should see 12 rows. -SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; query_text | query_status -----------------------------------------+--------------------- SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT @@ -162,14 +162,14 @@ SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_statu | QUERY_STATUS_DONE (12 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out b/gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out similarity index 64% rename from gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out rename to gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out index 3085cfa42e1..11a420839db 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out @@ -8,23 +8,23 @@ -- have its DONE event rejected, creating orphaned SUBMIT entries. -- This is due to query being actually executed between SUBMIT and DONE. -- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; -SELECT yagpcc.truncate_log(); +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); -- end_ignore CREATE OR REPLACE FUNCTION print_last_query(query text) RETURNS TABLE(query_status text) AS $$ SELECT query_status - FROM yagpcc.log + FROM gpsc.log WHERE segid = -1 AND query_text = query ORDER BY ccnt DESC $$ LANGUAGE sql; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.logging_mode TO 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; -- SET below disables utility logging and DONE must still be logged. -SET yagpcc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); query_status --------------------- QUERY_STATUS_SUBMIT @@ -33,14 +33,14 @@ SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); -- SELECT below adds current user to ignore list and DONE must still be logged. -- start_ignore -SELECT set_config('yagpcc.ignored_users_list', current_user, false); +SELECT set_config('gpsc.ignored_users_list', current_user, false); set_config ------------ gpadmin (1 row) -- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); query_status --------------------- QUERY_STATUS_SUBMIT @@ -50,8 +50,8 @@ SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', (4 rows) DROP FUNCTION print_last_query(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -RESET yagpcc.logging_mode; +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/expected/gpsc_locale.out b/gpcontrib/gp_stats_collector/expected/gpsc_locale.out new file mode 100644 index 00000000000..a01fe0648b9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/expected/gpsc_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale +CREATE EXTENSION gp_stats_collector; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_select.out b/gpcontrib/gp_stats_collector/expected/gpsc_select.out similarity index 67% rename from gpcontrib/yagp_hooks_collector/expected/yagp_select.out rename to gpcontrib/gp_stats_collector/expected/gpsc_select.out index af08f2d1def..3008c8f6d55 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_select.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_select.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,12 +12,12 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Basic SELECT tests -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT 1; ?column? ---------- @@ -30,8 +30,8 @@ SELECT COUNT(*) FROM generate_series(1,10); 10 (1 row) -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | SELECT 1; | QUERY_STATUS_SUBMIT @@ -44,13 +44,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Transaction test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; SELECT 1; ?column? @@ -59,8 +59,8 @@ SELECT 1; (1 row) COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+------------+--------------------- -1 | SELECT 1; | QUERY_STATUS_SUBMIT @@ -69,13 +69,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | SELECT 1; | QUERY_STATUS_DONE (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- CTE test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; WITH t AS (VALUES (1), (2)) SELECT * FROM t; column1 @@ -84,8 +84,8 @@ SELECT * FROM t; 2 (2 rows) -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-----------------------------+--------------------- -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT @@ -98,13 +98,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | SELECT * FROM t; | (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Prepared statement test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; PREPARE test_stmt AS SELECT 1; EXECUTE test_stmt; ?column? @@ -113,8 +113,8 @@ EXECUTE test_stmt; (1 row) DEALLOCATE test_stmt; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------+--------------------- -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT @@ -123,14 +123,14 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/expected/gpsc_uds.out b/gpcontrib/gp_stats_collector/expected/gpsc_uds.out new file mode 100644 index 00000000000..e8bca79e669 --- /dev/null +++ b/gpcontrib/gp_stats_collector/expected/gpsc_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +\set UDS_PATH '/tmp/gpsc_test.sock' +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out b/gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out similarity index 65% rename from gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out rename to gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out index 9de126dd882..db3949f3152 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out @@ -1,24 +1,24 @@ -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) RETURNS TEXT AS $$ SELECT query_text - FROM yagpcc.log + FROM gpsc.log WHERE query_text LIKE '%' || marker || '%' ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; -- Test 1: 1 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test1*/ 'HelloWorld'; ?column? ------------ HelloWorld (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; correct_length ---------------- @@ -26,15 +26,15 @@ SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; (1 row) -- Test 2: 2 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test2*/ 'РУССКИЙЯЗЫК'; ?column? ------------- РУССКИЙЯЗЫК (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Character 'Р' has two bytes and cut in the middle => not included. SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; correct_length @@ -43,15 +43,15 @@ SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; (1 row) -- Test 3: 4 byte chars -SET yagpcc.max_text_size to 21; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; SELECT /*test3*/ '😀'; ?column? ---------- 😀 (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Emoji has 4 bytes and cut before the last byte => not included. SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; correct_length @@ -61,8 +61,8 @@ SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; -- Cleanup DROP FUNCTION get_marked_query(TEXT); -RESET yagpcc.max_text_size; -RESET yagpcc.logging_mode; -RESET yagpcc.enable; -RESET yagpcc.ignored_users_list; -DROP EXTENSION yagp_hooks_collector; +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out b/gpcontrib/gp_stats_collector/expected/gpsc_utility.out similarity index 57% rename from gpcontrib/yagp_hooks_collector/expected/yagp_utility.out rename to gpcontrib/gp_stats_collector/expected/gpsc_utility.out index 7df1d2816eb..e8e28614370 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_utility.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,19 +12,19 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE test_table (a int, b text); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE INDEX test_idx ON test_table(a); ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; DROP TABLE test_table; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT @@ -35,24 +35,24 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT -1 | DROP TABLE test_table; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Partitioning -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE pt_test (a int, b int) DISTRIBUTED BY (a) PARTITION BY RANGE (a) (START (0) END (100) EVERY (50)); DROP TABLE pt_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT @@ -65,23 +65,23 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util | (START (0) END (100) EVERY (50)); | -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Views and Functions -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE VIEW test_view AS SELECT 1 AS a; CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; DROP VIEW test_view; DROP FUNCTION test_func(int); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT @@ -92,17 +92,17 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP VIEW test_view; | QUERY_STATUS_DONE -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Transaction Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; SAVEPOINT sp1; ROLLBACK TO sp1; @@ -112,37 +112,37 @@ SAVEPOINT sp2; ABORT; BEGIN; ROLLBACK; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (18 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- DML Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE dml_test (a int, b text); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -150,33 +150,33 @@ INSERT INTO dml_test VALUES (1, 'test'); UPDATE dml_test SET b = 'updated' WHERE a = 1; DELETE FROM dml_test WHERE a = 1; DROP TABLE dml_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- COPY Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE copy_test (a int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. COPY (SELECT 1) TO STDOUT; 1 DROP TABLE copy_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------+--------------------- -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT @@ -185,23 +185,23 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Prepared Statements and error during execute -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; PREPARE test_prep(int) AS SELECT $1/0 AS value; EXECUTE test_prep(0::int); ERROR: division by zero DEALLOCATE test_prep; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT @@ -210,39 +210,39 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- GUC Settings -SET yagpcc.logging_mode to 'TBL'; -SET yagpcc.report_nested_queries TO FALSE; -RESET yagpcc.report_nested_queries; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------------------+--------------------- - -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT - -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE - -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE +SET gpsc.logging_mode to 'TBL'; +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------+--------------------- + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql new file mode 100644 index 00000000000..4e0157117e9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql @@ -0,0 +1,113 @@ +/* gp_stats_collector--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_stats_collector UPDATE TO '1.1'" to load this file. \quit + +CREATE SCHEMA gpsc; + +-- Unlink existing objects from extension. +ALTER EXTENSION gp_stats_collector DROP VIEW gpsc_stat_messages; +ALTER EXTENSION gp_stats_collector DROP FUNCTION gpsc_stat_messages_reset(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_f_on_segments(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_f_on_master(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_reset_f_on_segments(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); + +-- Now drop the objects. +DROP VIEW gpsc_stat_messages; +DROP FUNCTION gpsc_stat_messages_reset(); +DROP FUNCTION __gpsc_stat_messages_f_on_segments(); +DROP FUNCTION __gpsc_stat_messages_f_on_master(); +DROP FUNCTION __gpsc_stat_messages_reset_f_on_segments(); +DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); + +-- Recreate functions and view in new schema. +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +-- Create new objects. +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments + ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql new file mode 100644 index 00000000000..ec902b02e02 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql @@ -0,0 +1,55 @@ +/* gp_stats_collector--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE FUNCTION __gpsc_stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc_stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT __gpsc_stat_messages_reset_f_on_master(); + SELECT __gpsc_stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc_stat_messages AS + SELECT C.* + FROM __gpsc_stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM __gpsc_stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql new file mode 100644 index 00000000000..6e24207e913 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql @@ -0,0 +1,110 @@ +/* gp_stats_collector--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE SCHEMA gpsc; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION gpsc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' +LANGUAGE C EXECUTE ON MASTER; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector.control b/gpcontrib/gp_stats_collector/gp_stats_collector.control new file mode 100644 index 00000000000..4aea2bd49b8 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector.control @@ -0,0 +1,5 @@ +# gp_stats_collector extension +comment = 'Intercept query and plan execution hooks and report them to Cloudberry monitor agents' +default_version = '1.1' +module_pathname = '$libdir/gp_stats_collector' +superuser = true diff --git a/gpcontrib/yagp_hooks_collector/metric.md b/gpcontrib/gp_stats_collector/metric.md similarity index 94% rename from gpcontrib/yagp_hooks_collector/metric.md rename to gpcontrib/gp_stats_collector/metric.md index 5df56877edb..6f168d8cd98 100644 --- a/gpcontrib/yagp_hooks_collector/metric.md +++ b/gpcontrib/gp_stats_collector/metric.md @@ -1,4 +1,23 @@ -## YAGP Hooks Collector Metrics + + +## GP Stats Collector Metrics ### States A Postgres process goes through 4 executor functions to execute a query: @@ -7,7 +26,7 @@ A Postgres process goes through 4 executor functions to execute a query: 3) `ExecutorFinish()` - cleanup. 4) `ExecutorEnd()` - cleanup. -yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +gp-stats-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: ``` submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done ``` @@ -67,8 +86,8 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `temp_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks written | | `blk_read_time` | double | E, D | ABS | + | Node | + | + | seconds | Time reading data blocks | | `blk_write_time` | double | E, D | ABS | + | Node | + | + | seconds | Time writing data blocks | -| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (YAGPCC-specific) | -| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (YAGPCC-specific) | +| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (GPSC-specific) | +| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (GPSC-specific) | | **NetworkStat (sent)** | | | | | | | | | | | `sent.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes sent, including headers | | `sent.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data sent | diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto similarity index 97% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index 91ac0c4941a..a9e26471839 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -1,8 +1,6 @@ syntax = "proto3"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCM"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; +package gpsc; enum QueryStatus { QUERY_STATUS_UNSPECIFIED = 0; diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto similarity index 98% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_plan.proto index 962fab4bbdd..5a7269edd20 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto @@ -1,8 +1,6 @@ syntax = "proto3"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCP"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; +package gpsc; message MetricPlan { GpdbNodeType type = 1; diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto similarity index 86% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto index 0b9e34df49d..4cd795424ab 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto @@ -2,12 +2,10 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -import "protos/yagpcc_metrics.proto"; -import "protos/yagpcc_plan.proto"; +import "protos/gpsc_metrics.proto"; +import "protos/gpsc_plan.proto"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCAS"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; +package gpsc; service SetQueryInfo { rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} diff --git a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out new file mode 100644 index 00000000000..282d9ac49e1 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out @@ -0,0 +1,163 @@ +CREATE EXTENSION gp_stats_collector; +CREATE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +-- DECLARE +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- DECLARE WITH HOLD +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(14 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- ROLLBACK +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(12 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- FETCH +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; + ?column? +---------- + 2 +(1 row) + +FETCH 1 IN cursor_stats_6; + ?column? +---------- + 3 +(1 row) + +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(18 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_dist.out b/gpcontrib/gp_stats_collector/results/gpsc_dist.out new file mode 100644 index 00000000000..92e8678767b --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_dist.out @@ -0,0 +1,175 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; +-- Hash distributed table +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; +SET gpsc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; + id +---- + 1 +(1 row) + +RESET optimizer_enable_direct_dispatch; +RESET gpsc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +SET gpsc.logging_mode to 'TBL'; +-- Scan all segments. +select * from test_hash_dist; + id +---- + 1 +(1 row) + +DROP TABLE test_hash_dist; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------+--------------------- + -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist; | QUERY_STATUS_START + -1 | select * from test_hash_dist; | QUERY_STATUS_END + -1 | select * from test_hash_dist; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; +SET gpsc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); + count +------- + 3 +(1 row) + +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------------------+--------------------- + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); +SET gpsc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; + count +------- + 100 +(1 row) + +RESET gpsc.logging_mode; +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + query_text | query_status +-----------------------------------------+--------------------- + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE +(12 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out new file mode 100644 index 00000000000..19c4774575d --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out @@ -0,0 +1,61 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); + truncate_log +-------------- +(0 rows) + +-- end_ignore +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM gpsc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; +-- SET below disables utility logging and DONE must still be logged. +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_DONE +(2 rows) + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('gpsc.ignored_users_list', current_user, false); + set_config +------------ + gpadmin +(1 row) + +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_START + QUERY_STATUS_END + QUERY_STATUS_DONE +(4 rows) + +DROP FUNCTION print_last_query(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_locale.out b/gpcontrib/gp_stats_collector/results/gpsc_locale.out new file mode 100644 index 00000000000..a01fe0648b9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale +CREATE EXTENSION gp_stats_collector; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_select.out b/gpcontrib/gp_stats_collector/results/gpsc_select.out new file mode 100644 index 00000000000..3008c8f6d55 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_select.out @@ -0,0 +1,136 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; +-- Basic SELECT tests +SET gpsc.logging_mode to 'TBL'; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +SELECT COUNT(*) FROM generate_series(1,10); + count +------- + 10 +(1 row) + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Transaction test +SET gpsc.logging_mode to 'TBL'; +BEGIN; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- CTE test +SET gpsc.logging_mode to 'TBL'; +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + column1 +--------- + 1 + 2 +(2 rows) + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-----------------------------+--------------------- + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE + | SELECT * FROM t; | +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Prepared statement test +SET gpsc.logging_mode to 'TBL'; +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; + ?column? +---------- + 1 +(1 row) + +DEALLOCATE test_stmt; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------+--------------------- + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_uds.out b/gpcontrib/gp_stats_collector/results/gpsc_uds.out new file mode 100644 index 00000000000..e8bca79e669 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +\set UDS_PATH '/tmp/gpsc_test.sock' +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out new file mode 100644 index 00000000000..db3949f3152 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out @@ -0,0 +1,68 @@ +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM gpsc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +-- Test 1: 1 byte chars +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; + ?column? +------------ + HelloWorld +(1 row) + +RESET gpsc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 2: 2 byte chars +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; + ?column? +------------- + РУССКИЙЯЗЫК +(1 row) + +RESET gpsc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 3: 4 byte chars +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; + ?column? +---------- + 😀 +(1 row) + +RESET gpsc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utility.out b/gpcontrib/gp_stats_collector/results/gpsc_utility.out new file mode 100644 index 00000000000..e8e28614370 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_utility.out @@ -0,0 +1,248 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE test_table (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------------------+--------------------- + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE + -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE test_table; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Partitioning +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------+--------------------- + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Views and Functions +SET gpsc.logging_mode to 'TBL'; +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------------------------------------------------+--------------------- + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE + -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT + -1 | DROP VIEW test_view; | QUERY_STATUS_DONE + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Transaction Operations +SET gpsc.logging_mode to 'TBL'; +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; +BEGIN; +SAVEPOINT sp2; +ABORT; +BEGIN; +ROLLBACK; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(18 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- DML Operations +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE dml_test (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------+--------------------- + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE + -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- COPY Operations +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE copy_test (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +COPY (SELECT 1) TO STDOUT; +1 +DROP TABLE copy_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Prepared Statements and error during execute +SET gpsc.logging_mode to 'TBL'; +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +ERROR: division by zero +DEALLOCATE test_prep; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------+--------------------- + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR + -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT + -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- GUC Settings +SET gpsc.logging_mode to 'TBL'; +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------+--------------------- + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql b/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql new file mode 100644 index 00000000000..8361f7b678d --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql @@ -0,0 +1,85 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; + +-- DECLARE +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- DECLARE WITH HOLD +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- ROLLBACK +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- FETCH +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; +FETCH 1 IN cursor_stats_6; +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql b/gpcontrib/gp_stats_collector/sql/gpsc_dist.sql similarity index 53% rename from gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_dist.sql index d5519d0cd96..46b531a70ca 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_dist.sql @@ -1,6 +1,6 @@ -CREATE EXTENSION yagp_hooks_collector; +CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -14,36 +14,36 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Hash distributed table CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); INSERT INTO test_hash_dist SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SET optimizer_enable_direct_dispatch TO TRUE; -- Direct dispatch is used here, only one segment is scanned. select * from test_hash_dist where id = 1; RESET optimizer_enable_direct_dispatch; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Should see 8 rows. -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; -- Scan all segments. select * from test_hash_dist; DROP TABLE test_hash_dist; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -- Replicated table CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ @@ -55,14 +55,14 @@ $$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; INSERT INTO test_replicated SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_replicated, force_segments(); DROP TABLE test_replicated; DROP FUNCTION force_segments(); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -- Partially distributed table (2 numsegments) SET allow_system_table_mods = ON; @@ -70,19 +70,19 @@ CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_partial_dist; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; DROP TABLE test_partial_dist; RESET allow_system_table_mods; -- Should see 12 rows. -SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql b/gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql similarity index 58% rename from gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql index 9e6de69d61e..6aff2ad5cf6 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql @@ -8,36 +8,36 @@ -- have its DONE event rejected, creating orphaned SUBMIT entries. -- This is due to query being actually executed between SUBMIT and DONE. -- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; -SELECT yagpcc.truncate_log(); +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); -- end_ignore CREATE OR REPLACE FUNCTION print_last_query(query text) RETURNS TABLE(query_status text) AS $$ SELECT query_status - FROM yagpcc.log + FROM gpsc.log WHERE segid = -1 AND query_text = query ORDER BY ccnt DESC $$ LANGUAGE sql; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.logging_mode TO 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; -- SET below disables utility logging and DONE must still be logged. -SET yagpcc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); -- SELECT below adds current user to ignore list and DONE must still be logged. -- start_ignore -SELECT set_config('yagpcc.ignored_users_list', current_user, false); +SELECT set_config('gpsc.ignored_users_list', current_user, false); -- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); DROP FUNCTION print_last_query(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -RESET yagpcc.logging_mode; +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql b/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql new file mode 100644 index 00000000000..6321c93f5ab --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql @@ -0,0 +1,29 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. + +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore + +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale + +CREATE EXTENSION gp_stats_collector; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; + +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; + +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_select.sql b/gpcontrib/gp_stats_collector/sql/gpsc_select.sql new file mode 100644 index 00000000000..673cbee0c10 --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_select.sql @@ -0,0 +1,69 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; + +-- Basic SELECT tests +SET gpsc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Transaction test +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +SELECT 1; +COMMIT; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- CTE test +SET gpsc.logging_mode to 'TBL'; + +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Prepared statement test +SET gpsc.logging_mode to 'TBL'; + +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; +DEALLOCATE test_stmt; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql b/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql new file mode 100644 index 00000000000..14377b15c8c --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql @@ -0,0 +1,31 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore + +\set UDS_PATH '/tmp/gpsc_test.sock' + +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; + +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + +-- Send +SELECT 1; + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql b/gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql similarity index 58% rename from gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql index c3053e4af0c..a3f8a376d55 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql @@ -1,45 +1,45 @@ -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) RETURNS TEXT AS $$ SELECT query_text - FROM yagpcc.log + FROM gpsc.log WHERE query_text LIKE '%' || marker || '%' ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; -- Test 1: 1 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test1*/ 'HelloWorld'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; -- Test 2: 2 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test2*/ 'РУССКИЙЯЗЫК'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Character 'Р' has two bytes and cut in the middle => not included. SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; -- Test 3: 4 byte chars -SET yagpcc.max_text_size to 21; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; SELECT /*test3*/ '😀'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Emoji has 4 bytes and cut before the last byte => not included. SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; -- Cleanup DROP FUNCTION get_marked_query(TEXT); -RESET yagpcc.max_text_size; -RESET yagpcc.logging_mode; -RESET yagpcc.enable; -RESET yagpcc.ignored_users_list; +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; -DROP EXTENSION yagp_hooks_collector; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql b/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql new file mode 100644 index 00000000000..9abb965db37 --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql @@ -0,0 +1,135 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; + +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE test_table (a int, b text); +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Partitioning +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Views and Functions +SET gpsc.logging_mode to 'TBL'; + +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Transaction Operations +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; + +BEGIN; +SAVEPOINT sp2; +ABORT; + +BEGIN; +ROLLBACK; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- DML Operations +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE dml_test (a int, b text); +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- COPY Operations +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE copy_test (a int); +COPY (SELECT 1) TO STDOUT; +DROP TABLE copy_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Prepared Statements and error during execute +SET gpsc.logging_mode to 'TBL'; + +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +DEALLOCATE test_prep; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- GUC Settings +SET gpsc.logging_mode to 'TBL'; + +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp similarity index 79% rename from gpcontrib/yagp_hooks_collector/src/Config.cpp rename to gpcontrib/gp_stats_collector/src/Config.cpp index 62c16e91d1f..e117aa941fd 100644 --- a/gpcontrib/yagp_hooks_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -20,7 +20,7 @@ * Config.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/Config.cpp + * gpcontrib/gp_stats_collector/src/Config.cpp * *------------------------------------------------------------------------- */ @@ -62,63 +62,63 @@ static void assign_ignored_users_hook(const char *, void *) { void Config::init_gucs() { DefineCustomStringVariable( - "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, - &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, + "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, + "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable_analyze", "Collect analyze metrics in yagpcc", 0LL, + "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, &guc_enable_analyze, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, + "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, + "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - DefineCustomStringVariable("yagpcc.ignored_users_list", - "Make yagpcc ignore queries issued by given users", + DefineCustomStringVariable("gpsc.ignored_users_list", + "Make gpsc ignore queries issued by given users", 0LL, &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, assign_ignored_users_hook, 0LL); DefineCustomIntVariable( - "yagpcc.max_text_size", - "Make yagpcc trim query texts longer than configured size in bytes", NULL, + "gpsc.max_text_size", + "Make gpsc trim query texts longer than configured size in bytes", NULL, &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); DefineCustomIntVariable( - "yagpcc.max_plan_size", - "Make yagpcc trim plan longer than configured size", NULL, + "gpsc.max_plan_size", + "Make gpsc trim plan longer than configured size", NULL, &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); DefineCustomIntVariable( - "yagpcc.min_analyze_time", + "gpsc.min_analyze_time", "Sets the minimum execution time above which plans will be logged.", "Zero prints all plans. -1 turns this feature off.", &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomEnumVariable( - "yagpcc.logging_mode", "Logging mode: UDS or PG Table", NULL, + "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, NULL); DefineCustomBoolVariable( - "yagpcc.enable_utility", "Collect utility statement stats", NULL, + "gpsc.enable_utility", "Collect utility statement stats", NULL, &guc_enable_utility, false, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } @@ -127,27 +127,27 @@ void Config::update_ignored_users(const char *new_guc_ignored_users) { auto new_ignored_users_set = std::make_unique(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); + "invalid list syntax in parameter gpsc.ignored_users_list"))); return; } foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); } ignored_users_ = std::move(new_ignored_users_set); } diff --git a/gpcontrib/yagp_hooks_collector/src/Config.h b/gpcontrib/gp_stats_collector/src/Config.h similarity index 97% rename from gpcontrib/yagp_hooks_collector/src/Config.h rename to gpcontrib/gp_stats_collector/src/Config.h index 01ae5ea328e..91a1ffe44f2 100644 --- a/gpcontrib/yagp_hooks_collector/src/Config.h +++ b/gpcontrib/gp_stats_collector/src/Config.h @@ -20,7 +20,7 @@ * Config.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/Config.h + * gpcontrib/gp_stats_collector/src/Config.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp similarity index 86% rename from gpcontrib/yagp_hooks_collector/src/EventSender.cpp rename to gpcontrib/gp_stats_collector/src/EventSender.cpp index 6993814ffbf..b28ceba175a 100644 --- a/gpcontrib/yagp_hooks_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -20,7 +20,7 @@ * EventSender.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/EventSender.cpp + * gpcontrib/gp_stats_collector/src/EventSender.cpp * *------------------------------------------------------------------------- */ @@ -106,7 +106,7 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, return true; } -bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, +bool EventSender::log_query_req(const gpsc::SetQueryReq &req, const std::string &event, bool utility) { bool clear_big_fields = false; switch (config.logging_mode()) { @@ -114,7 +114,7 @@ bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, clear_big_fields = UDSConnector::report_query(req, event, config); break; case LOG_MODE_TBL: - ya_gpdb::insert_log(req, utility); + gpdb::insert_log(req, utility); clear_big_fields = false; break; default: @@ -170,7 +170,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - ya_gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -192,12 +192,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { // context so it will go away at executor_end. if (query_desc->totaltime == NULL) { MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL, false); - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); + gpdb::mem_ctx_switch_to(oldcxt); } } - yagpcc::GPMetrics stats; + gpsc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); if (log_query_req(*query_msg, "started", false /* utility */)) { clear_big_fields(query_msg); @@ -233,7 +233,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { submit_query(query_desc); auto &query = get_query(query_desc); auto *query_msg = query.message.get(); - *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg); set_qi_nesting_level(query_msg, nesting_level); @@ -256,27 +256,27 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, QueryMetricsStatus status, bool utility, ErrorData *edata) { - yagpcc::QueryStatus query_status; + gpsc::QueryStatus query_status; std::string msg; switch (status) { case METRICS_QUERY_DONE: case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; msg = "done"; break; case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; msg = "error"; break; case METRICS_QUERY_CANCELING: // at the moment we don't track this event, but I`ll leave this code // here just in case Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; msg = "cancelling"; break; case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; msg = "cancelled"; break; default: @@ -285,15 +285,15 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, } auto prev_state = query.state; update_query_state(query, QueryState::DONE, utility, - query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); + query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query.message.get(); query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { bool error_flushed = elog_message() == NULL; if (error_flushed && (edata == NULL || edata->message == NULL)) { - ereport(WARNING, (errmsg("YAGPCC missing error message"))); + ereport(WARNING, (errmsg("GPSC missing error message"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } else { set_qi_error_message( query_msg, error_flushed ? edata->message : elog_message(), config); @@ -324,18 +324,18 @@ void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, // Skip sending done message if query errored before submit. if (!qdesc_submitted(query_desc)) { if (status != METRICS_QUERY_ERROR) { - ereport(WARNING, (errmsg("YAGPCC trying to process DONE hook for " + ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " "unsubmitted and unerrored query"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } return; } if (queries.empty()) { - ereport(WARNING, (errmsg("YAGPCC cannot find query to process DONE hook"))); + ereport(WARNING, (errmsg("GPSC cannot find query to process DONE hook"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); return; } auto &query = get_query(query_desc); @@ -346,8 +346,8 @@ void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, update_nested_counters(query_desc); queries.erase(QueryKey::from_qdesc(query_desc)); - pfree(query_desc->yagp_query_key); - query_desc->yagp_query_key = NULL; + pfree(query_desc->gpsc_query_key); + query_desc->gpsc_query_key = NULL; } void EventSender::ic_metrics_collect() { @@ -395,7 +395,7 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - ya_gpdb::instr_end_loop(query_desc->totaltime); + gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= config.min_analyze_time()) { @@ -424,7 +424,7 @@ EventSender::EventSender() { EventSender::~EventSender() { for (const auto &[qkey, _] : queries) { - ereport(LOG, (errmsg("YAGPCC query with missing done event: " + ereport(LOG, (errmsg("GPSC query with missing done event: " "tmid=%d ssid=%d ccnt=%d nlvl=%d", qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } @@ -440,7 +440,7 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, break; case QueryState::START: if (query.state == QueryState::SUBMIT) { - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_START); } else { Assert(false); } @@ -449,11 +449,11 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, // Example of below assert triggering: CURSOR closes before ever being // executed Assert(query->state == QueryState::START || // IsAbortInProgress()); - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: Assert(query.state == QueryState::END || !success || utility); - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_DONE); break; default: Assert(false); @@ -464,28 +464,28 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { if (!qdesc_submitted(query_desc)) { ereport(WARNING, - (errmsg("YAGPCC attempting to get query that was not submitted"))); + (errmsg("GPSC attempting to get query that was not submitted"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); throw std::runtime_error("Attempting to get query that was not submitted"); } return queries.find(QueryKey::from_qdesc(query_desc))->second; } void EventSender::submit_query(QueryDesc *query_desc) { - if (query_desc->yagp_query_key) { + if (query_desc->gpsc_query_key) { ereport(WARNING, - (errmsg("YAGPCC trying to submit already submitted query"))); + (errmsg("GPSC trying to submit already submitted query"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } QueryKey::register_qkey(query_desc, nesting_level); auto key = QueryKey::from_qdesc(query_desc); auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); if (!inserted) { - ereport(WARNING, (errmsg("YAGPCC duplicate query submit detected"))); + ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } } @@ -498,16 +498,16 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { if (end_time >= start_time) { nested_timing += end_time - start_time; } else { - ereport(WARNING, (errmsg("YAGPCC query start_time > end_time (%f > %f)", + ereport(WARNING, (errmsg("GPSC query start_time > end_time (%f > %f)", start_time, end_time))); ereport(DEBUG3, - (errmsg("YAGPCC nested query text %s", query_desc->sourceText))); + (errmsg("GPSC nested query text %s", query_desc->sourceText))); } } } bool EventSender::qdesc_submitted(QueryDesc *query_desc) { - if (query_desc->yagp_query_key == NULL) { + if (query_desc->gpsc_query_key == NULL) { return false; } return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); @@ -528,4 +528,4 @@ bool EventSender::filter_query(QueryDesc *query_desc) { } EventSender::QueryItem::QueryItem(QueryState st) - : message(std::make_unique()), state(st) {} + : message(std::make_unique()), state(st) {} diff --git a/gpcontrib/yagp_hooks_collector/src/EventSender.h b/gpcontrib/gp_stats_collector/src/EventSender.h similarity index 84% rename from gpcontrib/yagp_hooks_collector/src/EventSender.h rename to gpcontrib/gp_stats_collector/src/EventSender.h index ef7dcb0bf8c..154c2c0dceb 100644 --- a/gpcontrib/yagp_hooks_collector/src/EventSender.h +++ b/gpcontrib/gp_stats_collector/src/EventSender.h @@ -20,7 +20,7 @@ * EventSender.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/EventSender.h + * gpcontrib/gp_stats_collector/src/EventSender.h * *------------------------------------------------------------------------- */ @@ -45,7 +45,7 @@ extern "C" { class UDSConnector; struct QueryDesc; -namespace yagpcc { +namespace gpsc { class SetQueryReq; } @@ -67,24 +67,24 @@ struct QueryKey { } static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { - query_desc->yagp_query_key = - (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); + query_desc->gpsc_query_key = + (GpscQueryKey *)gpdb::palloc0(sizeof(GpscQueryKey)); int32 tmid; gp_gettmid(&tmid); - query_desc->yagp_query_key->tmid = tmid; - query_desc->yagp_query_key->ssid = gp_session_id; - query_desc->yagp_query_key->ccnt = gp_command_count; - query_desc->yagp_query_key->nesting_level = nesting_level; - query_desc->yagp_query_key->query_desc_addr = (uintptr_t)query_desc; + query_desc->gpsc_query_key->tmid = tmid; + query_desc->gpsc_query_key->ssid = gp_session_id; + query_desc->gpsc_query_key->ccnt = gp_command_count; + query_desc->gpsc_query_key->nesting_level = nesting_level; + query_desc->gpsc_query_key->query_desc_addr = (uintptr_t)query_desc; } static QueryKey from_qdesc(QueryDesc *query_desc) { return { - .tmid = query_desc->yagp_query_key->tmid, - .ssid = query_desc->yagp_query_key->ssid, - .ccnt = query_desc->yagp_query_key->ccnt, - .nesting_level = query_desc->yagp_query_key->nesting_level, - .query_desc_addr = query_desc->yagp_query_key->query_desc_addr, + .tmid = query_desc->gpsc_query_key->tmid, + .ssid = query_desc->gpsc_query_key->ssid, + .ccnt = query_desc->gpsc_query_key->ccnt, + .nesting_level = query_desc->gpsc_query_key->nesting_level, + .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, }; } }; @@ -130,13 +130,13 @@ class EventSender { enum QueryState { SUBMIT, START, END, DONE }; struct QueryItem { - std::unique_ptr message; + std::unique_ptr message; QueryState state; explicit QueryItem(QueryState st); }; - bool log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, + bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, bool utility); bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); void update_query_state(QueryItem &query, QueryState new_state, bool utility, diff --git a/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp b/gpcontrib/gp_stats_collector/src/GpscStat.cpp similarity index 78% rename from gpcontrib/yagp_hooks_collector/src/YagpStat.cpp rename to gpcontrib/gp_stats_collector/src/GpscStat.cpp index 3a760b6ea97..c4029f085cf 100644 --- a/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp +++ b/gpcontrib/gp_stats_collector/src/GpscStat.cpp @@ -17,15 +17,15 @@ * specific language governing permissions and limitations * under the License. * - * YagpStat.cpp + * GpscStat.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/YagpStat.cpp + * gpcontrib/gp_stats_collector/src/GpscStat.cpp * *------------------------------------------------------------------------- */ -#include "YagpStat.h" +#include "GpscStat.h" #include @@ -41,21 +41,21 @@ extern "C" { namespace { struct ProtectedData { slock_t mutex; - YagpStat::Data data; + GpscStat::Data data; }; shmem_startup_hook_type prev_shmem_startup_hook = NULL; ProtectedData *data = nullptr; -void yagp_shmem_startup() { +void gpsc_shmem_startup() { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); bool found; data = reinterpret_cast( - ShmemInitStruct("yagp_stat_messages", sizeof(ProtectedData), &found)); + ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); if (!found) { SpinLockInit(&data->mutex); - data->data = YagpStat::Data(); + data->data = GpscStat::Data(); } LWLockRelease(AddinShmemInitLock); } @@ -70,49 +70,49 @@ class LockGuard { }; } // namespace -void YagpStat::init() { +void GpscStat::init() { if (!process_shared_preload_libraries_in_progress) return; RequestAddinShmemSpace(sizeof(ProtectedData)); prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = yagp_shmem_startup; + shmem_startup_hook = gpsc_shmem_startup; } -void YagpStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } +void GpscStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } -void YagpStat::reset() { +void GpscStat::reset() { LockGuard lg(&data->mutex); - data->data = YagpStat::Data(); + data->data = GpscStat::Data(); } -void YagpStat::report_send(int32_t msg_size) { +void GpscStat::report_send(int32_t msg_size) { LockGuard lg(&data->mutex); data->data.total++; data->data.max_message_size = std::max(msg_size, data->data.max_message_size); } -void YagpStat::report_bad_connection() { +void GpscStat::report_bad_connection() { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_connects++; } -void YagpStat::report_bad_send(int32_t msg_size) { +void GpscStat::report_bad_send(int32_t msg_size) { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_sends++; data->data.max_message_size = std::max(msg_size, data->data.max_message_size); } -void YagpStat::report_error() { +void GpscStat::report_error() { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_other++; } -YagpStat::Data YagpStat::get_stats() { +GpscStat::Data GpscStat::get_stats() { LockGuard lg(&data->mutex); return data->data; } -bool YagpStat::loaded() { return data != nullptr; } +bool GpscStat::loaded() { return data != nullptr; } diff --git a/gpcontrib/yagp_hooks_collector/src/YagpStat.h b/gpcontrib/gp_stats_collector/src/GpscStat.h similarity index 94% rename from gpcontrib/yagp_hooks_collector/src/YagpStat.h rename to gpcontrib/gp_stats_collector/src/GpscStat.h index 57fc90cd4d1..af1a1261776 100644 --- a/gpcontrib/yagp_hooks_collector/src/YagpStat.h +++ b/gpcontrib/gp_stats_collector/src/GpscStat.h @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * YagpStat.h + * GpscStat.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/YagpStat.h + * gpcontrib/gp_stats_collector/src/GpscStat.h * *------------------------------------------------------------------------- */ @@ -29,7 +29,7 @@ #include -class YagpStat { +class GpscStat { public: struct Data { int64_t total, failed_sends, failed_connects, failed_other; diff --git a/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp b/gpcontrib/gp_stats_collector/src/PgUtils.cpp similarity index 83% rename from gpcontrib/yagp_hooks_collector/src/PgUtils.cpp rename to gpcontrib/gp_stats_collector/src/PgUtils.cpp index ed4bf4d7e64..3dbee97061b 100644 --- a/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/PgUtils.cpp @@ -20,7 +20,7 @@ * PgUtils.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/PgUtils.cpp + * gpcontrib/gp_stats_collector/src/PgUtils.cpp * *------------------------------------------------------------------------- */ @@ -37,31 +37,31 @@ extern "C" { std::string get_user_name() { // username is allocated on stack, we don't need to pfree it. const char *username = - ya_gpdb::get_config_option("session_authorization", false, false); + gpdb::get_config_option("session_authorization", false, false); return username ? std::string(username) : ""; } std::string get_db_name() { - char *dbname = ya_gpdb::get_database_name(MyDatabaseId); + char *dbname = gpdb::get_database_name(MyDatabaseId); if (dbname) { std::string result(dbname); - ya_gpdb::pfree(dbname); + gpdb::pfree(dbname); return result; } return ""; } std::string get_rg_name() { - auto groupId = ya_gpdb::get_rg_id_by_session_id(MySessionState->sessionId); + auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); if (!OidIsValid(groupId)) return ""; - char *rgname = ya_gpdb::get_rg_name_for_id(groupId); + char *rgname = gpdb::get_rg_name_for_id(groupId); if (rgname == nullptr) return ""; std::string result(rgname); - ya_gpdb::pfree(rgname); + gpdb::pfree(rgname); return result; } @@ -77,7 +77,7 @@ std::string get_rg_name() { * segment. An example would be `select a from tbl where is_good_value(b);`. In * this case master will issue one top-level statement, but segments will change * contexts for UDF execution and execute is_good_value(b) once for each tuple - * as a nested query. Creating massive load on gpcc agent. + * as a nested query. Creating massive load on external agent. * * Hence, here is a decision: * 1) ignore all queries that are nested on segments @@ -87,8 +87,8 @@ std::string get_rg_name() { */ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - if (query_desc->yagp_query_key == NULL) { + if (query_desc->gpsc_query_key == NULL) { return nesting_level == 0; } - return query_desc->yagp_query_key->nesting_level == 0; + return query_desc->gpsc_query_key->nesting_level == 0; } diff --git a/gpcontrib/yagp_hooks_collector/src/PgUtils.h b/gpcontrib/gp_stats_collector/src/PgUtils.h similarity index 96% rename from gpcontrib/yagp_hooks_collector/src/PgUtils.h rename to gpcontrib/gp_stats_collector/src/PgUtils.h index 5113fadbff2..d9f673e7cbc 100644 --- a/gpcontrib/yagp_hooks_collector/src/PgUtils.h +++ b/gpcontrib/gp_stats_collector/src/PgUtils.h @@ -20,7 +20,7 @@ * PgUtils.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/PgUtils.h + * gpcontrib/gp_stats_collector/src/PgUtils.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp b/gpcontrib/gp_stats_collector/src/ProcStats.cpp similarity index 92% rename from gpcontrib/yagp_hooks_collector/src/ProcStats.cpp rename to gpcontrib/gp_stats_collector/src/ProcStats.cpp index 72a12e8ca00..9c557879fc6 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp +++ b/gpcontrib/gp_stats_collector/src/ProcStats.cpp @@ -20,13 +20,13 @@ * ProcStats.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProcStats.cpp + * gpcontrib/gp_stats_collector/src/ProcStats.cpp * *------------------------------------------------------------------------- */ #include "ProcStats.h" -#include "yagpcc_metrics.pb.h" +#include "gpsc_metrics.pb.h" #include #include #include @@ -42,7 +42,7 @@ namespace { proc_stat >> tmp >> stat_name; \ stats->set_##stat_name(stat_name - stats->stat_name()); -void fill_io_stats(yagpcc::SystemStat *stats) { +void fill_io_stats(gpsc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/io"); std::string tmp; FILL_IO_STAT(rchar); @@ -54,7 +54,7 @@ void fill_io_stats(yagpcc::SystemStat *stats) { FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(yagpcc::SystemStat *stats) { +void fill_cpu_stats(gpsc::SystemStat *stats) { static const int UTIME_ID = 13; static const int STIME_ID = 14; static const int VSIZE_ID = 22; @@ -92,7 +92,7 @@ void fill_cpu_stats(yagpcc::SystemStat *stats) { } } -void fill_status_stats(yagpcc::SystemStat *stats) { +void fill_status_stats(gpsc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/status"); std::string key, measure; while (proc_stat >> key) { @@ -118,7 +118,7 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } } // namespace -void fill_self_stats(yagpcc::SystemStat *stats) { +void fill_self_stats(gpsc::SystemStat *stats) { fill_io_stats(stats); fill_cpu_stats(stats); fill_status_stats(stats); diff --git a/gpcontrib/yagp_hooks_collector/src/ProcStats.h b/gpcontrib/gp_stats_collector/src/ProcStats.h similarity index 89% rename from gpcontrib/yagp_hooks_collector/src/ProcStats.h rename to gpcontrib/gp_stats_collector/src/ProcStats.h index 7629edd0aea..4473125f875 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProcStats.h +++ b/gpcontrib/gp_stats_collector/src/ProcStats.h @@ -20,15 +20,15 @@ * ProcStats.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProcStats.h + * gpcontrib/gp_stats_collector/src/ProcStats.h * *------------------------------------------------------------------------- */ #pragma once -namespace yagpcc { +namespace gpsc { class SystemStat; } -void fill_self_stats(yagpcc::SystemStat *stats); \ No newline at end of file +void fill_self_stats(gpsc::SystemStat *stats); \ No newline at end of file diff --git a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp similarity index 85% rename from gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp rename to gpcontrib/gp_stats_collector/src/ProtoUtils.cpp index b449ae20900..c9ceff4739b 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp @@ -20,7 +20,7 @@ * ProtoUtils.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp + * gpcontrib/gp_stats_collector/src/ProtoUtils.cpp * *------------------------------------------------------------------------- */ @@ -74,7 +74,7 @@ google::protobuf::Timestamp current_ts() { return current_ts; } -void set_query_key(yagpcc::QueryKey *key) { +void set_query_key(gpsc::QueryKey *key) { key->set_ccnt(gp_command_count); key->set_ssid(gp_session_id); int32 tmid = 0; @@ -82,7 +82,7 @@ void set_query_key(yagpcc::QueryKey *key) { key->set_tmid(tmid); } -void set_segment_key(yagpcc::SegmentKey *key) { +void set_segment_key(gpsc::SegmentKey *key) { key->set_dbid(GpIdentity.dbid); key->set_segindex(GpIdentity.segindex); } @@ -109,51 +109,51 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { return std::string(str, cut_pos); } -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_explain_state(query_desc, true); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); - StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); if (norm_plan) { *qi->mutable_template_plan_text() = trim_str_shrink_utf8( norm_plan->data, norm_plan->len, config.max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - ya_gpdb::pfree(norm_plan->data); + gpdb::pfree(norm_plan->data); } qi->set_query_id(query_desc->plannedstmt->queryId); - ya_gpdb::pfree(es.str->data); + gpdb::pfree(es.str->data); } - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(oldcxt); } } -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), config.max_text_size()); - char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); if (norm_query) { *qi->mutable_template_query_text() = trim_str_shrink_utf8( norm_query, strlen(norm_query), config.max_text_size()); - ya_gpdb::pfree(norm_query); + gpdb::pfree(norm_query); } } } -void clear_big_fields(yagpcc::SetQueryReq *req) { +void clear_big_fields(gpsc::SetQueryReq *req) { if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); @@ -164,7 +164,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { } } -void set_query_info(yagpcc::SetQueryReq *req) { +void set_query_info(gpsc::SetQueryReq *req) { if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); @@ -174,24 +174,24 @@ void set_query_info(yagpcc::SetQueryReq *req) { } } -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { +void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) { auto aqi = req->mutable_add_info(); aqi->set_nested_level(nesting_level); } -void set_qi_slice_id(yagpcc::SetQueryReq *req) { +void set_qi_slice_id(gpsc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, +void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, const Config &config) { auto aqi = req->mutable_add_info(); *aqi->mutable_error_message() = trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, +void set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, QueryDesc *query_desc, int nested_calls, double nested_time) { auto instrument = query_desc->planstate->instrument; @@ -233,7 +233,7 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, metrics->set_inherited_time(nested_time); } -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, +void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, @@ -256,7 +256,7 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, metrics->mutable_interconnect()->proto_name() <= \ ic_statistics->stat_name) -void set_ic_stats(yagpcc::MetricInstrumentation *metrics, +void set_ic_stats(gpsc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics) { #ifdef IC_TEARDOWN_HOOK UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); @@ -279,8 +279,8 @@ void set_ic_stats(yagpcc::MetricInstrumentation *metrics, #endif } -yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { - yagpcc::SetQueryReq req; +gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status) { + gpsc::SetQueryReq req; req.set_query_status(status); *req.mutable_datetime() = current_ts(); set_query_key(req.mutable_query_key()); @@ -292,7 +292,7 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, +void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, const Config &config) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. @@ -300,10 +300,10 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, return; } MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_analyze_state( + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state( query_desc, query_desc->instrument_options && config.enable_analyze()); - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { @@ -312,6 +312,6 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); - ya_gpdb::pfree(es.str->data); + gpdb::pfree(es.str->data); } } diff --git a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h b/gpcontrib/gp_stats_collector/src/ProtoUtils.h similarity index 65% rename from gpcontrib/yagp_hooks_collector/src/ProtoUtils.h rename to gpcontrib/gp_stats_collector/src/ProtoUtils.h index c954545494f..5ddcd42d308 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.h @@ -20,35 +20,35 @@ * ProtoUtils.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProtoUtils.h + * gpcontrib/gp_stats_collector/src/ProtoUtils.h * *------------------------------------------------------------------------- */ #pragma once -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" struct QueryDesc; struct ICStatistics; class Config; google::protobuf::Timestamp current_ts(); -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config); -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config); -void clear_big_fields(yagpcc::SetQueryReq *req); -void set_query_info(yagpcc::SetQueryReq *req); -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); -void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, +void clear_big_fields(gpsc::SetQueryReq *req); +void set_query_info(gpsc::SetQueryReq *req); +void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level); +void set_qi_slice_id(gpsc::SetQueryReq *req); +void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, const Config &config); -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, +void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); -void set_ic_stats(yagpcc::MetricInstrumentation *metrics, +void set_ic_stats(gpsc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); -yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); +gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message, +void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *message, const Config &config); diff --git a/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp similarity index 87% rename from gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp rename to gpcontrib/gp_stats_collector/src/UDSConnector.cpp index d13a82a5ca9..9a01d4033d0 100644 --- a/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -20,14 +20,14 @@ * UDSConnector.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp + * gpcontrib/gp_stats_collector/src/UDSConnector.cpp * *------------------------------------------------------------------------- */ #include "UDSConnector.h" #include "Config.h" -#include "YagpStat.h" +#include "GpscStat.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" @@ -44,14 +44,14 @@ extern "C" { #include "postgres.h" } -static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, +static void inline log_tracing_failure(const gpsc::SetQueryReq &req, const std::string &event) { ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", req.query_key().tmid(), req.query_key().ssid(), req.query_key().ccnt(), event.c_str()))); } -bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, +bool UDSConnector::report_query(const gpsc::SetQueryReq &req, const std::string &event, const Config &config) { sockaddr_un address{}; @@ -60,7 +60,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); - YagpStat::report_error(); + GpscStat::report_error(); return false; } strcpy(address.sun_path, uds_path.c_str()); @@ -68,7 +68,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd == -1) { log_tracing_failure(req, event); - YagpStat::report_error(); + GpscStat::report_error(); return false; } @@ -83,24 +83,24 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, // visible to an end-user and admins. ereport(WARNING, (errmsg("Unable to create non-blocking socket connection %m"))); - YagpStat::report_error(); + GpscStat::report_error(); return false; } if (connect(sockfd, reinterpret_cast(&address), sizeof(address)) == -1) { log_tracing_failure(req, event); - YagpStat::report_bad_connection(); + GpscStat::report_bad_connection(); return false; } const auto data_size = req.ByteSizeLong(); const auto total_size = data_size + sizeof(uint32_t); - auto *buf = static_cast(ya_gpdb::palloc(total_size)); + auto *buf = static_cast(gpdb::palloc(total_size)); // Free buf automatically on error path. struct BufGuard { void *p; - ~BufGuard() { ya_gpdb::pfree(p); } + ~BufGuard() { gpdb::pfree(p); } } buf_guard{buf}; *reinterpret_cast(buf) = data_size; @@ -121,10 +121,10 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (sent < 0) { log_tracing_failure(req, event); - YagpStat::report_bad_send(total_size); + GpscStat::report_bad_send(total_size); return false; } - YagpStat::report_send(total_size); + GpscStat::report_send(total_size); return true; } diff --git a/gpcontrib/yagp_hooks_collector/src/UDSConnector.h b/gpcontrib/gp_stats_collector/src/UDSConnector.h similarity index 88% rename from gpcontrib/yagp_hooks_collector/src/UDSConnector.h rename to gpcontrib/gp_stats_collector/src/UDSConnector.h index be5ab1ef413..a91d22f9df1 100644 --- a/gpcontrib/yagp_hooks_collector/src/UDSConnector.h +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.h @@ -20,19 +20,19 @@ * UDSConnector.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/UDSConnector.h + * gpcontrib/gp_stats_collector/src/UDSConnector.h * *------------------------------------------------------------------------- */ #pragma once -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" class Config; class UDSConnector { public: - bool static report_query(const yagpcc::SetQueryReq &req, + bool static report_query(const gpsc::SetQueryReq &req, const std::string &event, const Config &config); }; diff --git a/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c similarity index 79% rename from gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c rename to gpcontrib/gp_stats_collector/src/gp_stats_collector.c index 271bceee178..d930f72246d 100644 --- a/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * yagp_hooks_collector.c + * gp_stats_collector.c * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c + * gpcontrib/gp_stats_collector/src/gp_stats_collector.c * *------------------------------------------------------------------------- */ @@ -36,14 +36,14 @@ PG_MODULE_MAGIC; void _PG_init(void); void _PG_fini(void); -PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); -PG_FUNCTION_INFO_V1(yagp_stat_messages); -PG_FUNCTION_INFO_V1(yagp_init_log); -PG_FUNCTION_INFO_V1(yagp_truncate_log); +PG_FUNCTION_INFO_V1(gpsc_stat_messages_reset); +PG_FUNCTION_INFO_V1(gpsc_stat_messages); +PG_FUNCTION_INFO_V1(gpsc_init_log); +PG_FUNCTION_INFO_V1(gpsc_truncate_log); -PG_FUNCTION_INFO_V1(yagp_test_uds_start_server); -PG_FUNCTION_INFO_V1(yagp_test_uds_receive); -PG_FUNCTION_INFO_V1(yagp_test_uds_stop_server); +PG_FUNCTION_INFO_V1(gpsc_test_uds_start_server); +PG_FUNCTION_INFO_V1(gpsc_test_uds_receive); +PG_FUNCTION_INFO_V1(gpsc_test_uds_stop_server); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) @@ -55,23 +55,23 @@ void _PG_fini(void) { hooks_deinit(); } -Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { +Datum gpsc_stat_messages_reset(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { funcctx = SRF_FIRSTCALL_INIT(); - yagp_functions_reset(); + gpsc_functions_reset(); } funcctx = SRF_PERCALL_SETUP(); SRF_RETURN_DONE(funcctx); } -Datum yagp_stat_messages(PG_FUNCTION_ARGS) { - return yagp_functions_get(fcinfo); +Datum gpsc_stat_messages(PG_FUNCTION_ARGS) { + return gpsc_functions_get(fcinfo); } -Datum yagp_init_log(PG_FUNCTION_ARGS) { +Datum gpsc_init_log(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -83,7 +83,7 @@ Datum yagp_init_log(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_truncate_log(PG_FUNCTION_ARGS) { +Datum gpsc_truncate_log(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -95,7 +95,7 @@ Datum yagp_truncate_log(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_start_server(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -109,7 +109,7 @@ Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_receive(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; int64 *result; @@ -137,7 +137,7 @@ Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_stop_server(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { diff --git a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp similarity index 84% rename from gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp rename to gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index cb4970d60d9..0a40b4cb359 100644 --- a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -20,7 +20,7 @@ * hook_wrappers.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp + * gpcontrib/gp_stats_collector/src/hook_wrappers.cpp * *------------------------------------------------------------------------- */ @@ -48,7 +48,7 @@ extern "C" { #undef typeid #include "Config.h" -#include "YagpStat.h" +#include "GpscStat.h" #include "EventSender.h" #include "hook_wrappers.h" #include "memory/gpdbwrappers.h" @@ -67,20 +67,20 @@ static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; -static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, +static void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, uint64 count, bool execute_once); -static void ya_ExecutorFinish_hook(QueryDesc *query_desc); -static void ya_ExecutorEnd_hook(QueryDesc *query_desc); -static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static void gpsc_ExecutorFinish_hook(QueryDesc *query_desc); +static void gpsc_ExecutorEnd_hook(QueryDesc *query_desc); +static void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg); #ifdef IC_TEARDOWN_HOOK -static void ya_ic_teardown_hook(ChunkTransportState *transportStates, +static void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); +static void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, +static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, @@ -108,34 +108,34 @@ R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { try { return (obj->*func)(args...); } catch (const std::exception &e) { - ereport(FATAL, (errmsg("Unexpected exception in yagpcc %s", e.what()))); + ereport(FATAL, (errmsg("Unexpected exception in gpsc %s", e.what()))); } } void hooks_init() { Config::init_gucs(); - YagpStat::init(); + GpscStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = ya_ExecutorStart_hook; + ExecutorStart_hook = gpsc_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; - ExecutorRun_hook = ya_ExecutorRun_hook; + ExecutorRun_hook = gpsc_ExecutorRun_hook; previous_ExecutorFinish_hook = ExecutorFinish_hook; - ExecutorFinish_hook = ya_ExecutorFinish_hook; + ExecutorFinish_hook = gpsc_ExecutorFinish_hook; previous_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = ya_ExecutorEnd_hook; + ExecutorEnd_hook = gpsc_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; - query_info_collect_hook = ya_query_info_collect_hook; + query_info_collect_hook = gpsc_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK previous_ic_teardown_hook = ic_teardown_hook; - ic_teardown_hook = ya_ic_teardown_hook; + ic_teardown_hook = gpsc_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK previous_analyze_stats_collect_hook = analyze_stats_collect_hook; - analyze_stats_collect_hook = ya_analyze_stats_collect_hook; + analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; #endif stat_statements_parser_init(); previous_ProcessUtility_hook = ProcessUtility_hook; - ProcessUtility_hook = ya_process_utility_hook; + ProcessUtility_hook = gpsc_process_utility_hook; } void hooks_deinit() { @@ -154,11 +154,11 @@ void hooks_deinit() { if (sender) { delete sender; } - YagpStat::deinit(); + GpscStat::deinit(); ProcessUtility_hook = previous_ProcessUtility_hook; } -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { +void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, eflags); if (previous_ExecutorStart_hook) { @@ -170,7 +170,7 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { eflags); } -void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, +void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, uint64 count, bool execute_once) { get_sender()->incr_depth(); PG_TRY(); @@ -189,7 +189,7 @@ void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) { +void gpsc_ExecutorFinish_hook(QueryDesc *query_desc) { get_sender()->incr_depth(); PG_TRY(); { @@ -207,7 +207,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { PG_END_TRY(); } -void ya_ExecutorEnd_hook(QueryDesc *query_desc) { +void gpsc_ExecutorEnd_hook(QueryDesc *query_desc) { cpp_call(get_sender(), &EventSender::executor_end, query_desc); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); @@ -216,7 +216,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } } -void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { +void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) { cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); if (previous_query_info_collect_hook) { @@ -225,7 +225,7 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } #ifdef IC_TEARDOWN_HOOK -void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { +void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { cpp_call(get_sender(), &EventSender::ic_metrics_collect); if (previous_ic_teardown_hook) { (*previous_ic_teardown_hook)(transportStates, hasErrors); @@ -234,7 +234,7 @@ void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { +void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) { cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); if (previous_analyze_stats_collect_hook) { (*previous_analyze_stats_collect_hook)(query_desc); @@ -242,7 +242,7 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif -static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, +static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, @@ -296,22 +296,22 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, } static void check_stats_loaded() { - if (!YagpStat::loaded()) { + if (!GpscStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("yagp_hooks_collector must be loaded via " + errmsg("gp_stats_collector must be loaded via " "shared_preload_libraries"))); } } -void yagp_functions_reset() { +void gpsc_functions_reset() { check_stats_loaded(); - YagpStat::reset(); + GpscStat::reset(); } -Datum yagp_functions_get(FunctionCallInfo fcinfo) { +Datum gpsc_functions_get(FunctionCallInfo fcinfo) { const int ATTNUM = 6; check_stats_loaded(); - auto stats = YagpStat::get_stats(); + auto stats = GpscStat::get_stats(); TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, 0 /* attdim */); @@ -335,7 +335,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } diff --git a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h b/gpcontrib/gp_stats_collector/src/hook_wrappers.h similarity index 89% rename from gpcontrib/yagp_hooks_collector/src/hook_wrappers.h rename to gpcontrib/gp_stats_collector/src/hook_wrappers.h index 443406a5259..06c8d064404 100644 --- a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.h @@ -20,7 +20,7 @@ * hook_wrappers.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/hook_wrappers.h + * gpcontrib/gp_stats_collector/src/hook_wrappers.h * *------------------------------------------------------------------------- */ @@ -33,8 +33,8 @@ extern "C" { extern void hooks_init(); extern void hooks_deinit(); -extern void yagp_functions_reset(); -extern Datum yagp_functions_get(FunctionCallInfo fcinfo); +extern void gpsc_functions_reset(); +extern Datum gpsc_functions_get(FunctionCallInfo fcinfo); extern void init_log(); extern void truncate_log(); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp similarity index 91% rename from gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp rename to gpcontrib/gp_stats_collector/src/log/LogOps.cpp index e8c927ece84..ef4f39c0749 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp @@ -20,12 +20,12 @@ * LogOps.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp + * gpcontrib/gp_stats_collector/src/log/LogOps.cpp * *------------------------------------------------------------------------- */ -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" #include "LogOps.h" #include "LogSchema.h" @@ -82,14 +82,14 @@ void init_log() { /* Table can be dropped only via DROP EXTENSION */ recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); } else { - ereport(NOTICE, (errmsg("YAGPCC failed to create log table or schema"))); + ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); } /* Make changes visible */ CommandCounterIncrement(); } -void insert_log(const yagpcc::SetQueryReq &req, bool utility) { +void insert_log(const gpsc::SetQueryReq &req, bool utility) { Oid namespaceId; Oid relationId; Relation rel; @@ -112,15 +112,15 @@ void insert_log(const yagpcc::SetQueryReq &req, bool utility) { return; } - bool nulls[natts_yagp_log]; - Datum values[natts_yagp_log]; + bool nulls[natts_gpsc_log]; + Datum values[natts_gpsc_log]; memset(nulls, true, sizeof(nulls)); memset(values, 0, sizeof(values)); extract_query_req(req, "", values, nulls); - nulls[attnum_yagp_log_utility] = false; - values[attnum_yagp_log_utility] = BoolGetDatum(utility); + nulls[attnum_gpsc_log_utility] = false; + values[attnum_gpsc_log_utility] = BoolGetDatum(utility); rel = heap_open(relationId, RowExclusiveLock); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogOps.h b/gpcontrib/gp_stats_collector/src/log/LogOps.h similarity index 83% rename from gpcontrib/yagp_hooks_collector/src/log/LogOps.h rename to gpcontrib/gp_stats_collector/src/log/LogOps.h index 1fc30c21030..f784270bb8f 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogOps.h +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.h @@ -20,7 +20,7 @@ * LogOps.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogOps.h + * gpcontrib/gp_stats_collector/src/log/LogOps.h * *------------------------------------------------------------------------- */ @@ -35,12 +35,12 @@ extern "C" { } extern "C" { -/* CREATE TABLE yagpcc.__log (...); */ +/* CREATE TABLE gpsc.__log (...); */ void init_log(); -/* TRUNCATE yagpcc.__log */ +/* TRUNCATE gpsc.__log */ void truncate_log(); } -/* INSERT INTO yagpcc.__log VALUES (...) */ -void insert_log(const yagpcc::SetQueryReq &req, bool utility); +/* INSERT INTO gpsc.__log VALUES (...) */ +void insert_log(const gpsc::SetQueryReq &req, bool utility); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp similarity index 94% rename from gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp rename to gpcontrib/gp_stats_collector/src/log/LogSchema.cpp index a391b1a2209..f9f43fac2fd 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp @@ -20,7 +20,7 @@ * LogSchema.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp + * gpcontrib/gp_stats_collector/src/log/LogSchema.cpp * *------------------------------------------------------------------------- */ @@ -36,7 +36,7 @@ const std::unordered_map &proto_name_to_col_idx() { std::unordered_map map; map.reserve(log_tbl_desc.size()); - for (size_t idx = 0; idx < natts_yagp_log; ++idx) { + for (size_t idx = 0; idx < natts_gpsc_log; ++idx) { map.emplace(log_tbl_desc[idx].proto_field_name, idx); } @@ -46,9 +46,9 @@ const std::unordered_map &proto_name_to_col_idx() { } TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log); + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); - for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { + for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) { TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, 0 /* attdim */); @@ -104,7 +104,7 @@ void process_field(const google::protobuf::FieldDescriptor *field, if (it == proto_idx_map.end()) { ereport(NOTICE, - (errmsg("YAGPCC protobuf field %s is not registered in log table", + (errmsg("GPSC protobuf field %s is not registered in log table", field_name.c_str()))); return; } diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h b/gpcontrib/gp_stats_collector/src/log/LogSchema.h similarity index 98% rename from gpcontrib/yagp_hooks_collector/src/log/LogSchema.h rename to gpcontrib/gp_stats_collector/src/log/LogSchema.h index f78acec7ce9..8754741823a 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.h @@ -20,7 +20,7 @@ * LogSchema.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogSchema.h + * gpcontrib/gp_stats_collector/src/log/LogSchema.h * *------------------------------------------------------------------------- */ @@ -50,7 +50,7 @@ class Timestamp; } // namespace protobuf } // namespace google -inline constexpr std::string_view schema_name = "yagpcc"; +inline constexpr std::string_view schema_name = "gpsc"; inline constexpr std::string_view log_relname = "__log"; struct LogDesc { @@ -165,8 +165,8 @@ inline constexpr std::array log_tbl_desc = { }; /* clang-format on */ -inline constexpr size_t natts_yagp_log = log_tbl_desc.size(); -inline constexpr size_t attnum_yagp_log_utility = natts_yagp_log - 1; +inline constexpr size_t natts_gpsc_log = log_tbl_desc.size(); +inline constexpr size_t attnum_gpsc_log_utility = natts_gpsc_log - 1; const std::unordered_map &proto_name_to_col_idx(); diff --git a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp similarity index 81% rename from gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp rename to gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp index 22083e8bdaf..4e3f6dae99f 100644 --- a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp @@ -20,7 +20,7 @@ * gpdbwrappers.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp + * gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp * *------------------------------------------------------------------------- */ @@ -125,22 +125,22 @@ auto wrap_noexcept(Func &&func, Args &&...args) noexcept } } // namespace -void *ya_gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } +void *gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } -void *ya_gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } +void *gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } -char *ya_gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } +char *gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } -char *ya_gpdb::get_database_name(Oid dbid) noexcept { +char *gpdb::get_database_name(Oid dbid) noexcept { return wrap_noexcept(::get_database_name, dbid); } -bool ya_gpdb::split_identifier_string(char *rawstring, char separator, +bool gpdb::split_identifier_string(char *rawstring, char separator, List **namelist) noexcept { return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, +ExplainState gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept { return wrap_noexcept([&]() { ExplainState *es = NewExplainState(); @@ -154,7 +154,7 @@ ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, +ExplainState gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept { return wrap_noexcept([&]() { ExplainState *es = NewExplainState(); @@ -174,12 +174,12 @@ ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, }); } -Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options, +Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options, bool async_mode) { return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } -HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, +HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull) { if (!tupleDescriptor || !values || !isnull) throw std::runtime_error( @@ -188,7 +188,7 @@ HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void ya_gpdb::pfree(void *pointer) noexcept { +void gpdb::pfree(void *pointer) noexcept { // Note that ::pfree asserts that pointer != NULL. if (!pointer) return; @@ -196,11 +196,11 @@ void ya_gpdb::pfree(void *pointer) noexcept { wrap_noexcept(::pfree, pointer); } -MemoryContext ya_gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { +MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { return MemoryContextSwitchTo(context); } -const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, +const char *gpdb::get_config_option(const char *name, bool missing_ok, bool restrict_superuser) noexcept { if (!name) return nullptr; @@ -208,7 +208,7 @@ const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void ya_gpdb::list_free(List *list) noexcept { +void gpdb::list_free(List *list) noexcept { if (!list) return; @@ -216,7 +216,7 @@ void ya_gpdb::list_free(List *list) noexcept { } CdbExplain_ShowStatCtx * -ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime) { if (!query_desc) throw std::runtime_error("Invalid query descriptor"); @@ -224,29 +224,29 @@ ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void ya_gpdb::instr_end_loop(Instrumentation *instr) { +void gpdb::instr_end_loop(Instrumentation *instr) { if (!instr) throw std::runtime_error("Invalid instrumentation pointer"); wrap_throw(::InstrEndLoop, instr); } -char *ya_gpdb::gen_normquery(const char *query) noexcept { +char *gpdb::gen_normquery(const char *query) noexcept { return wrap_noexcept(::gen_normquery, query); } -StringInfo ya_gpdb::gen_normplan(const char *exec_plan) noexcept { +StringInfo gpdb::gen_normplan(const char *exec_plan) noexcept { return wrap_noexcept(::gen_normplan, exec_plan); } -char *ya_gpdb::get_rg_name_for_id(Oid group_id) { +char *gpdb::get_rg_name_for_id(Oid group_id) { return wrap_throw(GetResGroupNameForId, group_id); } -Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { +Oid gpdb::get_rg_id_by_session_id(int session_id) { return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } -void ya_gpdb::insert_log(const yagpcc::SetQueryReq &req, bool utility) { +void gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) { return wrap_throw(::insert_log, req, utility); } diff --git a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h similarity index 92% rename from gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h rename to gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h index fe9b3ba0487..576007f6c7c 100644 --- a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h @@ -20,7 +20,7 @@ * gpdbwrappers.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h + * gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h * *------------------------------------------------------------------------- */ @@ -43,11 +43,11 @@ extern "C" { #include #include -namespace yagpcc { +namespace gpsc { class SetQueryReq; -} // namespace yagpcc +} // namespace gpsc -namespace ya_gpdb { +namespace gpdb { // Functions that call palloc(). // Make sure correct memory context is set. @@ -68,7 +68,7 @@ void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query) noexcept; StringInfo gen_normplan(const char *executionPlan) noexcept; char *get_rg_name_for_id(Oid group_id); -void insert_log(const yagpcc::SetQueryReq &req, bool utility); +void insert_log(const gpsc::SetQueryReq &req, bool utility); // Palloc-free functions. void pfree(void *pointer) noexcept; @@ -78,4 +78,4 @@ const char *get_config_option(const char *name, bool missing_ok, void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace ya_gpdb +} // namespace gpdb diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md b/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md new file mode 100644 index 00000000000..927189474fe --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md @@ -0,0 +1,20 @@ + + +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c similarity index 99% rename from gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 7404208055f..e24f53536a4 100644 --- a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -20,7 +20,7 @@ * pg_stat_statements_ya_parser.c * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h similarity index 93% rename from gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h index 96c6a776dba..a613ba04259 100644 --- a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -20,7 +20,7 @@ * pg_stat_statements_ya_parser.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/README.md b/gpcontrib/yagp_hooks_collector/README.md deleted file mode 100644 index 9f465a190cb..00000000000 --- a/gpcontrib/yagp_hooks_collector/README.md +++ /dev/null @@ -1,28 +0,0 @@ -## YAGP Hooks Collector - -An extension for collecting greenplum query execution metrics and reporting them to an external agent. - -### Collected Statistics - -#### 1. Query Lifecycle -- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. -- **GUC:** `yagpcc.enable`. - -#### 2. `EXPLAIN` data -- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. -- **GUC:** `yagpcc.enable`. - -#### 3. `EXPLAIN ANALYZE` data -- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. -- **GUCs:** `yagpcc.enable`, `yagpcc.min_analyze_time`, `yagpcc.enable_cdbstats`(ANALYZE), `yagpcc.enable_analyze`(BUFFERS, TIMING, VERBOSE). - -#### 4. Other Metrics -- **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics. -- **GUC:** `yagpcc.enable`. - -### General Configuration -- **Nested Queries:** When `yagpcc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. -- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `yagpcc.uds_path`. -- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `yagpcc.ignored_users_list`. -- **Trimming plans:** Query texts and execution plans are trimmed based on `yagpcc.max_text_size` and `yagpcc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. -- **Analyze collection:** Analyze is sent if execution time exceeds `yagpcc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `yagpcc.enable_analyze` is true. diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out b/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out deleted file mode 100644 index 6689b6a4ed3..00000000000 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out +++ /dev/null @@ -1,23 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. --- start_ignore -DROP DATABASE IF EXISTS yagp_test_locale; --- end_ignore -CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c yagp_test_locale -CREATE EXTENSION yagp_hooks_collector; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.enable TO TRUE; -CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -DROP TABLE yagp_hi_안녕세계; -DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out b/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out deleted file mode 100644 index d04929ffb4a..00000000000 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out +++ /dev/null @@ -1,42 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; --- end_ignore -\set UDS_PATH '/tmp/yagpcc_test.sock' --- Configure extension to send via UDS -SET yagpcc.uds_path TO :'UDS_PATH'; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.logging_mode TO 'UDS'; --- Start receiver -SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); - __test_uds_start_server -------------------------- -(0 rows) - --- Send -SELECT 1; - ?column? ----------- - 1 -(1 row) - --- Receive -SELECT yagpcc.__test_uds_receive() > 0 as received; - received ----------- - t -(1 row) - --- Stop receiver -SELECT yagpcc.__test_uds_stop_server(); - __test_uds_stop_server ------------------------- -(0 rows) - --- Cleanup -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.uds_path; -RESET yagpcc.ignored_users_list; -RESET yagpcc.enable; -RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql deleted file mode 100644 index f56351e0d43..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql +++ /dev/null @@ -1,85 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; - --- DECLARE -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_0 CURSOR FOR SELECT 0; -CLOSE cursor_stats_0; -COMMIT; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- DECLARE WITH HOLD -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; -CLOSE cursor_stats_1; -DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; -CLOSE cursor_stats_2; -COMMIT; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- ROLLBACK -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_3 CURSOR FOR SELECT 1; -CLOSE cursor_stats_3; -DECLARE cursor_stats_4 CURSOR FOR SELECT 1; -ROLLBACK; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- FETCH -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; -DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; -FETCH 1 IN cursor_stats_5; -FETCH 1 IN cursor_stats_6; -CLOSE cursor_stats_5; -CLOSE cursor_stats_6; -COMMIT; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql deleted file mode 100644 index 65d867d1680..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql +++ /dev/null @@ -1,29 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. - --- start_ignore -DROP DATABASE IF EXISTS yagp_test_locale; --- end_ignore - -CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c yagp_test_locale - -CREATE EXTENSION yagp_hooks_collector; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.enable TO TRUE; - -CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; - -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -DROP TABLE yagp_hi_안녕세계; -DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql deleted file mode 100644 index 90e972ae4c1..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql +++ /dev/null @@ -1,69 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE OR REPLACE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; - --- Basic SELECT tests -SET yagpcc.logging_mode to 'TBL'; - -SELECT 1; -SELECT COUNT(*) FROM generate_series(1,10); - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Transaction test -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -SELECT 1; -COMMIT; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- CTE test -SET yagpcc.logging_mode to 'TBL'; - -WITH t AS (VALUES (1), (2)) -SELECT * FROM t; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Prepared statement test -SET yagpcc.logging_mode to 'TBL'; - -PREPARE test_stmt AS SELECT 1; -EXECUTE test_stmt; -DEALLOCATE test_stmt; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql deleted file mode 100644 index 3eef697a4e7..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; --- end_ignore - -\set UDS_PATH '/tmp/yagpcc_test.sock' - --- Configure extension to send via UDS -SET yagpcc.uds_path TO :'UDS_PATH'; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.logging_mode TO 'UDS'; - --- Start receiver -SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); - --- Send -SELECT 1; - --- Receive -SELECT yagpcc.__test_uds_receive() > 0 as received; - --- Stop receiver -SELECT yagpcc.__test_uds_stop_server(); - --- Cleanup -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.uds_path; -RESET yagpcc.ignored_users_list; -RESET yagpcc.enable; -RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql deleted file mode 100644 index cf9c1d253d0..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql +++ /dev/null @@ -1,135 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE OR REPLACE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; - -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE test_table (a int, b text); -CREATE INDEX test_idx ON test_table(a); -ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; -DROP TABLE test_table; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Partitioning -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE pt_test (a int, b int) -DISTRIBUTED BY (a) -PARTITION BY RANGE (a) -(START (0) END (100) EVERY (50)); -DROP TABLE pt_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Views and Functions -SET yagpcc.logging_mode to 'TBL'; - -CREATE VIEW test_view AS SELECT 1 AS a; -CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; -DROP VIEW test_view; -DROP FUNCTION test_func(int); - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Transaction Operations -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -SAVEPOINT sp1; -ROLLBACK TO sp1; -COMMIT; - -BEGIN; -SAVEPOINT sp2; -ABORT; - -BEGIN; -ROLLBACK; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- DML Operations -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE dml_test (a int, b text); -INSERT INTO dml_test VALUES (1, 'test'); -UPDATE dml_test SET b = 'updated' WHERE a = 1; -DELETE FROM dml_test WHERE a = 1; -DROP TABLE dml_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- COPY Operations -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE copy_test (a int); -COPY (SELECT 1) TO STDOUT; -DROP TABLE copy_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Prepared Statements and error during execute -SET yagpcc.logging_mode to 'TBL'; - -PREPARE test_prep(int) AS SELECT $1/0 AS value; -EXECUTE test_prep(0::int); -DEALLOCATE test_prep; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- GUC Settings -SET yagpcc.logging_mode to 'TBL'; - -SET yagpcc.report_nested_queries TO FALSE; -RESET yagpcc.report_nested_queries; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md deleted file mode 100644 index 291e31a3099..00000000000 --- a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql deleted file mode 100644 index 8684ca73915..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql +++ /dev/null @@ -1,113 +0,0 @@ -/* yagp_hooks_collector--1.0--1.1.sql */ - --- complain if script is sourced in psql, rather than via ALTER EXTENSION -\echo Use "ALTER EXTENSION yagp_hooks_collector UPDATE TO '1.1'" to load this file. \quit - -CREATE SCHEMA yagpcc; - --- Unlink existing objects from extension. -ALTER EXTENSION yagp_hooks_collector DROP VIEW yagp_stat_messages; -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION yagp_stat_messages_reset(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_segments(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_master(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); - --- Now drop the objects. -DROP VIEW yagp_stat_messages; -DROP FUNCTION yagp_stat_messages_reset(); -DROP FUNCTION __yagp_stat_messages_f_on_segments(); -DROP FUNCTION __yagp_stat_messages_f_on_master(); -DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); -DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); - --- Recreate functions and view in new schema. -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT yagpcc.__stat_messages_reset_f_on_master(); - SELECT yagpcc.__stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagpcc.stat_messages AS - SELECT C.* - FROM yagpcc.__stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM yagpcc.__stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; - --- Create new objects. -CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - --- Creates log table inside yagpcc schema. -SELECT yagpcc.__init_log_on_master(); -SELECT yagpcc.__init_log_on_segments(); - -CREATE VIEW yagpcc.log AS - SELECT * FROM yagpcc.__log -- master - UNION ALL - SELECT * FROM gp_dist_random('yagpcc.__log') -- segments - ORDER BY tmid, ssid, ccnt; - -CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.truncate_log() -RETURNS SETOF void AS $$ -BEGIN - PERFORM yagpcc.__truncate_log_on_master(); - PERFORM yagpcc.__truncate_log_on_segments(); -END; -$$ LANGUAGE plpgsql VOLATILE; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql deleted file mode 100644 index 270cab92382..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql +++ /dev/null @@ -1,55 +0,0 @@ -/* yagp_hooks_collector--1.0.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit - -CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagp_stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT __yagp_stat_messages_reset_f_on_master(); - SELECT __yagp_stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagp_stat_messages AS - SELECT C.* - FROM __yagp_stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM __yagp_stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql deleted file mode 100644 index 83bfb553638..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql +++ /dev/null @@ -1,110 +0,0 @@ -/* yagp_hooks_collector--1.1.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit - -CREATE SCHEMA yagpcc; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT yagpcc.__stat_messages_reset_f_on_master(); - SELECT yagpcc.__stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagpcc.stat_messages AS - SELECT C.* - FROM yagpcc.__stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM yagpcc.__stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; - -CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - --- Creates log table inside yagpcc schema. -SELECT yagpcc.__init_log_on_master(); -SELECT yagpcc.__init_log_on_segments(); - -CREATE VIEW yagpcc.log AS - SELECT * FROM yagpcc.__log -- master - UNION ALL - SELECT * FROM gp_dist_random('yagpcc.__log') -- segments -ORDER BY tmid, ssid, ccnt; - -CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.truncate_log() -RETURNS SETOF void AS $$ -BEGIN - PERFORM yagpcc.__truncate_log_on_master(); - PERFORM yagpcc.__truncate_log_on_segments(); -END; -$$ LANGUAGE plpgsql VOLATILE; - -CREATE FUNCTION yagpcc.__test_uds_start_server(path text) -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_test_uds_start_server' -LANGUAGE C STRICT EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__test_uds_receive(timeout_ms int DEFAULT 2000) -RETURNS SETOF bigint -AS 'MODULE_PATHNAME', 'yagp_test_uds_receive' -LANGUAGE C STRICT EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__test_uds_stop_server() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_test_uds_stop_server' -LANGUAGE C EXECUTE ON MASTER; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control deleted file mode 100644 index cb5906a1302..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control +++ /dev/null @@ -1,5 +0,0 @@ -# yagp_hooks_collector extension -comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' -default_version = '1.1' -module_pathname = '$libdir/yagp_hooks_collector' -superuser = true diff --git a/pom.xml b/pom.xml index 2af9aa19a35..05bd00966ed 100644 --- a/pom.xml +++ b/pom.xml @@ -154,12 +154,6 @@ code or new licensing patterns. gpcontrib/gp_exttable_fdw/gp_exttable_fdw.control gpcontrib/diskquota/** - gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control - gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto - gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto - gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto - gpcontrib/yagp_hooks_collector/.clang-format - gpcontrib/yagp_hooks_collector/Makefile getversion .git-blame-ignore-revs @@ -1276,6 +1270,16 @@ code or new licensing patterns. src/include/task/task_states.h src/include/task/job_metadata.h + + gpcontrib/gp_stats_collector/gp_stats_collector.control + gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto + gpcontrib/gp_stats_collector/protos/gpsc_plan.proto + gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto + gpcontrib/gp_stats_collector/.clang-format + gpcontrib/gp_stats_collector/Makefile + diff --git a/src/Makefile.global.in b/src/Makefile.global.in index d520faeaf85..1393e4fb0ff 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -271,7 +271,7 @@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ EVENT_LIBS = @EVENT_LIBS@ -with_yagp_hooks_collector = @with_yagp_hooks_collector@ +with_gp_stats_collector = @with_gp_stats_collector@ ########################################################################## # diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 553830e8599..0ea5874e884 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -376,7 +376,7 @@ PortalCleanup(Portal portal) CurrentResourceOwner = saveResourceOwner; } else { /* GPDB hook for collecting query info */ - if (queryDesc->yagp_query_key && query_info_collect_hook) + if (queryDesc->gpsc_query_key && query_info_collect_hook) (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); } } diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 7c1dbc480bc..e5512bb8271 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -127,8 +127,8 @@ CreateQueryDesc(PlannedStmt *plannedstmt, if (Gp_role != GP_ROLE_EXECUTE) increment_command_count(); - /* null this field until set by YAGP Hooks collector */ - qd->yagp_query_key = NULL; + /* null this field until set by GP Stats Collector */ + qd->gpsc_query_key = NULL; return qd; } diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index e469945a4c5..d50d3e48f6b 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -22,14 +22,14 @@ struct CdbExplain_ShowStatCtx; /* private, in "cdb/cdbexplain.c" */ -typedef struct YagpQueryKey +typedef struct GpscQueryKey { int tmid; /* transaction time */ int ssid; /* session id */ int ccnt; /* command count */ int nesting_level; uintptr_t query_desc_addr; -} YagpQueryKey; +} GpscQueryKey; /* * SerializedParams is used to serialize external query parameters @@ -339,8 +339,8 @@ typedef struct QueryDesc /* This is always set NULL by the core system, but plugins can change it */ struct Instrumentation *totaltime; /* total time spent in ExecutorRun */ - /* YAGP Hooks collector */ - YagpQueryKey *yagp_query_key; + /* GP Stats Collector */ + GpscQueryKey *gpsc_query_key; } QueryDesc; /* in pquery.c */ From 9dbd12b58877a6c82507a56216adb5030bec74cd Mon Sep 17 00:00:00 2001 From: NJrslv Date: Thu, 26 Mar 2026 11:21:35 +0300 Subject: [PATCH 079/128] [gp_stats_collector] Simplify Makefile and add -Wno-unused-but-set-variable --- gpcontrib/gp_stats_collector/Makefile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/gpcontrib/gp_stats_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile index c8f7b3c30fe..43255ca1955 100644 --- a/gpcontrib/gp_stats_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -10,13 +10,8 @@ C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) -override CXXFLAGS = -Werror -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ - -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ - -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ - -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD - -PG_CXXFLAGS += -Isrc -Iinclude -SHLIB_LINK += -lprotobuf -lpthread -lstdc++ +PG_CXXFLAGS += -Werror -Wall -Wno-unused-but-set-variable -std=c++17 -Isrc/protos -Isrc -Iinclude -DGPBUILD +SHLIB_LINK += -lprotobuf -lstdc++ EXTRA_CLEAN = src/protos ifdef USE_PGXS @@ -30,10 +25,11 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto +src/protos/.done: $(wildcard protos/*.proto) @mkdir -p src/protos protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ - mv src/protos/$*.pb.cc src/protos/$*.pb.cpp + for f in src/protos/*.pb.cc; do mv "$$f" "$${f%.cc}.cpp"; done + touch $@ -$(CPP_OBJS): src/protos/gpsc_metrics.pb.h src/protos/gpsc_plan.pb.h src/protos/gpsc_set_service.pb.h -src/protos/gpsc_set_service.pb.o: src/protos/gpsc_metrics.pb.h +src/protos/%.pb.cpp src/protos/%.pb.h: src/protos/.done ; +$(CPP_OBJS): src/protos/.done From 25d66ff1342983abfab272b9c7cb76a01c5b2cbe Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 08:34:53 +0300 Subject: [PATCH 080/128] [gp_stats_collector] Build by default with extension disabled via GUCs Enable building gp_stats_collector by default in configure. Add missing check in verify_query() to ensure the extension does not execute main code while disabled. Always verify protobuf version once the shared library is preloaded. --- .github/workflows/build-cloudberry-rocky8.yml | 3 +-- .github/workflows/build-cloudberry.yml | 3 +-- .github/workflows/build-deb-cloudberry.yml | 3 +-- .../cloudberry/scripts/configure-cloudberry.sh | 1 + gpcontrib/gp_stats_collector/src/Config.cpp | 6 +++--- gpcontrib/gp_stats_collector/src/EventSender.cpp | 16 +++++++++------- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 39175753a99..dd4d10ab115 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -544,11 +544,10 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index cbd4fd753dc..cc99a997c3f 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -539,11 +539,10 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index bf85a107b31..52db1819194 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -452,14 +452,13 @@ jobs: shell: bash env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail export BUILD_DESTINATION=${SRC_DIR}/debian/build chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index d30a0b794f0..a9086a434fb 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -163,6 +163,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ + --with-gp-stats-collector \ --with-gssapi \ --with-ldap \ --with-libxml \ diff --git a/gpcontrib/gp_stats_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp index e117aa941fd..2f40b30e922 100644 --- a/gpcontrib/gp_stats_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -40,7 +40,7 @@ extern "C" { static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; -static bool guc_enable_collector = true; +static bool guc_enable_collector = false; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1 << 20; // in bytes (1MB) @@ -68,7 +68,7 @@ void Config::init_gucs() { DefineCustomBoolVariable( "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, - true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + false, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, @@ -88,7 +88,7 @@ void Config::init_gucs() { DefineCustomStringVariable("gpsc.ignored_users_list", "Make gpsc ignore queries issued by given users", 0LL, &guc_ignored_users, - "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + "", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, assign_ignored_users_hook, 0LL); diff --git a/gpcontrib/gp_stats_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp index b28ceba175a..c0faaf0ad0e 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -68,6 +68,10 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, // not executed yet, causing DONE to be skipped/added. config.sync(); + if (!config.enable_collector()) { + return false; + } + if (utility && !config.enable_utility()) { return false; } @@ -409,13 +413,11 @@ EventSender::EventSender() { // Perform initial sync to get default GUC values config.sync(); - if (config.enable_collector()) { - try { - GOOGLE_PROTOBUF_VERIFY_VERSION; - proto_verified = true; - } catch (const std::exception &e) { - ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); - } + try { + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; + } catch (const std::exception &e) { + ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", e.what()))); } #ifdef IC_TEARDOWN_HOOK memset(&ic_statistics, 0, sizeof(ICStatistics)); From 1bcb523149a781ac5c56b428300cabd5e4dde0a5 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 08:47:13 +0300 Subject: [PATCH 081/128] [gp_stats_collector] Code quality cleanup Delete stale .gitignore. Add Apache headers to .proto files. Change #pragma once to #ifndef guards. Remove test result files from tree. Change ereport(FATAL) to ereport(ERROR). Remove internal naming suffixes. Apply clang-format from gporca. --- .gitignore | 2 +- gpcontrib/gp_stats_collector/.clang-format | 180 +++- gpcontrib/gp_stats_collector/.gitignore | 5 - .../protos/gpsc_metrics.proto | 18 + .../gp_stats_collector/protos/gpsc_plan.proto | 18 + .../protos/gpsc_set_service.proto | 18 + .../results/gpsc_cursors.out | 163 --- .../gp_stats_collector/results/gpsc_dist.out | 175 ---- .../results/gpsc_guc_cache.out | 61 -- .../results/gpsc_locale.out | 23 - .../results/gpsc_select.out | 136 --- .../gp_stats_collector/results/gpsc_uds.out | 42 - .../results/gpsc_utf8_trim.out | 68 -- .../results/gpsc_utility.out | 248 ----- gpcontrib/gp_stats_collector/src/Config.cpp | 248 ++--- gpcontrib/gp_stats_collector/src/Config.h | 98 +- .../gp_stats_collector/src/EventSender.cpp | 976 ++++++++++-------- .../gp_stats_collector/src/EventSender.h | 243 +++-- gpcontrib/gp_stats_collector/src/GpscStat.cpp | 142 ++- gpcontrib/gp_stats_collector/src/GpscStat.h | 36 +- gpcontrib/gp_stats_collector/src/PgUtils.cpp | 68 +- .../gp_stats_collector/src/ProcStats.cpp | 179 ++-- gpcontrib/gp_stats_collector/src/ProcStats.h | 9 +- .../gp_stats_collector/src/ProtoUtils.cpp | 492 +++++---- gpcontrib/gp_stats_collector/src/ProtoUtils.h | 17 +- .../gp_stats_collector/src/UDSConnector.cpp | 166 +-- .../gp_stats_collector/src/UDSConnector.h | 12 +- .../src/gp_stats_collector.c | 169 +-- .../gp_stats_collector/src/hook_wrappers.cpp | 627 ++++++----- .../gp_stats_collector/src/hook_wrappers.h | 6 +- .../gp_stats_collector/src/log/LogOps.cpp | 199 ++-- gpcontrib/gp_stats_collector/src/log/LogOps.h | 5 +- .../gp_stats_collector/src/log/LogSchema.cpp | 261 ++--- .../gp_stats_collector/src/log/LogSchema.h | 38 +- .../src/memory/gpdbwrappers.cpp | 386 ++++--- .../src/memory/gpdbwrappers.h | 33 +- ...a_parser.c => pg_stat_statements_parser.c} | 94 +- ...a_parser.h => pg_stat_statements_parser.h} | 12 +- pom.xml | 3 - 39 files changed, 2774 insertions(+), 2902 deletions(-) delete mode 100644 gpcontrib/gp_stats_collector/.gitignore delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_cursors.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_dist.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_locale.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_select.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_uds.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utility.out rename gpcontrib/gp_stats_collector/src/stat_statements_parser/{pg_stat_statements_ya_parser.c => pg_stat_statements_parser.c} (82%) rename gpcontrib/gp_stats_collector/src/stat_statements_parser/{pg_stat_statements_ya_parser.h => pg_stat_statements_parser.h} (87%) diff --git a/.gitignore b/.gitignore index 7f5110d5c8e..5c21989c4ab 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,4 @@ lib*.pc /compile_commands.json /tmp_install/ /.cache/ -/install/ +/install/ \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/.clang-format b/gpcontrib/gp_stats_collector/.clang-format index 99130575c9a..eb90ff33671 100644 --- a/gpcontrib/gp_stats_collector/.clang-format +++ b/gpcontrib/gp_stats_collector/.clang-format @@ -1,2 +1,178 @@ -BasedOnStyle: LLVM -SortIncludes: false +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false +AlignConsecutiveAssignments: false +AlignConsecutiveBitFields: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: All +AlwaysBreakAfterReturnType: AllDefinitions +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: true + AfterClass: true + AfterControlStatement: Always + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: false + BeforeCatch: true + BeforeElse: true + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^<.*' + Priority: 1 + - Regex: '"protos/.*\.pb\.h"' + Priority: 2 + - Regex: '"postgres\.h"' + Priority: 3 + - Regex: '.*' + Priority: 4 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 3 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Right +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + - ParseTestProto + - ParsePartialTestProto + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: true +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Always +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE +... + + diff --git a/gpcontrib/gp_stats_collector/.gitignore b/gpcontrib/gp_stats_collector/.gitignore deleted file mode 100644 index e8dfe855dad..00000000000 --- a/gpcontrib/gp_stats_collector/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.o -*.so -src/protos/ -.vscode -compile_commands.json diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index a9e26471839..7853dc58db7 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; package gpsc; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto index 5a7269edd20..c1632478464 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; package gpsc; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto index 4cd795424ab..bcf09074ed7 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; import "google/protobuf/timestamp.proto"; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out deleted file mode 100644 index 282d9ac49e1..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out +++ /dev/null @@ -1,163 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.report_nested_queries TO TRUE; --- DECLARE -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_0 CURSOR FOR SELECT 0; -CLOSE cursor_stats_0; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- DECLARE WITH HOLD -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; -CLOSE cursor_stats_1; -DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; -CLOSE cursor_stats_2; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(14 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- ROLLBACK -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_3 CURSOR FOR SELECT 1; -CLOSE cursor_stats_3; -DECLARE cursor_stats_4 CURSOR FOR SELECT 1; -ROLLBACK; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(12 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- FETCH -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; -DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; -FETCH 1 IN cursor_stats_5; - ?column? ----------- - 2 -(1 row) - -FETCH 1 IN cursor_stats_6; - ?column? ----------- - 3 -(1 row) - -CLOSE cursor_stats_5; -CLOSE cursor_stats_6; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE - -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT - -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE - -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT - -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(18 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_dist.out b/gpcontrib/gp_stats_collector/results/gpsc_dist.out deleted file mode 100644 index 92e8678767b..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_dist.out +++ /dev/null @@ -1,175 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.enable_utility TO FALSE; --- Hash distributed table -CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); -INSERT INTO test_hash_dist SELECT 1; -SET gpsc.logging_mode to 'TBL'; -SET optimizer_enable_direct_dispatch TO TRUE; --- Direct dispatch is used here, only one segment is scanned. -select * from test_hash_dist where id = 1; - id ----- - 1 -(1 row) - -RESET optimizer_enable_direct_dispatch; -RESET gpsc.logging_mode; --- Should see 8 rows. -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------------------+--------------------- - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -SET gpsc.logging_mode to 'TBL'; --- Scan all segments. -select * from test_hash_dist; - id ----- - 1 -(1 row) - -DROP TABLE test_hash_dist; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------+--------------------- - -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT - -1 | select * from test_hash_dist; | QUERY_STATUS_START - -1 | select * from test_hash_dist; | QUERY_STATUS_END - -1 | select * from test_hash_dist; | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE - 2 | | QUERY_STATUS_SUBMIT - 2 | | QUERY_STATUS_START - 2 | | QUERY_STATUS_END - 2 | | QUERY_STATUS_DONE - | | QUERY_STATUS_SUBMIT - | | QUERY_STATUS_START - | | QUERY_STATUS_END - | | QUERY_STATUS_DONE -(16 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Replicated table -CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ -BEGIN - RETURN NEXT 'seg'; -END; -$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; -CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; -INSERT INTO test_replicated SELECT 1; -SET gpsc.logging_mode to 'TBL'; -SELECT COUNT(*) FROM test_replicated, force_segments(); - count -------- - 3 -(1 row) - -DROP TABLE test_replicated; -DROP FUNCTION force_segments(); -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------------------+--------------------- - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE - 2 | | QUERY_STATUS_SUBMIT - 2 | | QUERY_STATUS_START - 2 | | QUERY_STATUS_END - 2 | | QUERY_STATUS_DONE - | | QUERY_STATUS_SUBMIT - | | QUERY_STATUS_START - | | QUERY_STATUS_END - | | QUERY_STATUS_DONE -(16 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Partially distributed table (2 numsegments) -SET allow_system_table_mods = ON; -CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); -UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; -INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET gpsc.logging_mode to 'TBL'; -SELECT COUNT(*) FROM test_partial_dist; - count -------- - 100 -(1 row) - -RESET gpsc.logging_mode; -DROP TABLE test_partial_dist; -RESET allow_system_table_mods; --- Should see 12 rows. -SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - query_text | query_status ------------------------------------------+--------------------- - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE - | QUERY_STATUS_SUBMIT - | QUERY_STATUS_START - | QUERY_STATUS_END - | QUERY_STATUS_DONE - | QUERY_STATUS_SUBMIT - | QUERY_STATUS_START - | QUERY_STATUS_END - | QUERY_STATUS_DONE -(12 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out deleted file mode 100644 index 19c4774575d..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out +++ /dev/null @@ -1,61 +0,0 @@ --- --- Test GUC caching for query lifecycle consistency. --- --- The extension logs SUBMIT and DONE events for each query. --- GUC values that control logging (enable_utility, ignored_users_list, ...) --- must be cached at SUBMIT time to ensure DONE uses the same filtering --- criteria. Otherwise, a SET command that modifies these GUCs would --- have its DONE event rejected, creating orphaned SUBMIT entries. --- This is due to query being actually executed between SUBMIT and DONE. --- start_ignore -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; -SELECT gpsc.truncate_log(); - truncate_log --------------- -(0 rows) - --- end_ignore -CREATE OR REPLACE FUNCTION print_last_query(query text) -RETURNS TABLE(query_status text) AS $$ - SELECT query_status - FROM gpsc.log - WHERE segid = -1 AND query_text = query - ORDER BY ccnt DESC -$$ LANGUAGE sql; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.logging_mode TO 'TBL'; --- SET below disables utility logging and DONE must still be logged. -SET gpsc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); - query_status ---------------------- - QUERY_STATUS_SUBMIT - QUERY_STATUS_DONE -(2 rows) - --- SELECT below adds current user to ignore list and DONE must still be logged. --- start_ignore -SELECT set_config('gpsc.ignored_users_list', current_user, false); - set_config ------------- - gpadmin -(1 row) - --- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); - query_status ---------------------- - QUERY_STATUS_SUBMIT - QUERY_STATUS_START - QUERY_STATUS_END - QUERY_STATUS_DONE -(4 rows) - -DROP FUNCTION print_last_query(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; -RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_locale.out b/gpcontrib/gp_stats_collector/results/gpsc_locale.out deleted file mode 100644 index a01fe0648b9..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_locale.out +++ /dev/null @@ -1,23 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. --- start_ignore -DROP DATABASE IF EXISTS gpsc_test_locale; --- end_ignore -CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c gpsc_test_locale -CREATE EXTENSION gp_stats_collector; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable_utility TO TRUE; -SET gpsc.enable TO TRUE; -CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; -RESET gpsc.enable; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; -DROP TABLE gpsc_hi_안녕세계; -DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_select.out b/gpcontrib/gp_stats_collector/results/gpsc_select.out deleted file mode 100644 index 3008c8f6d55..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_select.out +++ /dev/null @@ -1,136 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.enable_utility TO FALSE; --- Basic SELECT tests -SET gpsc.logging_mode to 'TBL'; -SELECT 1; - ?column? ----------- - 1 -(1 row) - -SELECT COUNT(*) FROM generate_series(1,10); - count -------- - 10 -(1 row) - -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | SELECT 1; | QUERY_STATUS_SUBMIT - -1 | SELECT 1; | QUERY_STATUS_START - -1 | SELECT 1; | QUERY_STATUS_END - -1 | SELECT 1; | QUERY_STATUS_DONE - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Transaction test -SET gpsc.logging_mode to 'TBL'; -BEGIN; -SELECT 1; - ?column? ----------- - 1 -(1 row) - -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------+--------------------- - -1 | SELECT 1; | QUERY_STATUS_SUBMIT - -1 | SELECT 1; | QUERY_STATUS_START - -1 | SELECT 1; | QUERY_STATUS_END - -1 | SELECT 1; | QUERY_STATUS_DONE -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- CTE test -SET gpsc.logging_mode to 'TBL'; -WITH t AS (VALUES (1), (2)) -SELECT * FROM t; - column1 ---------- - 1 - 2 -(2 rows) - -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------+--------------------- - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE - | SELECT * FROM t; | -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Prepared statement test -SET gpsc.logging_mode to 'TBL'; -PREPARE test_stmt AS SELECT 1; -EXECUTE test_stmt; - ?column? ----------- - 1 -(1 row) - -DEALLOCATE test_stmt; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------+--------------------- - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_uds.out b/gpcontrib/gp_stats_collector/results/gpsc_uds.out deleted file mode 100644 index e8bca79e669..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_uds.out +++ /dev/null @@ -1,42 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; --- end_ignore -\set UDS_PATH '/tmp/gpsc_test.sock' --- Configure extension to send via UDS -SET gpsc.uds_path TO :'UDS_PATH'; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.logging_mode TO 'UDS'; --- Start receiver -SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); - __test_uds_start_server -------------------------- -(0 rows) - --- Send -SELECT 1; - ?column? ----------- - 1 -(1 row) - --- Receive -SELECT gpsc.__test_uds_receive() > 0 as received; - received ----------- - t -(1 row) - --- Stop receiver -SELECT gpsc.__test_uds_stop_server(); - __test_uds_stop_server ------------------------- -(0 rows) - --- Cleanup -DROP EXTENSION gp_stats_collector; -RESET gpsc.uds_path; -RESET gpsc.ignored_users_list; -RESET gpsc.enable; -RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out deleted file mode 100644 index db3949f3152..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out +++ /dev/null @@ -1,68 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; -CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) -RETURNS TEXT AS $$ - SELECT query_text - FROM gpsc.log - WHERE query_text LIKE '%' || marker || '%' - ORDER BY datetime DESC - LIMIT 1 -$$ LANGUAGE sql VOLATILE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; --- Test 1: 1 byte chars -SET gpsc.max_text_size to 19; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test1*/ 'HelloWorld'; - ?column? ------------- - HelloWorld -(1 row) - -RESET gpsc.logging_mode; -SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Test 2: 2 byte chars -SET gpsc.max_text_size to 19; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test2*/ 'РУССКИЙЯЗЫК'; - ?column? -------------- - РУССКИЙЯЗЫК -(1 row) - -RESET gpsc.logging_mode; --- Character 'Р' has two bytes and cut in the middle => not included. -SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Test 3: 4 byte chars -SET gpsc.max_text_size to 21; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test3*/ '😀'; - ?column? ----------- - 😀 -(1 row) - -RESET gpsc.logging_mode; --- Emoji has 4 bytes and cut before the last byte => not included. -SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Cleanup -DROP FUNCTION get_marked_query(TEXT); -RESET gpsc.max_text_size; -RESET gpsc.logging_mode; -RESET gpsc.enable; -RESET gpsc.ignored_users_list; -DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utility.out b/gpcontrib/gp_stats_collector/results/gpsc_utility.out deleted file mode 100644 index e8e28614370..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_utility.out +++ /dev/null @@ -1,248 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE test_table (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -CREATE INDEX test_idx ON test_table(a); -ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; -DROP TABLE test_table; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------------------------------+--------------------- - -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE - -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT - -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE - -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT - -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE - -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE test_table; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Partitioning -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE pt_test (a int, b int) -DISTRIBUTED BY (a) -PARTITION BY RANGE (a) -(START (0) END (100) EVERY (50)); -DROP TABLE pt_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------+--------------------- - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Views and Functions -SET gpsc.logging_mode to 'TBL'; -CREATE VIEW test_view AS SELECT 1 AS a; -CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; -DROP VIEW test_view; -DROP FUNCTION test_func(int); -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------------------------------------------------------------------------------+--------------------- - -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT - -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE - -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT - -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE - -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT - -1 | DROP VIEW test_view; | QUERY_STATUS_DONE - -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT - -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Transaction Operations -SET gpsc.logging_mode to 'TBL'; -BEGIN; -SAVEPOINT sp1; -ROLLBACK TO sp1; -COMMIT; -BEGIN; -SAVEPOINT sp2; -ABORT; -BEGIN; -ROLLBACK; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(18 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- DML Operations -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE dml_test (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO dml_test VALUES (1, 'test'); -UPDATE dml_test SET b = 'updated' WHERE a = 1; -DELETE FROM dml_test WHERE a = 1; -DROP TABLE dml_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------------------+--------------------- - -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE - -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- COPY Operations -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE copy_test (a int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -COPY (SELECT 1) TO STDOUT; -1 -DROP TABLE copy_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------+--------------------- - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Prepared Statements and error during execute -SET gpsc.logging_mode to 'TBL'; -PREPARE test_prep(int) AS SELECT $1/0 AS value; -EXECUTE test_prep(0::int); -ERROR: division by zero -DEALLOCATE test_prep; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------+--------------------- - -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT - -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE - -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT - -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR - -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT - -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- GUC Settings -SET gpsc.logging_mode to 'TBL'; -SET gpsc.report_nested_queries TO FALSE; -RESET gpsc.report_nested_queries; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------------------------------------+--------------------- - -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT - -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE - -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp index 2f40b30e922..08a8d8cff86 100644 --- a/gpcontrib/gp_stats_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -26,11 +26,11 @@ */ #include "Config.h" -#include "memory/gpdbwrappers.h" #include #include #include #include +#include "memory/gpdbwrappers.h" extern "C" { #include "postgres.h" @@ -43,135 +43,149 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = false; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1 << 20; // in bytes (1MB) -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = 10000; // in ms +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms static int guc_logging_mode = LOG_MODE_UDS; static bool guc_enable_utility = false; static const struct config_enum_entry logging_mode_options[] = { - {"uds", LOG_MODE_UDS, false /* hidden */}, - {"tbl", LOG_MODE_TBL, false}, - {NULL, 0, false}}; + {"uds", LOG_MODE_UDS, false /* hidden */}, + {"tbl", LOG_MODE_TBL, false}, + {NULL, 0, false}}; static bool ignored_users_guc_dirty = false; -static void assign_ignored_users_hook(const char *, void *) { - ignored_users_guc_dirty = true; +static void +assign_ignored_users_hook(const char *, void *) +{ + ignored_users_guc_dirty = true; } -void Config::init_gucs() { - DefineCustomStringVariable( - "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, - &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, - false, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, - &guc_enable_analyze, true, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, - &guc_enable_cdbstats, true, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, - &guc_report_nested_queries, true, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomStringVariable("gpsc.ignored_users_list", - "Make gpsc ignore queries issued by given users", - 0LL, &guc_ignored_users, - "", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, - assign_ignored_users_hook, 0LL); - - DefineCustomIntVariable( - "gpsc.max_text_size", - "Make gpsc trim query texts longer than configured size in bytes", NULL, - &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); - - DefineCustomIntVariable( - "gpsc.max_plan_size", - "Make gpsc trim plan longer than configured size", NULL, - &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); - - DefineCustomIntVariable( - "gpsc.min_analyze_time", - "Sets the minimum execution time above which plans will be logged.", - "Zero prints all plans. -1 turns this feature off.", - &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); - - DefineCustomEnumVariable( - "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, - &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, - NULL); - - DefineCustomBoolVariable( - "gpsc.enable_utility", "Collect utility statement stats", NULL, - &guc_enable_utility, false, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); +void +Config::init_gucs() +{ + DefineCustomStringVariable( + "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable("gpsc.enable", "Enable metrics collector", 0LL, + &guc_enable_collector, false, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, + 0LL); + + DefineCustomBoolVariable( + "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, + &guc_enable_analyze, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, + &guc_enable_cdbstats, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, + &guc_report_nested_queries, true, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomStringVariable("gpsc.ignored_users_list", + "Make gpsc ignore queries issued by given users", + 0LL, &guc_ignored_users, "", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, + assign_ignored_users_hook, 0LL); + + DefineCustomIntVariable( + "gpsc.max_text_size", + "Make gpsc trim query texts longer than configured size in bytes", NULL, + &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); + + DefineCustomIntVariable( + "gpsc.max_plan_size", "Make gpsc trim plan longer than configured size", + NULL, &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "gpsc.min_analyze_time", + "Sets the minimum execution time above which plans will be logged.", + "Zero prints all plans. -1 turns this feature off.", + &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); + + DefineCustomEnumVariable( + "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, + &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, + NULL); + + DefineCustomBoolVariable( + "gpsc.enable_utility", "Collect utility statement stats", NULL, + &guc_enable_utility, false, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } -void Config::update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = std::make_unique(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = gpdb::pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { - /* syntax error in list */ - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter gpsc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); - } - ignored_users_ = std::move(new_ignored_users_set); +void +Config::update_ignored_users(const char *new_guc_ignored_users) +{ + auto new_ignored_users_set = std::make_unique(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') + { + /* Need a modifiable copy of string */ + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) + { + /* syntax error in list */ + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter gpsc.ignored_users_list"))); + return; + } + foreach (l, elemlist) + { + new_ignored_users_set->insert((char *) lfirst(l)); + } + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); + } + ignored_users_ = std::move(new_ignored_users_set); } -bool Config::filter_user(const std::string &username) const { - if (!ignored_users_) { - return true; - } - return ignored_users_->find(username) != ignored_users_->end(); +bool +Config::filter_user(const std::string &username) const +{ + if (!ignored_users_) + { + return true; + } + return ignored_users_->find(username) != ignored_users_->end(); } -void Config::sync() { - if (ignored_users_guc_dirty) { - update_ignored_users(guc_ignored_users); - ignored_users_guc_dirty = false; - } - uds_path_ = guc_uds_path; - enable_analyze_ = guc_enable_analyze; - enable_cdbstats_ = guc_enable_cdbstats; - enable_collector_ = guc_enable_collector; - enable_utility_ = guc_enable_utility; - report_nested_queries_ = guc_report_nested_queries; - max_text_size_ = guc_max_text_size; - max_plan_size_ = guc_max_plan_size; - min_analyze_time_ = guc_min_analyze_time; - logging_mode_ = guc_logging_mode; +void +Config::sync() +{ + if (ignored_users_guc_dirty) + { + update_ignored_users(guc_ignored_users); + ignored_users_guc_dirty = false; + } + uds_path_ = guc_uds_path; + enable_analyze_ = guc_enable_analyze; + enable_cdbstats_ = guc_enable_cdbstats; + enable_collector_ = guc_enable_collector; + enable_utility_ = guc_enable_utility; + report_nested_queries_ = guc_report_nested_queries; + max_text_size_ = guc_max_text_size; + max_plan_size_ = guc_max_plan_size; + min_analyze_time_ = guc_min_analyze_time; + logging_mode_ = guc_logging_mode; } diff --git a/gpcontrib/gp_stats_collector/src/Config.h b/gpcontrib/gp_stats_collector/src/Config.h index 91a1ffe44f2..259799e5135 100644 --- a/gpcontrib/gp_stats_collector/src/Config.h +++ b/gpcontrib/gp_stats_collector/src/Config.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef CONFIG_H +#define CONFIG_H #include #include @@ -36,36 +37,79 @@ using IgnoredUsers = std::unordered_set; -class Config { +class Config +{ public: - static void init_gucs(); + static void init_gucs(); - void sync(); + void sync(); - const std::string &uds_path() const { return uds_path_; } - bool enable_analyze() const { return enable_analyze_; } - bool enable_cdbstats() const { return enable_cdbstats_; } - bool enable_collector() const { return enable_collector_; } - bool enable_utility() const { return enable_utility_; } - bool report_nested_queries() const { return report_nested_queries_; } - int max_text_size() const { return max_text_size_; } - int max_plan_size() const { return max_plan_size_ * 1024; } - int min_analyze_time() const { return min_analyze_time_; } - int logging_mode() const { return logging_mode_; } - bool filter_user(const std::string &username) const; + const std::string & + uds_path() const + { + return uds_path_; + } + bool + enable_analyze() const + { + return enable_analyze_; + } + bool + enable_cdbstats() const + { + return enable_cdbstats_; + } + bool + enable_collector() const + { + return enable_collector_; + } + bool + enable_utility() const + { + return enable_utility_; + } + bool + report_nested_queries() const + { + return report_nested_queries_; + } + int + max_text_size() const + { + return max_text_size_; + } + int + max_plan_size() const + { + return max_plan_size_ * 1024; + } + int + min_analyze_time() const + { + return min_analyze_time_; + } + int + logging_mode() const + { + return logging_mode_; + } + bool filter_user(const std::string &username) const; private: - void update_ignored_users(const char *new_guc_ignored_users); + void update_ignored_users(const char *new_guc_ignored_users); - std::unique_ptr ignored_users_; - std::string uds_path_; - bool enable_analyze_; - bool enable_cdbstats_; - bool enable_collector_; - bool enable_utility_; - bool report_nested_queries_; - int max_text_size_; - int max_plan_size_; - int min_analyze_time_; - int logging_mode_; + std::unique_ptr ignored_users_; + std::string uds_path_; + bool enable_analyze_; + bool enable_cdbstats_; + bool enable_collector_; + bool enable_utility_; + bool report_nested_queries_; + int max_text_size_; + int max_plan_size_; + int min_analyze_time_; + int logging_mode_; }; + +#endif /* CONFIG_H */ diff --git a/gpcontrib/gp_stats_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp index c0faaf0ad0e..0bc44c1198d 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -26,8 +26,8 @@ */ #include "UDSConnector.h" -#include "memory/gpdbwrappers.h" #include "log/LogOps.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid extern "C" { @@ -47,487 +47,599 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" -#define need_collect_analyze() \ - (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ - config.enable_analyze()) - -bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, - bool utility) { - if (!proto_verified) { - return false; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { - return false; - } - - switch (state) { - case QueryState::SUBMIT: - // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent - // states. Without caching, a query that unsets/sets filtering GUCs would - // see different filter criteria at DONE, because at SUBMIT the query was - // not executed yet, causing DONE to be skipped/added. - config.sync(); - - if (!config.enable_collector()) { - return false; - } - - if (utility && !config.enable_utility()) { - return false; - } - - // Register qkey for a nested query we won't report, - // so we can detect nesting_level > 0 and skip reporting at end/done. - if (!need_report_nested_query() && nesting_level > 0) { - QueryKey::register_qkey(query_desc, nesting_level); - return false; - } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; - } - break; - case QueryState::START: - if (!qdesc_submitted(query_desc)) { - collect_query_submit(query_desc, false /* utility */); - } - break; - case QueryState::DONE: - if (utility && !config.enable_utility()) { - return false; - } - default: - break; - } - - if (filter_query(query_desc)) { - return false; - } - if (!nesting_is_valid(query_desc, nesting_level)) { - return false; - } - - return true; +#define need_collect_analyze() \ + (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ + config.enable_analyze()) + +bool +EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool utility) +{ + if (!proto_verified) + { + return false; + } + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) + { + return false; + } + + switch (state) + { + case QueryState::SUBMIT: + // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent + // states. Without caching, a query that unsets/sets filtering GUCs would + // see different filter criteria at DONE, because at SUBMIT the query was + // not executed yet, causing DONE to be skipped/added. + config.sync(); + + if (!config.enable_collector()) + { + return false; + } + + if (utility && !config.enable_utility()) + { + return false; + } + + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) + { + QueryKey::register_qkey(query_desc, nesting_level); + return false; + } + if (is_top_level_query(query_desc, nesting_level)) + { + nested_timing = 0; + nested_calls = 0; + } + break; + case QueryState::START: + if (!qdesc_submitted(query_desc)) + { + collect_query_submit(query_desc, false /* utility */); + } + break; + case QueryState::DONE: + if (utility && !config.enable_utility()) + { + return false; + } + default: + break; + } + + if (filter_query(query_desc)) + { + return false; + } + if (!nesting_is_valid(query_desc, nesting_level)) + { + return false; + } + + return true; } -bool EventSender::log_query_req(const gpsc::SetQueryReq &req, - const std::string &event, bool utility) { - bool clear_big_fields = false; - switch (config.logging_mode()) { - case LOG_MODE_UDS: - clear_big_fields = UDSConnector::report_query(req, event, config); - break; - case LOG_MODE_TBL: - gpdb::insert_log(req, utility); - clear_big_fields = false; - break; - default: - Assert(false); - } - return clear_big_fields; +bool +EventSender::log_query_req(const gpsc::SetQueryReq &req, + const std::string &event, bool utility) +{ + bool clear_big_fields = false; + switch (config.logging_mode()) + { + case LOG_MODE_UDS: + clear_big_fields = UDSConnector::report_query(req, event, config); + break; + case LOG_MODE_TBL: + gpdb::insert_log(req, utility); + clear_big_fields = false; + break; + default: + Assert(false); + } + return clear_big_fields; } -void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, - bool utility, ErrorData *edata) { - auto *query_desc = reinterpret_cast(arg); - switch (status) { - case METRICS_PLAN_NODE_INITIALIZE: - case METRICS_PLAN_NODE_EXECUTING: - case METRICS_PLAN_NODE_FINISHED: - // TODO - break; - case METRICS_QUERY_SUBMIT: - collect_query_submit(query_desc, utility); - break; - case METRICS_QUERY_START: - // no-op: executor_after_start is enough - break; - case METRICS_QUERY_CANCELING: - // it appears we're only interested in the actual CANCELED event. - // for now we will ignore CANCELING state unless otherwise requested from - // end users - break; - case METRICS_QUERY_DONE: - case METRICS_QUERY_ERROR: - case METRICS_QUERY_CANCELED: - case METRICS_INNER_QUERY_DONE: - collect_query_done(query_desc, utility, status, edata); - break; - default: - ereport(FATAL, (errmsg("Unknown query status: %d", status))); - } +void +EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata) +{ + auto *query_desc = reinterpret_cast(arg); + switch (status) + { + case METRICS_PLAN_NODE_INITIALIZE: + case METRICS_PLAN_NODE_EXECUTING: + case METRICS_PLAN_NODE_FINISHED: + // TODO + break; + case METRICS_QUERY_SUBMIT: + collect_query_submit(query_desc, utility); + break; + case METRICS_QUERY_START: + // no-op: executor_after_start is enough + break; + case METRICS_QUERY_CANCELING: + // it appears we're only interested in the actual CANCELED event. + // for now we will ignore CANCELING state unless otherwise requested from + // end users + break; + case METRICS_QUERY_DONE: + case METRICS_QUERY_ERROR: + case METRICS_QUERY_CANCELED: + case METRICS_INNER_QUERY_DONE: + collect_query_done(query_desc, utility, status, edata); + break; + default: + ereport(ERROR, (errmsg("Unknown query status: %d", status))); + } } -void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { - if (!verify_query(query_desc, QueryState::START, false /* utility*/)) { - return; - } - - if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && - (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - if (config.enable_cdbstats()) { - query_desc->instrument_options |= INSTRUMENT_CDB; - if (!query_desc->showstatctx) { - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); - } - } - } +void +EventSender::executor_before_start(QueryDesc *query_desc, int eflags) +{ + if (!verify_query(query_desc, QueryState::START, false /* utility*/)) + { + return; + } + + if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && + (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + { + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + if (config.enable_cdbstats()) + { + query_desc->instrument_options |= INSTRUMENT_CDB; + if (!query_desc->showstatctx) + { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + } + } + } } -void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!verify_query(query_desc, QueryState::START, false /* utility */)) { - return; - } - - auto &query = get_query(query_desc); - auto query_msg = query.message.get(); - *query_msg->mutable_start_time() = current_ts(); - update_query_state(query, QueryState::START, false /* utility */); - set_query_plan(query_msg, query_desc, config); - if (need_collect_analyze()) { - // Set up to track total elapsed time during query run. - // Make sure the space is allocated in the per-query - // context so it will go away at executor_end. - if (query_desc->totaltime == NULL) { - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); - gpdb::mem_ctx_switch_to(oldcxt); - } - } - gpsc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (log_query_req(*query_msg, "started", false /* utility */)) { - clear_big_fields(query_msg); - } - std::swap(stats, *query_msg->mutable_query_metrics()); +void +EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) +{ + if (!verify_query(query_desc, QueryState::START, false /* utility */)) + { + return; + } + + auto &query = get_query(query_desc); + auto query_msg = query.message.get(); + *query_msg->mutable_start_time() = current_ts(); + update_query_state(query, QueryState::START, false /* utility */); + set_query_plan(query_msg, query_desc, config); + if (need_collect_analyze()) + { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) + { + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); + gpdb::mem_ctx_switch_to(oldcxt); + } + } + gpsc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (log_query_req(*query_msg, "started", false /* utility */)) + { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } -void EventSender::executor_end(QueryDesc *query_desc) { - if (!verify_query(query_desc, QueryState::END, false /* utility */)) { - return; - } - - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg->mutable_end_time() = current_ts(); - update_query_state(query, QueryState::END, false /* utility */); - if (is_top_level_query(query_desc, nesting_level)) { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, - nested_timing); - } else { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); - } - if (log_query_req(*query_msg, "ended", false /* utility */)) { - clear_big_fields(query_msg); - } +void +EventSender::executor_end(QueryDesc *query_desc) +{ + if (!verify_query(query_desc, QueryState::END, false /* utility */)) + { + return; + } + + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg->mutable_end_time() = current_ts(); + update_query_state(query, QueryState::END, false /* utility */); + if (is_top_level_query(query_desc, nesting_level)) + { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } + else + { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (log_query_req(*query_msg, "ended", false /* utility */)) + { + clear_big_fields(query_msg); + } } -void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { - if (!verify_query(query_desc, QueryState::SUBMIT, utility)) { - return; - } - - submit_query(query_desc); - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); - *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg); - set_qi_nesting_level(query_msg, nesting_level); - set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc, config); - if (log_query_req(*query_msg, "submit", utility)) { - clear_big_fields(query_msg); - } - // take initial metrics snapshot so that we can safely take diff afterwards - // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); +void +EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) +{ + if (!verify_query(query_desc, QueryState::SUBMIT, utility)) + { + return; + } + + submit_query(query_desc); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg); + set_qi_nesting_level(query_msg, nesting_level); + set_qi_slice_id(query_msg); + set_query_text(query_msg, query_desc, config); + if (log_query_req(*query_msg, "submit", utility)) + { + clear_big_fields(query_msg); + } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); #ifdef IC_TEARDOWN_HOOK - // same for interconnect statistics - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif } -void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status, bool utility, - ErrorData *edata) { - gpsc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code - // here just in case - Assert(false); - query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, - (errmsg("Unexpected query status in query_done hook: %d", status))); - } - auto prev_state = query.state; - update_query_state(query, QueryState::DONE, utility, - query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query.message.get(); - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - bool error_flushed = elog_message() == NULL; - if (error_flushed && (edata == NULL || edata->message == NULL)) { - ereport(WARNING, (errmsg("GPSC missing error message"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } else { - set_qi_error_message( - query_msg, error_flushed ? edata->message : elog_message(), config); - } - } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, - nested_timing); - } +void +EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status, bool utility, + ErrorData *edata) +{ + gpsc::QueryStatus query_status; + std::string msg; + switch (status) + { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(ERROR, + (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + auto prev_state = query.state; + update_query_state(query, QueryState::DONE, utility, + query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query.message.get(); + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) + { + bool error_flushed = elog_message() == NULL; + if (error_flushed && (edata == NULL || edata->message == NULL)) + { + ereport(WARNING, (errmsg("GPSC missing error message"))); + ereport(DEBUG3, (errmsg("GPSC query sourceText: %s", + query_desc->sourceText))); + } + else + { + set_qi_error_message( + query_msg, error_flushed ? edata->message : elog_message(), + config); + } + } + if (prev_state == START) + { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } #ifdef IC_TEARDOWN_HOOK - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif - (void)log_query_req(*query_msg, msg, utility); + (void) log_query_req(*query_msg, msg, utility); } -void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, - QueryMetricsStatus status, - ErrorData *edata) { - if (!verify_query(query_desc, QueryState::DONE, utility)) { - return; - } - - // Skip sending done message if query errored before submit. - if (!qdesc_submitted(query_desc)) { - if (status != METRICS_QUERY_ERROR) { - ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " - "unsubmitted and unerrored query"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } - return; - } - - if (queries.empty()) { - ereport(WARNING, (errmsg("GPSC cannot find query to process DONE hook"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - return; - } - auto &query = get_query(query_desc); - - report_query_done(query_desc, query, status, utility, edata); - - if (need_report_nested_query()) - update_nested_counters(query_desc); - - queries.erase(QueryKey::from_qdesc(query_desc)); - pfree(query_desc->gpsc_query_key); - query_desc->gpsc_query_key = NULL; +void +EventSender::collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata) +{ + if (!verify_query(query_desc, QueryState::DONE, utility)) + { + return; + } + + // Skip sending done message if query errored before submit. + if (!qdesc_submitted(query_desc)) + { + if (status != METRICS_QUERY_ERROR) + { + ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " + "unsubmitted and unerrored query"))); + ereport(DEBUG3, (errmsg("GPSC query sourceText: %s", + query_desc->sourceText))); + } + return; + } + + if (queries.empty()) + { + ereport(WARNING, + (errmsg("GPSC cannot find query to process DONE hook"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + return; + } + auto &query = get_query(query_desc); + + report_query_done(query_desc, query, status, utility, edata); + + if (need_report_nested_query()) + update_nested_counters(query_desc); + + queries.erase(QueryKey::from_qdesc(query_desc)); + pfree(query_desc->gpsc_query_key); + query_desc->gpsc_query_key = NULL; } -void EventSender::ic_metrics_collect() { +void +EventSender::ic_metrics_collect() +{ #ifdef IC_TEARDOWN_HOOK - if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { - return; - } - if (!proto_verified || gp_command_count == 0 || !config.enable_collector() || - config.filter_user(get_user_name())) { - return; - } - // we also would like to know nesting level here and filter queries BUT we - // don't have this kind of information from this callback. Will have to - // collect stats anyways and throw it away later, if necessary - auto metrics = UDPIFCGetICStats(); - ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; - ic_statistics.recvQueueSizeCountingTime += metrics.recvQueueSizeCountingTime; - ic_statistics.totalCapacity += metrics.totalCapacity; - ic_statistics.capacityCountingTime += metrics.capacityCountingTime; - ic_statistics.totalBuffers += metrics.totalBuffers; - ic_statistics.bufferCountingTime += metrics.bufferCountingTime; - ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; - ic_statistics.retransmits += metrics.retransmits; - ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; - ic_statistics.mismatchNum += metrics.mismatchNum; - ic_statistics.crcErrors += metrics.crcErrors; - ic_statistics.sndPktNum += metrics.sndPktNum; - ic_statistics.recvPktNum += metrics.recvPktNum; - ic_statistics.disorderedPktNum += metrics.disorderedPktNum; - ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; - ic_statistics.recvAckNum += metrics.recvAckNum; - ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; + if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) + { + return; + } + if (!proto_verified || gp_command_count == 0 || + !config.enable_collector() || config.filter_user(get_user_name())) + { + return; + } + // we also would like to know nesting level here and filter queries BUT we + // don't have this kind of information from this callback. Will have to + // collect stats anyways and throw it away later, if necessary + auto metrics = UDPIFCGetICStats(); + ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; + ic_statistics.recvQueueSizeCountingTime += + metrics.recvQueueSizeCountingTime; + ic_statistics.totalCapacity += metrics.totalCapacity; + ic_statistics.capacityCountingTime += metrics.capacityCountingTime; + ic_statistics.totalBuffers += metrics.totalBuffers; + ic_statistics.bufferCountingTime += metrics.bufferCountingTime; + ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; + ic_statistics.retransmits += metrics.retransmits; + ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; + ic_statistics.mismatchNum += metrics.mismatchNum; + ic_statistics.crcErrors += metrics.crcErrors; + ic_statistics.sndPktNum += metrics.sndPktNum; + ic_statistics.recvPktNum += metrics.recvPktNum; + ic_statistics.disorderedPktNum += metrics.disorderedPktNum; + ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; + ic_statistics.recvAckNum += metrics.recvAckNum; + ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; #endif } -void EventSender::analyze_stats_collect(QueryDesc *query_desc) { - if (!verify_query(query_desc, QueryState::END, false /* utility */)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH) { - return; - } - if (!query_desc->totaltime || !need_collect_analyze()) { - return; - } - // Make sure stats accumulation is done. - // (Note: it's okay if several levels of hook all do this.) - gpdb::instr_end_loop(query_desc->totaltime); - - double ms = query_desc->totaltime->total * 1000.0; - if (ms >= config.min_analyze_time()) { - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - set_analyze_plan_text(query_desc, query_msg, config); - } +void +EventSender::analyze_stats_collect(QueryDesc *query_desc) +{ + if (!verify_query(query_desc, QueryState::END, false /* utility */)) + { + return; + } + if (Gp_role != GP_ROLE_DISPATCH) + { + return; + } + if (!query_desc->totaltime || !need_collect_analyze()) + { + return; + } + // Make sure stats accumulation is done. + // (Note: it's okay if several levels of hook all do this.) + gpdb::instr_end_loop(query_desc->totaltime); + + double ms = query_desc->totaltime->total * 1000.0; + if (ms >= config.min_analyze_time()) + { + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + set_analyze_plan_text(query_desc, query_msg, config); + } } -EventSender::EventSender() { - // Perform initial sync to get default GUC values - config.sync(); - - try { - GOOGLE_PROTOBUF_VERIFY_VERSION; - proto_verified = true; - } catch (const std::exception &e) { - ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", e.what()))); - } +EventSender::EventSender() +{ + // Perform initial sync to get default GUC values + config.sync(); + + try + { + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; + } + catch (const std::exception &e) + { + ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", + e.what()))); + } #ifdef IC_TEARDOWN_HOOK - memset(&ic_statistics, 0, sizeof(ICStatistics)); + memset(&ic_statistics, 0, sizeof(ICStatistics)); #endif } -EventSender::~EventSender() { - for (const auto &[qkey, _] : queries) { - ereport(LOG, (errmsg("GPSC query with missing done event: " - "tmid=%d ssid=%d ccnt=%d nlvl=%d", - qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); - } +EventSender::~EventSender() +{ + for (const auto &[qkey, _] : queries) + { + ereport(LOG, + (errmsg("GPSC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); + } } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP -void EventSender::update_query_state(QueryItem &query, QueryState new_state, - bool utility, bool success) { - switch (new_state) { - case QueryState::SUBMIT: - Assert(false); - break; - case QueryState::START: - if (query.state == QueryState::SUBMIT) { - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_START); - } else { - Assert(false); - } - break; - case QueryState::END: - // Example of below assert triggering: CURSOR closes before ever being - // executed Assert(query->state == QueryState::START || - // IsAbortInProgress()); - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_END); - break; - case QueryState::DONE: - Assert(query.state == QueryState::END || !success || utility); - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_DONE); - break; - default: - Assert(false); - } - query.state = new_state; +void +EventSender::update_query_state(QueryItem &query, QueryState new_state, + bool utility, bool success) +{ + switch (new_state) + { + case QueryState::SUBMIT: + Assert(false); + break; + case QueryState::START: + if (query.state == QueryState::SUBMIT) + { + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_START); + } + else + { + Assert(false); + } + break; + case QueryState::END: + // Example of below assert triggering: CURSOR closes before ever being + // executed Assert(query->state == QueryState::START || + // IsAbortInProgress()); + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_END); + break; + case QueryState::DONE: + Assert(query.state == QueryState::END || !success || utility); + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_DONE); + break; + default: + Assert(false); + } + query.state = new_state; } -EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { - if (!qdesc_submitted(query_desc)) { - ereport(WARNING, - (errmsg("GPSC attempting to get query that was not submitted"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - throw std::runtime_error("Attempting to get query that was not submitted"); - } - return queries.find(QueryKey::from_qdesc(query_desc))->second; +EventSender::QueryItem & +EventSender::get_query(QueryDesc *query_desc) +{ + if (!qdesc_submitted(query_desc)) + { + ereport( + WARNING, + (errmsg("GPSC attempting to get query that was not submitted"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + throw std::runtime_error( + "Attempting to get query that was not submitted"); + } + return queries.find(QueryKey::from_qdesc(query_desc))->second; } -void EventSender::submit_query(QueryDesc *query_desc) { - if (query_desc->gpsc_query_key) { - ereport(WARNING, - (errmsg("GPSC trying to submit already submitted query"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } - QueryKey::register_qkey(query_desc, nesting_level); - auto key = QueryKey::from_qdesc(query_desc); - auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); - if (!inserted) { - ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } +void +EventSender::submit_query(QueryDesc *query_desc) +{ + if (query_desc->gpsc_query_key) + { + ereport(WARNING, + (errmsg("GPSC trying to submit already submitted query"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + } + QueryKey::register_qkey(query_desc, nesting_level); + auto key = QueryKey::from_qdesc(query_desc); + auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); + if (!inserted) + { + ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + } } -void EventSender::update_nested_counters(QueryDesc *query_desc) { - if (!is_top_level_query(query_desc, nesting_level)) { - auto &query = get_query(query_desc); - nested_calls++; - double end_time = protots_to_double(query.message->end_time()); - double start_time = protots_to_double(query.message->start_time()); - if (end_time >= start_time) { - nested_timing += end_time - start_time; - } else { - ereport(WARNING, (errmsg("GPSC query start_time > end_time (%f > %f)", - start_time, end_time))); - ereport(DEBUG3, - (errmsg("GPSC nested query text %s", query_desc->sourceText))); - } - } +void +EventSender::update_nested_counters(QueryDesc *query_desc) +{ + if (!is_top_level_query(query_desc, nesting_level)) + { + auto &query = get_query(query_desc); + nested_calls++; + double end_time = protots_to_double(query.message->end_time()); + double start_time = protots_to_double(query.message->start_time()); + if (end_time >= start_time) + { + nested_timing += end_time - start_time; + } + else + { + ereport(WARNING, + (errmsg("GPSC query start_time > end_time (%f > %f)", + start_time, end_time))); + ereport(DEBUG3, (errmsg("GPSC nested query text %s", + query_desc->sourceText))); + } + } } -bool EventSender::qdesc_submitted(QueryDesc *query_desc) { - if (query_desc->gpsc_query_key == NULL) { - return false; - } - return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); +bool +EventSender::qdesc_submitted(QueryDesc *query_desc) +{ + if (query_desc->gpsc_query_key == NULL) + { + return false; + } + return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); } -bool EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); +bool +EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) +{ + return need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); } -bool EventSender::need_report_nested_query() { - return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; +bool +EventSender::need_report_nested_query() +{ + return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; } -bool EventSender::filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !config.enable_collector() || config.filter_user(get_user_name()); +bool +EventSender::filter_query(QueryDesc *query_desc) +{ + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !config.enable_collector() || config.filter_user(get_user_name()); } EventSender::QueryItem::QueryItem(QueryState st) - : message(std::make_unique()), state(st) {} + : message(std::make_unique()), state(st) +{ +} diff --git a/gpcontrib/gp_stats_collector/src/EventSender.h b/gpcontrib/gp_stats_collector/src/EventSender.h index 154c2c0dceb..2651a020593 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.h +++ b/gpcontrib/gp_stats_collector/src/EventSender.h @@ -25,11 +25,12 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef EVENTSENDER_H +#define EVENTSENDER_H #include -#include #include +#include #define typeid __typeid extern "C" { @@ -40,12 +41,13 @@ extern "C" { } #undef typeid -#include "memory/gpdbwrappers.h" #include "Config.h" +#include "memory/gpdbwrappers.h" class UDSConnector; struct QueryDesc; -namespace gpsc { +namespace gpsc +{ class SetQueryReq; } @@ -53,116 +55,149 @@ class SetQueryReq; extern void gp_gettmid(int32 *); -struct QueryKey { - int tmid; - int ssid; - int ccnt; - int nesting_level; - uintptr_t query_desc_addr; - - bool operator==(const QueryKey &other) const { - return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == - std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, - other.query_desc_addr); - } - - static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { - query_desc->gpsc_query_key = - (GpscQueryKey *)gpdb::palloc0(sizeof(GpscQueryKey)); - int32 tmid; - gp_gettmid(&tmid); - query_desc->gpsc_query_key->tmid = tmid; - query_desc->gpsc_query_key->ssid = gp_session_id; - query_desc->gpsc_query_key->ccnt = gp_command_count; - query_desc->gpsc_query_key->nesting_level = nesting_level; - query_desc->gpsc_query_key->query_desc_addr = (uintptr_t)query_desc; - } - - static QueryKey from_qdesc(QueryDesc *query_desc) { - return { - .tmid = query_desc->gpsc_query_key->tmid, - .ssid = query_desc->gpsc_query_key->ssid, - .ccnt = query_desc->gpsc_query_key->ccnt, - .nesting_level = query_desc->gpsc_query_key->nesting_level, - .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, - }; - } +struct QueryKey +{ + int tmid; + int ssid; + int ccnt; + int nesting_level; + uintptr_t query_desc_addr; + + bool + operator==(const QueryKey &other) const + { + return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == + std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, + other.query_desc_addr); + } + + static void + register_qkey(QueryDesc *query_desc, size_t nesting_level) + { + query_desc->gpsc_query_key = + (GpscQueryKey *) gpdb::palloc0(sizeof(GpscQueryKey)); + int32 tmid; + gp_gettmid(&tmid); + query_desc->gpsc_query_key->tmid = tmid; + query_desc->gpsc_query_key->ssid = gp_session_id; + query_desc->gpsc_query_key->ccnt = gp_command_count; + query_desc->gpsc_query_key->nesting_level = nesting_level; + query_desc->gpsc_query_key->query_desc_addr = (uintptr_t) query_desc; + } + + static QueryKey + from_qdesc(QueryDesc *query_desc) + { + return { + .tmid = query_desc->gpsc_query_key->tmid, + .ssid = query_desc->gpsc_query_key->ssid, + .ccnt = query_desc->gpsc_query_key->ccnt, + .nesting_level = query_desc->gpsc_query_key->nesting_level, + .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, + }; + } }; // https://www.boost.org/doc/libs/1_35_0/doc/html/boost/hash_combine_id241013.html -template inline void hash_combine(std::size_t &seed, const T &v) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +template +inline void +hash_combine(std::size_t &seed, const T &v) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); } -namespace std { -template <> struct hash { - size_t operator()(const QueryKey &k) const noexcept { - size_t seed = hash{}(k.tmid); - hash_combine(seed, k.ssid); - hash_combine(seed, k.ccnt); - hash_combine(seed, k.nesting_level); - uintptr_t addr = k.query_desc_addr; - if constexpr (SIZE_MAX < UINTPTR_MAX) { - addr %= SIZE_MAX; - } - hash_combine(seed, addr); - return seed; - } +namespace std +{ +template <> +struct hash +{ + size_t + operator()(const QueryKey &k) const noexcept + { + size_t seed = hash{}(k.tmid); + hash_combine(seed, k.ssid); + hash_combine(seed, k.ccnt); + hash_combine(seed, k.nesting_level); + uintptr_t addr = k.query_desc_addr; + if constexpr (SIZE_MAX < UINTPTR_MAX) + { + addr %= SIZE_MAX; + } + hash_combine(seed, addr); + return seed; + } }; -} // namespace std +} // namespace std -class EventSender { +class EventSender +{ public: - void executor_before_start(QueryDesc *query_desc, int eflags); - void executor_after_start(QueryDesc *query_desc, int eflags); - void executor_end(QueryDesc *query_desc); - void query_metrics_collect(QueryMetricsStatus status, void *arg, bool utility, - ErrorData *edata = NULL); - void ic_metrics_collect(); - void analyze_stats_collect(QueryDesc *query_desc); - void incr_depth() { nesting_level++; } - void decr_depth() { nesting_level--; } - EventSender(); - ~EventSender(); + void executor_before_start(QueryDesc *query_desc, int eflags); + void executor_after_start(QueryDesc *query_desc, int eflags); + void executor_end(QueryDesc *query_desc); + void query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata = NULL); + void ic_metrics_collect(); + void analyze_stats_collect(QueryDesc *query_desc); + void + incr_depth() + { + nesting_level++; + } + void + decr_depth() + { + nesting_level--; + } + EventSender(); + ~EventSender(); private: - enum QueryState { SUBMIT, START, END, DONE }; - - struct QueryItem { - std::unique_ptr message; - QueryState state; - - explicit QueryItem(QueryState st); - }; - - bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, - bool utility); - bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); - void update_query_state(QueryItem &query, QueryState new_state, bool utility, - bool success = true); - QueryItem &get_query(QueryDesc *query_desc); - void submit_query(QueryDesc *query_desc); - void collect_query_submit(QueryDesc *query_desc, bool utility); - void report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status, bool utility, - ErrorData *edata = NULL); - void collect_query_done(QueryDesc *query_desc, bool utility, - QueryMetricsStatus status, ErrorData *edata = NULL); - void update_nested_counters(QueryDesc *query_desc); - bool qdesc_submitted(QueryDesc *query_desc); - bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); - bool need_report_nested_query(); - bool filter_query(QueryDesc *query_desc); - - bool proto_verified = false; - int nesting_level = 0; - int64_t nested_calls = 0; - double nested_timing = 0; + enum QueryState + { + SUBMIT, + START, + END, + DONE + }; + + struct QueryItem + { + std::unique_ptr message; + QueryState state; + + explicit QueryItem(QueryState st); + }; + + bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, + bool utility); + bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); + void update_query_state(QueryItem &query, QueryState new_state, + bool utility, bool success = true); + QueryItem &get_query(QueryDesc *query_desc); + void submit_query(QueryDesc *query_desc); + void collect_query_submit(QueryDesc *query_desc, bool utility); + void report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status, bool utility, + ErrorData *edata = NULL); + void collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata = NULL); + void update_nested_counters(QueryDesc *query_desc); + bool qdesc_submitted(QueryDesc *query_desc); + bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); + bool need_report_nested_query(); + bool filter_query(QueryDesc *query_desc); + + bool proto_verified = false; + int nesting_level = 0; + int64_t nested_calls = 0; + double nested_timing = 0; #ifdef IC_TEARDOWN_HOOK - ICStatistics ic_statistics; + ICStatistics ic_statistics; #endif - std::unordered_map queries; + std::unordered_map queries; - Config config; -}; \ No newline at end of file + Config config; +}; +#endif /* EVENTSENDER_H */ diff --git a/gpcontrib/gp_stats_collector/src/GpscStat.cpp b/gpcontrib/gp_stats_collector/src/GpscStat.cpp index c4029f085cf..151cfd87c02 100644 --- a/gpcontrib/gp_stats_collector/src/GpscStat.cpp +++ b/gpcontrib/gp_stats_collector/src/GpscStat.cpp @@ -38,81 +38,117 @@ extern "C" { #include "storage/spin.h" } -namespace { -struct ProtectedData { - slock_t mutex; - GpscStat::Data data; +namespace +{ +struct ProtectedData +{ + slock_t mutex; + GpscStat::Data data; }; shmem_startup_hook_type prev_shmem_startup_hook = NULL; ProtectedData *data = nullptr; -void gpsc_shmem_startup() { - if (prev_shmem_startup_hook) - prev_shmem_startup_hook(); - LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - bool found; - data = reinterpret_cast( - ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); - if (!found) { - SpinLockInit(&data->mutex); - data->data = GpscStat::Data(); - } - LWLockRelease(AddinShmemInitLock); +void +gpsc_shmem_startup() +{ + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + bool found; + data = reinterpret_cast( + ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); + if (!found) + { + SpinLockInit(&data->mutex); + data->data = GpscStat::Data(); + } + LWLockRelease(AddinShmemInitLock); } -class LockGuard { +class LockGuard +{ public: - LockGuard(slock_t *mutex) : mutex_(mutex) { SpinLockAcquire(mutex_); } - ~LockGuard() { SpinLockRelease(mutex_); } + LockGuard(slock_t *mutex) : mutex_(mutex) + { + SpinLockAcquire(mutex_); + } + ~LockGuard() + { + SpinLockRelease(mutex_); + } private: - slock_t *mutex_; + slock_t *mutex_; }; -} // namespace - -void GpscStat::init() { - if (!process_shared_preload_libraries_in_progress) - return; - RequestAddinShmemSpace(sizeof(ProtectedData)); - prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = gpsc_shmem_startup; +} // namespace + +void +GpscStat::init() +{ + if (!process_shared_preload_libraries_in_progress) + return; + RequestAddinShmemSpace(sizeof(ProtectedData)); + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = gpsc_shmem_startup; } -void GpscStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } +void +GpscStat::deinit() +{ + shmem_startup_hook = prev_shmem_startup_hook; +} -void GpscStat::reset() { - LockGuard lg(&data->mutex); - data->data = GpscStat::Data(); +void +GpscStat::reset() +{ + LockGuard lg(&data->mutex); + data->data = GpscStat::Data(); } -void GpscStat::report_send(int32_t msg_size) { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +void +GpscStat::report_send(int32_t msg_size) +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.max_message_size = + std::max(msg_size, data->data.max_message_size); } -void GpscStat::report_bad_connection() { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_connects++; +void +GpscStat::report_bad_connection() +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_connects++; } -void GpscStat::report_bad_send(int32_t msg_size) { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_sends++; - data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +void +GpscStat::report_bad_send(int32_t msg_size) +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_sends++; + data->data.max_message_size = + std::max(msg_size, data->data.max_message_size); } -void GpscStat::report_error() { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_other++; +void +GpscStat::report_error() +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_other++; } -GpscStat::Data GpscStat::get_stats() { - LockGuard lg(&data->mutex); - return data->data; +GpscStat::Data +GpscStat::get_stats() +{ + LockGuard lg(&data->mutex); + return data->data; } -bool GpscStat::loaded() { return data != nullptr; } +bool +GpscStat::loaded() +{ + return data != nullptr; +} diff --git a/gpcontrib/gp_stats_collector/src/GpscStat.h b/gpcontrib/gp_stats_collector/src/GpscStat.h index af1a1261776..d82930c7b5b 100644 --- a/gpcontrib/gp_stats_collector/src/GpscStat.h +++ b/gpcontrib/gp_stats_collector/src/GpscStat.h @@ -25,24 +25,28 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef GPSCSTAT_H +#define GPSCSTAT_H #include -class GpscStat { +class GpscStat +{ public: - struct Data { - int64_t total, failed_sends, failed_connects, failed_other; - int32_t max_message_size; - }; + struct Data + { + int64_t total, failed_sends, failed_connects, failed_other; + int32_t max_message_size; + }; - static void init(); - static void deinit(); - static void reset(); - static void report_send(int32_t msg_size); - static void report_bad_connection(); - static void report_bad_send(int32_t msg_size); - static void report_error(); - static Data get_stats(); - static bool loaded(); -}; \ No newline at end of file + static void init(); + static void deinit(); + static void reset(); + static void report_send(int32_t msg_size); + static void report_bad_connection(); + static void report_bad_send(int32_t msg_size); + static void report_error(); + static Data get_stats(); + static bool loaded(); +}; +#endif /* GPSCSTAT_H */ diff --git a/gpcontrib/gp_stats_collector/src/PgUtils.cpp b/gpcontrib/gp_stats_collector/src/PgUtils.cpp index 3dbee97061b..c473cc383f2 100644 --- a/gpcontrib/gp_stats_collector/src/PgUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/PgUtils.cpp @@ -30,39 +30,46 @@ #include "memory/gpdbwrappers.h" extern "C" { -#include "commands/resgroupcmds.h" #include "cdb/cdbvars.h" +#include "commands/resgroupcmds.h" } -std::string get_user_name() { - // username is allocated on stack, we don't need to pfree it. - const char *username = - gpdb::get_config_option("session_authorization", false, false); - return username ? std::string(username) : ""; +std::string +get_user_name() +{ + // username is allocated on stack, we don't need to pfree it. + const char *username = + gpdb::get_config_option("session_authorization", false, false); + return username ? std::string(username) : ""; } -std::string get_db_name() { - char *dbname = gpdb::get_database_name(MyDatabaseId); - if (dbname) { - std::string result(dbname); - gpdb::pfree(dbname); - return result; - } - return ""; +std::string +get_db_name() +{ + char *dbname = gpdb::get_database_name(MyDatabaseId); + if (dbname) + { + std::string result(dbname); + gpdb::pfree(dbname); + return result; + } + return ""; } -std::string get_rg_name() { - auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); - if (!OidIsValid(groupId)) - return ""; +std::string +get_rg_name() +{ + auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); + if (!OidIsValid(groupId)) + return ""; - char *rgname = gpdb::get_rg_name_for_id(groupId); - if (rgname == nullptr) - return ""; + char *rgname = gpdb::get_rg_name_for_id(groupId); + if (rgname == nullptr) + return ""; - std::string result(rgname); - gpdb::pfree(rgname); - return result; + std::string result(rgname); + gpdb::pfree(rgname); + return result; } /** @@ -86,9 +93,12 @@ std::string get_rg_name() { * segment sees those as top-level. */ -bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - if (query_desc->gpsc_query_key == NULL) { - return nesting_level == 0; - } - return query_desc->gpsc_query_key->nesting_level == 0; +bool +is_top_level_query(QueryDesc *query_desc, int nesting_level) +{ + if (query_desc->gpsc_query_key == NULL) + { + return nesting_level == 0; + } + return query_desc->gpsc_query_key->nesting_level == 0; } diff --git a/gpcontrib/gp_stats_collector/src/ProcStats.cpp b/gpcontrib/gp_stats_collector/src/ProcStats.cpp index 9c557879fc6..e308b30dfa5 100644 --- a/gpcontrib/gp_stats_collector/src/ProcStats.cpp +++ b/gpcontrib/gp_stats_collector/src/ProcStats.cpp @@ -26,100 +26,119 @@ */ #include "ProcStats.h" -#include "gpsc_metrics.pb.h" -#include #include +#include #include +#include "gpsc_metrics.pb.h" extern "C" { #include "postgres.h" #include "utils/elog.h" } -namespace { -#define FILL_IO_STAT(stat_name) \ - uint64_t stat_name; \ - proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name - stats->stat_name()); +namespace +{ +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name - stats->stat_name()); -void fill_io_stats(gpsc::SystemStat *stats) { - std::ifstream proc_stat("/proc/self/io"); - std::string tmp; - FILL_IO_STAT(rchar); - FILL_IO_STAT(wchar); - FILL_IO_STAT(syscr); - FILL_IO_STAT(syscw); - FILL_IO_STAT(read_bytes); - FILL_IO_STAT(write_bytes); - FILL_IO_STAT(cancelled_write_bytes); +void +fill_io_stats(gpsc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(gpsc::SystemStat *stats) { - static const int UTIME_ID = 13; - static const int STIME_ID = 14; - static const int VSIZE_ID = 22; - static const int RSS_ID = 23; - static const double tps = sysconf(_SC_CLK_TCK); +void +fill_cpu_stats(gpsc::SystemStat *stats) +{ + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); - std::ifstream proc_stat("/proc/self/stat"); - std::string trash; - for (int i = 0; i <= RSS_ID; ++i) { - switch (i) { - case UTIME_ID: - double utime; - proc_stat >> utime; - stats->set_usertimeseconds(utime / tps - stats->usertimeseconds()); - break; - case STIME_ID: - double stime; - proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps - stats->kerneltimeseconds()); - break; - case VSIZE_ID: - uint64_t vsize; - proc_stat >> vsize; - stats->set_vsize(vsize); - break; - case RSS_ID: - uint64_t rss; - proc_stat >> rss; - // NOTE: this is a double AFAIU, need to double-check - stats->set_rss(rss); - break; - default: - proc_stat >> trash; - } - } + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + for (int i = 0; i <= RSS_ID; ++i) + { + switch (i) + { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps - + stats->usertimeseconds()); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps - + stats->kerneltimeseconds()); + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; + } + } } -void fill_status_stats(gpsc::SystemStat *stats) { - std::ifstream proc_stat("/proc/self/status"); - std::string key, measure; - while (proc_stat >> key) { - if (key == "VmPeak:") { - uint64_t value; - proc_stat >> value; - stats->set_vmpeakkb(value); - proc_stat >> measure; - if (measure != "kB") { - throw std::runtime_error("Expected memory sizes in kB, but got in " + - measure); - } - } else if (key == "VmSize:") { - uint64_t value; - proc_stat >> value; - stats->set_vmsizekb(value); - if (measure != "kB") { - throw std::runtime_error("Expected memory sizes in kB, but got in " + - measure); - } - } - } +void +fill_status_stats(gpsc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) + { + if (key == "VmPeak:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") + { + throw std::runtime_error( + "Expected memory sizes in kB, but got in " + measure); + } + } + else if (key == "VmSize:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") + { + throw std::runtime_error( + "Expected memory sizes in kB, but got in " + measure); + } + } + } } -} // namespace +} // namespace -void fill_self_stats(gpsc::SystemStat *stats) { - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); +void +fill_self_stats(gpsc::SystemStat *stats) +{ + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/ProcStats.h b/gpcontrib/gp_stats_collector/src/ProcStats.h index 4473125f875..8b83dbfef02 100644 --- a/gpcontrib/gp_stats_collector/src/ProcStats.h +++ b/gpcontrib/gp_stats_collector/src/ProcStats.h @@ -25,10 +25,13 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef PROCSTATS_H +#define PROCSTATS_H -namespace gpsc { +namespace gpsc +{ class SystemStat; } -void fill_self_stats(gpsc::SystemStat *stats); \ No newline at end of file +void fill_self_stats(gpsc::SystemStat *stats); +#endif /* PROCSTATS_H */ diff --git a/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp index c9ceff4739b..b22f580303e 100644 --- a/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp @@ -26,9 +26,9 @@ */ #include "ProtoUtils.h" +#include "Config.h" #include "PgUtils.h" #include "ProcStats.h" -#include "Config.h" #include "memory/gpdbwrappers.h" #define typeid __typeid @@ -53,265 +53,323 @@ extern "C" { extern void gp_gettmid(int32 *); -namespace { +namespace +{ constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); constexpr uint8_t UTF8_MAX_SYMBOL_BYTES = 4; // Returns true if byte is the starting byte of utf8 // character, false if byte is the continuation (10xxxxxx). -inline bool utf8_start_byte(uint8_t byte) { - return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; +inline bool +utf8_start_byte(uint8_t byte) +{ + return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; } -} // namespace +} // namespace -google::protobuf::Timestamp current_ts() { - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; +google::protobuf::Timestamp +current_ts() +{ + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; } -void set_query_key(gpsc::QueryKey *key) { - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gp_gettmid(&tmid); - key->set_tmid(tmid); +void +set_query_key(gpsc::QueryKey *key) +{ + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gp_gettmid(&tmid); + key->set_tmid(tmid); } -void set_segment_key(gpsc::SegmentKey *key) { - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); +void +set_segment_key(gpsc::SegmentKey *key) +{ + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); } -std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { - if (unlikely(str == nullptr)) { - return std::string(); - } - if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) { - return std::string(str, std::min(len, lim)); - } +std::string +trim_str_shrink_utf8(const char *str, size_t len, size_t lim) +{ + if (unlikely(str == nullptr)) + { + return std::string(); + } + if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) + { + return std::string(str, std::min(len, lim)); + } - // Handle trimming of utf8 correctly, do not cut multi-byte characters. - size_t cut_pos = lim; - size_t visited_bytes = 1; - while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) { - if (utf8_start_byte(static_cast(str[cut_pos]))) { - break; - } - ++visited_bytes; - --cut_pos; - } + // Handle trimming of utf8 correctly, do not cut multi-byte characters. + size_t cut_pos = lim; + size_t visited_bytes = 1; + while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) + { + if (utf8_start_byte(static_cast(str[cut_pos]))) + { + break; + } + ++visited_bytes; + --cut_pos; + } - return std::string(str, cut_pos); + return std::string(str, cut_pos); } -void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config) { - if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { - auto qi = req->mutable_query_info(); - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_explain_state(query_desc, true); - if (es.str) { - *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, - config.max_plan_size()); - StringInfo norm_plan = gpdb::gen_normplan(es.str->data); - if (norm_plan) { - *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, config.max_plan_size()); - qi->set_plan_id( - hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - gpdb::pfree(norm_plan->data); - } - qi->set_query_id(query_desc->plannedstmt->queryId); - gpdb::pfree(es.str->data); - } - gpdb::mem_ctx_switch_to(oldcxt); - } +void +set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) +{ + if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) + { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); + if (es.str) + { + *qi->mutable_plan_text() = trim_str_shrink_utf8( + es.str->data, es.str->len, config.max_plan_size()); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + if (norm_plan) + { + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( + norm_plan->data, norm_plan->len, config.max_plan_size()); + qi->set_plan_id(hash_any((unsigned char *) norm_plan->data, + norm_plan->len)); + gpdb::pfree(norm_plan->data); + } + qi->set_query_id(query_desc->plannedstmt->queryId); + gpdb::pfree(es.str->data); + } + gpdb::mem_ctx_switch_to(oldcxt); + } } -void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config) { - if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { - auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = trim_str_shrink_utf8( - query_desc->sourceText, strlen(query_desc->sourceText), - config.max_text_size()); - char *norm_query = gpdb::gen_normquery(query_desc->sourceText); - if (norm_query) { - *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), config.max_text_size()); - gpdb::pfree(norm_query); - } - } +void +set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) +{ + if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) + { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = trim_str_shrink_utf8( + query_desc->sourceText, strlen(query_desc->sourceText), + config.max_text_size()); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); + if (norm_query) + { + *qi->mutable_template_query_text() = trim_str_shrink_utf8( + norm_query, strlen(norm_query), config.max_text_size()); + gpdb::pfree(norm_query); + } + } } -void clear_big_fields(gpsc::SetQueryReq *req) { - if (Gp_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->clear_plan_text(); - qi->clear_template_plan_text(); - qi->clear_query_text(); - qi->clear_template_query_text(); - qi->clear_analyze_text(); - } +void +clear_big_fields(gpsc::SetQueryReq *req) +{ + if (Gp_role == GP_ROLE_DISPATCH) + { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_template_plan_text(); + qi->clear_query_text(); + qi->clear_template_query_text(); + qi->clear_analyze_text(); + } } -void set_query_info(gpsc::SetQueryReq *req) { - if (Gp_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->set_username(get_user_name()); - if (IsTransactionState()) - qi->set_databasename(get_db_name()); - qi->set_rsgname(get_rg_name()); - } +void +set_query_info(gpsc::SetQueryReq *req) +{ + if (Gp_role == GP_ROLE_DISPATCH) + { + auto qi = req->mutable_query_info(); + qi->set_username(get_user_name()); + if (IsTransactionState()) + qi->set_databasename(get_db_name()); + qi->set_rsgname(get_rg_name()); + } } -void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) { - auto aqi = req->mutable_add_info(); - aqi->set_nested_level(nesting_level); +void +set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) +{ + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); } -void set_qi_slice_id(gpsc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - aqi->set_slice_id(currentSliceId); +void +set_qi_slice_id(gpsc::SetQueryReq *req) +{ + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, - const Config &config) { - auto aqi = req->mutable_add_info(); - *aqi->mutable_error_message() = - trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); +void +set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, + const Config &config) +{ + auto aqi = req->mutable_add_info(); + *aqi->mutable_error_message() = + trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } -void set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, - QueryDesc *query_desc, int nested_calls, - double nested_time) { - auto instrument = query_desc->planstate->instrument; - if (instrument) { - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time( - INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); - } - if (query_desc->estate && query_desc->estate->motionlayer_context) { - MotionLayerState *mlstate = - (MotionLayerState *)query_desc->estate->motionlayer_context; - metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); - metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); - metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); - metrics->mutable_received()->set_total_bytes( - mlstate->stat_total_bytes_recvd); - metrics->mutable_received()->set_tuple_bytes( - mlstate->stat_tuple_bytes_recvd); - metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); - } - metrics->set_inherited_calls(nested_calls); - metrics->set_inherited_time(nested_time); +void +set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, + QueryDesc *query_desc, int nested_calls, + double nested_time) +{ + auto instrument = query_desc->planstate->instrument; + if (instrument) + { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } + if (query_desc->estate && query_desc->estate->motionlayer_context) + { + MotionLayerState *mlstate = + (MotionLayerState *) query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes( + mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes( + mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks( + mlstate->stat_total_chunks_recvd); + } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); } -void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time) { - if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, - nested_calls, nested_time); - } - fill_self_stats(metrics->mutable_systemstat()); - metrics->mutable_systemstat()->set_runningtimeseconds( - time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); - metrics->mutable_spill()->set_filecount( - WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); - metrics->mutable_spill()->set_totalbytes( - WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); +void +set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) +{ + if (query_desc->planstate && query_desc->planstate->instrument) + { + set_metric_instrumentation(metrics->mutable_instrumentation(), + query_desc, nested_calls, nested_time); + } + fill_self_stats(metrics->mutable_systemstat()); + metrics->mutable_systemstat()->set_runningtimeseconds( + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } -#define UPDATE_IC_STATS(proto_name, stat_name) \ - metrics->mutable_interconnect()->set_##proto_name( \ - ic_statistics->stat_name - \ - metrics->mutable_interconnect()->proto_name()); \ - Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ - metrics->mutable_interconnect()->proto_name() <= \ - ic_statistics->stat_name) +#define UPDATE_IC_STATS(proto_name, stat_name) \ + metrics->mutable_interconnect()->set_##proto_name( \ + ic_statistics->stat_name - \ + metrics->mutable_interconnect()->proto_name()); \ + Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ + metrics->mutable_interconnect()->proto_name() <= \ + ic_statistics->stat_name) -void set_ic_stats(gpsc::MetricInstrumentation *metrics, - const ICStatistics *ic_statistics) { +void +set_ic_stats(gpsc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics) +{ #ifdef IC_TEARDOWN_HOOK - UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); - UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); - UPDATE_IC_STATS(total_capacity, totalCapacity); - UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); - UPDATE_IC_STATS(total_buffers, totalBuffers); - UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); - UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); - UPDATE_IC_STATS(retransmits, retransmits); - UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); - UPDATE_IC_STATS(mismatch_num, mismatchNum); - UPDATE_IC_STATS(crc_errors, crcErrors); - UPDATE_IC_STATS(snd_pkt_num, sndPktNum); - UPDATE_IC_STATS(recv_pkt_num, recvPktNum); - UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); - UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); - UPDATE_IC_STATS(recv_ack_num, recvAckNum); - UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); + UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); + UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); + UPDATE_IC_STATS(total_capacity, totalCapacity); + UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); + UPDATE_IC_STATS(total_buffers, totalBuffers); + UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); + UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); + UPDATE_IC_STATS(retransmits, retransmits); + UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); + UPDATE_IC_STATS(mismatch_num, mismatchNum); + UPDATE_IC_STATS(crc_errors, crcErrors); + UPDATE_IC_STATS(snd_pkt_num, sndPktNum); + UPDATE_IC_STATS(recv_pkt_num, recvPktNum); + UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); + UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); + UPDATE_IC_STATS(recv_ack_num, recvAckNum); + UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); #endif } -gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status) { - gpsc::SetQueryReq req; - req.set_query_status(status); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key()); - set_segment_key(req.mutable_segment_key()); - return req; +gpsc::SetQueryReq +create_query_req(gpsc::QueryStatus status) +{ + gpsc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key()); + set_segment_key(req.mutable_segment_key()); + return req; } -double protots_to_double(const google::protobuf::Timestamp &ts) { - return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +double +protots_to_double(const google::protobuf::Timestamp &ts) +{ + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, - const Config &config) { - // Make sure it is a valid txn and it is not an utility - // statement for ExplainPrintPlan() later. - if (!IsTransactionState() || !query_desc->plannedstmt) { - return; - } - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_analyze_state( - query_desc, query_desc->instrument_options && config.enable_analyze()); - gpdb::mem_ctx_switch_to(oldcxt); - if (es.str) { - // Remove last line break. - if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { - es.str->data[--es.str->len] = '\0'; - } - auto trimmed_analyze = - trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); - req->mutable_query_info()->set_analyze_text(trimmed_analyze); - gpdb::pfree(es.str->data); - } +void +set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, + const Config &config) +{ + // Make sure it is a valid txn and it is not an utility + // statement for ExplainPrintPlan() later. + if (!IsTransactionState() || !query_desc->plannedstmt) + { + return; + } + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state( + query_desc, query_desc->instrument_options && config.enable_analyze()); + gpdb::mem_ctx_switch_to(oldcxt); + if (es.str) + { + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') + { + es.str->data[--es.str->len] = '\0'; + } + auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, + config.max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + gpdb::pfree(es.str->data); + } } diff --git a/gpcontrib/gp_stats_collector/src/ProtoUtils.h b/gpcontrib/gp_stats_collector/src/ProtoUtils.h index 5ddcd42d308..6b38097fbcc 100644 --- a/gpcontrib/gp_stats_collector/src/ProtoUtils.h +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef PROTOUTILS_H +#define PROTOUTILS_H #include "protos/gpsc_set_service.pb.h" @@ -35,20 +36,22 @@ class Config; google::protobuf::Timestamp current_ts(); void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config); + const Config &config); void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config); + const Config &config); void clear_big_fields(gpsc::SetQueryReq *req); void set_query_info(gpsc::SetQueryReq *req); void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(gpsc::SetQueryReq *req); void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, - const Config &config); + const Config &config); void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time); + int nested_calls, double nested_time); void set_ic_stats(gpsc::MetricInstrumentation *metrics, - const ICStatistics *ic_statistics); + const ICStatistics *ic_statistics); gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *message, - const Config &config); + const Config &config); + +#endif /* PROTOUTILS_H */ diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp index 9a01d4033d0..16344366456 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -28,103 +28,119 @@ #include "UDSConnector.h" #include "Config.h" #include "GpscStat.h" -#include "memory/gpdbwrappers.h" #include "log/LogOps.h" +#include "memory/gpdbwrappers.h" +#include #include -#include +#include #include -#include #include -#include -#include +#include #include +#include extern "C" { #include "postgres.h" } static void inline log_tracing_failure(const gpsc::SetQueryReq &req, - const std::string &event) { - ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str()))); + const std::string &event) +{ + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str()))); } -bool UDSConnector::report_query(const gpsc::SetQueryReq &req, - const std::string &event, - const Config &config) { - sockaddr_un address{}; - address.sun_family = AF_UNIX; - const auto &uds_path = config.uds_path(); +bool +UDSConnector::report_query(const gpsc::SetQueryReq &req, + const std::string &event, const Config &config) +{ + sockaddr_un address{}; + address.sun_family = AF_UNIX; + const auto &uds_path = config.uds_path(); - if (uds_path.size() >= sizeof(address.sun_path)) { - ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); - GpscStat::report_error(); - return false; - } - strcpy(address.sun_path, uds_path.c_str()); + if (uds_path.size() >= sizeof(address.sun_path)) + { + ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); + GpscStat::report_error(); + return false; + } + strcpy(address.sun_path, uds_path.c_str()); - const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd == -1) { - log_tracing_failure(req, event); - GpscStat::report_error(); - return false; - } + const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) + { + log_tracing_failure(req, event); + GpscStat::report_error(); + return false; + } - // Close socket automatically on error path. - struct SockGuard { - int fd; - ~SockGuard() { close(fd); } - } sock_guard{sockfd}; + // Close socket automatically on error path. + struct SockGuard + { + int fd; + ~SockGuard() + { + close(fd); + } + } sock_guard{sockfd}; - if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) { - // That's a very important error that should never happen, so make it - // visible to an end-user and admins. - ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %m"))); - GpscStat::report_error(); - return false; - } + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) + { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %m"))); + GpscStat::report_error(); + return false; + } - if (connect(sockfd, reinterpret_cast(&address), - sizeof(address)) == -1) { - log_tracing_failure(req, event); - GpscStat::report_bad_connection(); - return false; - } + if (connect(sockfd, reinterpret_cast(&address), + sizeof(address)) == -1) + { + log_tracing_failure(req, event); + GpscStat::report_bad_connection(); + return false; + } - const auto data_size = req.ByteSizeLong(); - const auto total_size = data_size + sizeof(uint32_t); - auto *buf = static_cast(gpdb::palloc(total_size)); - // Free buf automatically on error path. - struct BufGuard { - void *p; - ~BufGuard() { gpdb::pfree(p); } - } buf_guard{buf}; + const auto data_size = req.ByteSizeLong(); + const auto total_size = data_size + sizeof(uint32_t); + auto *buf = static_cast(gpdb::palloc(total_size)); + // Free buf automatically on error path. + struct BufGuard + { + void *p; + ~BufGuard() + { + gpdb::pfree(p); + } + } buf_guard{buf}; - *reinterpret_cast(buf) = data_size; - req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + *reinterpret_cast(buf) = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); - int64_t sent = 0, sent_total = 0; - do { - sent = - send(sockfd, buf + sent_total, total_size - sent_total, MSG_DONTWAIT); - if (sent > 0) - sent_total += sent; - } while (sent > 0 && size_t(sent_total) != total_size && - // the line below is a small throttling hack: - // if a message does not fit a single packet, we take a nap - // before sending the next one. - // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + int64_t sent = 0, sent_total = 0; + do + { + sent = send(sockfd, buf + sent_total, total_size - sent_total, + MSG_DONTWAIT); + if (sent > 0) + sent_total += sent; + } while (sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); - if (sent < 0) { - log_tracing_failure(req, event); - GpscStat::report_bad_send(total_size); - return false; - } + if (sent < 0) + { + log_tracing_failure(req, event); + GpscStat::report_bad_send(total_size); + return false; + } - GpscStat::report_send(total_size); - return true; + GpscStat::report_send(total_size); + return true; } diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.h b/gpcontrib/gp_stats_collector/src/UDSConnector.h index a91d22f9df1..ac56dd54f44 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.h +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.h @@ -25,14 +25,18 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef UDSCONNECTOR_H +#define UDSCONNECTOR_H #include "protos/gpsc_set_service.pb.h" class Config; -class UDSConnector { +class UDSConnector +{ public: - bool static report_query(const gpsc::SetQueryReq &req, - const std::string &event, const Config &config); + bool static report_query(const gpsc::SetQueryReq &req, + const std::string &event, const Config &config); }; + +#endif /* UDSCONNECTOR_H */ diff --git a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c index d930f72246d..d295e37b396 100644 --- a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -45,106 +45,131 @@ PG_FUNCTION_INFO_V1(gpsc_test_uds_start_server); PG_FUNCTION_INFO_V1(gpsc_test_uds_receive); PG_FUNCTION_INFO_V1(gpsc_test_uds_stop_server); -void _PG_init(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) - hooks_init(); +void +_PG_init(void) +{ + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + hooks_init(); } -void _PG_fini(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) - hooks_deinit(); +void +_PG_fini(void) +{ + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + hooks_deinit(); } -Datum gpsc_stat_messages_reset(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_stat_messages_reset(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - gpsc_functions_reset(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + gpsc_functions_reset(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_stat_messages(PG_FUNCTION_ARGS) { - return gpsc_functions_get(fcinfo); +Datum +gpsc_stat_messages(PG_FUNCTION_ARGS) +{ + return gpsc_functions_get(fcinfo); } -Datum gpsc_init_log(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_init_log(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - init_log(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + init_log(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_truncate_log(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_truncate_log(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - truncate_log(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + truncate_log(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_start_server(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); - test_uds_start_server(path); - pfree(path); - } - - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); +Datum +gpsc_test_uds_start_server(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + test_uds_start_server(path); + pfree(path); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_receive(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - int64 *result; +Datum +gpsc_test_uds_receive(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + int64 *result; - if (SRF_IS_FIRSTCALL()) { - MemoryContext oldcontext; + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcontext; - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - result = (int64 *)palloc(sizeof(int64)); - funcctx->user_fctx = result; - funcctx->max_calls = 1; - MemoryContextSwitchTo(oldcontext); + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + result = (int64 *) palloc(sizeof(int64)); + funcctx->user_fctx = result; + funcctx->max_calls = 1; + MemoryContextSwitchTo(oldcontext); - int timeout_ms = PG_GETARG_INT32(0); - *result = test_uds_receive(timeout_ms); - } + int timeout_ms = PG_GETARG_INT32(0); + *result = test_uds_receive(timeout_ms); + } - funcctx = SRF_PERCALL_SETUP(); + funcctx = SRF_PERCALL_SETUP(); - if (funcctx->call_cntr < funcctx->max_calls) { - result = (int64 *)funcctx->user_fctx; - SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); - } + if (funcctx->call_cntr < funcctx->max_calls) + { + result = (int64 *) funcctx->user_fctx; + SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); + } - SRF_RETURN_DONE(funcctx); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - test_uds_stop_server(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + test_uds_stop_server(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index 0a40b4cb359..3f19d4d9930 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -28,28 +28,28 @@ #define typeid __typeid extern "C" { #include "postgres.h" -#include "funcapi.h" -#include "executor/executor.h" -#include "executor/execUtils.h" -#include "utils/elog.h" -#include "utils/builtins.h" -#include "utils/metrics_utils.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" +#include "executor/execUtils.h" +#include "executor/executor.h" +#include "funcapi.h" +#include "stat_statements_parser/pg_stat_statements_parser.h" #include "tcop/utility.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" +#include +#include #include #include #include -#include -#include } #undef typeid #include "Config.h" -#include "GpscStat.h" #include "EventSender.h" +#include "GpscStat.h" #include "hook_wrappers.h" #include "memory/gpdbwrappers.h" @@ -60,7 +60,7 @@ static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; #ifdef ANALYZE_STATS_COLLECT_HOOK static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = - nullptr; + nullptr; #endif #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; @@ -68,24 +68,23 @@ static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - uint64 count, bool execute_once); +static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, + ScanDirection direction, uint64 count, + bool execute_once); static void gpsc_ExecutorFinish_hook(QueryDesc *query_desc); static void gpsc_ExecutorEnd_hook(QueryDesc *query_desc); static void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg); #ifdef IC_TEARDOWN_HOOK static void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, - bool hasErrors); + bool hasErrors); #endif #ifdef ANALYZE_STATS_COLLECT_HOOK static void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, - bool readOnlyTree, - ProcessUtilityContext context, - ParamListInfo params, - QueryEnvironment *queryEnv, - DestReceiver *dest, QueryCompletion *qc); +static void gpsc_process_utility_hook( + PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, + ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); #define TEST_MAX_CONNECTIONS 4 #define TEST_RCV_BUF_SIZE 8192 @@ -96,319 +95,379 @@ static char *test_sock_path = NULL; static EventSender *sender = nullptr; -static inline EventSender *get_sender() { - if (!sender) { - sender = new EventSender(); - } - return sender; +static inline EventSender * +get_sender() +{ + if (!sender) + { + sender = new EventSender(); + } + return sender; } template -R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { - try { - return (obj->*func)(args...); - } catch (const std::exception &e) { - ereport(FATAL, (errmsg("Unexpected exception in gpsc %s", e.what()))); - } +R +cpp_call(T *obj, R (T::*func)(Args...), Args... args) +{ + try + { + return (obj->*func)(args...); + } + catch (const std::exception &e) + { + ereport(ERROR, (errmsg("Unexpected exception in gpsc %s", e.what()))); + } } -void hooks_init() { - Config::init_gucs(); - GpscStat::init(); - previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = gpsc_ExecutorStart_hook; - previous_ExecutorRun_hook = ExecutorRun_hook; - ExecutorRun_hook = gpsc_ExecutorRun_hook; - previous_ExecutorFinish_hook = ExecutorFinish_hook; - ExecutorFinish_hook = gpsc_ExecutorFinish_hook; - previous_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = gpsc_ExecutorEnd_hook; - previous_query_info_collect_hook = query_info_collect_hook; - query_info_collect_hook = gpsc_query_info_collect_hook; +void +hooks_init() +{ + Config::init_gucs(); + GpscStat::init(); + previous_ExecutorStart_hook = ExecutorStart_hook; + ExecutorStart_hook = gpsc_ExecutorStart_hook; + previous_ExecutorRun_hook = ExecutorRun_hook; + ExecutorRun_hook = gpsc_ExecutorRun_hook; + previous_ExecutorFinish_hook = ExecutorFinish_hook; + ExecutorFinish_hook = gpsc_ExecutorFinish_hook; + previous_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = gpsc_ExecutorEnd_hook; + previous_query_info_collect_hook = query_info_collect_hook; + query_info_collect_hook = gpsc_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK - previous_ic_teardown_hook = ic_teardown_hook; - ic_teardown_hook = gpsc_ic_teardown_hook; + previous_ic_teardown_hook = ic_teardown_hook; + ic_teardown_hook = gpsc_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK - previous_analyze_stats_collect_hook = analyze_stats_collect_hook; - analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; + previous_analyze_stats_collect_hook = analyze_stats_collect_hook; + analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; #endif - stat_statements_parser_init(); - previous_ProcessUtility_hook = ProcessUtility_hook; - ProcessUtility_hook = gpsc_process_utility_hook; + stat_statements_parser_init(); + previous_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = gpsc_process_utility_hook; } -void hooks_deinit() { - ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorEnd_hook = previous_ExecutorEnd_hook; - ExecutorRun_hook = previous_ExecutorRun_hook; - ExecutorFinish_hook = previous_ExecutorFinish_hook; - query_info_collect_hook = previous_query_info_collect_hook; +void +hooks_deinit() +{ + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; + ExecutorRun_hook = previous_ExecutorRun_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; + query_info_collect_hook = previous_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK - ic_teardown_hook = previous_ic_teardown_hook; + ic_teardown_hook = previous_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK - analyze_stats_collect_hook = previous_analyze_stats_collect_hook; + analyze_stats_collect_hook = previous_analyze_stats_collect_hook; #endif - stat_statements_parser_deinit(); - if (sender) { - delete sender; - } - GpscStat::deinit(); - ProcessUtility_hook = previous_ProcessUtility_hook; + stat_statements_parser_deinit(); + if (sender) + { + delete sender; + } + GpscStat::deinit(); + ProcessUtility_hook = previous_ProcessUtility_hook; } -void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, - eflags); - if (previous_ExecutorStart_hook) { - (*previous_ExecutorStart_hook)(query_desc, eflags); - } else { - standard_ExecutorStart(query_desc, eflags); - } - cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, - eflags); +void +gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) +{ + cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, + eflags); + if (previous_ExecutorStart_hook) + { + (*previous_ExecutorStart_hook)(query_desc, eflags); + } + else + { + standard_ExecutorStart(query_desc, eflags); + } + cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, + eflags); } -void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - uint64 count, bool execute_once) { - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ExecutorRun_hook) - previous_ExecutorRun_hook(query_desc, direction, count, execute_once); - else - standard_ExecutorRun(query_desc, direction, count, execute_once); - get_sender()->decr_depth(); - } - PG_CATCH(); - { - get_sender()->decr_depth(); - PG_RE_THROW(); - } - PG_END_TRY(); +void +gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + uint64 count, bool execute_once) +{ + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorRun_hook) + previous_ExecutorRun_hook(query_desc, direction, count, + execute_once); + else + standard_ExecutorRun(query_desc, direction, count, execute_once); + get_sender()->decr_depth(); + } + PG_CATCH(); + { + get_sender()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); } -void gpsc_ExecutorFinish_hook(QueryDesc *query_desc) { - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ExecutorFinish_hook) - previous_ExecutorFinish_hook(query_desc); - else - standard_ExecutorFinish(query_desc); - get_sender()->decr_depth(); - } - PG_CATCH(); - { - get_sender()->decr_depth(); - PG_RE_THROW(); - } - PG_END_TRY(); +void +gpsc_ExecutorFinish_hook(QueryDesc *query_desc) +{ + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorFinish_hook) + previous_ExecutorFinish_hook(query_desc); + else + standard_ExecutorFinish(query_desc); + get_sender()->decr_depth(); + } + PG_CATCH(); + { + get_sender()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); } -void gpsc_ExecutorEnd_hook(QueryDesc *query_desc) { - cpp_call(get_sender(), &EventSender::executor_end, query_desc); - if (previous_ExecutorEnd_hook) { - (*previous_ExecutorEnd_hook)(query_desc); - } else { - standard_ExecutorEnd(query_desc); - } +void +gpsc_ExecutorEnd_hook(QueryDesc *query_desc) +{ + cpp_call(get_sender(), &EventSender::executor_end, query_desc); + if (previous_ExecutorEnd_hook) + { + (*previous_ExecutorEnd_hook)(query_desc); + } + else + { + standard_ExecutorEnd(query_desc); + } } -void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - cpp_call(get_sender(), &EventSender::query_metrics_collect, status, - arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); - if (previous_query_info_collect_hook) { - (*previous_query_info_collect_hook)(status, arg); - } +void +gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) +{ + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, + arg /* queryDesc */, false /* utility */, (ErrorData *) NULL); + if (previous_query_info_collect_hook) + { + (*previous_query_info_collect_hook)(status, arg); + } } #ifdef IC_TEARDOWN_HOOK -void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { - cpp_call(get_sender(), &EventSender::ic_metrics_collect); - if (previous_ic_teardown_hook) { - (*previous_ic_teardown_hook)(transportStates, hasErrors); - } +void +gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) +{ + cpp_call(get_sender(), &EventSender::ic_metrics_collect); + if (previous_ic_teardown_hook) + { + (*previous_ic_teardown_hook)(transportStates, hasErrors); + } } #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) { - cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); - if (previous_analyze_stats_collect_hook) { - (*previous_analyze_stats_collect_hook)(query_desc); - } +void +gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) +{ + cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); + if (previous_analyze_stats_collect_hook) + { + (*previous_analyze_stats_collect_hook)(query_desc); + } } #endif -static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, - bool readOnlyTree, - ProcessUtilityContext context, - ParamListInfo params, - QueryEnvironment *queryEnv, - DestReceiver *dest, QueryCompletion *qc) { - /* Project utility data on QueryDesc to use existing logic */ - QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); - query_desc->sourceText = queryString; - - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_SUBMIT, (void *)query_desc, true /* utility */, - (ErrorData *)NULL); - - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ProcessUtility_hook) { - (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, - params, queryEnv, dest, qc); - } else { - standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, - queryEnv, dest, qc); - } - - get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_DONE, (void *)query_desc, true /* utility */, - (ErrorData *)NULL); - - pfree(query_desc); - } - PG_CATCH(); - { - ErrorData *edata; - MemoryContext oldctx; - - oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - FlushErrorState(); - MemoryContextSwitchTo(oldctx); - - get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_ERROR, (void *)query_desc, true /* utility */, - edata); - - pfree(query_desc); - ReThrowError(edata); - } - PG_END_TRY(); +static void +gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, + ParamListInfo params, QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) +{ + /* Project utility data on QueryDesc to use existing logic */ + QueryDesc *query_desc = (QueryDesc *) palloc0(sizeof(QueryDesc)); + query_desc->sourceText = queryString; + + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_SUBMIT, (void *) query_desc, true /* utility */, + (ErrorData *) NULL); + + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ProcessUtility_hook) + { + (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, + qc); + } + else + { + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); + } + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_DONE, (void *) query_desc, true /* utility */, + (ErrorData *) NULL); + + pfree(query_desc); + } + PG_CATCH(); + { + ErrorData *edata; + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(oldctx); + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_ERROR, (void *) query_desc, true /* utility */, + edata); + + pfree(query_desc); + ReThrowError(edata); + } + PG_END_TRY(); } -static void check_stats_loaded() { - if (!GpscStat::loaded()) { - ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("gp_stats_collector must be loaded via " - "shared_preload_libraries"))); - } +static void +check_stats_loaded() +{ + if (!GpscStat::loaded()) + { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("gp_stats_collector must be loaded via " + "shared_preload_libraries"))); + } } -void gpsc_functions_reset() { - check_stats_loaded(); - GpscStat::reset(); +void +gpsc_functions_reset() +{ + check_stats_loaded(); + GpscStat::reset(); } -Datum gpsc_functions_get(FunctionCallInfo fcinfo) { - const int ATTNUM = 6; - check_stats_loaded(); - auto stats = GpscStat::get_stats(); - TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); - TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, - 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)3, "send_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)4, "connection_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)5, "other_errors", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)6, "max_message_size", INT4OID, - -1 /* typmod */, 0 /* attdim */); - tupdesc = BlessTupleDesc(tupdesc); - Datum values[ATTNUM]; - bool nulls[ATTNUM]; - MemSet(nulls, 0, sizeof(nulls)); - values[0] = Int32GetDatum(GpIdentity.segindex); - values[1] = Int64GetDatum(stats.total); - values[2] = Int64GetDatum(stats.failed_sends); - values[3] = Int64GetDatum(stats.failed_connects); - values[4] = Int64GetDatum(stats.failed_other); - values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); - Datum result = HeapTupleGetDatum(tuple); - PG_RETURN_DATUM(result); +Datum +gpsc_functions_get(FunctionCallInfo fcinfo) +{ + const int ATTNUM = 6; + check_stats_loaded(); + auto stats = GpscStat::get_stats(); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segid", INT4OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "total_messages", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "send_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "connection_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "other_errors", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "max_message_size", INT4OID, + -1 /* typmod */, 0 /* attdim */); + tupdesc = BlessTupleDesc(tupdesc); + Datum values[ATTNUM]; + bool nulls[ATTNUM]; + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(GpIdentity.segindex); + values[1] = Int64GetDatum(stats.total); + values[2] = Int64GetDatum(stats.failed_sends); + values[3] = Int64GetDatum(stats.failed_connects); + values[4] = Int64GetDatum(stats.failed_other); + values[5] = Int32GetDatum(stats.max_message_size); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + PG_RETURN_DATUM(result); } -void test_uds_stop_server() { - if (test_server_fd >= 0) { - close(test_server_fd); - test_server_fd = -1; - } - if (test_sock_path) { - unlink(test_sock_path); - pfree(test_sock_path); - test_sock_path = NULL; - } +void +test_uds_stop_server() +{ + if (test_server_fd >= 0) + { + close(test_server_fd); + test_server_fd = -1; + } + if (test_sock_path) + { + unlink(test_sock_path); + pfree(test_sock_path); + test_sock_path = NULL; + } } -void test_uds_start_server(const char *path) { - struct sockaddr_un addr = {.sun_family = AF_UNIX}; +void +test_uds_start_server(const char *path) +{ + struct sockaddr_un addr = {.sun_family = AF_UNIX}; - if (strlen(path) >= sizeof(addr.sun_path)) - ereport(ERROR, (errmsg("path too long"))); + if (strlen(path) >= sizeof(addr.sun_path)) + ereport(ERROR, (errmsg("path too long"))); - test_uds_stop_server(); + test_uds_stop_server(); - strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); - test_sock_path = MemoryContextStrdup(TopMemoryContext, path); - unlink(path); + strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); + test_sock_path = MemoryContextStrdup(TopMemoryContext, path); + unlink(path); - if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || - bind(test_server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0 || - listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) { - test_uds_stop_server(); - ereport(ERROR, (errmsg("socket setup failed: %m"))); - } + if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || + bind(test_server_fd, (struct sockaddr *) &addr, sizeof(addr)) < 0 || + listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) + { + test_uds_stop_server(); + ereport(ERROR, (errmsg("socket setup failed: %m"))); + } } -int64 test_uds_receive(int timeout_ms) { - char buf[TEST_RCV_BUF_SIZE]; - int rc; - struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; - int64 total = 0; - - if (test_server_fd < 0) - ereport(ERROR, (errmsg("server not started"))); - - for (;;) { - CHECK_FOR_INTERRUPTS(); - rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); - if (rc > 0) - break; - if (rc < 0 && errno != EINTR) - ereport(ERROR, (errmsg("poll: %m"))); - timeout_ms -= TEST_POLL_TIMEOUT_MS; - if (timeout_ms <= 0) - return total; - } - - if (pfd.revents & POLLIN) { - int client = accept(test_server_fd, NULL, NULL); - ssize_t n; - - if (client < 0) - ereport(ERROR, (errmsg("accept: %m"))); - - while ((n = recv(client, buf, sizeof(buf), 0)) != 0) { - if (n > 0) - total += n; - else if (errno != EINTR) - break; - } - - close(client); - } - - return total; +int64 +test_uds_receive(int timeout_ms) +{ + char buf[TEST_RCV_BUF_SIZE]; + int rc; + struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; + int64 total = 0; + + if (test_server_fd < 0) + ereport(ERROR, (errmsg("server not started"))); + + for (;;) + { + CHECK_FOR_INTERRUPTS(); + rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); + if (rc > 0) + break; + if (rc < 0 && errno != EINTR) + ereport(ERROR, (errmsg("poll: %m"))); + timeout_ms -= TEST_POLL_TIMEOUT_MS; + if (timeout_ms <= 0) + return total; + } + + if (pfd.revents & POLLIN) + { + int client = accept(test_server_fd, NULL, NULL); + ssize_t n; + + if (client < 0) + ereport(ERROR, (errmsg("accept: %m"))); + + while ((n = recv(client, buf, sizeof(buf), 0)) != 0) + { + if (n > 0) + total += n; + else if (errno != EINTR) + break; + } + + close(client); + } + + return total; } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.h b/gpcontrib/gp_stats_collector/src/hook_wrappers.h index 06c8d064404..a04f5a95144 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.h +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef HOOK_WRAPPERS_H +#define HOOK_WRAPPERS_H #ifdef __cplusplus extern "C" { @@ -45,4 +46,5 @@ extern void test_uds_stop_server(); #ifdef __cplusplus } -#endif \ No newline at end of file +#endif +#endif /* HOOK_WRAPPERS_H */ diff --git a/gpcontrib/gp_stats_collector/src/log/LogOps.cpp b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp index ef4f39c0749..865e0f6ce3f 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogOps.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp @@ -43,8 +43,8 @@ extern "C" { #include "catalog/pg_type.h" #include "cdb/cdbvars.h" #include "commands/tablecmds.h" -#include "funcapi.h" #include "fmgr.h" +#include "funcapi.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/rel.h" @@ -52,107 +52,122 @@ extern "C" { #include "utils/timestamp.h" } -void init_log() { - Oid namespaceId; - Oid relationId; - ObjectAddress tableAddr; - ObjectAddress schemaAddr; - - namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); - - /* Create table */ - relationId = heap_create_with_catalog( - log_relname.data() /* relname */, namespaceId /* namespace */, - 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, - InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, - DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, RELKIND_RELATION, - RELPERSISTENCE_PERMANENT, false /* shared_relation */, false /* mapped_relation */, ONCOMMIT_NOOP, - NULL /* GP Policy */, (Datum)0 /* reloptions */, false /* use_user_acl */, true /* allow_system_table_mods */, true /* is_internal */, - InvalidOid /* relrewrite */, NULL /* typaddress */, - false /* valid_opts */); - - /* Make the table visible */ - CommandCounterIncrement(); - - /* Record dependency of the table on the schema */ - if (OidIsValid(relationId) && OidIsValid(namespaceId)) { - ObjectAddressSet(tableAddr, RelationRelationId, relationId); - ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); - - /* Table can be dropped only via DROP EXTENSION */ - recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); - } else { - ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); - } - - /* Make changes visible */ - CommandCounterIncrement(); +void +init_log() +{ + Oid namespaceId; + Oid relationId; + ObjectAddress tableAddr; + ObjectAddress schemaAddr; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + + /* Create table */ + relationId = heap_create_with_catalog( + log_relname.data() /* relname */, namespaceId /* namespace */, + 0 /* tablespace */, InvalidOid /* relid */, + InvalidOid /* reltype oid */, InvalidOid /* reloftypeid */, + GetUserId() /* owner */, HEAP_TABLE_AM_OID, + DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, + RELKIND_RELATION, RELPERSISTENCE_PERMANENT, false /* shared_relation */, + false /* mapped_relation */, ONCOMMIT_NOOP, NULL /* GP Policy */, + (Datum) 0 /* reloptions */, false /* use_user_acl */, + true /* allow_system_table_mods */, true /* is_internal */, + InvalidOid /* relrewrite */, NULL /* typaddress */, + false /* valid_opts */); + + /* Make the table visible */ + CommandCounterIncrement(); + + /* Record dependency of the table on the schema */ + if (OidIsValid(relationId) && OidIsValid(namespaceId)) + { + ObjectAddressSet(tableAddr, RelationRelationId, relationId); + ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); + + /* Table can be dropped only via DROP EXTENSION */ + recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); + } + else + { + ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); + } + + /* Make changes visible */ + CommandCounterIncrement(); } -void insert_log(const gpsc::SetQueryReq &req, bool utility) { - Oid namespaceId; - Oid relationId; - Relation rel; - HeapTuple tuple; - - /* Return if xact is not valid (needed for catalog lookups). */ - if (!IsTransactionState()) { - return; - } - - /* Return if extension was not loaded */ - namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); - if (!OidIsValid(namespaceId)) { - return; - } - - /* Return if the table was not created yet */ - relationId = get_relname_relid(log_relname.data(), namespaceId); - if (!OidIsValid(relationId)) { - return; - } - - bool nulls[natts_gpsc_log]; - Datum values[natts_gpsc_log]; - - memset(nulls, true, sizeof(nulls)); - memset(values, 0, sizeof(values)); - - extract_query_req(req, "", values, nulls); - nulls[attnum_gpsc_log_utility] = false; - values[attnum_gpsc_log_utility] = BoolGetDatum(utility); - - rel = heap_open(relationId, RowExclusiveLock); - - /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls +void +insert_log(const gpsc::SetQueryReq &req, bool utility) +{ + Oid namespaceId; + Oid relationId; + Relation rel; + HeapTuple tuple; + + /* Return if xact is not valid (needed for catalog lookups). */ + if (!IsTransactionState()) + { + return; + } + + /* Return if extension was not loaded */ + namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); + if (!OidIsValid(namespaceId)) + { + return; + } + + /* Return if the table was not created yet */ + relationId = get_relname_relid(log_relname.data(), namespaceId); + if (!OidIsValid(relationId)) + { + return; + } + + bool nulls[natts_gpsc_log]; + Datum values[natts_gpsc_log]; + + memset(nulls, true, sizeof(nulls)); + memset(values, 0, sizeof(values)); + + extract_query_req(req, "", values, nulls); + nulls[attnum_gpsc_log_utility] = false; + values[attnum_gpsc_log_utility] = BoolGetDatum(utility); + + rel = heap_open(relationId, RowExclusiveLock); + + /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls * back or aborts */ - tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); - frozen_heap_insert(rel, tuple); + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + frozen_heap_insert(rel, tuple); - heap_freetuple(tuple); - /* Keep lock on rel until end of xact */ - heap_close(rel, NoLock); + heap_freetuple(tuple); + /* Keep lock on rel until end of xact */ + heap_close(rel, NoLock); - /* Make changes visible */ - CommandCounterIncrement(); + /* Make changes visible */ + CommandCounterIncrement(); } -void truncate_log() { - Oid namespaceId; - Oid relationId; - Relation relation; +void +truncate_log() +{ + Oid namespaceId; + Oid relationId; + Relation relation; - namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); - relationId = get_relname_relid(log_relname.data(), namespaceId); + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + relationId = get_relname_relid(log_relname.data(), namespaceId); - relation = heap_open(relationId, AccessExclusiveLock); + relation = heap_open(relationId, AccessExclusiveLock); - /* Truncate the main table */ - heap_truncate_one_rel(relation); + /* Truncate the main table */ + heap_truncate_one_rel(relation); - /* Keep lock on rel until end of xact */ - heap_close(relation, NoLock); + /* Keep lock on rel until end of xact */ + heap_close(relation, NoLock); - /* Make changes visible */ - CommandCounterIncrement(); + /* Make changes visible */ + CommandCounterIncrement(); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/log/LogOps.h b/gpcontrib/gp_stats_collector/src/log/LogOps.h index f784270bb8f..45d79cd4560 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogOps.h +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef LOGOPS_H +#define LOGOPS_H #include @@ -44,3 +45,5 @@ void truncate_log(); /* INSERT INTO gpsc.__log VALUES (...) */ void insert_log(const gpsc::SetQueryReq &req, bool utility); + +#endif /* LOGOPS_H */ diff --git a/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp index f9f43fac2fd..254b1b04af4 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp @@ -25,138 +25,165 @@ *------------------------------------------------------------------------- */ -#include "google/protobuf/reflection.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/reflection.h" #include "google/protobuf/timestamp.pb.h" #include "LogSchema.h" -const std::unordered_map &proto_name_to_col_idx() { - static const auto name_col_idx = [] { - std::unordered_map map; - map.reserve(log_tbl_desc.size()); - - for (size_t idx = 0; idx < natts_gpsc_log; ++idx) { - map.emplace(log_tbl_desc[idx].proto_field_name, idx); - } - - return map; - }(); - return name_col_idx; +const std::unordered_map & +proto_name_to_col_idx() +{ + static const auto name_col_idx = [] { + std::unordered_map map; + map.reserve(log_tbl_desc.size()); + + for (size_t idx = 0; idx < natts_gpsc_log; ++idx) + { + map.emplace(log_tbl_desc[idx].proto_field_name, idx); + } + + return map; + }(); + return name_col_idx; } -TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); +TupleDesc +DescribeTuple() +{ + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); - for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) { - TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), - log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, - 0 /* attdim */); - } + for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) + { + TupleDescInitEntry( + tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), + log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, 0 /* attdim */); + } - return tupdesc; + return tupdesc; } -Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts) { - TimestampTz pgtimestamp = - (TimestampTz)ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); - pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; - return TimestampTzGetDatum(pgtimestamp); +Datum +protots_to_timestamptz(const google::protobuf::Timestamp &ts) +{ + TimestampTz pgtimestamp = + (TimestampTz) ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); + pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; + return TimestampTzGetDatum(pgtimestamp); } -Datum field_to_datum(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg) { - using namespace google::protobuf; - - switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - return Int32GetDatum(reflection->GetInt32(msg, field)); - case FieldDescriptor::CPPTYPE_INT64: - return Int64GetDatum(reflection->GetInt64(msg, field)); - case FieldDescriptor::CPPTYPE_UINT32: - return Int64GetDatum(reflection->GetUInt32(msg, field)); - case FieldDescriptor::CPPTYPE_UINT64: - return Int64GetDatum( - static_cast(reflection->GetUInt64(msg, field))); - case FieldDescriptor::CPPTYPE_DOUBLE: - return Float8GetDatum(reflection->GetDouble(msg, field)); - case FieldDescriptor::CPPTYPE_FLOAT: - return Float4GetDatum(reflection->GetFloat(msg, field)); - case FieldDescriptor::CPPTYPE_BOOL: - return BoolGetDatum(reflection->GetBool(msg, field)); - case FieldDescriptor::CPPTYPE_ENUM: - return CStringGetTextDatum(reflection->GetEnum(msg, field)->name().data()); - case FieldDescriptor::CPPTYPE_STRING: - return CStringGetTextDatum(reflection->GetString(msg, field).c_str()); - default: - return (Datum)0; - } +Datum +field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg) +{ + using namespace google::protobuf; + + switch (field->cpp_type()) + { + case FieldDescriptor::CPPTYPE_INT32: + return Int32GetDatum(reflection->GetInt32(msg, field)); + case FieldDescriptor::CPPTYPE_INT64: + return Int64GetDatum(reflection->GetInt64(msg, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return Int64GetDatum(reflection->GetUInt32(msg, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return Int64GetDatum( + static_cast(reflection->GetUInt64(msg, field))); + case FieldDescriptor::CPPTYPE_DOUBLE: + return Float8GetDatum(reflection->GetDouble(msg, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return Float4GetDatum(reflection->GetFloat(msg, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return BoolGetDatum(reflection->GetBool(msg, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return CStringGetTextDatum( + reflection->GetEnum(msg, field)->name().data()); + case FieldDescriptor::CPPTYPE_STRING: + return CStringGetTextDatum( + reflection->GetString(msg, field).c_str()); + default: + return (Datum) 0; + } } -void process_field(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg, - const std::string &field_name, Datum *values, bool *nulls) { - - auto proto_idx_map = proto_name_to_col_idx(); - auto it = proto_idx_map.find(field_name); - - if (it == proto_idx_map.end()) { - ereport(NOTICE, - (errmsg("GPSC protobuf field %s is not registered in log table", - field_name.c_str()))); - return; - } - - int idx = it->second; - - if (!reflection->HasField(msg, field)) { - nulls[idx] = true; - return; - } - - if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && - field->message_type()->full_name() == "google.protobuf.Timestamp") { - const auto &ts = static_cast( - reflection->GetMessage(msg, field)); - values[idx] = protots_to_timestamptz(ts); - } else { - values[idx] = field_to_datum(field, reflection, msg); - } - nulls[idx] = false; - - return; +void +process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls) +{ + auto proto_idx_map = proto_name_to_col_idx(); + auto it = proto_idx_map.find(field_name); + + if (it == proto_idx_map.end()) + { + ereport(NOTICE, + (errmsg("GPSC protobuf field %s is not registered in log table", + field_name.c_str()))); + return; + } + + int idx = it->second; + + if (!reflection->HasField(msg, field)) + { + nulls[idx] = true; + return; + } + + if (field->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() == "google.protobuf.Timestamp") + { + const auto &ts = static_cast( + reflection->GetMessage(msg, field)); + values[idx] = protots_to_timestamptz(ts); + } + else + { + values[idx] = field_to_datum(field, reflection, msg); + } + nulls[idx] = false; + + return; } -void extract_query_req(const google::protobuf::Message &msg, - const std::string &prefix, Datum *values, bool *nulls) { - using namespace google::protobuf; - - const Descriptor *descriptor = msg.GetDescriptor(); - const Reflection *reflection = msg.GetReflection(); - - for (int i = 0; i < descriptor->field_count(); ++i) { - const FieldDescriptor *field = descriptor->field(i); - - // For now, we do not log any repeated fields plus they need special - // treatment. - if (field->is_repeated()) { - continue; - } - - std::string curr_pref = prefix.empty() ? "" : prefix + "."; - std::string field_name = curr_pref + field->name().data(); - - if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && - field->message_type()->full_name() != "google.protobuf.Timestamp") { - - if (reflection->HasField(msg, field)) { - const Message &nested = reflection->GetMessage(msg, field); - extract_query_req(nested, field_name, values, nulls); - } - } else { - process_field(field, reflection, msg, field_name, values, nulls); - } - } +void +extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls) +{ + using namespace google::protobuf; + + const Descriptor *descriptor = msg.GetDescriptor(); + const Reflection *reflection = msg.GetReflection(); + + for (int i = 0; i < descriptor->field_count(); ++i) + { + const FieldDescriptor *field = descriptor->field(i); + + // For now, we do not log any repeated fields plus they need special + // treatment. + if (field->is_repeated()) + { + continue; + } + + std::string curr_pref = prefix.empty() ? "" : prefix + "."; + std::string field_name = curr_pref + field->name().data(); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() != "google.protobuf.Timestamp") + { + if (reflection->HasField(msg, field)) + { + const Message &nested = reflection->GetMessage(msg, field); + extract_query_req(nested, field_name, values, nulls); + } + } + else + { + process_field(field, reflection, msg, field_name, values, nulls); + } + } } diff --git a/gpcontrib/gp_stats_collector/src/log/LogSchema.h b/gpcontrib/gp_stats_collector/src/log/LogSchema.h index 8754741823a..f6c2247370a 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogSchema.h +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef LOGSCHEMA_H +#define LOGSCHEMA_H #include #include @@ -37,26 +38,29 @@ extern "C" { #include "access/htup_details.h" #include "access/tupdesc.h" #include "catalog/pg_type.h" -#include "utils/timestamp.h" #include "utils/builtins.h" +#include "utils/timestamp.h" } -namespace google { -namespace protobuf { +namespace google +{ +namespace protobuf +{ class FieldDescriptor; class Message; class Reflection; class Timestamp; -} // namespace protobuf -} // namespace google +} // namespace protobuf +} // namespace google inline constexpr std::string_view schema_name = "gpsc"; inline constexpr std::string_view log_relname = "__log"; -struct LogDesc { - std::string_view pg_att_name; - std::string_view proto_field_name; - Oid type_oid; +struct LogDesc +{ + std::string_view pg_att_name; + std::string_view proto_field_name; + Oid type_oid; }; /* @@ -175,14 +179,14 @@ TupleDesc DescribeTuple(); Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts); Datum field_to_datum(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg); + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg); /* Process a single proto field and store in values/nulls arrays */ void process_field(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg, - const std::string &field_name, Datum *values, bool *nulls); + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls); /* * Extracts values from msg into values/nulls arrays. Caller must @@ -190,4 +194,6 @@ void process_field(const google::protobuf::FieldDescriptor *field, * to true for nested messages if parent message is missing). */ void extract_query_req(const google::protobuf::Message &msg, - const std::string &prefix, Datum *values, bool *nulls); + const std::string &prefix, Datum *values, bool *nulls); + +#endif /* LOGSCHEMA_H */ diff --git a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp index 4e3f6dae99f..de54a716016 100644 --- a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp @@ -30,223 +30,287 @@ extern "C" { #include "postgres.h" -#include "utils/guc.h" +#include "access/htup.h" +#include "access/tupdesc.h" +#include "cdb/cdbexplain.h" #include "commands/dbcommands.h" -#include "commands/resgroupcmds.h" -#include "utils/builtins.h" -#include "utils/varlena.h" -#include "nodes/pg_list.h" #include "commands/explain.h" +#include "commands/resgroupcmds.h" #include "executor/instrument.h" -#include "access/tupdesc.h" -#include "access/htup.h" +#include "nodes/pg_list.h" +#include "stat_statements_parser/pg_stat_statements_parser.h" +#include "utils/builtins.h" #include "utils/elog.h" -#include "cdb/cdbexplain.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "utils/guc.h" +#include "utils/varlena.h" } -namespace { +namespace +{ template -auto wrap(Func &&func, Args &&...args) noexcept(!Throws) - -> decltype(func(std::forward(args)...)) { - - using RetType = decltype(func(std::forward(args)...)); - - // Empty struct for void return type. - struct VoidResult {}; - using ResultHolder = std::conditional_t, VoidResult, - std::optional>; - - bool success; - ErrorData *edata; - ResultHolder result_holder; - - PG_TRY(); - { - if constexpr (!std::is_void_v) { - result_holder.emplace(func(std::forward(args)...)); - } else { - func(std::forward(args)...); - } - edata = NULL; - success = true; - } - PG_CATCH(); - { - MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - MemoryContextSwitchTo(oldctx); - FlushErrorState(); - success = false; - } - PG_END_TRY(); - - if (!success) { - std::string err; - if (edata && edata->message) { - err = std::string(edata->message); - } else { - err = "Unknown error occurred"; - } - - if (edata) { - FreeErrorData(edata); - } - - if constexpr (Throws) { - throw std::runtime_error(err); - } - - if constexpr (!std::is_void_v) { - return RetType{}; - } else { - return; - } - } - - if constexpr (!std::is_void_v) { - return *std::move(result_holder); - } else { - return; - } +auto +wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) +{ + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult + { + }; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) + { + result_holder.emplace(func(std::forward(args)...)); + } + else + { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) + { + std::string err; + if (edata && edata->message) + { + err = std::string(edata->message); + } + else + { + err = "Unknown error occurred"; + } + + if (edata) + { + FreeErrorData(edata); + } + + if constexpr (Throws) + { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) + { + return RetType{}; + } + else + { + return; + } + } + + if constexpr (!std::is_void_v) + { + return *std::move(result_holder); + } + else + { + return; + } } template -auto wrap_throw(Func &&func, Args &&...args) - -> decltype(func(std::forward(args)...)) { - return wrap(std::forward(func), std::forward(args)...); +auto +wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) +{ + return wrap(std::forward(func), std::forward(args)...); } template -auto wrap_noexcept(Func &&func, Args &&...args) noexcept - -> decltype(func(std::forward(args)...)) { - return wrap(std::forward(func), std::forward(args)...); +auto +wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) +{ + return wrap(std::forward(func), std::forward(args)...); } -} // namespace +} // namespace -void *gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } +void * +gpdb::palloc(Size size) +{ + return wrap_throw(::palloc, size); +} -void *gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } +void * +gpdb::palloc0(Size size) +{ + return wrap_throw(::palloc0, size); +} -char *gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } +char * +gpdb::pstrdup(const char *str) +{ + return wrap_throw(::pstrdup, str); +} -char *gpdb::get_database_name(Oid dbid) noexcept { - return wrap_noexcept(::get_database_name, dbid); +char * +gpdb::get_database_name(Oid dbid) noexcept +{ + return wrap_noexcept(::get_database_name, dbid); } -bool gpdb::split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept { - return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); +bool +gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept +{ + return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState gpdb::get_explain_state(QueryDesc *query_desc, - bool costs) noexcept { - return wrap_noexcept([&]() { - ExplainState *es = NewExplainState(); - es->costs = costs; - es->verbose = true; - es->format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(es); - ExplainPrintPlan(es, query_desc); - ExplainEndOutput(es); - return *es; - }); +ExplainState +gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept +{ + return wrap_noexcept([&]() { + ExplainState *es = NewExplainState(); + es->costs = costs; + es->verbose = true; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + ExplainPrintPlan(es, query_desc); + ExplainEndOutput(es); + return *es; + }); } -ExplainState gpdb::get_analyze_state(QueryDesc *query_desc, - bool analyze) noexcept { - return wrap_noexcept([&]() { - ExplainState *es = NewExplainState(); - es->analyze = analyze; - es->verbose = true; - es->buffers = es->analyze; - es->timing = es->analyze; - es->summary = es->analyze; - es->format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(es); - if (analyze) { - ExplainPrintPlan(es, query_desc); - ExplainPrintExecStatsEnd(es, query_desc); - } - ExplainEndOutput(es); - return *es; - }); +ExplainState +gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept +{ + return wrap_noexcept([&]() { + ExplainState *es = NewExplainState(); + es->analyze = analyze; + es->verbose = true; + es->buffers = es->analyze; + es->timing = es->analyze; + es->summary = es->analyze; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + if (analyze) + { + ExplainPrintPlan(es, query_desc); + ExplainPrintExecStatsEnd(es, query_desc); + } + ExplainEndOutput(es); + return *es; + }); } -Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options, - bool async_mode) { - return wrap_throw(InstrAlloc, n, instrument_options, async_mode); +Instrumentation * +gpdb::instr_alloc(size_t n, int instrument_options, bool async_mode) +{ + return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } -HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull) { - if (!tupleDescriptor || !values || !isnull) - throw std::runtime_error( - "Invalid input parameters for heap tuple formation"); +HeapTuple +gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull) +{ + if (!tupleDescriptor || !values || !isnull) + throw std::runtime_error( + "Invalid input parameters for heap tuple formation"); - return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); + return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void gpdb::pfree(void *pointer) noexcept { - // Note that ::pfree asserts that pointer != NULL. - if (!pointer) - return; +void +gpdb::pfree(void *pointer) noexcept +{ + // Note that ::pfree asserts that pointer != NULL. + if (!pointer) + return; - wrap_noexcept(::pfree, pointer); + wrap_noexcept(::pfree, pointer); } -MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { - return MemoryContextSwitchTo(context); +MemoryContext +gpdb::mem_ctx_switch_to(MemoryContext context) noexcept +{ + return MemoryContextSwitchTo(context); } -const char *gpdb::get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept { - if (!name) - return nullptr; +const char * +gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept +{ + if (!name) + return nullptr; - return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); + return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void gpdb::list_free(List *list) noexcept { - if (!list) - return; +void +gpdb::list_free(List *list) noexcept +{ + if (!list) + return; - wrap_noexcept(::list_free, list); + wrap_noexcept(::list_free, list); } CdbExplain_ShowStatCtx * -gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime) { - if (!query_desc) - throw std::runtime_error("Invalid query descriptor"); +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime) +{ + if (!query_desc) + throw std::runtime_error("Invalid query descriptor"); - return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); + return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void gpdb::instr_end_loop(Instrumentation *instr) { - if (!instr) - throw std::runtime_error("Invalid instrumentation pointer"); +void +gpdb::instr_end_loop(Instrumentation *instr) +{ + if (!instr) + throw std::runtime_error("Invalid instrumentation pointer"); - wrap_throw(::InstrEndLoop, instr); + wrap_throw(::InstrEndLoop, instr); } -char *gpdb::gen_normquery(const char *query) noexcept { - return wrap_noexcept(::gen_normquery, query); +char * +gpdb::gen_normquery(const char *query) noexcept +{ + return wrap_noexcept(::gen_normquery, query); } -StringInfo gpdb::gen_normplan(const char *exec_plan) noexcept { - return wrap_noexcept(::gen_normplan, exec_plan); +StringInfo +gpdb::gen_normplan(const char *exec_plan) noexcept +{ + return wrap_noexcept(::gen_normplan, exec_plan); } -char *gpdb::get_rg_name_for_id(Oid group_id) { - return wrap_throw(GetResGroupNameForId, group_id); +char * +gpdb::get_rg_name_for_id(Oid group_id) +{ + return wrap_throw(GetResGroupNameForId, group_id); } -Oid gpdb::get_rg_id_by_session_id(int session_id) { - return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +Oid +gpdb::get_rg_id_by_session_id(int session_id) +{ + return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } -void gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) { - return wrap_throw(::insert_log, req, utility); +void +gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) +{ + return wrap_throw(::insert_log, req, utility); } diff --git a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h index 576007f6c7c..5237b6be68a 100644 --- a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h @@ -25,29 +25,32 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef GPDBWRAPPERS_H +#define GPDBWRAPPERS_H extern "C" { #include "postgres.h" -#include "nodes/pg_list.h" +#include "access/htup.h" #include "commands/explain.h" #include "executor/instrument.h" -#include "access/htup.h" +#include "nodes/pg_list.h" #include "utils/elog.h" #include "utils/memutils.h" } -#include -#include #include -#include +#include #include +#include +#include -namespace gpsc { +namespace gpsc +{ class SetQueryReq; -} // namespace gpsc +} // namespace gpsc -namespace gpdb { +namespace gpdb +{ // Functions that call palloc(). // Make sure correct memory context is set. @@ -56,14 +59,14 @@ void *palloc0(Size size); char *pstrdup(const char *str); char *get_database_name(Oid dbid) noexcept; bool split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept; + List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; Instrumentation *instr_alloc(size_t n, int instrument_options, bool async_mode); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull); + bool *isnull); CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime); + instr_time starttime); void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query) noexcept; StringInfo gen_normplan(const char *executionPlan) noexcept; @@ -74,8 +77,10 @@ void insert_log(const gpsc::SetQueryReq &req, bool utility); void pfree(void *pointer) noexcept; MemoryContext mem_ctx_switch_to(MemoryContext context) noexcept; const char *get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept; + bool restrict_superuser) noexcept; void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace gpdb +} // namespace gpdb + +#endif /* GPDBWRAPPERS_H */ diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c similarity index 82% rename from gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c index e24f53536a4..8e7bd917541 100644 --- a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * pg_stat_statements_ya_parser.c + * pg_stat_statements_parser.c * * IDENTIFICATION - * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c * *------------------------------------------------------------------------- */ @@ -42,7 +42,7 @@ #include "utils/memutils.h" #include "utils/queryjumble.h" -#include "pg_stat_statements_ya_parser.h" +#include "pg_stat_statements_parser.h" #ifndef FCONST #define FCONST 260 @@ -67,12 +67,14 @@ static bool need_replace(int token); static char *generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding); -void stat_statements_parser_init(void) +void +stat_statements_parser_init(void) { EnableQueryId(); } -void stat_statements_parser_deinit(void) +void +stat_statements_parser_deinit(void) { /* NO-OP */ } @@ -81,7 +83,8 @@ void stat_statements_parser_deinit(void) static bool need_replace(int token) { - return (token == FCONST) || (token == ICONST) || (token == SCONST) || (token == BCONST) || (token == XCONST); + return (token == FCONST) || (token == ICONST) || (token == SCONST) || + (token == BCONST) || (token == XCONST); } /* @@ -103,14 +106,11 @@ gen_normplan(const char *execution_plan) StringInfo plan_out = makeStringInfo(); ; - yyscanner = scanner_init(execution_plan, - &yyextra, + yyscanner = scanner_init(execution_plan, &yyextra, #if PG_VERSION_NUM >= 120000 - &ScanKeywords, - ScanKeywordTokens + &ScanKeywords, ScanKeywordTokens #else - ScanKeywords, - NumScanKeywords + ScanKeywords, NumScanKeywords #endif ); @@ -137,7 +137,8 @@ gen_normplan(const char *execution_plan) else { /* do not change - just copy as-is */ - tmp_str = strndup((char *)execution_plan + last_yylloc, yylloc - last_yylloc); + tmp_str = strndup((char *) execution_plan + last_yylloc, + yylloc - last_yylloc); appendStringInfoString(plan_out, tmp_str); free(tmp_str); } @@ -159,8 +160,8 @@ gen_normplan(const char *execution_plan) static int comp_location(const void *a, const void *b) { - int l = ((const LocationLen *) a)->location; - int r = ((const LocationLen *) b)->location; + int l = ((const LocationLen *) a)->location; + int r = ((const LocationLen *) b)->location; if (l < r) return -1; @@ -199,35 +200,32 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) core_yyscan_t yyscanner; core_yy_extra_type yyextra; core_YYSTYPE yylval; - YYLTYPE yylloc; - int last_loc = -1; - int i; + YYLTYPE yylloc; + int last_loc = -1; + int i; /* * Sort the records by location so that we can process them in order while * scanning the query text. */ if (jstate->clocations_count > 1) - qsort(jstate->clocations, jstate->clocations_count, - sizeof(LocationLen), comp_location); + qsort(jstate->clocations, jstate->clocations_count, sizeof(LocationLen), + comp_location); locs = jstate->clocations; /* initialize the flex scanner --- should match raw_parser() */ - yyscanner = scanner_init(query, - &yyextra, - &ScanKeywords, - ScanKeywordTokens); + yyscanner = scanner_init(query, &yyextra, &ScanKeywords, ScanKeywordTokens); /* Search for each constant, in sequence */ for (i = 0; i < jstate->clocations_count; i++) { - int loc = locs[i].location; - int tok; + int loc = locs[i].location; + int tok; Assert(loc >= 0); if (loc <= last_loc) - continue; /* Duplicate constant, ignore */ + continue; /* Duplicate constant, ignore */ /* Lex tokens until we find the desired constant */ for (;;) @@ -236,7 +234,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) /* We should not hit end-of-string, but if we do, behave sanely */ if (tok == 0) - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ /* * We should find the token position exactly, but if we somehow @@ -260,7 +258,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) */ tok = core_yylex(&yylval, &yylloc, yyscanner); if (tok == 0) - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ } /* @@ -268,7 +266,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) * byte after the text of the current token in scanbuf. */ locs[i].length = strlen(yyextra.scanbuf + loc); - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ } } @@ -299,14 +297,13 @@ static char * generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding) { - char *norm_query; - int query_len = *query_len_p; - int i, - len_to_wrt, /* Length (in bytes) to write */ - quer_loc = 0, /* Source query byte location */ - n_quer_loc = 0, /* Normalized query byte location */ - last_off = 0, /* Offset from start for previous tok */ - last_tok_len = 0; /* Length (in bytes) of that tok */ + char *norm_query; + int query_len = *query_len_p; + int i, len_to_wrt, /* Length (in bytes) to write */ + quer_loc = 0, /* Source query byte location */ + n_quer_loc = 0, /* Normalized query byte location */ + last_off = 0, /* Offset from start for previous tok */ + last_tok_len = 0; /* Length (in bytes) of that tok */ /* * Get constants' lengths (core system only gives us locations). Note @@ -319,14 +316,14 @@ generate_normalized_query(JumbleState *jstate, const char *query, for (i = 0; i < jstate->clocations_count; i++) { - int off, /* Offset from start for cur tok */ - tok_len; /* Length (in bytes) of that tok */ + int off, /* Offset from start for cur tok */ + tok_len; /* Length (in bytes) of that tok */ off = jstate->clocations[i].location; tok_len = jstate->clocations[i].length; if (tok_len < 0) - continue; /* ignore any duplicates */ + continue; /* ignore any duplicates */ /* Copy next chunk (what precedes the next constant) */ len_to_wrt = off - last_off; @@ -361,18 +358,21 @@ generate_normalized_query(JumbleState *jstate, const char *query, return norm_query; } -char *gen_normquery(const char *query) +char * +gen_normquery(const char *query) { - if (!query) { + if (!query) + { return NULL; } JumbleState jstate; - jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble = (unsigned char *) palloc(JUMBLE_SIZE); jstate.jumble_len = 0; jstate.clocations_buf_size = 32; - jstate.clocations = (LocationLen *) - palloc(jstate.clocations_buf_size * sizeof(LocationLen)); + jstate.clocations = (LocationLen *) palloc(jstate.clocations_buf_size * + sizeof(LocationLen)); jstate.clocations_count = 0; int query_len = strlen(query); - return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); + return generate_normalized_query(&jstate, query, &query_len, + GetDatabaseEncoding()); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h similarity index 87% rename from gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h index a613ba04259..b6c5dea7b36 100644 --- a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h @@ -17,19 +17,19 @@ * specific language governing permissions and limitations * under the License. * - * pg_stat_statements_ya_parser.h + * pg_stat_statements_parser.h * * IDENTIFICATION - * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h * *------------------------------------------------------------------------- */ -#pragma once +#ifndef PG_STAT_STATEMENTS_PARSER_H +#define PG_STAT_STATEMENTS_PARSER_H #ifdef __cplusplus -extern "C" -{ +extern "C" { #endif extern void stat_statements_parser_init(void); @@ -41,3 +41,5 @@ char *gen_normquery(const char *query); #ifdef __cplusplus } #endif + +#endif /* PG_STAT_STATEMENTS_PARSER_H */ diff --git a/pom.xml b/pom.xml index 05bd00966ed..77ca5e3e3d8 100644 --- a/pom.xml +++ b/pom.xml @@ -1274,9 +1274,6 @@ code or new licensing patterns. introduced by Cloudberry. --> gpcontrib/gp_stats_collector/gp_stats_collector.control - gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto - gpcontrib/gp_stats_collector/protos/gpsc_plan.proto - gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto gpcontrib/gp_stats_collector/.clang-format gpcontrib/gp_stats_collector/Makefile From 796083d014b379e49bc686e680db29bcb1162f9b Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 14:15:58 +0300 Subject: [PATCH 082/128] [gp_stats_collector] Wrap hook call in try/catch on error path Add PG_TRY/PG_CATCH around query_info_collect_hook in PortalCleanup error path to prevent exceptions from propagating during cleanup. --- src/backend/commands/portalcmds.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 0ea5874e884..e23dc1d9c43 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -374,10 +374,22 @@ PortalCleanup(Portal portal) FreeQueryDesc(queryDesc); CurrentResourceOwner = saveResourceOwner; - } else { + } + else + { /* GPDB hook for collecting query info */ if (queryDesc->gpsc_query_key && query_info_collect_hook) - (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); + { + PG_TRY(); + { + (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); + } + PG_CATCH(); + { + FlushErrorState(); + } + PG_END_TRY(); + } } } From a7279b9e89448a944c834340f94b2495704f391a Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 14:33:20 +0300 Subject: [PATCH 083/128] [gp_stats_collector] Adapt namings for Cloudberry Rename ON MASTER to ON COORDINATOR in test SQL. Prefer pg_usleep() over std::this_thread::sleep_for(). Add pg_unreachable() after ereport(ERROR). Widen motion stats fields to uint64. --- .../gp_stats_collector--1.0--1.1.sql | 10 +++++----- .../gp_stats_collector--1.0.sql | 6 +++--- .../gp_stats_collector--1.1.sql | 16 ++++++++-------- .../gp_stats_collector/protos/gpsc_metrics.proto | 6 +++--- .../gp_stats_collector/src/UDSConnector.cpp | 4 +--- .../gp_stats_collector/src/hook_wrappers.cpp | 1 + 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql index 4e0157117e9..398f03b4fa9 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql @@ -25,7 +25,7 @@ DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -39,12 +39,12 @@ $$ SELECT gpsc.__stat_messages_reset_f_on_master(); SELECT gpsc.__stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_segments() RETURNS SETOF record @@ -77,7 +77,7 @@ ORDER BY segid; CREATE FUNCTION gpsc.__init_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__init_log_on_segments() RETURNS SETOF void @@ -97,7 +97,7 @@ CREATE VIEW gpsc.log AS CREATE FUNCTION gpsc.__truncate_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__truncate_log_on_segments() RETURNS SETOF void diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql index ec902b02e02..e4a50aa2133 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql @@ -6,7 +6,7 @@ CREATE FUNCTION __gpsc_stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -20,12 +20,12 @@ $$ SELECT __gpsc_stat_messages_reset_f_on_master(); SELECT __gpsc_stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_f_on_segments() RETURNS SETOF record diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql index 6e24207e913..3ebdad14b06 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql @@ -8,7 +8,7 @@ CREATE SCHEMA gpsc; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -22,12 +22,12 @@ $$ SELECT gpsc.__stat_messages_reset_f_on_master(); SELECT gpsc.__stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_segments() RETURNS SETOF record @@ -59,7 +59,7 @@ ORDER BY segid; CREATE FUNCTION gpsc.__init_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__init_log_on_segments() RETURNS SETOF void @@ -79,7 +79,7 @@ ORDER BY tmid, ssid, ccnt; CREATE FUNCTION gpsc.__truncate_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__truncate_log_on_segments() RETURNS SETOF void @@ -97,14 +97,14 @@ $$ LANGUAGE plpgsql VOLATILE; CREATE FUNCTION gpsc.__test_uds_start_server(path text) RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' -LANGUAGE C STRICT EXECUTE ON MASTER; +LANGUAGE C STRICT EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) RETURNS SETOF bigint AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' -LANGUAGE C STRICT EXECUTE ON MASTER; +LANGUAGE C STRICT EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__test_uds_stop_server() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index 7853dc58db7..10991301557 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -113,9 +113,9 @@ message SystemStat { } message NetworkStat { - uint32 total_bytes = 1; - uint32 tuple_bytes = 2; - uint32 chunks = 3; + uint64 total_bytes = 1; + uint64 tuple_bytes = 2; + uint64 chunks = 3; } message InterconnectStat { diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp index 16344366456..056fa9071a5 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -31,13 +31,11 @@ #include "log/LogOps.h" #include "memory/gpdbwrappers.h" -#include #include #include #include #include #include -#include #include extern "C" { @@ -132,7 +130,7 @@ UDSConnector::report_query(const gpsc::SetQueryReq &req, // if a message does not fit a single packet, we take a nap // before sending the next one. // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + (pg_usleep(1000), true)); if (sent < 0) { diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index 3f19d4d9930..38ea117bda2 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -116,6 +116,7 @@ cpp_call(T *obj, R (T::*func)(Args...), Args... args) catch (const std::exception &e) { ereport(ERROR, (errmsg("Unexpected exception in gpsc %s", e.what()))); + pg_unreachable(); } } From 0c986919045f64d60806d1a93ee0515cc0be5d94 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 1 Apr 2026 10:39:45 +0300 Subject: [PATCH 084/128] [gp_stats_collector] Remove unnecessary CONFIGURE_EXTRA_OPTS param --- .../automation/cloudberry/scripts/configure-cloudberry.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index a9086a434fb..90f0614bfe8 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -53,7 +53,6 @@ # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) -# CONFIGURE_EXTRA_OPTS - Args to pass to configure command # ENABLE_DEBUG - Enable debug build options (true/false, defaults to # false) # @@ -179,8 +178,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-uuid=e2fs \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib \ - ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" # Capture version information From 96caa8d357d6facdedcc93979c34f19d7d1a9682 Mon Sep 17 00:00:00 2001 From: zhangyue Date: Fri, 3 Apr 2026 12:36:41 +0800 Subject: [PATCH 085/128] Fix sed -i compatibility in configure.ac Apply the same macOS BSD sed fix from 7e867f6 to configure.ac, so that users regenerating configure from source also get the compatibility fix. --- configure.ac | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9664aa86e57..c28bf48eba3 100644 --- a/configure.ac +++ b/configure.ac @@ -3212,4 +3212,11 @@ AC_OUTPUT # The configure args contain '-Wl,-rpath,\$$ORIGIN`, when it falls # as a C literal string, it's invalid, so converting `\` to `\\` # to be correct for C program. -sed -i '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h +case $build_os in +darwin*) + sed -i '' '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h + ;; +*) + sed -i '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h + ;; +esac From 2ecd058a815e456c0d79b8b9a0b9d957a43cd6b3 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 31 Mar 2026 17:56:30 +0800 Subject: [PATCH 086/128] Disable mdblocales by default in configure Changed the default value of `--with-mdblocales` from 'yes' to 'no' in configure.ac. This prevents enabling custom locales by default, which require additional packages and may cause compatibility issues across different system environments (e.g., Rocky Linux 8 vs 9 with different libc versions). Users who need mdblocales can explicitly enable it with `--with-mdblocales` option. Regenerated configure file using autoconf 2.69 on Rocky Linux 8. --- configure | 103 ++++++++++++++++++++++++++------------------------- configure.ac | 2 +- 2 files changed, 53 insertions(+), 52 deletions(-) diff --git a/configure b/configure index 5127039e72b..16f511852ee 100755 --- a/configure +++ b/configure @@ -1709,7 +1709,7 @@ Optional Packages: --without-libcurl do not use libcurl --with-apr-config=PATH path to apr-1-config utility --with-gnu-ld assume the C compiler uses GNU ld [default=no] - --without-mdblocales build without MDB locales + --with-mdblocales build with MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2926,6 +2926,7 @@ PG_PACKAGE_VERSION=14.7 + ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -13007,56 +13008,6 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } -fi - -if test "$with_mdblocales" = yes; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 -$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } -if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lmdblocales $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char mdb_setlocale (); -int -main () -{ -return mdb_setlocale (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_mdblocales_mdb_setlocale=yes -else - ac_cv_lib_mdblocales_mdb_setlocale=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 -$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } -if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBMDBLOCALES 1 -_ACEOF - - LIBS="-lmdblocales $LIBS" - -else - as_fn_error $? "mdblocales library not found" "$LINENO" 5 -fi - fi if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then @@ -14981,6 +14932,56 @@ fi fi +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + +fi + if test "$enable_external_fts" = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for jansson_version_str in -ljansson" >&5 $as_echo_n "checking for jansson_version_str in -ljansson... " >&6; } diff --git a/configure.ac b/configure.ac index c28bf48eba3..1ee0958a425 100644 --- a/configure.ac +++ b/configure.ac @@ -1487,7 +1487,7 @@ AC_SUBST(install_bin) # MDB locales # -PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], +PGAC_ARG_BOOL(with, mdblocales, no, [build with MDB locales], [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) AC_SUBST(USE_MDBLOCALES) From 1acbfbee9baf94fe0b0776734459c866448b390e Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 31 Mar 2026 17:56:30 +0800 Subject: [PATCH 087/128] Regenerate configure for gp_stats_clolector Regenerated configure file using autoconf 2.69 on Rocky Linux 8. --- configure | 180 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 27 deletions(-) diff --git a/configure b/configure index 16f511852ee..ca76bad684c 100755 --- a/configure +++ b/configure @@ -721,8 +721,9 @@ GREP with_apr_config with_libcurl with_rt -with_zstd +PROTOC with_gp_stats_collector +with_zstd with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -1705,6 +1706,8 @@ Optional Packages: --with-lz4 build with LZ4 support --without-libbz2 do not use bzip2 --without-zstd do not build with Zstandard + --with-gp_stats_collector + build with stats collector extension --without-rt do not use Realtime Library --without-libcurl do not use libcurl --with-apr-config=PATH path to apr-1-config utility @@ -11160,6 +11163,155 @@ fi $as_echo "$with_zstd" >&6; } +# +# gp_stats_collector +# + + + +# Check whether --with-gp_stats_collector was given. +if test "${with_gp_stats_collector+set}" = set; then : + withval=$with_gp_stats_collector; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-gp_stats_collector option" "$LINENO" 5 + ;; + esac + +else + with_gp_stats_collector=no + +fi + + + + +if test "$with_gp_stats_collector" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for protobuf >= 3.0.0" >&5 +$as_echo_n "checking for protobuf >= 3.0.0... " >&6; } + +if test -n "$PROTOBUF_CFLAGS"; then + pkg_cv_PROTOBUF_CFLAGS="$PROTOBUF_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"protobuf >= 3.0.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "protobuf >= 3.0.0") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PROTOBUF_CFLAGS=`$PKG_CONFIG --cflags "protobuf >= 3.0.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PROTOBUF_LIBS"; then + pkg_cv_PROTOBUF_LIBS="$PROTOBUF_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"protobuf >= 3.0.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "protobuf >= 3.0.0") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PROTOBUF_LIBS=`$PKG_CONFIG --libs "protobuf >= 3.0.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PROTOBUF_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "protobuf >= 3.0.0" 2>&1` + else + PROTOBUF_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "protobuf >= 3.0.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PROTOBUF_PKG_ERRORS" >&5 + + as_fn_error $? "protobuf >= 3.0.0 is required for gp_stats_collector" "$LINENO" 5 + +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "protobuf >= 3.0.0 is required for gp_stats_collector" "$LINENO" 5 + +else + PROTOBUF_CFLAGS=$pkg_cv_PROTOBUF_CFLAGS + PROTOBUF_LIBS=$pkg_cv_PROTOBUF_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + # Extract the first word of "protoc", so it can be a program name with args. +set dummy protoc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PROTOC+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PROTOC in + [\\/]* | ?:[\\/]*) + ac_cv_path_PROTOC="$PROTOC" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PROTOC="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_path_PROTOC" && ac_cv_path_PROTOC="no" + ;; +esac +fi +PROTOC=$ac_cv_path_PROTOC +if test -n "$PROTOC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROTOC" >&5 +$as_echo "$PROTOC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test "$PROTOC" = no; then + as_fn_error $? "protoc is required for gp_stats_collector but was not found in PATH" "$LINENO" 5 + fi +fi + if test "$with_zstd" = yes; then pkg_failed=no @@ -11254,32 +11406,6 @@ $as_echo "yes" >&6; } fi fi -# -# gp_stats_collector -# - - - -# Check whether --with-gp-stats-collector was given. -if test "${with_gp_stats_collector+set}" = set; then : - withval=$with_gp_stats_collector; - case $withval in - yes) - : - ;; - no) - : - ;; - *) - as_fn_error $? "no argument expected for --with-gp-stats-collector option" "$LINENO" 5 - ;; - esac - -else - with_gp_stats_collector=no - -fi - # # Realtime library # From 198329cb9aca150cd4b6dea3c3682d6df7ae9cd8 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 2 Apr 2026 19:11:43 +0800 Subject: [PATCH 088/128] Add missing runtime dependencies for minimal installations Add essential command-line utilities as runtime dependencies to ensure proper operation in minimal system installations such as container environments. Added dependencies: - which/debianutils: Required by gp_bash_functions.sh to locate binaries - hostname: Used by various scripts to identify the system - less: Required for paging output in interactive sessions These utilities may not be present in minimal installations (e.g., Oracle Linux 9 containers, Ubuntu minimal images), causing runtime failures during database initialization and operation. Changes: - RPM spec: Add hostname, less, and which to Requires - DEB control (Ubuntu 22.04/24.04): Add hostname and debianutils to Depends (less already present) --- devops/build/packaging/deb/ubuntu22.04/control | 2 ++ devops/build/packaging/deb/ubuntu24.04/control | 2 ++ .../build/packaging/rpm/apache-cloudberry-db-incubating.spec | 3 +++ 3 files changed, 7 insertions(+) diff --git a/devops/build/packaging/deb/ubuntu22.04/control b/devops/build/packaging/deb/ubuntu22.04/control index 4bc5d90b84d..6b05863b780 100644 --- a/devops/build/packaging/deb/ubuntu22.04/control +++ b/devops/build/packaging/deb/ubuntu22.04/control @@ -46,6 +46,8 @@ Provides: apache-cloudberry-db Architecture: any Depends: curl, cgroup-tools, + debianutils, + hostname, iputils-ping, iproute2, keyutils, diff --git a/devops/build/packaging/deb/ubuntu24.04/control b/devops/build/packaging/deb/ubuntu24.04/control index a561d8a4386..9e2c3eab451 100644 --- a/devops/build/packaging/deb/ubuntu24.04/control +++ b/devops/build/packaging/deb/ubuntu24.04/control @@ -46,6 +46,8 @@ Provides: apache-cloudberry-db Architecture: amd64 Depends: curl, cgroup-tools, + debianutils, + hostname, iputils-ping, iproute2, keyutils, diff --git a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec index 03fa0a34570..517b35212bf 100644 --- a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec +++ b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec @@ -52,12 +52,15 @@ Prefix: %{cloudberry_install_dir} # List runtime dependencies Requires: bash +Requires: hostname Requires: iproute Requires: iputils +Requires: less Requires: openssh Requires: openssh-clients Requires: openssh-server Requires: rsync +Requires: which %if 0%{?rhel} == 8 Requires: apr From 8712d2ba4985e6de2cd4d2904b0b556b86bba06c Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Fri, 3 Apr 2026 21:17:59 +0800 Subject: [PATCH 089/128] Fix compliance issues for RPM and DEB packages As an Apache incubating project, convenience binaries must include LICENSE, NOTICE, and DISCLAIMER files. This commit adds these mandatory compliance files into the spec and rules definitions to ensure they are properly distributed with the binary RPM and DEB packages. Additionally, this commit: - Automates the cp of the .spec file into the ~/rpmbuild tree to prevent build failures for new users. - Dynamically locates debian metadata from OS-specific directories and copies to project root for dpkg-buildpackage. - Generates debian/copyright file by combining LICENSE and NOTICE to meet Debian policy requirements. --- devops/build/packaging/deb/build-deb.sh | 47 +++++++++++++++---- devops/build/packaging/deb/ubuntu22.04/rules | 17 ++++++- .../rpm/apache-cloudberry-db-incubating.spec | 8 +++- devops/build/packaging/rpm/build-rpm.sh | 40 +++++++++++++++- 4 files changed, 99 insertions(+), 13 deletions(-) diff --git a/devops/build/packaging/deb/build-deb.sh b/devops/build/packaging/deb/build-deb.sh index 1f5aef2258a..61a29e50fc9 100755 --- a/devops/build/packaging/deb/build-deb.sh +++ b/devops/build/packaging/deb/build-deb.sh @@ -109,7 +109,7 @@ export CBDB_FULL_VERSION=$VERSION # Set version if not provided if [ -z "${VERSION}" ]; then - export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) + export CBDB_FULL_VERSION=$(./getversion 2>/dev/null | cut -d'-' -f 1 | cut -d'+' -f 1 || echo "unknown") fi if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then @@ -127,22 +127,48 @@ fi # Detect OS distribution (e.g., ubuntu22.04, debian12) if [ -z ${OS_DISTRO+x} ]; then if [ -f /etc/os-release ]; then + # Temporarily disable unbound variable check for sourcing os-release + set +u . /etc/os-release - OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') + set -u + # Ensure ID and VERSION_ID are set before using them + OS_DISTRO=$(echo "${ID:-unknown}${VERSION_ID:-}" | tr '[:upper:]' '[:lower:]') else OS_DISTRO="unknown" fi fi +# Ensure OS_DISTRO is exported and not empty +export OS_DISTRO=${OS_DISTRO:-unknown} + export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} # Check if required commands are available check_commands -# Define the control file path -CONTROL_FILE=debian/control +# Find project root (assumed to be four levels up from scripts directory: devops/build/packaging/deb/) +PROJECT_ROOT="$(cd "$(dirname "$0")/../../../../" && pwd)" + +# Define where the debian metadata is located +DEBIAN_SRC_DIR="$(dirname "$0")/${OS_DISTRO}" + +# Prepare the debian directory at the project root (required by dpkg-buildpackage) +if [ -d "$DEBIAN_SRC_DIR" ]; then + echo "Preparing debian directory from $DEBIAN_SRC_DIR..." + mkdir -p "$PROJECT_ROOT/debian" + # Use /. to copy directory contents if target exists instead of nested directories + cp -rf "$DEBIAN_SRC_DIR"/. "$PROJECT_ROOT/debian/" +else + if [ ! -d "$PROJECT_ROOT/debian" ]; then + echo "Error: Debian metadata not found at $DEBIAN_SRC_DIR and no debian/ directory exists at root." + exit 1 + fi +fi + +# Define the control file path (at the project root) +CONTROL_FILE="$PROJECT_ROOT/debian/control" -# Check if the spec file exists +# Check if the control file exists if [ ! -f "$CONTROL_FILE" ]; then echo "Error: Control file not found at $CONTROL_FILE." exit 1 @@ -160,10 +186,15 @@ if [ "${DRY_RUN:-false}" = true ]; then exit 0 fi -# Run debbuild with the provided options -echo "Building DEB with Version $CBDB_FULL_VERSION ..." +# Run debbuild from the project root +echo "Building DEB with Version $CBDB_FULL_VERSION in $PROJECT_ROOT ..." + +print_changelog > "$PROJECT_ROOT/debian/changelog" -print_changelog > debian/changelog +# Only cd if we are not already at the project root +if [ "$(pwd)" != "$PROJECT_ROOT" ]; then + cd "$PROJECT_ROOT" +fi if ! eval "$DEBBUILD_CMD"; then echo "Error: deb build failed." diff --git a/devops/build/packaging/deb/ubuntu22.04/rules b/devops/build/packaging/deb/ubuntu22.04/rules index cb387d209e6..463486cf03f 100755 --- a/devops/build/packaging/deb/ubuntu22.04/rules +++ b/devops/build/packaging/deb/ubuntu22.04/rules @@ -19,7 +19,22 @@ include /usr/share/dpkg/default.mk dh $@ --parallel gpinstall: - make install DESTDIR=${DEBIAN_DESTINATION} prefix= + # If the build staging directory is empty, copy from the pre-installed location. + # In CI, BUILD_DESTINATION already points here so it will be populated. + # For local manual packaging, copy from the installed Cloudberry path. + @mkdir -p ${DEBIAN_DESTINATION} + @if [ -z "$$(ls -A ${DEBIAN_DESTINATION} 2>/dev/null)" ]; then \ + echo "Copying pre-built binaries from ${CBDB_BIN_PATH} to ${DEBIAN_DESTINATION}..."; \ + cp -a ${CBDB_BIN_PATH}/* ${DEBIAN_DESTINATION}/; \ + else \ + echo "Build staging directory already populated, skipping copy."; \ + fi + # Copy Apache compliance files into the build staging directory + cp -a LICENSE NOTICE DISCLAIMER ${DEBIAN_DESTINATION}/ + cp -a licenses ${DEBIAN_DESTINATION}/ + # Create debian/copyright for Debian policy compliance + mkdir -p $(shell pwd)/debian + cat LICENSE NOTICE > $(shell pwd)/debian/copyright override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. diff --git a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec index 517b35212bf..e228f8fe76a 100644 --- a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec +++ b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec @@ -155,6 +155,12 @@ mkdir -p %{buildroot}%{cloudberry_install_dir}-%{version} cp -R %{cloudberry_install_dir}/* %{buildroot}%{cloudberry_install_dir}-%{version} +# Copy Apache mandatory compliance files from the SOURCES directory into the installation directory +cp %{_sourcedir}/LICENSE %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp %{_sourcedir}/NOTICE %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp %{_sourcedir}/DISCLAIMER %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp -R %{_sourcedir}/licenses %{buildroot}%{cloudberry_install_dir}-%{version}/ + # Create the symbolic link ln -sfn %{cloudberry_install_dir}-%{version} %{buildroot}%{cloudberry_install_dir} @@ -162,8 +168,6 @@ ln -sfn %{cloudberry_install_dir}-%{version} %{buildroot}%{cloudberry_install_di %{prefix}-%{version} %{prefix} -%license %{cloudberry_install_dir}-%{version}/LICENSE - %debug_package %post diff --git a/devops/build/packaging/rpm/build-rpm.sh b/devops/build/packaging/rpm/build-rpm.sh index ceb7d18d392..2c490166f45 100755 --- a/devops/build/packaging/rpm/build-rpm.sh +++ b/devops/build/packaging/rpm/build-rpm.sh @@ -118,10 +118,46 @@ fi # Check if required commands are available check_commands -# Define the spec file path +# Define the source spec file path (assuming it is in the same directory as the script) +SOURCE_SPEC_FILE="$(dirname "$0")/apache-cloudberry-db-incubating.spec" + +# Ensure rpmbuild SPECS and SOURCES directories exist +mkdir -p ~/rpmbuild/SPECS +mkdir -p ~/rpmbuild/SOURCES + +# Find project root (assumed to be four levels up from scripts directory: devops/build/packaging/rpm/) +PROJECT_ROOT="$(cd "$(dirname "$0")/../../../../" && pwd)" + +# Define the target spec file path SPEC_FILE=~/rpmbuild/SPECS/apache-cloudberry-db-incubating.spec -# Check if the spec file exists +# Copy the spec file to rpmbuild/SPECS if the source exists and is different +if [ -f "$SOURCE_SPEC_FILE" ]; then + # Avoid copying if SPEC_FILE is already a symlink/file pointing to SOURCE_SPEC_FILE (common in CI) + if [ ! "$SOURCE_SPEC_FILE" -ef "$SPEC_FILE" ]; then + cp -f "$SOURCE_SPEC_FILE" "$SPEC_FILE" + fi +else + echo "Warning: Source spec file not found at $SOURCE_SPEC_FILE, assuming it is already in ~/rpmbuild/SPECS/" +fi + +# Copy Apache mandatory compliance files to rpmbuild/SOURCES +echo "Copying compliance files from $PROJECT_ROOT to ~/rpmbuild/SOURCES..." +for f in LICENSE NOTICE DISCLAIMER; do + if [ -f "$PROJECT_ROOT/$f" ]; then + cp -af "$PROJECT_ROOT/$f" ~/rpmbuild/SOURCES/ + else + echo "Warning: $f not found in $PROJECT_ROOT" + fi +done + +if [ -d "$PROJECT_ROOT/licenses" ]; then + cp -af "$PROJECT_ROOT/licenses" ~/rpmbuild/SOURCES/ +else + echo "Warning: licenses directory not found in $PROJECT_ROOT" +fi + +# Check if the spec file exists at the target location before proceeding if [ ! -f "$SPEC_FILE" ]; then echo "Error: Spec file not found at $SPEC_FILE." exit 1 From fef42484a9789c9c75fbcbfa85fd790aa7e32957 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 7 Apr 2026 11:40:07 +0800 Subject: [PATCH 090/128] Build: make diskquota installation opt-in diskquota is currently built and installed by default from gpcontrib, which makes it available in all standard builds without an explicit decision to enable it. Change diskquota to an opt-in installation model by adding a dedicated `--with-diskquota` configure option that defaults to `no`. Only build, install, and run installcheck for diskquota when that option is explicitly enabled. This makes diskquota adoption an explicit choice and avoids shipping the extension by default. --- configure | 32 +++++++++++++++++++ configure.ac | 7 ++++ .../scripts/configure-cloudberry.sh | 1 + gpcontrib/Makefile | 12 ++++--- src/Makefile.global.in | 1 + 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/configure b/configure index ca76bad684c..74d1415d637 100755 --- a/configure +++ b/configure @@ -723,6 +723,7 @@ with_libcurl with_rt PROTOC with_gp_stats_collector +with_diskquota with_zstd with_libbz2 LZ4_LIBS @@ -945,6 +946,7 @@ with_zlib with_lz4 with_libbz2 with_zstd +with_diskquota with_gp_stats_collector with_rt with_libcurl @@ -1706,6 +1708,7 @@ Optional Packages: --with-lz4 build with LZ4 support --without-libbz2 do not use bzip2 --without-zstd do not build with Zstandard + --with-diskquota build with diskquota extension --with-gp_stats_collector build with stats collector extension --without-rt do not use Realtime Library @@ -11163,6 +11166,35 @@ fi $as_echo "$with_zstd" >&6; } +# +# diskquota +# + + + +# Check whether --with-diskquota was given. +if test "${with_diskquota+set}" = set; then : + withval=$with_diskquota; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-diskquota option" "$LINENO" 5 + ;; + esac + +else + with_diskquota=no + +fi + + + + # # gp_stats_collector # diff --git a/configure.ac b/configure.ac index 1ee0958a425..f0584d65076 100644 --- a/configure.ac +++ b/configure.ac @@ -1368,6 +1368,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# diskquota +# +PGAC_ARG_BOOL(with, diskquota, no, + [build with diskquota extension]) +AC_SUBST(with_diskquota) + # # gp_stats_collector # diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 90f0614bfe8..80575309092 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -162,6 +162,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ + --with-diskquota \ --with-gp-stats-collector \ --with-gssapi \ --with-ldap \ diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 956cb470477..2969194cfac 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -22,8 +22,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_legacy_string_agg \ gp_replica_check \ gp_toolkit \ - pg_hint_plan \ - diskquota + pg_hint_plan else recurse_targets = gp_sparse_vector \ gp_distribution_policy \ @@ -31,8 +30,11 @@ else gp_legacy_string_agg \ gp_exttable_fdw \ gp_toolkit \ - pg_hint_plan \ - diskquota + pg_hint_plan +endif + +ifeq "$(with_diskquota)" "yes" + recurse_targets += diskquota endif ifeq "$(with_gp_stats_collector)" "yes" @@ -102,4 +104,4 @@ installcheck: $(MAKE) -C gp_sparse_vector installcheck $(MAKE) -C gp_toolkit installcheck $(MAKE) -C gp_exttable_fdw installcheck - $(MAKE) -C diskquota installcheck + if [ "$(with_diskquota)" = "yes" ]; then $(MAKE) -C diskquota installcheck; fi diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 1393e4fb0ff..d413bd86761 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -271,6 +271,7 @@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ EVENT_LIBS = @EVENT_LIBS@ +with_diskquota = @with_diskquota@ with_gp_stats_collector = @with_gp_stats_collector@ ########################################################################## From c7c96aa7cce7e4d01f76444c165012890a16a72b Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Fri, 3 Apr 2026 17:43:02 +0300 Subject: [PATCH 091/128] pg_dump/psql: properly recognize GP During a batch rebranding from Greenplum to Cloudberry we've lost ability to work with Greenplum from pg_dump and psql --- src/bin/pg_dump/pg_dump.c | 16 +++++++-------- src/bin/psql/describe.c | 42 +++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index e85e39cf9f5..f1c2644bfdd 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -385,17 +385,17 @@ static char *nextToken(register char **stringp, register const char *delim); static void addDistributedBy(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo, int actual_atts); static void addDistributedByOld(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo, int actual_atts); static void addSchedule(Archive *fout, PQExpBuffer q, const TableInfo *tbinfo); -static bool isGPDB(Archive *fout); +static bool isMPP(Archive *fout); static bool isGPDB5000OrLater(Archive *fout); static bool isGPDB6000OrLater(Archive *fout); /* END MPP ADDITION */ /* - * Check if we are talking to GPDB + * Check if we are talking to Greenplum or Cloudberry */ static bool -isGPDB(Archive *fout) +isMPP(Archive *fout) { static int value = -1; /* -1 = not known yet, 0 = no, 1 = yes */ @@ -409,7 +409,7 @@ isGPDB(Archive *fout) res = ExecuteSqlQuery(fout, query, PGRES_TUPLES_OK); ver = (PQgetvalue(res, 0, 0)); - if (strstr(ver, "Cloudberry") != NULL) + if (strstr(ver, "Cloudberry") != NULL || strstr(ver, "Greenplum") != NULL) value = 1; else value = 0; @@ -423,8 +423,8 @@ isGPDB(Archive *fout) static bool isGPDB5000OrLater(Archive *fout) { - if (!isGPDB(fout)) - return false; /* Not Cloudberry at all. */ + if (!isMPP(fout)) + return false; /* Not GP-based at all. */ /* GPDB 5 is based on PostgreSQL 8.3 */ return fout->remoteVersion >= 80300; @@ -434,8 +434,8 @@ isGPDB5000OrLater(Archive *fout) static bool isGPDB6000OrLater(Archive *fout) { - if (!isGPDB(fout)) - return false; /* Not Cloudberry at all. */ + if (!isMPP(fout)) + return false; /* Not GP-based at all. */ /* GPDB 6 is based on PostgreSQL 9.4 */ return fout->remoteVersion >= 90400; diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 90c8bb777fa..7423459e690 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -52,7 +52,7 @@ static bool describeOneTSConfig(const char *oid, const char *nspname, static void printACLColumn(PQExpBuffer buf, const char *colname); static bool listOneExtensionContents(const char *extname, const char *oid); -static bool isGPDB(void); +static bool isMPP(void); static bool isGPDB4200OrLater(void); static bool isGPDB5000OrLater(void); static bool isGPDB6000OrLater(void); @@ -64,7 +64,7 @@ static bool validateSQLNamePattern(PQExpBuffer buf, const char *pattern, const char *visibilityrule, bool *added_clause, int maxparts); -static bool isGPDB(void) +static bool isMPP(void) { static enum { @@ -86,7 +86,7 @@ static bool isGPDB(void) return false; ver = PQgetvalue(res, 0, 0); - if (strstr(ver, "Cloudberry") != NULL) + if (strstr(ver, "Cloudberry") != NULL || strstr(ver, "Greenplum") != NULL) { PQclear(res); talking_to_gpdb = gpdb_yes; @@ -113,7 +113,7 @@ static bool isGPDB4200OrLater(void) { bool retValue = false; - if (isGPDB() == true) + if (isMPP() == true) { PGresult *result; @@ -134,7 +134,7 @@ isGPDB4300OrLater(void) { bool retValue = false; - if (isGPDB() == true) + if (isMPP() == true) { PGresult *result; @@ -157,7 +157,7 @@ static bool isGPDB5000OrLater(void) { bool retValue = false; - if (isGPDB() == true) + if (isMPP() == true) { PGresult *res; @@ -171,8 +171,8 @@ static bool isGPDB5000OrLater(void) static bool isGPDB6000OrLater(void) { - if (!isGPDB()) - return false; /* Not Cloudberry at all. */ + if (!isMPP()) + return false; /* Not GP-based at all. */ /* GPDB 6 is based on PostgreSQL 9.4 */ return pset.sversion >= 90400; @@ -181,8 +181,8 @@ isGPDB6000OrLater(void) static bool isGPDB6000OrBelow(void) { - if (!isGPDB()) - return false; /* Not Cloudberry at all. */ + if (!isMPP()) + return false; /* Not GP-based at all. */ /* GPDB 6 is based on PostgreSQL 9.4 */ return pset.sversion <= 90400; @@ -191,8 +191,8 @@ isGPDB6000OrBelow(void) static bool isGPDB7000OrLater(void) { - if (!isGPDB()) - return false; /* Not Cloudberry at all. */ + if (!isMPP()) + return false; /* Not GP-based at all. */ /* GPDB 7 is based on PostgreSQL v12 */ return pset.sversion >= 120000; @@ -2007,7 +2007,7 @@ describeOneTableDetails(const char *schemaname, "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "c.relstorage" : "'h'"), + (isMPP() ? "c.relstorage" : "'h'"), oid); } else if (pset.sversion >= 90400) @@ -2027,7 +2027,7 @@ describeOneTableDetails(const char *schemaname, "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "c.relstorage" : "'h'"), + (isMPP() ? "c.relstorage" : "'h'"), oid); } else if (pset.sversion >= 90100) @@ -2047,7 +2047,7 @@ describeOneTableDetails(const char *schemaname, "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "c.relstorage" : "'h'"), + (isMPP() ? "c.relstorage" : "'h'"), oid); } else if (pset.sversion >= 90000) @@ -2066,7 +2066,7 @@ describeOneTableDetails(const char *schemaname, "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "c.relstorage" : "'h'"), + (isMPP() ? "c.relstorage" : "'h'"), oid); } else if (pset.sversion >= 80400) @@ -2084,7 +2084,7 @@ describeOneTableDetails(const char *schemaname, "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "c.relstorage" : "'h'"), + (isMPP() ? "c.relstorage" : "'h'"), oid); } else if (pset.sversion >= 80200) @@ -2098,7 +2098,7 @@ describeOneTableDetails(const char *schemaname, (verbose ? "pg_catalog.array_to_string(reloptions, E', ')" : "''"), /* GPDB Only: relstorage */ - (isGPDB() ? "relstorage" : "'h'"), + (isMPP() ? "relstorage" : "'h'"), oid); } else if (pset.sversion >= 80000) @@ -2161,7 +2161,7 @@ describeOneTableDetails(const char *schemaname, tableinfo.isdynamic = strcmp(PQgetvalue(res, 0, 16), "t") == 0; /* GPDB Only: relstorage */ - if (pset.sversion < 120000 && isGPDB()) + if (pset.sversion < 120000 && isMPP()) tableinfo.relstorage = *(PQgetvalue(res, 0, PQfnumber(res, "relstorage"))); else tableinfo.relstorage = 'h'; @@ -3840,7 +3840,7 @@ describeOneTableDetails(const char *schemaname, * listing them. */ tgdef = PQgetvalue(result, i, 1); - if (isGPDB() && strstr(tgdef, "RI_FKey_") != NULL) + if (isMPP() && strstr(tgdef, "RI_FKey_") != NULL) list_trigger = false; break; @@ -5145,7 +5145,7 @@ listTables(const char *tabtypes, const char *pattern, bool verbose, bool showSys cols_so_far = 4; /* Show Storage type for tables */ - if (showTables && isGPDB()) + if (showTables && isMPP()) { if (isGPDB7000OrLater()) { From 6801342a06594ded221827c48dc07964d0112ce7 Mon Sep 17 00:00:00 2001 From: FairyFar Date: Thu, 9 Apr 2026 20:17:24 +0800 Subject: [PATCH 092/128] Improve the SQL tab-completion feature for resource group (#1669) 1. For the time being, we will not handle the tab-completion feature of resource queue because there is a trend for resource group to replace resource queue. 2. Correct the description of the gp_resource_manager parameter. --- src/backend/utils/misc/guc_gp.c | 2 +- src/bin/psql/tab-complete.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 42b61dfbbbb..f846cc24aa3 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -4936,7 +4936,7 @@ struct config_string ConfigureNamesString_gp[] = { {"gp_resource_manager", PGC_POSTMASTER, RESOURCES, gettext_noop("Sets the type of resource manager."), - gettext_noop("Only support \"queue\" and \"group\" for now.") + gettext_noop("Only support \"queue\", \"group\" and \"group-v2\" for now.") }, &gp_resource_manager_str, "queue", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index dc401f503b1..905ad740555 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1614,6 +1614,12 @@ psql_completion(const char *text, int start, int end) NULL }; + static const char *const list_resource_group_type[] = { + "CONCURRENCY", "CPU_MAX_PERCENT", "CPUSET", "CPU_WEIGHT", + "MEMORY_QUOTA", "MIN_COST", "IO_LIMIT", + NULL + }; + /* * Temporary workaround for a bug in recent (2019) libedit: it incorrectly * de-escapes the input "text", causing us to fail to recognize backslash @@ -3115,8 +3121,8 @@ psql_completion(const char *text, int start, int end) else if (Matches("CREATE", "ROLE|USER|GROUP", MatchAny, "IN")) COMPLETE_WITH("GROUP", "ROLE"); -/* CREATE/DROP RESOURCE GROUP/QUEUE */ - else if (Matches("CREATE|DROP", "RESOURCE")) +/* CREATE/DROP/ALTER RESOURCE GROUP/QUEUE */ + else if (Matches("CREATE|DROP|ALTER", "RESOURCE")) { static const char *const list_CREATERESOURCEGROUP[] = {"GROUP", "QUEUE", NULL}; @@ -3130,19 +3136,19 @@ psql_completion(const char *text, int start, int end) else if (Matches("CREATE", "PROFILE", MatchAny, "LIMIT")) COMPLETE_WITH("FAILED_LOGIN_ATTEMPTS", "PASSWORD_REUSE_MAX", "PASSWORD_LOCK_TIME"); - /* CREATE/DROP RESOURCE GROUP */ - else if (TailMatches("CREATE|DROP", "RESOURCE", "GROUP")) + /* CREATE/DROP/ALTER RESOURCE GROUP */ + else if (TailMatches("CREATE|DROP|ALTER", "RESOURCE", "GROUP")) COMPLETE_WITH_QUERY(Query_for_list_of_resgroups); /* CREATE RESOURCE GROUP */ else if (TailMatches("CREATE|DROP", "RESOURCE", "GROUP", MatchAny)) COMPLETE_WITH("WITH ("); + /* ALTER RESOURCE GROUP */ + else if (TailMatches("ALTER", "RESOURCE", "GROUP", MatchAny)) + COMPLETE_WITH("SET"); + else if (TailMatches("ALTER", "RESOURCE", "GROUP", MatchAny, "SET")) + COMPLETE_WITH_LIST(list_resource_group_type); else if (TailMatches("RESOURCE", "GROUP", MatchAny, "WITH", "(")) - { - static const char *const list_CREATERESOURCEGROUP[] = - {"CONCURRENCY", "CPU_MAX_PERCENT", "CPUSET", "CPU_WEIGHT", "MEMORY_QUOTA", "MIN_COST", "IO_LIMIT", NULL}; - - COMPLETE_WITH_LIST(list_CREATERESOURCEGROUP); - } + COMPLETE_WITH_LIST(list_resource_group_type); /* CREATE TYPE */ else if (Matches("CREATE", "TYPE", MatchAny)) From eea9250df034deaee5f7df049016649cd4d29492 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 8 Apr 2026 22:47:38 +0800 Subject: [PATCH 093/128] Fix aoco_relation_size() using wrong snapshot to read pg_aocsseg aoco_relation_size() used GetLatestSnapshot() to read pg_aocsseg catalog metadata. During ALTER TABLE SET DISTRIBUTED BY on AOCO tables, the reader gang's GetLatestSnapshot() cannot see pg_aocsseg rows written by the writer gang within the same distributed transaction (uncommitted local xid), causing the function to return 0 bytes. This led to relpages=0 being passed to vac_update_relstats() alongside a non-zero totalrows from sampling (which correctly uses GetCatalogSnapshot()), triggering an assertion failure: FailedAssertion: "Gp_role == GP_ROLE_UTILITY", vacuum.c:1738 Fix by passing NULL to GetAllAOCSFileSegInfo() so that systable_beginscan() uses GetCatalogSnapshot() internally, consistent with appendonly_relation_size() for AO row tables. --- src/backend/access/aocs/aocsam_handler.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/backend/access/aocs/aocsam_handler.c b/src/backend/access/aocs/aocsam_handler.c index c2faa538e10..4b3cd2a52ef 100644 --- a/src/backend/access/aocs/aocsam_handler.c +++ b/src/backend/access/aocs/aocsam_handler.c @@ -2044,15 +2044,25 @@ static uint64 aoco_relation_size(Relation rel, ForkNumber forkNumber) { AOCSFileSegInfo **allseg; - Snapshot snapshot; uint64 totalbytes = 0; int totalseg; if (forkNumber != MAIN_FORKNUM) return totalbytes; - snapshot = RegisterSnapshot(GetLatestSnapshot()); - allseg = GetAllAOCSFileSegInfo(rel, snapshot, &totalseg, NULL); + /* + * Pass NULL as snapshot so that GetAllAOCSFileSegInfo -> systable_beginscan + * uses GetCatalogSnapshot() internally. This is consistent with + * appendonly_relation_size() for AO row tables and ensures pg_aocsseg + * entries are visible even when called within the same transaction that + * populated them (e.g. ALTER TABLE SET DISTRIBUTED BY). + * + * Using GetLatestSnapshot() here previously caused the metadata to be + * invisible on QE segments during in-transaction redistribution, leading + * to a zero return value and a subsequent assertion failure in + * vac_update_relstats(). + */ + allseg = GetAllAOCSFileSegInfo(rel, NULL, &totalseg, NULL); for (int seg = 0; seg < totalseg; seg++) { for (int attr = 0; attr < RelationGetNumberOfAttributes(rel); attr++) @@ -2079,7 +2089,6 @@ aoco_relation_size(Relation rel, ForkNumber forkNumber) FreeAllAOCSSegFileInfo(allseg, totalseg); pfree(allseg); } - UnregisterSnapshot(snapshot); return totalbytes; } From 392fbcfe720a06f283a29d5a3cf0b3ae4d80df89 Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Fri, 10 Apr 2026 06:23:35 -0700 Subject: [PATCH 094/128] CI: fix 'Check and Display Regression Diffs' step to use bash shell (#1673) * CI: fix 'Check and Display Regression Diffs' step to use bash shell The step used bash-specific [[ ]] syntax but lacked shell: bash {0}, causing failures when the runner defaulted to sh. --- .github/workflows/build-cloudberry-rocky8.yml | 1 + .github/workflows/build-cloudberry.yml | 1 + .github/workflows/build-deb-cloudberry.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index dd4d10ab115..4986eae11b2 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -1676,6 +1676,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index cc99a997c3f..b0191a2d0e9 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -1679,6 +1679,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 52db1819194..f8eadee3c8f 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -1618,6 +1618,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) From 42dc916d89cefbf1d62b81a179b281800f0dd85e Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Thu, 2 Apr 2026 18:21:09 +0800 Subject: [PATCH 095/128] Fix SIGSEGV in fsm_extend when vacuuming tables in non-default tablespace The commit df1e2ff ("Prevent CREATE TABLE from using dangling tablespace") added a call to TablespaceLockTuple() inside TablespaceCreateDbspace() for every non-default tablespace, including the common path where the per-database directory already exists. That lock acquisition calls LockSharedObject(), which calls AcceptInvalidationMessages(), allowing pending sinval messages to be processed at an unexpected point deep inside smgrcreate(). This creates a crash window during VACUUM of a heap table (including auxiliary tables such as the aoseg table of an AO relation) that lives in a non-default tablespace and has never been vacuumed before (so no FSM or VM fork exists yet): lazy_scan_heap -> visibilitymap_pin -> vm_readbuf -> vm_extend smgrcreate(VM fork) + CacheInvalidateSmgr() <- queues SHAREDINVALSMGR_ID -> RecordPageWithFreeSpace -> fsm_readbuf -> fsm_extend smgrcreate(FSM fork) -> mdcreate -> TablespaceCreateDbspace [CBDB-specific for non-default tablespaces] -> TablespaceLockTuple -> LockSharedObject -> AcceptInvalidationMessages() -> smgrclosenode() -> rel->rd_smgr = NULL rel->rd_smgr->smgr_cached_nblocks[FSM_FORKNUM] = ... -> SIGSEGV (NULL dereference) at freespace.c:637 Fix: add RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() (freespace.c) and vm_extend() (visibilitymap.c). RelationOpenSmgr is a no-op when rd_smgr is still valid, and re-opens the smgr handle if the sinval handler has closed it. The identical guard already exists earlier in both functions for the LockRelationForExtension path. Add a regression test (vacuum_fsm_nondefault_tablespace) covering both a plain heap table and an AO table in a non-default tablespace. --- src/backend/access/heap/visibilitymap.c | 8 +++ src/backend/storage/freespace/freespace.c | 8 +++ src/test/regress/expected/.gitignore | 1 + src/test/regress/greenplum_schedule | 1 + .../vacuum_fsm_nondefault_tablespace.source | 54 ++++++++++++++ .../vacuum_fsm_nondefault_tablespace.source | 70 +++++++++++++++++++ src/test/regress/sql/.gitignore | 1 + 7 files changed, 143 insertions(+) create mode 100644 src/test/regress/input/vacuum_fsm_nondefault_tablespace.source create mode 100644 src/test/regress/output/vacuum_fsm_nondefault_tablespace.source diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index e198df65d82..7d252fa94ab 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -645,6 +645,14 @@ vm_extend(Relation rel, BlockNumber vm_nblocks) !smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM)) smgrcreate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, false); + /* + * Might have to re-open if smgrcreate triggered AcceptInvalidationMessages + * (via TablespaceCreateDbspace -> LockSharedObject for non-default + * tablespaces), which may have processed a pending SHAREDINVALSMGR_ID + * message and closed our smgr entry. + */ + RelationOpenSmgr(rel); + /* Invalidate cache so that smgrnblocks() asks the kernel. */ rel->rd_smgr->smgr_cached_nblocks[VISIBILITYMAP_FORKNUM] = InvalidBlockNumber; vm_nblocks_now = smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM); diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index 796b915156b..ee97e757115 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -633,6 +633,14 @@ fsm_extend(Relation rel, BlockNumber fsm_nblocks) !smgrexists(rel->rd_smgr, FSM_FORKNUM)) smgrcreate(rel->rd_smgr, FSM_FORKNUM, false); + /* + * Might have to re-open if smgrcreate triggered AcceptInvalidationMessages + * (via TablespaceCreateDbspace -> LockSharedObject for non-default + * tablespaces), which may have processed a pending SHAREDINVALSMGR_ID + * message and closed our smgr entry. + */ + RelationOpenSmgr(rel); + /* Invalidate cache so that smgrnblocks() asks the kernel. */ rel->rd_smgr->smgr_cached_nblocks[FSM_FORKNUM] = InvalidBlockNumber; fsm_nblocks_now = smgrnblocks(rel->rd_smgr, FSM_FORKNUM); diff --git a/src/test/regress/expected/.gitignore b/src/test/regress/expected/.gitignore index c837ca324d5..f625061dbd6 100644 --- a/src/test/regress/expected/.gitignore +++ b/src/test/regress/expected/.gitignore @@ -70,3 +70,4 @@ /ao_unique_index_partition.out /bfv_copy.out /copy_encoding_error.out +/vacuum_fsm_nondefault_tablespace.out diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 3c8f7965b28..604616791c8 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -167,6 +167,7 @@ test: instr_in_shmem_verify # hold locks. test: partition_locking test: vacuum_gp +test: vacuum_fsm_nondefault_tablespace test: resource_queue_stat # background analyze may affect pgstat test: pg_stat diff --git a/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source b/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source new file mode 100644 index 00000000000..adce9ab77de --- /dev/null +++ b/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source @@ -0,0 +1,54 @@ +-- Test: VACUUM on a table in a non-default tablespace does not crash on first run. +-- +-- Bug: SIGSEGV in fsm_extend() (freespace.c:637) when vacuuming a heap table +-- (or an AO table's aoseg auxiliary table) that resides in a non-default +-- tablespace for the very first time. +-- +-- Root cause: commit "Prevent CREATE TABLE from using dangling tablespace" +-- added TablespaceLockTuple() in TablespaceCreateDbspace() for non-default +-- tablespaces. That call reaches AcceptInvalidationMessages() via +-- LockSharedObject(), which processes a pending SHAREDINVALSMGR_ID message +-- that vm_extend() had queued via CacheInvalidateSmgr(), nullifying +-- rel->rd_smgr before fsm_extend() dereferences it at freespace.c:637. +-- +-- Fix: added RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() +-- (freespace.c) and vm_extend() (visibilitymap.c) so that rd_smgr is +-- re-opened if the sinval handler closed it. + +CREATE TABLESPACE fsm_ts_test LOCATION '@testtablespace@'; + +-- Case 1: plain heap table in non-default tablespace, first VACUUM. +-- Before the fix this crashed with SIGSEGV at freespace.c:637: +-- vm_extend() -> CacheInvalidateSmgr (queues SHAREDINVALSMGR_ID) +-- fsm_extend() -> smgrcreate -> TablespaceCreateDbspace +-- -> TablespaceLockTuple -> AcceptInvalidationMessages +-- -> processes SHAREDINVALSMGR_ID -> rel->rd_smgr = NULL +-- -> rel->rd_smgr->smgr_cached_nblocks[...] = ... SIGSEGV +CREATE TABLE fsm_ts_heap (id int, val text) + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_heap SELECT i, repeat('x', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; +-- Second VACUUM must also succeed (FSM/VM now exist, different code path). +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; +DROP TABLE fsm_ts_heap; + +-- Case 2: AO table in non-default tablespace. +-- The crash occurs inside the recursive vacuum of the aoseg auxiliary table +-- (which is also a heap table stored in the same non-default tablespace). +CREATE TABLE fsm_ts_ao (id int, val text) + USING ao_row + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_ao SELECT i, repeat('y', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; +-- Second VACUUM must also succeed. +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; +DROP TABLE fsm_ts_ao; + +-- Cleanup. +DROP TABLESPACE fsm_ts_test; diff --git a/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source b/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source new file mode 100644 index 00000000000..fc2ff245691 --- /dev/null +++ b/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source @@ -0,0 +1,70 @@ +-- Test: VACUUM on a table in a non-default tablespace does not crash on first run. +-- +-- Bug: SIGSEGV in fsm_extend() (freespace.c:637) when vacuuming a heap table +-- (or an AO table's aoseg auxiliary table) that resides in a non-default +-- tablespace for the very first time. +-- +-- Root cause: commit "Prevent CREATE TABLE from using dangling tablespace" +-- added TablespaceLockTuple() in TablespaceCreateDbspace() for non-default +-- tablespaces. That call reaches AcceptInvalidationMessages() via +-- LockSharedObject(), which processes a pending SHAREDINVALSMGR_ID message +-- that vm_extend() had queued via CacheInvalidateSmgr(), nullifying +-- rel->rd_smgr before fsm_extend() dereferences it at freespace.c:637. +-- +-- Fix: added RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() +-- (freespace.c) and vm_extend() (visibilitymap.c) so that rd_smgr is +-- re-opened if the sinval handler closed it. +CREATE TABLESPACE fsm_ts_test LOCATION '@testtablespace@'; +-- Case 1: plain heap table in non-default tablespace, first VACUUM. +-- Before the fix this crashed with SIGSEGV at freespace.c:637: +-- vm_extend() -> CacheInvalidateSmgr (queues SHAREDINVALSMGR_ID) +-- fsm_extend() -> smgrcreate -> TablespaceCreateDbspace +-- -> TablespaceLockTuple -> AcceptInvalidationMessages +-- -> processes SHAREDINVALSMGR_ID -> rel->rd_smgr = NULL +-- -> rel->rd_smgr->smgr_cached_nblocks[...] = ... SIGSEGV +CREATE TABLE fsm_ts_heap (id int, val text) + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_heap SELECT i, repeat('x', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; + count +------- + 500 +(1 row) + +-- Second VACUUM must also succeed (FSM/VM now exist, different code path). +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; + count +------- + 500 +(1 row) + +DROP TABLE fsm_ts_heap; +-- Case 2: AO table in non-default tablespace. +-- The crash occurs inside the recursive vacuum of the aoseg auxiliary table +-- (which is also a heap table stored in the same non-default tablespace). +CREATE TABLE fsm_ts_ao (id int, val text) + USING ao_row + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_ao SELECT i, repeat('y', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; + count +------- + 500 +(1 row) + +-- Second VACUUM must also succeed. +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; + count +------- + 500 +(1 row) + +DROP TABLE fsm_ts_ao; +-- Cleanup. +DROP TABLESPACE fsm_ts_test; diff --git a/src/test/regress/sql/.gitignore b/src/test/regress/sql/.gitignore index 9b5f3660fa7..3a340338616 100644 --- a/src/test/regress/sql/.gitignore +++ b/src/test/regress/sql/.gitignore @@ -64,3 +64,4 @@ /ao_unique_index_partition.sql /bfv_copy.sql /copy_encoding_error.sql +/vacuum_fsm_nondefault_tablespace.sql From eb4fae1a0ba1b1804838908603d5ab6c8929c0d5 Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Thu, 16 Apr 2026 10:16:19 -0700 Subject: [PATCH 096/128] ORCA: fall back to Postgres planner for KNN ORDER BY queries (#1653) ORCA is unaware of amcanorderbyop, so it plans "ORDER BY col <-> val" queries with a full Seq Scan + Sort instead of a native KNN ordered index scan. Detect this pattern by checking whether any ORDER BY target is an operator with amoppurpose = AMOP_ORDER in pg_amop and at least one direct Var argument, then raise ExmiQuery2DXLUnsupportedFeature to hand the query off to the Postgres planner, which generates an efficient Index Only Scan with native KNN ordering. Queries where the ordering operator's arguments are entirely computed expressions (e.g. circle(col,1) <-> point(0,0)) are excluded from the fallback to avoid lossy-distance errors in index-only scans. Co-authored-by: reshke --- .../btree_gist/expected/cash_optimizer.out | 11 +- .../btree_gist/expected/date_optimizer.out | 7 +- .../btree_gist/expected/float4_optimizer.out | 7 +- .../btree_gist/expected/float8_optimizer.out | 7 +- .../btree_gist/expected/int2_optimizer.out | 7 +- .../btree_gist/expected/int4_optimizer.out | 7 +- .../btree_gist/expected/int8_optimizer.out | 7 +- .../expected/interval_optimizer.out | 24 ++-- .../btree_gist/expected/time_optimizer.out | 7 +- .../expected/timestamp_optimizer.out | 7 +- .../expected/timestamptz_optimizer.out | 7 +- .../pg_trgm/expected/pg_trgm_optimizer.out | 23 ++-- src/backend/gpopt/gpdbwrappers.cpp | 11 ++ .../gpopt/translate/CTranslatorQueryToDXL.cpp | 9 ++ src/backend/optimizer/util/walkers.c | 112 ++++++++++++++++++ src/include/gpopt/gpdbwrappers.h | 3 + src/include/optimizer/walkers.h | 1 + .../expected/create_index_optimizer.out | 45 ++++--- src/test/regress/expected/gist_optimizer.out | 63 +++++----- 19 files changed, 242 insertions(+), 123 deletions(-) diff --git a/contrib/btree_gist/expected/cash_optimizer.out b/contrib/btree_gist/expected/cash_optimizer.out index 171dec7e511..f2c9ac07420 100644 --- a/contrib/btree_gist/expected/cash_optimizer.out +++ b/contrib/btree_gist/expected/cash_optimizer.out @@ -77,12 +77,11 @@ SELECT a, a <-> '21472.79' FROM moneytmp ORDER BY a <-> '21472.79' LIMIT 3; QUERY PLAN ------------------------------------------------------------ Limit - -> Sort - Sort Key: ((a <-> '$21,472.79'::money)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Seq Scan on moneytmp - Optimizer: GPORCA + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((a <-> '$21,472.79'::money)) + -> Limit + -> Index Only Scan using moneyidx on moneytmp + Order By: (a <-> '$21,472.79'::money) (7 rows) SELECT a, a <-> '21472.79' FROM moneytmp ORDER BY a <-> '21472.79' LIMIT 3; diff --git a/contrib/btree_gist/expected/date_optimizer.out b/contrib/btree_gist/expected/date_optimizer.out index a77041f847f..12269cf169b 100644 --- a/contrib/btree_gist/expected/date_optimizer.out +++ b/contrib/btree_gist/expected/date_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '2001-02-13' FROM datetmp ORDER BY a <-> '2001-02-13' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '02-13-2001'::date)) - -> Sort - Sort Key: ((a <-> '02-13-2001'::date)) - -> Seq Scan on datetmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using dateidx on datetmp + Order By: (a <-> '02-13-2001'::date) (7 rows) SELECT a, a <-> '2001-02-13' FROM datetmp ORDER BY a <-> '2001-02-13' LIMIT 3; diff --git a/contrib/btree_gist/expected/float4_optimizer.out b/contrib/btree_gist/expected/float4_optimizer.out index cc40e9bd1ae..7b71a2f5112 100644 --- a/contrib/btree_gist/expected/float4_optimizer.out +++ b/contrib/btree_gist/expected/float4_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '-179'::real)) - -> Sort - Sort Key: ((a <-> '-179'::real)) - -> Seq Scan on float4tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using float4idx on float4tmp + Order By: (a <-> '-179'::real) (7 rows) SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; diff --git a/contrib/btree_gist/expected/float8_optimizer.out b/contrib/btree_gist/expected/float8_optimizer.out index 1bd96c44d3b..18e5c195286 100644 --- a/contrib/btree_gist/expected/float8_optimizer.out +++ b/contrib/btree_gist/expected/float8_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '-1890'::double precision)) - -> Sort - Sort Key: ((a <-> '-1890'::double precision)) - -> Seq Scan on float8tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using float8idx on float8tmp + Order By: (a <-> '-1890'::double precision) (7 rows) SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; diff --git a/contrib/btree_gist/expected/int2_optimizer.out b/contrib/btree_gist/expected/int2_optimizer.out index fdfc859097b..f8f6a428b93 100644 --- a/contrib/btree_gist/expected/int2_optimizer.out +++ b/contrib/btree_gist/expected/int2_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '237' FROM int2tmp ORDER BY a <-> '237' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '237'::smallint)) - -> Sort - Sort Key: ((a <-> '237'::smallint)) - -> Seq Scan on int2tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int2idx on int2tmp + Order By: (a <-> '237'::smallint) (7 rows) SELECT a, a <-> '237' FROM int2tmp ORDER BY a <-> '237' LIMIT 3; diff --git a/contrib/btree_gist/expected/int4_optimizer.out b/contrib/btree_gist/expected/int4_optimizer.out index 67107e63bfa..6877fb09af5 100644 --- a/contrib/btree_gist/expected/int4_optimizer.out +++ b/contrib/btree_gist/expected/int4_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '237' FROM int4tmp ORDER BY a <-> '237' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 237)) - -> Sort - Sort Key: ((a <-> 237)) - -> Seq Scan on int4tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int4idx on int4tmp + Order By: (a <-> 237) (7 rows) SELECT a, a <-> '237' FROM int4tmp ORDER BY a <-> '237' LIMIT 3; diff --git a/contrib/btree_gist/expected/int8_optimizer.out b/contrib/btree_gist/expected/int8_optimizer.out index ba8e21135e8..962dd314661 100644 --- a/contrib/btree_gist/expected/int8_optimizer.out +++ b/contrib/btree_gist/expected/int8_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '464571291354841' FROM int8tmp ORDER BY a <-> '464571291354841' Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '464571291354841'::bigint)) - -> Sort - Sort Key: ((a <-> '464571291354841'::bigint)) - -> Seq Scan on int8tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int8idx on int8tmp + Order By: (a <-> '464571291354841'::bigint) (7 rows) SELECT a, a <-> '464571291354841' FROM int8tmp ORDER BY a <-> '464571291354841' LIMIT 3; diff --git a/contrib/btree_gist/expected/interval_optimizer.out b/contrib/btree_gist/expected/interval_optimizer.out index f5afd17456b..f0a4e850aeb 100644 --- a/contrib/btree_gist/expected/interval_optimizer.out +++ b/contrib/btree_gist/expected/interval_optimizer.out @@ -74,15 +74,15 @@ SELECT count(*) FROM intervaltmp WHERE a > '199 days 21:21:23'::interval; EXPLAIN (COSTS OFF) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Sort - Sort Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Seq Scan on intervaltmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using intervalidx on intervaltmp + Order By: (a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval) + Optimizer: Postgres query optimizer (7 rows) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; @@ -96,15 +96,15 @@ SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21 SET enable_indexonlyscan=off; EXPLAIN (COSTS OFF) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Sort - Sort Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Seq Scan on intervaltmp - Optimizer: GPORCA + -> Limit + -> Index Scan using intervalidx on intervaltmp + Order By: (a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval) + Optimizer: Postgres query optimizer (7 rows) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; diff --git a/contrib/btree_gist/expected/time_optimizer.out b/contrib/btree_gist/expected/time_optimizer.out index 590ada880b9..40d49e79b02 100644 --- a/contrib/btree_gist/expected/time_optimizer.out +++ b/contrib/btree_gist/expected/time_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '10:57:11' FROM timetmp ORDER BY a <-> '10:57:11' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '10:57:11'::time without time zone)) - -> Sort - Sort Key: ((a <-> '10:57:11'::time without time zone)) - -> Seq Scan on timetmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timeidx on timetmp + Order By: (a <-> '10:57:11'::time without time zone) (7 rows) SELECT a, a <-> '10:57:11' FROM timetmp ORDER BY a <-> '10:57:11' LIMIT 3; diff --git a/contrib/btree_gist/expected/timestamp_optimizer.out b/contrib/btree_gist/expected/timestamp_optimizer.out index 1b8e709fe90..85c3a1a5e5d 100644 --- a/contrib/btree_gist/expected/timestamp_optimizer.out +++ b/contrib/btree_gist/expected/timestamp_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '2004-10-26 08:55:08' FROM timestamptmp ORDER BY a <-> '2004-10- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone)) - -> Sort - Sort Key: ((a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone)) - -> Seq Scan on timestamptmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timestampidx on timestamptmp + Order By: (a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone) (7 rows) SELECT a, a <-> '2004-10-26 08:55:08' FROM timestamptmp ORDER BY a <-> '2004-10-26 08:55:08' LIMIT 3; diff --git a/contrib/btree_gist/expected/timestamptz_optimizer.out b/contrib/btree_gist/expected/timestamptz_optimizer.out index 2173c5dca35..a9e043f98a6 100644 --- a/contrib/btree_gist/expected/timestamptz_optimizer.out +++ b/contrib/btree_gist/expected/timestamptz_optimizer.out @@ -199,10 +199,9 @@ SELECT a, a <-> '2018-12-18 10:59:54 GMT+2' FROM timestamptztmp ORDER BY a <-> ' Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone)) - -> Sort - Sort Key: ((a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone)) - -> Seq Scan on timestamptztmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timestamptzidx on timestamptztmp + Order By: (a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone) (7 rows) SELECT a, a <-> '2018-12-18 10:59:54 GMT+2' FROM timestamptztmp ORDER BY a <-> '2018-12-18 10:59:54 GMT+2' LIMIT 3; diff --git a/contrib/pg_trgm/expected/pg_trgm_optimizer.out b/contrib/pg_trgm/expected/pg_trgm_optimizer.out index 4597b8ca047..a1e9b3d299d 100644 --- a/contrib/pg_trgm/expected/pg_trgm_optimizer.out +++ b/contrib/pg_trgm/expected/pg_trgm_optimizer.out @@ -2351,6 +2351,7 @@ select t <-> 'q0987wertyu0988', t from test_trgm order by t <-> 'q0987wertyu0988 -> Limit -> Index Scan using trgm_idx on test_trgm Order By: (t <-> 'q0987wertyu0988'::text) + Optimizer: Postgres query optimizer (7 rows) select t <-> 'q0987wertyu0988', t from test_trgm order by t <-> 'q0987wertyu0988' limit 2; @@ -5003,8 +5004,8 @@ select * from test2 where t ~ '/\d+/-\d'; -- test = operator explain (costs off) select * from test2 where t = 'abcdef'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = 'abcdef'::text) @@ -5020,8 +5021,8 @@ select * from test2 where t = 'abcdef'; explain (costs off) select * from test2 where t = '%line%'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = '%line%'::text) @@ -5311,14 +5312,15 @@ select * from test2 where t ~ '/\d+/-\d'; -- test = operator explain (costs off) select * from test2 where t = 'abcdef'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = 'abcdef'::text) -> Bitmap Index Scan on test2_idx_gist Index Cond: (t = 'abcdef'::text) -(2 rows) + Optimizer: Postgres query optimizer +(6 rows) select * from test2 where t = 'abcdef'; t @@ -5328,13 +5330,14 @@ select * from test2 where t = 'abcdef'; explain (costs off) select * from test2 where t = '%line%'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = '%line%'::text) -> Bitmap Index Scan on test2_idx_gist Index Cond: (t = '%line%'::text) + Optimizer: Postgres query optimizer (6 rows) select * from test2 where t = '%line%'; @@ -5423,7 +5426,7 @@ SELECT DISTINCT city, similarity(city, 'Warsaw'), show_limit() -> Index Scan using restaurants_city_idx on restaurants Index Cond: (city % 'Warsaw'::text) Filter: (city % 'Warsaw'::text) - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (9 rows) SELECT set_limit(0.3); diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index aca95b2cc0a..4d8d8c59100 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -2012,6 +2012,17 @@ gpdb::CheckCollation(Node *node) return -1; } +bool +gpdb::HasOrderByOrderingOp(Query *query) +{ + GP_WRAP_START; + { + return has_orderby_ordering_op(query); + } + GP_WRAP_END; + return false; +} + Node * gpdb::CoerceToCommonType(ParseState *pstate, Node *node, Oid target_type, const char *context) diff --git a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp index 20cc6557c28..99d87917b38 100644 --- a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp @@ -324,6 +324,15 @@ CTranslatorQueryToDXL::CheckUnsupportedNodeTypes(Query *query) GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Non-default collation")); } + + // ORCA does not support amcanorderbyop (KNN ordered index scans). + // Fall back to the PostgreSQL planner for queries whose ORDER BY + // contains an ordering operator (e.g., <-> for distance). + if (gpdb::HasOrderByOrderingOp(query)) + { + GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, + GPOS_WSZ_LIT("ORDER BY with ordering operator (amcanorderbyop)")); + } } //--------------------------------------------------------------------------- diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c index 3b3d0311d06..be806f4daf7 100644 --- a/src/backend/optimizer/util/walkers.c +++ b/src/backend/optimizer/util/walkers.c @@ -8,11 +8,17 @@ #include "postgres.h" +#include "access/htup_details.h" +#include "catalog/pg_amop.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" #include "optimizer/walkers.h" +#include "utils/catcache.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" /** * Plan node walker related methods. @@ -1011,3 +1017,109 @@ check_collation_walker(Node *node, check_collation_context *context) } } +/* + * is_ordering_op + * + * Return true if the operator is registered as an ordering operator + * (amoppurpose = AMOP_ORDER) in any opfamily in pg_amop. + */ +static bool +is_ordering_op(Oid opno) +{ + CatCList *catlist = SearchSysCacheList1(AMOPOPID, + ObjectIdGetDatum(opno)); + + for (int i = 0; i < catlist->n_members; i++) + { + HeapTuple tp = &catlist->members[i]->tuple; + Form_pg_amop amop = (Form_pg_amop) GETSTRUCT(tp); + + if (amop->amoppurpose == AMOP_ORDER) + { + ReleaseSysCacheList(catlist); + return true; + } + } + ReleaseSysCacheList(catlist); + return false; +} + +/* + * has_plain_var_arg + * + * Return true if the OpExpr has at least one direct Var argument + * (not wrapped in a function or other expression). + * + * Implicit coercions such as RelabelType (binary-compatible casts, e.g. + * varchar -> text) are stripped before the check so that a column + * reference that was implicitly cast to match the operator's input type + * is still recognised as a plain Var. + */ +static bool +has_plain_var_arg(OpExpr *op) +{ + ListCell *arg_lc; + + foreach(arg_lc, op->args) + { + Node *arg = strip_implicit_coercions(lfirst(arg_lc)); + + if (IsA(arg, Var)) + return true; + } + return false; +} + +/* + * has_orderby_ordering_op + * + * Check if the query's ORDER BY uses ordering operators (amoppurpose = + * AMOP_ORDER in pg_amop) that the PostgreSQL planner can safely optimize + * with KNN-GiST index scans but ORCA cannot. + * + * Return true only when ALL ordering-operator expressions in ORDER BY + * have at least one direct Var (column reference) argument. Expressions + * like "circle(p,1) <-> point(0,0)" wrap the column in a function, + * which can cause "lossy distance functions are not supported in + * index-only scans" errors in the planner. In such cases we leave the + * query for ORCA to handle via Seq Scan + Sort. + */ +bool +has_orderby_ordering_op(Query *query) +{ + ListCell *lc; + bool found_ordering_op = false; + + if (query->sortClause == NIL) + return false; + + foreach(lc, query->sortClause) + { + SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); + TargetEntry *tle = get_sortgroupclause_tle(sgc, query->targetList); + Node *expr = (Node *) tle->expr; + + if (!IsA(expr, OpExpr)) + continue; + + OpExpr *opexpr = (OpExpr *) expr; + + if (!is_ordering_op(opexpr->opno)) + continue; + + /* + * Found an ordering operator. Check that at least one argument is + * a plain Var. If any ordering operator has only computed arguments + * (e.g., function calls wrapping columns), bail out immediately — + * falling back to the planner could produce lossy distance errors + * in index-only scans. + */ + found_ordering_op = true; + + if (!has_plain_var_arg(opexpr)) + return false; + } + + return found_ordering_op; +} + diff --git a/src/include/gpopt/gpdbwrappers.h b/src/include/gpopt/gpdbwrappers.h index 261cd28b5f0..9ef53169599 100644 --- a/src/include/gpopt/gpdbwrappers.h +++ b/src/include/gpopt/gpdbwrappers.h @@ -673,6 +673,9 @@ int FindNodes(Node *node, List *nodeTags); // look for nodes with non-default collation; returns 1 if any exist, -1 otherwise int CheckCollation(Node *node); +// check if ORDER BY uses an ordering operator (amcanorderbyop) unsupported by ORCA +bool HasOrderByOrderingOp(Query *query); + Node *CoerceToCommonType(ParseState *pstate, Node *node, Oid target_type, const char *context); diff --git a/src/include/optimizer/walkers.h b/src/include/optimizer/walkers.h index 6d0d38717f5..d29bc5551e8 100644 --- a/src/include/optimizer/walkers.h +++ b/src/include/optimizer/walkers.h @@ -43,5 +43,6 @@ extern List *extract_nodes_plan(Plan *pl, int nodeTag, bool descendIntoSubquerie extern List *extract_nodes_expression(Node *node, int nodeTag, bool descendIntoSubqueries); extern int find_nodes(Node *node, List *nodeTags); extern int check_collation(Node *node); +extern bool has_orderby_ordering_op(Query *query); #endif /* WALKERS_H_ */ diff --git a/src/test/regress/expected/create_index_optimizer.out b/src/test/regress/expected/create_index_optimizer.out index 65f5f92b8bd..aca6fbb1332 100644 --- a/src/test/regress/expected/create_index_optimizer.out +++ b/src/test/regress/expected/create_index_optimizer.out @@ -652,18 +652,16 @@ SELECT * FROM point_tblv WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; --SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; EXPLAIN (COSTS OFF) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Result - -> Sort - Sort Key: ((f1 <-> '(0,1)'::point)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Index Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) - Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision) AND (f1 <@ '(10,10),(-10,-10)'::box)) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((point_tbl.f1 <-> '(0,1)'::point)) + -> Index Only Scan using gpointind on point_tbl + Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Order By: (f1 <-> '(0,1)'::point) + Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision)) + Optimizer: Postgres query optimizer +(7 rows) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 @@ -767,18 +765,19 @@ SET enable_indexscan = OFF; SET enable_bitmapscan = ON; EXPLAIN (COSTS OFF) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Result + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((point_tbl.f1 <-> '(0,1)'::point)) -> Sort - Sort Key: ((f1 <-> '(0,1)'::point)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Index Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) - Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision) AND (f1 <@ '(10,10),(-10,-10)'::box)) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + Sort Key: ((point_tbl.f1 <-> '(0,1)'::point)) + -> Bitmap Heap Scan on point_tbl + Recheck Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision)) + -> Bitmap Index Scan on gpointind + Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Optimizer: Postgres query optimizer +(10 rows) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 diff --git a/src/test/regress/expected/gist_optimizer.out b/src/test/regress/expected/gist_optimizer.out index e9020c5db70..abb8b5524cf 100644 --- a/src/test/regress/expected/gist_optimizer.out +++ b/src/test/regress/expected/gist_optimizer.out @@ -98,18 +98,15 @@ select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)); explain (costs off) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by p <-> point(0.201, 0.201); - QUERY PLAN ---------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: ((p <-> '(0.201,0.201)'::point)) - -> Sort - Sort Key: ((p <-> '(0.201,0.201)'::point)) - -> Index Scan using gist_tbl_point_index on gist_tbl - Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) - Filter: (p <@ '(0.5,0.5),(0,0)'::box) - Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 -(9 rows) + QUERY PLAN +-------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((p <-> '(0.201,0.201)'::point)) + -> Index Only Scan using gist_tbl_point_index on gist_tbl + Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) + Order By: (p <-> '(0.201,0.201)'::point) + Optimizer: Postgres query optimizer +(6 rows) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by p <-> point(0.201, 0.201); @@ -132,18 +129,15 @@ order by p <-> point(0.201, 0.201); explain (costs off) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by point(0.101, 0.101) <-> p; - QUERY PLAN ---------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: (('(0.101,0.101)'::point <-> p)) - -> Sort - Sort Key: (('(0.101,0.101)'::point <-> p)) - -> Index Scan using gist_tbl_point_index on gist_tbl - Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) - Filter: (p <@ '(0.5,0.5),(0,0)'::box) - Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 -(9 rows) + QUERY PLAN +-------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: (('(0.101,0.101)'::point <-> p)) + -> Index Only Scan using gist_tbl_point_index on gist_tbl + Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) + Order By: (p <-> '(0.101,0.101)'::point) + Optimizer: Postgres query optimizer +(6 rows) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by point(0.101, 0.101) <-> p; @@ -248,18 +242,15 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6)); explain (costs off) select b from gist_tbl where b <@ box(point(5,5), point(6,6)) order by b <-> point(5.2, 5.91); - QUERY PLAN -------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: ((b <-> '(5.2,5.91)'::point)) - -> Sort - Sort Key: ((b <-> '(5.2,5.91)'::point)) - -> Index Scan using gist_tbl_box_index on gist_tbl - Index Cond: (b <@ '(6,6),(5,5)'::box) - Filter: (b <@ '(6,6),(5,5)'::box) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((b <-> '(5.2,5.91)'::point)) + -> Index Only Scan using gist_tbl_box_index on gist_tbl + Index Cond: (b <@ '(6,6),(5,5)'::box) + Order By: (b <-> '(5.2,5.91)'::point) + Optimizer: Postgres query optimizer +(6 rows) select b from gist_tbl where b <@ box(point(5,5), point(6,6)) order by b <-> point(5.2, 5.91); From 80ece001b353a7d2c3934d2bb004b8afe101a195 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 12 Nov 2025 14:28:23 +0800 Subject: [PATCH 097/128] Fix: Python build dependencies installation Improve wheel and cython dependency management in gpMgmt/bin/Makefile to handle Ubuntu 24.04's PEP 668 restrictions while maintaining compatibility with Rocky Linux and older Ubuntu versions. Changes: - Split wheel and cython dependency checks into separate commands - Add fallback to --break-system-packages flag for Ubuntu 24.04+ - Only install dependencies if not already present in the system - Maintain backward compatibility with existing build environments This resolves build failures on Ubuntu 24.04 where pip install --user is restricted by default, while preserving the existing behavior on Rocky Linux 8/9 and Ubuntu 20.04/22.04 systems. --- gpMgmt/bin/Makefile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index c5eb6ccba9c..7092700b784 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -111,8 +111,19 @@ download-python-deps: else \ echo "PyGreSQL-$(PYGRESQL_VERSION).tar.gz already exists, skipping download"; \ fi - # Install wheel and cython for PyYAML building - pip3 install --user wheel "cython<3.0.0" + # Install wheel and cython for PyYAML building (only if not exists) + @if python3 -c "import wheel" >/dev/null 2>&1; then \ + echo "wheel already exists, skipping installation"; \ + else \ + echo "Installing wheel..."; \ + pip3 install --user wheel 2>/dev/null || pip3 install --user --break-system-packages wheel; \ + fi + @if python3 -c "import cython" >/dev/null 2>&1; then \ + echo "cython already exists, skipping installation"; \ + else \ + echo "Installing cython..."; \ + pip3 install --user "cython<3.0.0" 2>/dev/null || pip3 install --user --break-system-packages "cython<3.0.0"; \ + fi # # PyGreSQL From c5b9fcbb24853b920e565af1c0ed9f29bec5502f Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Mon, 20 Apr 2026 21:43:32 -0700 Subject: [PATCH 098/128] ORCA: add optimizer_use_streaming_hashagg GUC (#1681) ORCA unconditionally sets stream_safe=true for all local HashAggs in FLocalHashAggStreamSafe, so the existing gp_use_streaming_hashagg GUC (which is only read by the Postgres planner path in cdbgroupingpaths.c) has no effect when optimizer=on. There was no way to disable streaming hash agg for ORCA plans. Introduce optimizer_use_streaming_hashagg (default on) and wire it through the standard CConfigParamMapping path: map it (negated) to a new EopttraceDisableStreamingHashAgg traceflag, and check that traceflag in FLocalHashAggStreamSafe. When the GUC is off, ORCA emits a non-streaming Partial HashAggregate that spills to disk and fully deduplicates. --- src/backend/gpopt/config/CConfigParamMapping.cpp | 7 ++++++- .../src/translate/CTranslatorExprToDXLUtils.cpp | 3 ++- .../include/naucrates/traceflags/traceflags.h | 3 +++ src/backend/utils/misc/guc_gp.c | 11 +++++++++++ src/include/utils/guc.h | 1 + src/include/utils/unsync_guc_name.h | 1 + 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/backend/gpopt/config/CConfigParamMapping.cpp b/src/backend/gpopt/config/CConfigParamMapping.cpp index 603855c50ec..62255364ba8 100644 --- a/src/backend/gpopt/config/CConfigParamMapping.cpp +++ b/src/backend/gpopt/config/CConfigParamMapping.cpp @@ -331,7 +331,12 @@ CConfigParamMapping::SConfigMappingElem CConfigParamMapping::m_elements[] = { false, // m_negate_param GPOS_WSZ_LIT( "Enable create window hash agg")}, - + + {EopttraceDisableStreamingHashAgg, &optimizer_use_streaming_hashagg, + true, // m_negate_param + GPOS_WSZ_LIT( + "Disable streaming hash agg in ORCA-generated local partial aggregations.")}, + }; //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp index 27f5cb688fe..cf497223553 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp @@ -1296,7 +1296,8 @@ CTranslatorExprToDXLUtils::FLocalHashAggStreamSafe(CExpression *pexprAgg) // is a local hash aggregate and it generates duplicates (therefore safe to stream) return (COperator::EgbaggtypeLocal == popAgg->Egbaggtype()) && - popAgg->FGeneratesDuplicates(); + popAgg->FGeneratesDuplicates() && + !GPOS_FTRACE(EopttraceDisableStreamingHashAgg); } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h index 2e489f214e5..8a18ace986a 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h @@ -250,6 +250,9 @@ enum EOptTraceFlag // Use the all key exclude the non-fixed key in AGG pds EopttraceAggRRSExcludeNonFixedKey = 103053, + // Disable streaming hash agg in ORCA-generated local partial aggregations + EopttraceDisableStreamingHashAgg = 103054, + /////////////////////////////////////////////////////// ///////////////////// statistics flags //////////////// ////////////////////////////////////////////////////// diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index f846cc24aa3..7a4433cfa98 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -154,6 +154,7 @@ bool enable_parallel_dedup_semi_join = true; bool enable_parallel_dedup_semi_reverse_join = true; bool parallel_query_use_streaming_hashagg = false; bool gp_use_streaming_hashagg = true; +bool optimizer_use_streaming_hashagg = true; int gp_appendonly_insert_files = 0; int gp_appendonly_insert_files_tuples_range = 0; int gp_random_insert_segments = 0; @@ -1909,6 +1910,16 @@ struct config_bool ConfigureNamesBool_gp[] = true, NULL, NULL }, + { + {"optimizer_use_streaming_hashagg", PGC_USERSET, DEVELOPER_OPTIONS, + gettext_noop("Use streaming hash agg in ORCA-generated local partial hash aggregations."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &optimizer_use_streaming_hashagg, + true, NULL, NULL + }, + { {"gp_force_random_redistribution", PGC_USERSET, CUSTOM_OPTIONS, gettext_noop("Force redistribution of insert for randomly-distributed."), diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index aa34138a4b5..652e0b451f3 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -295,6 +295,7 @@ extern bool enable_parallel_dedup_semi_join; extern bool enable_parallel_dedup_semi_reverse_join; extern bool parallel_query_use_streaming_hashagg; extern bool gp_use_streaming_hashagg; +extern bool optimizer_use_streaming_hashagg; extern int gp_appendonly_insert_files; extern int gp_appendonly_insert_files_tuples_range; extern int gp_random_insert_segments; diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 55a81df5bae..85ecb3548e6 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -501,6 +501,7 @@ "optimizer_skew_factor", "optimizer_use_external_constant_expression_evaluation_for_ints", "optimizer_use_gpdb_allocators", + "optimizer_use_streaming_hashagg", "optimizer_xform_bind_threshold", "parallel_leader_participation", "parallel_query_use_streaming_hashagg", From 5980dae3396fe09a6d96617e8b73dd6a85a9333e Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 18:19:14 +0800 Subject: [PATCH 099/128] Align release artifact naming with ASF incubator conventions Use base version (without -rcN suffix) for release tarball filename to align with Apache incubator release conventions. Changes: - Modified TAR_NAME to use ${BASE_VERSION} instead of ${TAG} - Tarball filename now follows pattern: apache-cloudberry-${BASE_VERSION}-src.tar.gz - Example: apache-cloudberry-2.0.0-incubating-src.tar.gz (instead of apache-cloudberry-2.0.0-incubating-rc1-src.tar.gz) Benefits: - Enables direct 'svn mv' to release repository after voting without renaming artifacts - Aligns with Apache release best practices where RC identifiers are used only for Git tags and voting process, not in final artifact names - Eliminates need to regenerate .sha512 files during promotion - Maintains consistency between tarball filename and extracted directory name - Simplifies release manager workflow The extracted directory name remains unchanged: apache-cloudberry-${BASE_VERSION}/ --- devops/release/cloudberry-release.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devops/release/cloudberry-release.sh b/devops/release/cloudberry-release.sh index 3ab044d5aab..fdc4809f2f8 100755 --- a/devops/release/cloudberry-release.sh +++ b/devops/release/cloudberry-release.sh @@ -565,9 +565,10 @@ section "Staging release: $TAG" # NOTE: For RC tags like "X.Y.Z-incubating-rcN", keep the tag as-is but # generate the tarball name and top-level directory using BASE_VERSION # (without "-rcN"). This allows promoting the voted bits without rebuilding. - # Keep -rcN in the artifact filename for RC voting, but keep the extracted - # top-level directory name as BASE_VERSION (without -rcN). - TAR_NAME="apache-cloudberry-${TAG}-src.tar.gz" + # Use BASE_VERSION for both tarball filename and extracted directory name + # to align with Apache incubator release conventions. This enables direct + # 'svn mv' to release repository after voting without renaming artifacts. + TAR_NAME="apache-cloudberry-${BASE_VERSION}-src.tar.gz" TMP_DIR=$(mktemp -d) trap 'rm -rf "$TMP_DIR"' EXIT From eb5a493fc532c5e5aa139fb57332b2b37ebe9bf6 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 16:48:12 +0800 Subject: [PATCH 100/128] CI: use commit hash for Docker actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace version tags with commit hashes for Docker GitHub Actions to comply with Apache organization security requirements. Changes: - docker/setup-qemu-action@v3 → @c7c53464625b32c7a7e944ae62b3e17d2b600130 (v3.7.0) - docker/login-action@v3 → @c94ce9fb468520275223c153574b00df6fe4bcc9 (v3.7.0) - docker/setup-buildx-action@v3 → @8d2750c68a42422c14e847fe6c8ac0403b4cbd6f (v3.12.0) - docker/build-push-action@v6 → @10e90e3645eae34f1e60eeb005ba3a3d33f178e8 (v6.19.2) Affected workflows: - .github/workflows/docker-cbdb-build-containers.yml - .github/workflows/docker-cbdb-test-containers.yml Fixes https://github.com/apache/cloudberry/issues/1687 --- .github/workflows/docker-cbdb-build-containers.yml | 8 ++++---- .github/workflows/docker-cbdb-test-containers.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index dd9ea9acd27..3ef8fae00a8 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -117,13 +117,13 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 # Login to DockerHub for pushing images # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set - name: Login to Docker Hub if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -132,7 +132,7 @@ jobs: # Enable debug mode for better troubleshooting - name: Set up Docker Buildx if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: buildkitd-flags: --debug @@ -172,7 +172,7 @@ jobs: # This creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/build/${{ matrix.platform }} push: true diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 1c8e1c8a9a2..efb98d2b7a6 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -106,12 +106,12 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 # Login to DockerHub for pushing images - name: Login to Docker Hub if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -119,7 +119,7 @@ jobs: # Setup Docker Buildx for efficient multi-architecture builds - name: Set up Docker Buildx if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: buildkitd-flags: --debug @@ -142,7 +142,7 @@ jobs: # Creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/test/${{ matrix.platform }} push: true From 9504b5757b8e95262c664064592378f7e989aa70 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 20 Apr 2026 18:44:48 +0800 Subject: [PATCH 101/128] CI: Add Ubuntu24.04 workflow with test matrix support This commit introduces a new GitHub Actions workflow for building and testing Apache Cloudberry on Ubuntu 24.04, enabling automated builds, DEB packaging, and regresssion testing. Triggers: - Push to main branch - Pull requests modifying this workflow file - Scheduled: Every Monday at 02:00 UTC - Manual workflow dispatch with optional test selection --- .../build-deb-cloudberry-ubuntu24.04.yml | 1892 +++++++++++++++++ 1 file changed, 1892 insertions(+) create mode 100644 .github/workflows/build-deb-cloudberry-ubuntu24.04.yml diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml new file mode 100644 index 00000000000..041eabc252b --- /dev/null +++ b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml @@ -0,0 +1,1892 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Apache Cloudberry Build Pipeline +# -------------------------------------------------------------------- +# Description: +# +# This workflow builds, tests, and packages Apache Cloudberry on +# Ubuntu 24.04. It ensures artifact integrity and performs installation +# tests. +# +# Workflow Overview: +# 1. **Build Job**: +# - Configures and builds Apache Cloudberry. +# - Supports debug build configuration via ENABLE_DEBUG flag. +# - Runs unit tests and verifies build artifacts. +# - Creates DEB packages (regular and debug), source tarball +# and additional files for dupload utility. +# - **Key Artifacts**: DEB package, source tarball, changes and dsc files, build logs. +# +# 2. **DEB Install Test Job**: +# - Verifies DEB integrity and installs Cloudberry. +# - Validates successful installation. +# - **Key Artifacts**: Installation logs, verification results. +# +# 3. **Report Job**: +# - Aggregates job results into a final report. +# - Sends failure notifications if any step fails. +# +# Execution Environment: +# - **Runs On**: ubuntu-22.04 with ubuntu-24.04 containers. +# - **Resource Requirements**: +# - Disk: Minimum 20GB free space. +# - Memory: Minimum 8GB RAM. +# - CPU: Recommended 4+ cores. +# +# Triggers: +# - Push to `main` branch. +# - Pull request that modifies this workflow file. +# - Scheduled: Every Monday at 02:00 UTC. +# - Manual workflow dispatch. +# +# Container Images: +# - **Build**: `apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest` +# - **Test**: `apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest` +# +# Artifacts: +# - DEB Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Changes and DSC files (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# +# Notes: +# - Supports concurrent job execution. +# - Supports debug builds with preserved symbols. +# -------------------------------------------------------------------- + +name: Apache Cloudberry Debian Build + +on: + push: + branches: [main, REL_2_STABLE] + pull_request: + paths: + - '.github/workflows/build-deb-cloudberry-ubuntu24.04.yml' + # We can enable the PR test when needed + # branches: [main, REL_2_STABLE] + # types: [opened, synchronize, reopened, edited] + schedule: + # Run every Monday at 02:00 UTC + - cron: '0 2 * * 1' + workflow_dispatch: # Manual trigger + inputs: + test_selection: + description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' + required: false + default: 'all' + type: string + reuse_artifacts_from_run_id: + description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' + required: false + default: '' + type: string + +# Note: Step details, logs, and artifacts require users to be logged into GitHub +# even for public repositories. This is a GitHub security feature and cannot +# be overridden by permissions. + +permissions: + # READ permissions allow viewing repository contents + contents: read # Required for checking out code and reading repository files + + # READ permissions for packages (Container registry, etc) + packages: read # Allows reading from GitHub package registry + + # WRITE permissions for actions includes read access to: + # - Workflow runs + # - Artifacts (requires GitHub login) + # - Logs (requires GitHub login) + actions: write + + # READ permissions for checks API: + # - Step details visibility (requires GitHub login) + # - Check run status and details + checks: read + + # READ permissions for pull request metadata: + # - PR status + # - Associated checks + # - Review states + pull-requests: read + +env: + LOG_RETENTION_DAYS: 7 + ENABLE_DEBUG: false + +jobs: + + ## ====================================================================== + ## Job: check-skip + ## ====================================================================== + + check-skip: + runs-on: ubuntu-22.04 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + steps: + - id: skip-check + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title || '' }} + PR_BODY: ${{ github.event.pull_request.body || '' }} + run: | + # Default to not skipping + echo "should_skip=false" >> "$GITHUB_OUTPUT" + + # Apply skip logic only for pull_request events + if [[ "$EVENT_NAME" == "pull_request" ]]; then + # Combine PR title and body for skip check + MESSAGE="${PR_TITLE}\n${PR_BODY}" + + # Escape special characters using printf %s + ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") + + echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" + + # Check for skip patterns + if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then + echo "should_skip=true" >> "$GITHUB_OUTPUT" + fi + else + echo "Skip logic is not applied for $EVENT_NAME events." + fi + + - name: Report Skip Status + if: steps.skip-check.outputs.should_skip == 'true' + run: | + echo "CI Skip flag detected in PR - skipping all checks." + exit 0 + + ## ====================================================================== + ## Job: prepare-test-matrix-deb + ## ====================================================================== + + prepare-test-matrix-deb: + runs-on: ubuntu-22.04 + needs: [check-skip] + if: needs.check-skip.outputs.should_skip != 'true' + outputs: + test-matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - id: set-matrix + run: | + echo "=== Matrix Preparation Diagnostics ===" + echo "Event type: ${{ github.event_name }}" + echo "Test selection input: '${{ github.event.inputs.test_selection }}'" + + # Define defaults + DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 + DEFAULT_ENABLE_CGROUPS=false + DEFAULT_ENABLE_CORE_CHECK=true + DEFAULT_PG_SETTINGS_OPTIMIZER="" + + # Define base test configurations + ALL_TESTS='{ + "include": [ + {"test":"ic-deb-good-opt-off", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"off"} + }, + {"test":"ic-deb-good-opt-on", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"on"} + }, + {"test":"pax-ic-deb-good-opt-off", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"off", + "default_table_access_method":"pax" + } + }, + {"test":"pax-ic-deb-good-opt-on", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"on", + "default_table_access_method":"pax" + } + }, + {"test":"ic-deb-contrib", + "make_configs":["contrib/auto_explain:installcheck", + "contrib/amcheck:installcheck", + "contrib/citext:installcheck", + "contrib/btree_gin:installcheck", + "contrib/btree_gist:installcheck", + "contrib/dblink:installcheck", + "contrib/dict_int:installcheck", + "contrib/dict_xsyn:installcheck", + "contrib/extprotocol:installcheck", + "contrib/file_fdw:installcheck", + "contrib/formatter_fixedwidth:installcheck", + "contrib/hstore:installcheck", + "contrib/indexscan:installcheck", + "contrib/pg_trgm:installcheck", + "contrib/indexscan:installcheck", + "contrib/pgcrypto:installcheck", + "contrib/pgstattuple:installcheck", + "contrib/tablefunc:installcheck", + "contrib/passwordcheck:installcheck", + "contrib/pg_buffercache:installcheck", + "contrib/sslinfo:installcheck"] + }, + {"test":"ic-deb-gpcontrib", + "make_configs":["gpcontrib/orafce:installcheck", + "gpcontrib/zstd:installcheck", + "gpcontrib/gp_sparse_vector:installcheck", + "gpcontrib/gp_toolkit:installcheck"] + }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, + {"test":"ic-cbdb-parallel", + "make_configs":["src/test/regress:installcheck-cbdb-parallel"] + } + ] + }' + + # Function to apply defaults + apply_defaults() { + echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ + --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ + --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ + --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ + 'def get_defaults: + { + num_primary_mirror_pairs: ($npm|tonumber), + enable_cgroups: $ec, + enable_core_check: $ecc, + pg_settings: { + optimizer: $opt + } + }; + get_defaults * .' + } + + # Extract all valid test names from ALL_TESTS + VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') + + # Parse input test selection + IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" + + # Default to all tests if selection is empty or 'all' + if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then + mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" + fi + + # Validate and filter selected tests + INVALID_TESTS=() + FILTERED_TESTS=() + for TEST in "${SELECTED_TESTS[@]}"; do + TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace + if echo "$VALID_TESTS" | grep -qw "$TEST"; then + FILTERED_TESTS+=("$TEST") + else + INVALID_TESTS+=("$TEST") + fi + done + + # Handle invalid tests + if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then + echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" + echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" + exit 1 + fi + + # Build result JSON with defaults applied + RESULT='{"include":[' + FIRST=true + for TEST in "${FILTERED_TESTS[@]}"; do + CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") + FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") + if [[ "$FIRST" == true ]]; then + FIRST=false + else + RESULT="${RESULT}," + fi + RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" + done + RESULT="${RESULT}]}" + + # Output the matrix for GitHub Actions + echo "Final matrix configuration:" + echo "$RESULT" | jq . + + # Fix: Use block redirection + { + echo "matrix<> "$GITHUB_OUTPUT" + + echo "=== Matrix Preparation Complete ===" + + ## ====================================================================== + ## Job: build-deb + ## ====================================================================== + + build-deb: + name: Build Apache Cloudberry DEB (Ubuntu 24.04) + env: + JOB_TYPE: build + needs: [check-skip] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + if: github.event.inputs.reuse_artifacts_from_run_id == '' + outputs: + build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Set build timestamp + id: set_timestamp # Add an ID to reference this step + run: | + timestamp=$(date +'%Y%m%d_%H%M%S') + echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs + echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable + + - name: Checkout Apache Cloudberry + uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: true + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Generate Build Job Summary Start + run: | + { + echo "# Build Job Summary (Ubuntu 24.04)" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" + echo "- OS Version: $(lsb_release -sd)" + echo "- GCC Version: $(gcc --version | head -n1)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run Apache Cloudberry configure script + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Run Apache Cloudberry build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Verify build artifacts + shell: bash + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + echo "Verifying build artifacts..." + { + echo "=== Build Artifacts Verification ===" + echo "Timestamp: $(date -u)" + + if [ ! -d "${BUILD_DESTINATION}" ]; then + echo "::error::Build artifacts directory not found" + exit 1 + fi + + # Verify critical binaries + critical_binaries=( + "${BUILD_DESTINATION}/bin/postgres" + "${BUILD_DESTINATION}/bin/psql" + ) + + echo "Checking critical binaries..." + for binary in "${critical_binaries[@]}"; do + if [ ! -f "$binary" ]; then + echo "::error::Critical binary missing: $binary" + exit 1 + fi + if [ ! -x "$binary" ]; then + echo "::error::Binary not executable: $binary" + exit 1 + fi + echo "Binary verified: $binary" + ls -l "$binary" + done + + # Test binary execution + echo "Testing binary execution..." + if ! ${BUILD_DESTINATION}/bin/postgres --version; then + echo "::error::postgres binary verification failed" + exit 1 + fi + if ! ${BUILD_DESTINATION}/bin/psql --version; then + echo "::error::psql binary verification failed" + exit 1 + fi + + echo "All build artifacts verified successfully" + } 2>&1 | tee -a build-logs/details/build-verification.log + + - name: Create Source tarball, create DEB and verify artifacts + shell: bash + env: + CBDB_VERSION: 99.0.0 + BUILD_NUMBER: 1 + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Artifact Creation Log ===" + echo "Timestamp: $(date -u)" + + cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu24.04/* debian/ + chown -R "$(whoami)" debian + chmod -x debian/*install + + # replace not supported symbols in version + CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/\//./g") + CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/_/-/g") + + echo "We will built ${CBDB_VERSION}" + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + if ! ${SRC_DIR}/devops/build/packaging/deb/build-deb.sh -v $CBDB_VERSION; then + echo "::error::Build script failed" + exit 1 + fi + + ARCH=$(dpkg --print-architecture) + # Detect OS distribution (e.g., ubuntu24.04, debian12) + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') + else + OS_DISTRO="unknown" + fi + CBDB_PKG_VERSION=${CBDB_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} + + echo "Produced artifacts" + ls -l ../ + + echo "Copy artifacts to subdirectory for sign/upload" + mkdir ${SRC_DIR}/deb + DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".deb + DBG_DEB_FILE="apache-cloudberry-db-incubating-dbgsym_${CBDB_PKG_VERSION}"_"${ARCH}".ddeb + CHANGES_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".changes + BUILDINFO_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".buildinfo + DSC_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".dsc + SOURCE_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".tar.xz + cp ../"${DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${DBG_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${CHANGES_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${BUILDINFO_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${DSC_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${SOURCE_FILE}" "${SRC_DIR}/deb" + mkdir "${SRC_DIR}/deb/debian" + cp debian/changelog "${SRC_DIR}/deb/debian" + + # Get package information + echo "Package Information:" + dpkg --info "${SRC_DIR}/deb/${DEB_FILE}" + dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" + + # Verify critical files in DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" | grep -c "${binary}$"; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + # Record checksums + echo "Calculating checksums..." + sha256sum "${SRC_DIR}/deb/${DEB_FILE}" | tee -a build-logs/details/checksums.log + + echo "Artifacts created and verified successfully" + + + } 2>&1 | tee -a build-logs/details/artifact-creation.log + + - name: Run Apache Cloudberry unittest script + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then + echo "::error::Unittest script failed" + exit 1 + fi + + - name: Generate Build Job Summary End + run: | + { + echo "## Build Results" + echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload build logs + uses: actions/upload-artifact@v4 + with: + name: build-logs-ubuntu24.04-${{ env.BUILD_TIMESTAMP }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Cloudberry DEB build artifacts + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + deb/*.deb + deb/*.ddeb + + - name: Upload Cloudberry deb source build artifacts + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-source-build-artifacts + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + deb/*.tar.xz + deb/*.changes + deb/*.dsc + deb/*.buildinfo + deb/debian/changelog + + ## ====================================================================== + ## Job: deb-install-test + ## ====================================================================== + + deb-install-test: + name: DEB Install Test Apache Cloudberry (Ubuntu 24.04) + needs: [check-skip, build-deb] + if: | + !cancelled() && + (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') && + github.event.inputs.reuse_artifacts_from_run_id == '' + runs-on: ubuntu-22.04 + timeout-minutes: 120 + + container: + image: apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "DEB install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Download Cloudberry DEB build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + path: ${{ github.workspace }}/deb_build_artifacts + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + merge-multiple: false + + - name: Cloudberry Environment Initialization + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + LOGS_DIR: install-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Verify DEB artifacts + id: verify-artifacts + shell: bash + run: | + set -eo pipefail + + DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) + if [ ! -f "${DEB_FILE}" ]; then + echo "::error::DEB file not found" + exit 1 + fi + + echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying DEB artifacts..." + { + echo "=== DEB Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + + # Get DEB metadata and verify contents + echo "Package Information:" + dpkg-deb -f "${DEB_FILE}" + + # Get key DEB attributes for verification + DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) + DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) + echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + echo "DEB Details:" + echo "- Version: ${DEB_VERSION}" + echo "- Release: ${DEB_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${DEB_FILE}" + + } 2>&1 | tee -a install-logs/details/deb-verification.log + + - name: Install Cloudberry DEB + shell: bash + env: + DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} + DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} + DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${DEB_FILE}" ]; then + echo "::error::DEB_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== DEB Installation Log ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + echo "Version: ${DEB_VERSION}" + echo "Release: ${DEB_RELEASE}" + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install DEB + echo "Starting installation..." + apt-get update + if ! apt-get -y install "${DEB_FILE}"; then + echo "::error::DEB installation failed" + exit 1 + fi + + # Change ownership back to gpadmin - it is needed for future tests + chown -R gpadmin:gpadmin /usr/local/cloudberry-db + + echo "Installation completed successfully" + dpkg-query -s apache-cloudberry-db-incubating + echo "Installed files:" + dpkg-query -L apache-cloudberry-db-incubating + } 2>&1 | tee -a install-logs/details/deb-installation.log + + - name: Upload install logs + uses: actions/upload-artifact@v4 + with: + name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + install-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Generate Install Test Job Summary End + if: always() + shell: bash {0} + run: | + { + echo "# Installed Package Summary (Ubuntu 24.04)" + echo "\`\`\`" + + dpkg-query -s apache-cloudberry-db-incubating + echo "\`\`\`" + } >> "$GITHUB_STEP_SUMMARY" || true + + ## ====================================================================== + ## Job: test-deb + ## ====================================================================== + + test-deb: + name: ${{ matrix.test }} (Ubuntu 24.04) + needs: [check-skip, build-deb, prepare-test-matrix-deb] + if: | + !cancelled() && + (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') + runs-on: ubuntu-22.04 + timeout-minutes: 120 + # actionlint-allow matrix[*].pg_settings + strategy: + fail-fast: false # Continue with other tests if one fails + matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} + + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest + options: >- + --privileged + --user root + --hostname cdw + --shm-size=2gb + --ulimit core=-1 + --cgroupns=host + -v /sys/fs/cgroup:/sys/fs/cgroup:rw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Use timestamp from previous job + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "Timestamp from output: ${{ needs.build-deb.outputs.build_timestamp }}" + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Setup cgroups + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + run: | + set -uxo pipefail + + if [ "${{ matrix.enable_cgroups }}" = "true" ]; then + + echo "Current mounts:" + mount | grep cgroup + + CGROUP_BASEDIR=/sys/fs/cgroup + + # 1. Basic setup with permissions + sudo chmod -R 777 ${CGROUP_BASEDIR}/ + sudo mkdir -p ${CGROUP_BASEDIR}/gpdb + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 2. Enable controllers + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true + + # 3. CPU settings + sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true + sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true + sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true + + # 4. Memory settings + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true + + # 5. IO settings + echo "Available block devices:" + lsblk + + sudo bash -c " + if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then + echo 'Detected IO devices:' + cat \${CGROUP_BASEDIR}/gpdb/io.stat + fi + echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true + " + + # 6. Fix permissions again after all writes + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 7. Check required files + echo "Checking required files:" + required_files=( + "cgroup.procs" + "cpu.max" + "cpu.pressure" + "cpu.weight" + "cpu.weight.nice" + "cpu.stat" + "cpuset.cpus" + "cpuset.mems" + "cpuset.cpus.effective" + "cpuset.mems.effective" + "memory.current" + "io.max" + ) + + for file in "${required_files[@]}"; do + if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then + echo "✓ $file exists" + ls -l "${CGROUP_BASEDIR}/gpdb/$file" + else + echo "✗ $file missing" + fi + done + + # 8. Test subdirectory creation + echo "Testing subdirectory creation..." + sudo -u gpadmin bash -c " + TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 + if mkdir -p \$TEST_DIR; then + echo 'Created test directory' + sudo chmod -R 777 \$TEST_DIR + if echo \$\$ > \$TEST_DIR/cgroup.procs; then + echo 'Successfully wrote to cgroup.procs' + cat \$TEST_DIR/cgroup.procs + # Move processes back to parent before cleanup + echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs + else + echo 'Failed to write to cgroup.procs' + ls -la \$TEST_DIR/cgroup.procs + fi + ls -la \$TEST_DIR/ + rmdir \$TEST_DIR || { + echo 'Moving all processes to parent before cleanup' + cat \$TEST_DIR/cgroup.procs | while read pid; do + echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true + done + rmdir \$TEST_DIR + } + else + echo 'Failed to create test directory' + fi + " + + # 9. Verify setup as gpadmin user + echo "Testing cgroup access as gpadmin..." + sudo -u gpadmin bash -c " + echo 'Checking mounts...' + mount | grep cgroup + + echo 'Checking /proc/self/mounts...' + cat /proc/self/mounts | grep cgroup + + if ! grep -q cgroup2 /proc/self/mounts; then + echo 'ERROR: cgroup2 mount NOT visible to gpadmin' + exit 1 + fi + echo 'SUCCESS: cgroup2 mount visible to gpadmin' + + if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then + echo 'ERROR: gpadmin cannot write to gpdb cgroup' + exit 1 + fi + echo 'SUCCESS: gpadmin can write to gpdb cgroup' + + echo 'Verifying key files content:' + echo 'cpu.max:' + cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' + echo 'cpuset.cpus:' + cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' + echo 'cgroup.subtree_control:' + cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' + " + + # 10. Show final state + echo "Final cgroup state:" + ls -la ${CGROUP_BASEDIR}/gpdb/ + echo "Cgroup setup completed successfully" + else + echo "Cgroup setup skipped" + fi + + - name: "Generate Test Job Summary Start: ${{ matrix.test }}" + if: always() + run: | + { + echo "# Test Job Summary: ${{ matrix.test }} (Ubuntu 24.04)" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## Skip Status" + echo "✓ Test execution skipped via CI skip flag" + else + echo "- OS Version: $(cat /etc/redhat-release)" + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download Cloudberry DEB build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + path: ${{ github.workspace }}/deb_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download Cloudberry Source build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-source-build-artifacts + path: ${{ github.workspace }}/source_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify DEB artifacts + if: needs.check-skip.outputs.should_skip != 'true' + id: verify-artifacts + shell: bash + run: | + set -eo pipefail + + SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-db-incubating_*.tar.xz) + if [ ! -f "${SRC_TARBALL_FILE}" ]; then + echo "::error::SRC TARBALL file not found" + exit 1 + fi + + echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying SRC TARBALL artifacts..." + { + echo "=== SRC TARBALL Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${SRC_TARBALL_FILE}" + + } 2>&1 | tee -a build-logs/details/src-tarball-verification.log + + DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) + if [ ! -f "${DEB_FILE}" ]; then + echo "::error::DEB file not found" + exit 1 + fi + + echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying DEB artifacts..." + { + echo "=== DEB Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + + # Get DEB metadata and verify contents + echo "Package Information:" + dpkg-deb -f "${DEB_FILE}" + + # Get key DEB attributes for verification + DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) + DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) + echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + echo "DEB Details:" + echo "- Version: ${DEB_VERSION}" + echo "- Release: ${DEB_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${DEB_FILE}" + + } 2>&1 | tee -a build-logs/details/deb-verification.log + + - name: Install Cloudberry DEB + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} + DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} + DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${DEB_FILE}" ]; then + echo "::error::DEB_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== DEB Installation Log ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + echo "Version: ${DEB_VERSION}" + echo "Release: ${DEB_RELEASE}" + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install DEB + echo "Starting installation..." + apt-get update + if ! apt-get -y install "${DEB_FILE}"; then + echo "::error::DEB installation failed" + exit 1 + fi + + # Change ownership back to gpadmin - it is needed for future tests + chown -R gpadmin:gpadmin /usr/local/cloudberry-db + + echo "Installation completed successfully" + dpkg-query -s apache-cloudberry-db-incubating + echo "Installed files:" + dpkg-query -L apache-cloudberry-db-incubating + } 2>&1 | tee -a build-logs/details/deb-installation.log + + - name: Extract source tarball + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Source Extraction Log ===" + echo "Timestamp: $(date -u)" + + echo "Starting extraction..." + file "${SRC_TARBALL_FILE}" + if ! time tar xf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then + echo "::error::Source extraction failed" + exit 1 + fi + + echo "Extraction completed successfully" + echo "Extracted contents:" + ls -la "${SRC_DIR}/../cloudberry" + echo "Directory size:" + du -sh "${SRC_DIR}/../cloudberry" + } 2>&1 | tee -a build-logs/details/source-extraction.log + + - name: Prepare DEB Environment + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + + # change ownership to gpadmin + chown -R gpadmin "${SRC_DIR}/../cloudberry" + touch build-logs/sections.log + chown gpadmin build-logs/sections.log + chmod 777 build-logs + + # configure link lib directory to temporary location, fix it + rm -rf "${SRC_DIR}"/debian/build/lib + ln -sf /usr/cloudberry-db/lib "${SRC_DIR}"/debian/build/lib + + # check if regress.so exists in src directory - it is needed for contrib/dblink tests + if [ ! -f ${SRC_DIR}/src/test/regress/regress.so ]; then + ln -sf /usr/cloudberry-db/lib/postgresql/regress.so ${SRC_DIR}/src/test/regress/regress.so + fi + + # FIXME + # temporary install gdb - delete after creating new docker build/test contaners + apt-get update + apt-get -y install gdb + + } 2>&1 | tee -a build-logs/details/prepare-deb-env.log + + - name: Create Apache Cloudberry demo cluster + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + + # Build BLDWRAP_POSTGRES_CONF_ADDONS for shared_preload_libraries if specified + EXTRA_CONF="" + if [[ -n "${{ matrix.shared_preload_libraries }}" ]]; then + EXTRA_CONF="shared_preload_libraries='${{ matrix.shared_preload_libraries }}'" + echo "Adding shared_preload_libraries: ${{ matrix.shared_preload_libraries }}" + fi + + if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' BLDWRAP_POSTGRES_CONF_ADDONS=\"${EXTRA_CONF}\" SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + + } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log + + - name: "Run Tests: ${{ matrix.test }}" + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: ${{ github.workspace }}/debian/build + shell: bash {0} + run: | + set -o pipefail + + # Initialize test status + overall_status=0 + + # Create logs directory structure + mkdir -p build-logs/details + + # Core file config + mkdir -p "/tmp/cloudberry-cores" + chmod 1777 "/tmp/cloudberry-cores" + sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" + sysctl kernel.core_pattern + su - gpadmin -c "ulimit -c" + + # WARNING: PostgreSQL Settings + # When adding new pg_settings key/value pairs: + # 1. Add a new check below for the setting + # 2. Follow the same pattern as optimizer + # 3. Update matrix entries to include the new setting + + + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + + # Set PostgreSQL options if defined + PG_OPTS="" + if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" + fi + + if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" + fi + + # Read configs into array + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting test execution for ${{ matrix.test }} ===" + echo "Number of configurations to execute: ${#configs[@]}" + echo "" + + # Execute each config separately + for ((i=0; i<${#configs[@]}; i++)); do + config="${configs[$i]}" + IFS=':' read -r dir target <<< "$config" + + echo "=== Executing configuration $((i+1))/${#configs[@]} ===" + echo "Make command: make -C $dir $target" + echo "Environment:" + echo "- PGOPTIONS: ${PG_OPTS}" + + # Create unique log file for this configuration + config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" + + # Clean up any existing core files + echo "Cleaning up existing core files..." + rm -f /tmp/cloudberry-cores/core-* + + # Execute test script with proper environment setup + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + MAKE_NAME='${{ matrix.test }}-config$i' \ + MAKE_TARGET='$target' \ + MAKE_DIRECTORY='-C $dir' \ + PGOPTIONS='${PG_OPTS}' \ + SRC_DIR='${SRC_DIR}' \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ + 2>&1 | tee "$config_log"; then + echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" + overall_status=1 + fi + + # Check for results directory + results_dir="${dir}/results" + + if [[ -d "$results_dir" ]]; then + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + + find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + else + echo "-----------------------------------------" + echo "Results directory $results_dir does not exit" + echo "-----------------------------------------" + fi + + # Analyze any core files generated by this test configuration + echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." + test_id="${{ matrix.test }}-config$i" + + # List the cores directory + echo "-----------------------------------------" + echo "Cores directory: /tmp/cloudberry-cores" + echo "Contents of cores directory:" + ls -Rl "/tmp/cloudberry-cores" + echo "-----------------------------------------" + + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" + core_analysis_rc=$? + case "$core_analysis_rc" in + 0) echo "No core dumps found for this configuration" ;; + 1) echo "Core dumps were found and analyzed successfully" ;; + 2) echo "::warning::Issues encountered during core dump analysis" ;; + *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; + esac + + echo "Log file: $config_log" + echo "=== End configuration $((i+1)) execution ===" + echo "" + done + + echo "=== Test execution completed ===" + echo "Log files:" + ls -l build-logs/details/ + + # Store number of configurations for parsing step + echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" + + # Report overall status + if [ $overall_status -eq 0 ]; then + echo "All test executions completed successfully" + else + echo "::warning::Some test executions failed, check individual logs for details" + fi + + exit $overall_status + + - name: "Parse Test Results: ${{ matrix.test }}" + id: test-results + if: always() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + shell: bash {0} + run: | + set -o pipefail + + overall_status=0 + + # Get configs array to create context for results + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting results parsing for ${{ matrix.test }} ===" + echo "Number of configurations to parse: ${#configs[@]}" + echo "" + + # Parse each configuration's results independently + for ((i=0; i "test_results.$i.txt" + overall_status=1 + continue + fi + + # Parse this configuration's results + + MAKE_NAME="${{ matrix.test }}-config$i" \ + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" + status_code=$? + + { + echo "SUITE_NAME=${{ matrix.test }}" + echo "DIR=${dir}" + echo "TARGET=${target}" + } >> test_results.txt + + # Process return code + case $status_code in + 0) # All tests passed + echo "All tests passed successfully" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + ;; + 1) # Tests failed but parsed successfully + echo "Test failures detected but properly parsed" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + overall_status=1 + ;; + 2) # Parse error or missing file + echo "::warning::Could not parse test results properly for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=parse_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + *) # Unexpected error + echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=unknown_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + esac + + echo "Results stored in test_results.$i.txt" + echo "=== End parsing for configuration $((i+1)) ===" + echo "" + done + + # Report status of results files + echo "=== Results file status ===" + echo "Generated results files:" + for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Test Metadata + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-metadata-${{ matrix.test }} + path: | + test_results*.txt + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test results files + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test regression logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + gpAux/gpdemo/datadirs/standby/log/ + gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ + gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ + gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + ## ====================================================================== + ## Job: report-deb + ## ====================================================================== + + report-deb: + name: Generate Apache Cloudberry Build Report (Ubuntu 24.04) + needs: [check-skip, build-deb, prepare-test-matrix-deb, deb-install-test, test-deb] + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Generate Final Report + run: | + { + echo "# Apache Cloudberry Build Pipeline Report" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## CI Skip Status" + echo "✅ CI checks skipped via skip flag" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + else + echo "## Job Status" + echo "- Build Job: ${{ needs.build-deb.result }}" + echo "- Test Job: ${{ needs.test-deb.result }}" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.build-deb.result }}" == "success" && "${{ needs.test-deb.result }}" == "success" ]]; then + echo "✅ Pipeline completed successfully" + else + echo "⚠️ Pipeline completed with failures" + + if [[ "${{ needs.build-deb.result }}" != "success" ]]; then + echo "### Build Job Failure" + echo "Check build logs for details" + fi + + if [[ "${{ needs.test-deb.result }}" != "success" ]]; then + echo "### Test Job Failure" + echo "Check test logs and regression files for details" + fi + fi + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Notify on failure + if: | + needs.check-skip.outputs.should_skip != 'true' && + (needs.build-deb.result != 'success' || needs.test-deb.result != 'success') + run: | + echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" + echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "Build Result: ${{ needs.build-deb.result }}" + echo "Test Result: ${{ needs.test-deb.result }}" From 08f0bd3f83a6ef255b0154d7853039b73648a2b9 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 10:41:47 +0800 Subject: [PATCH 102/128] Fix Python 3.12 SyntaxWarning in orphaned_toast_tables_check.py Use raw string literal (r""") for SQL query in orphaned_toast_tables_check.py to avoid SyntaxWarning on Python 3.12. The query contains `\d` for PostgreSQL regex which Python 3.12 incorrectly interprets as an invalid escape sequence, causing test failures on Ubuntu 24.04. See: https://github.com/apache/cloudberry/issues/1686 --- gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py index 21ec8d18047..789e1b139d2 100644 --- a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py @@ -25,7 +25,7 @@ def __init__(self): # pg_depend back to pg_class, and if the table oids don't match and/or # one is missing, the TOAST table is considered to be an orphan. # Note: Handles toast tables which is created/used by InitTempTableNamespace(). - self.orphaned_toast_tables_query = """ + self.orphaned_toast_tables_query = r""" SELECT gp_segment_id AS content_id, toast_table_oid, From 93a38e71c88d82594287c279c3546d5ba9d8ca87 Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Wed, 22 Apr 2026 20:38:59 -0700 Subject: [PATCH 103/128] Fix colNDVBySeg attnum index mismatch in column-specific ANALYZE (#1680) * Fix colNDVBySeg index mismatch in do_analyze_rel When ANALYZE is run on specific columns (e.g., ANALYZE t (col)) or when a table has dropped columns, the vacattrstats loop index `i` diverges from the attribute's actual attnum-1 index used by colNDVBySeg. Two fixes: 1. QD side (line 887): read colNDVBySeg[attnum-1] instead of colNDVBySeg[i] when storing stadistinctbyseg. 2. Segment side (line 1011): write ctx->stadistincts[attnum-1] instead of ctx->stadistincts[i] when collecting per-segment NDV. * Add regression test for colNDVBySeg index mismatch in do_analyze_rel ANALYZE t(b) puts column b at loop index i=0 on the QD, but b has attnum=2, so attnum-1=1 != i=0. The fix in do_analyze_rel (using attnum-1 instead of i to index colNDVBySeg) ensures stadistinctbyseg is read from the correct per-segment NDV slot. Test verifies stadistinctbyseg for column b equals 100 (all distinct) rather than ~5 (NDV of column a at index 0). --- src/backend/commands/analyze.c | 4 ++-- src/test/regress/expected/analyze.out | 27 +++++++++++++++++++++++++++ src/test/regress/sql/analyze.sql | 23 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index c15dcdc8213..6f8b5e3d0ec 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -884,7 +884,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, if (Gp_role == GP_ROLE_DISPATCH && GpPolicyIsPartitioned(onerel->rd_cdbpolicy)) { - stats->stadistinctbyseg = colNDVBySeg[i]; + stats->stadistinctbyseg = colNDVBySeg[stats->attr->attnum - 1]; } stats->tupDesc = onerel->rd_att; @@ -1008,7 +1008,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, if (Gp_role == GP_ROLE_EXECUTE) { Assert(ctx->stadistincts); - ctx->stadistincts[i] = Float8GetDatum(stats->stadistinct); + ctx->stadistincts[stats->attr->attnum - 1] = Float8GetDatum(stats->stadistinct); } MemoryContextResetAndDeleteChildren(col_context); diff --git a/src/test/regress/expected/analyze.out b/src/test/regress/expected/analyze.out index 843a728c9b6..74169660301 100644 --- a/src/test/regress/expected/analyze.out +++ b/src/test/regress/expected/analyze.out @@ -1314,3 +1314,30 @@ select * from pg_stats where tablename like 'part2'; (1 row) drop table multipart cascade; +-- +-- Test column-specific ANALYZE correctly uses attnum-based NDV index (not loop index). +-- When ANALYZE t(b) is run, the QD loop has i=0 for column b (attnum=2), +-- so attnum-1=1 != i=0. Without the fix, colNDVBySeg[i=0] reads column a's NDV +-- instead of column b's NDV. +-- +CREATE TABLE analyze_col_ndv_drop (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO analyze_col_ndv_drop SELECT i%5, i, i%50 FROM generate_series(1, 100) i; +-- ANALYZE specific column b: QD loop has i=0, b.attnum=2, so attnum-1=1 != i=0 +ANALYZE analyze_col_ndv_drop (b); +-- stadistinctbyseg for b should be 100 (all distinct), not ~5 (NDV of column a at index 0) +SELECT a.attname, + CASE WHEN s.stakind1 = 8 THEN array_to_string(s.stavalues1, ',') + WHEN s.stakind2 = 8 THEN array_to_string(s.stavalues2, ',') + WHEN s.stakind3 = 8 THEN array_to_string(s.stavalues3, ',') + WHEN s.stakind4 = 8 THEN array_to_string(s.stavalues4, ',') + WHEN s.stakind5 = 8 THEN array_to_string(s.stavalues5, ',') + END AS stadistinctbyseg +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'analyze_col_ndv_drop'::regclass AND a.attname = 'b'; + attname | stadistinctbyseg +---------+------------------ + b | 100 +(1 row) + +DROP TABLE analyze_col_ndv_drop; diff --git a/src/test/regress/sql/analyze.sql b/src/test/regress/sql/analyze.sql index fa2ce8b834f..6d1c7ddd159 100644 --- a/src/test/regress/sql/analyze.sql +++ b/src/test/regress/sql/analyze.sql @@ -677,3 +677,26 @@ analyze verbose p2; select * from pg_stats where tablename like 'part2'; drop table multipart cascade; + +-- +-- Test column-specific ANALYZE correctly uses attnum-based NDV index (not loop index). +-- When ANALYZE t(b) is run, the QD loop has i=0 for column b (attnum=2), +-- so attnum-1=1 != i=0. Without the fix, colNDVBySeg[i=0] reads column a's NDV +-- instead of column b's NDV. +-- +CREATE TABLE analyze_col_ndv_drop (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO analyze_col_ndv_drop SELECT i%5, i, i%50 FROM generate_series(1, 100) i; +-- ANALYZE specific column b: QD loop has i=0, b.attnum=2, so attnum-1=1 != i=0 +ANALYZE analyze_col_ndv_drop (b); +-- stadistinctbyseg for b should be 100 (all distinct), not ~5 (NDV of column a at index 0) +SELECT a.attname, + CASE WHEN s.stakind1 = 8 THEN array_to_string(s.stavalues1, ',') + WHEN s.stakind2 = 8 THEN array_to_string(s.stavalues2, ',') + WHEN s.stakind3 = 8 THEN array_to_string(s.stavalues3, ',') + WHEN s.stakind4 = 8 THEN array_to_string(s.stavalues4, ',') + WHEN s.stakind5 = 8 THEN array_to_string(s.stavalues5, ',') + END AS stadistinctbyseg +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'analyze_col_ndv_drop'::regclass AND a.attname = 'b'; +DROP TABLE analyze_col_ndv_drop; From 9476679637419ab4860eded5f1fe0d79a8c2e262 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 11:26:16 +0800 Subject: [PATCH 104/128] DevOps: upgrade Go to 1.24.13 in Docker build images Update Go installation in all Docker build containers to use the latest Go 1.24.13 release instead of 1.23.4, with corresponding SHA256 checksums for both amd64 and arm64 architectures. Affected files: - devops/deploy/docker/build/rocky8/Dockerfile - devops/deploy/docker/build/rocky9/Dockerfile - devops/deploy/docker/build/ubuntu22.04/Dockerfile - devops/deploy/docker/build/ubuntu24.04/Dockerfile Updated SHA256 checksums: - linux-amd64: 1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730 - linux-arm64: 74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3 --- devops/deploy/docker/build/rocky8/Dockerfile | 6 +++--- devops/deploy/docker/build/rocky9/Dockerfile | 6 +++--- devops/deploy/docker/build/ubuntu22.04/Dockerfile | 6 +++--- devops/deploy/docker/build/ubuntu24.04/Dockerfile | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index 45d6706e593..2c19236bd76 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -150,14 +150,14 @@ RUN dnf makecache && \ make -j$(nproc) && \ make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ rm -rf ~/xerces-c* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 26190109ef0..62289f51371 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -151,14 +151,14 @@ RUN dnf makecache && \ make -j$(nproc) && \ make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ rm -rf ~/xerces-c* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/ubuntu22.04/Dockerfile b/devops/deploy/docker/build/ubuntu22.04/Dockerfile index 3023a9fce67..8c0e4cf3bac 100644 --- a/devops/deploy/docker/build/ubuntu22.04/Dockerfile +++ b/devops/deploy/docker/build/ubuntu22.04/Dockerfile @@ -144,14 +144,14 @@ RUN apt-get update && \ quilt \ unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/ubuntu24.04/Dockerfile b/devops/deploy/docker/build/ubuntu24.04/Dockerfile index c4f4e646720..762456d0b84 100644 --- a/devops/deploy/docker/build/ubuntu24.04/Dockerfile +++ b/devops/deploy/docker/build/ubuntu24.04/Dockerfile @@ -144,14 +144,14 @@ RUN apt-get update && \ quilt \ unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ From 9a944742a95ba5a6a332f41dfcdb32162b71116e Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 14:24:20 +0800 Subject: [PATCH 105/128] CI: update docker/setup-qemu-action to v4.0.0 See: https://github.com/apache/infrastructure-actions/blob/2b6ec5f38ac73c7c5970f3b4f863e8d15bf12d7d/actions.yml#L326 --- .github/workflows/docker-cbdb-build-containers.yml | 2 +- .github/workflows/docker-cbdb-test-containers.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 3ef8fae00a8..d42139d0fae 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -117,7 +117,7 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Login to DockerHub for pushing images # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index efb98d2b7a6..36d320f0737 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -106,7 +106,7 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Login to DockerHub for pushing images - name: Login to Docker Hub From 3737bd8dfed2fa175fe56f5c3c94eef8e456ceb7 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 17:27:13 +0800 Subject: [PATCH 106/128] Sandbox: add Cloudberry 2.1.0 release support Main changes: - Update default CODEBASE_VERSION from 2.0.0 to 2.1.0 in .env - Update documentation examples to use version 2.1.0 in README.md - Update help message example version in run.sh - Switch to Apache mirror system for downloading release tarball using closer.lua for better download reliability and speed - Replace wget with curl for source download in Dockerfile This change ensures the sandbox environment defaults to the latest Apache Cloudberry 2.1.0 release and uses the recommended Apache mirror download method. --- devops/sandbox/.env | 2 +- devops/sandbox/Dockerfile.RELEASE.rockylinux9 | 52 ++++--------------- devops/sandbox/README.md | 4 +- devops/sandbox/run.sh | 2 +- 4 files changed, 13 insertions(+), 47 deletions(-) diff --git a/devops/sandbox/.env b/devops/sandbox/.env index 233d7c5b1b5..1ceec2e5fb7 100644 --- a/devops/sandbox/.env +++ b/devops/sandbox/.env @@ -17,5 +17,5 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -CODEBASE_VERSION=2.0.0 +CODEBASE_VERSION=2.1.0 OS_VERSION=rockylinux9 diff --git a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 index ac394c6cb60..215c32f452d 100644 --- a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 +++ b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 @@ -94,6 +94,7 @@ RUN dnf makecache && \ readline-devel \ zlib-devel && \ dnf install -y --enablerepo=crb \ + liburing-devel \ libuv-devel \ libyaml-devel \ perl-IPC-Run \ @@ -120,10 +121,12 @@ USER gpadmin WORKDIR /home/gpadmin # Release version to build (Apache official tarball) -ARG CB_RELEASE_VERSION=2.0.0-incubating +ARG CB_RELEASE_VERSION=2.1.0-incubating # Download and extract the specified release version from Apache -RUN wget -nv "https://downloads.apache.org/incubator/cloudberry/${CB_RELEASE_VERSION}/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz" -O /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz && \ +# Using Apache mirror system for better download reliability and speed +RUN curl -L -o /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz \ + "https://www.apache.org/dyn/closer.lua/incubator/cloudberry/${CB_RELEASE_VERSION}/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz?action=download" && \ tar -xzf /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz -C /home/gpadmin && \ rm -f /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz && \ mv /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION} /home/gpadmin/cloudberry @@ -131,47 +134,9 @@ RUN wget -nv "https://downloads.apache.org/incubator/cloudberry/${CB_RELEASE_VER # Build Cloudberry using the official build scripts RUN cd /home/gpadmin/cloudberry && \ export SRC_DIR=/home/gpadmin/cloudberry && \ - mkdir -p "${SRC_DIR}/build-logs" && \ - # Ensure Cloudberry lib dir exists and has Xerces libs available - sudo rm -rf /usr/local/cloudberry-db && \ - sudo mkdir -p /usr/local/cloudberry-db/lib && \ - sudo cp -v /usr/local/xerces-c/lib/libxerces-c.so \ - /usr/local/xerces-c/lib/libxerces-c-3.*.so \ - /usr/local/cloudberry-db/lib/ && \ - sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db && \ - # Configure with required features and paths - export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:$LD_LIBRARY_PATH && \ - ./configure --prefix=/usr/local/cloudberry-db \ - --disable-external-fts \ - --enable-debug \ - --enable-cassert \ - --enable-debug-extensions \ - --enable-gpcloud \ - --enable-ic-proxy \ - --enable-mapreduce \ - --enable-orafce \ - --enable-orca \ - --enable-pax \ - --disable-pxf \ - --enable-tap-tests \ - --with-gssapi \ - --with-ldap \ - --with-libxml \ - --with-lz4 \ - --with-pam \ - --with-perl \ - --with-pgport=5432 \ - --with-python \ - --with-pythonsrc-ext \ - --with-ssl=openssl \ - --with-uuid=e2fs \ - --with-includes=/usr/local/xerces-c/include \ - --with-libraries=/usr/local/cloudberry-db/lib && \ - # Build and install - make -j$(nproc) --directory ${SRC_DIR} && \ - make -j$(nproc) --directory ${SRC_DIR}/contrib && \ - make install --directory ${SRC_DIR} && \ - make install --directory "${SRC_DIR}/contrib" + mkdir -p ${SRC_DIR}/build-logs && \ + ./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh && \ + ./devops/build/automation/cloudberry/scripts/build-cloudberry.sh # -------------------------------------------------------------------- # Runtime stage: Rocky Linux 9 runtime with required dependencies @@ -192,6 +157,7 @@ RUN dnf -y update && \ krb5-libs \ libevent \ libicu \ + liburing \ libuuid \ libxml2 \ libyaml \ diff --git a/devops/sandbox/README.md b/devops/sandbox/README.md index 9f475977835..fb6a5ef80c3 100644 --- a/devops/sandbox/README.md +++ b/devops/sandbox/README.md @@ -92,14 +92,14 @@ Build and deploy steps: ```shell cd cloudberry/devops/sandbox - ./run.sh -c 2.0.0 + ./run.sh -c 2.1.0 ``` - For latest Apache Cloudberry release running across multiple containers ```shell cd cloudberry/devops/sandbox - ./run.sh -c 2.0.0 -m + ./run.sh -c 2.1.0 -m ``` - For latest main branch running on a single container diff --git a/devops/sandbox/run.sh b/devops/sandbox/run.sh index 7c266b8f64c..705442d98e1 100755 --- a/devops/sandbox/run.sh +++ b/devops/sandbox/run.sh @@ -38,7 +38,7 @@ PIP_INDEX_URL_VAR="${PIP_INDEX_URL_VAR:-$DEFAULT_PIP_INDEX_URL_VAR}" # Function to display help message function usage() { echo "Usage: $0 [-o ] [-c ] [-b] [-m]" - echo " -c Codebase version (valid values: main, local, or other available version like 2.0.0)" + echo " -c Codebase version (valid values: main, local, or other available version like 2.1.0)" echo " -t Timezone (default: America/Los_Angeles, or set via TIMEZONE_VAR environment variable)" echo " -p Python Package Index (PyPI) (default: https://pypi.org/simple, or set via PIP_INDEX_URL_VAR environment variable)" echo " -b Build only, do not run the container (default: false, or set via BUILD_ONLY environment variable)" From 1179057b0c61655d17eee07823be43fd87f4e908 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Sat, 25 Apr 2026 04:50:37 +0800 Subject: [PATCH 107/128] Fix: init missing PlannedStmt fields in orca Those fields are missed by orca which are needed by the pg_stat_statements to identify the query. Without initialization of those fields, pg_stat_statements won't track those queries. --- src/backend/optimizer/plan/orca.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/optimizer/plan/orca.c b/src/backend/optimizer/plan/orca.c index 514385cc2e9..a083fcda6c7 100644 --- a/src/backend/optimizer/plan/orca.c +++ b/src/backend/optimizer/plan/orca.c @@ -405,6 +405,10 @@ optimize_query(Query *parse, int cursorOptions, ParamListInfo boundParams, Optim result->oneoffPlan = glob->oneoffPlan; result->transientPlan = glob->transientPlan; + result->queryId = parse->queryId; + result->stmt_location = parse->stmt_location; + result->stmt_len = parse->stmt_len; + return result; } From 423f77fc010ec72c51fbaaa98cbe294f42bd94fd Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:55:56 +0300 Subject: [PATCH 108/128] Add Rocky Linux 10 Docker containers (#1676) This is the copy or Rocky9 containers. Changes comparing to Rocky9: - Get rid here of packages not available right now under Rocky10 repositories (for example rocky-release-hpc) - Move to the Java 21, it's default to Rocky Linux 10 Co-authored-by: Leonid Borchuk --- .../docker-cbdb-build-containers.yml | 5 +- .../workflows/docker-cbdb-test-containers.yml | 5 +- devops/deploy/docker/build/rocky10/Dockerfile | 216 +++++++++++++++++ .../build/rocky10/configs/90-cbdb-limits | 32 +++ .../build/rocky10/configs/gpinitsystem.conf | 89 +++++++ .../build/rocky10/configs/init_system.sh | 192 +++++++++++++++ .../build/rocky10/tests/requirements.txt | 3 + .../tests/testinfra/test_cloudberry_db_env.py | 127 ++++++++++ devops/deploy/docker/test/rocky10/Dockerfile | 135 +++++++++++ .../test/rocky10/configs/90-cbdb-limits | 32 +++ .../test/rocky10/configs/gpinitsystem.conf | 87 +++++++ .../test/rocky10/configs/init_system.sh | 221 ++++++++++++++++++ pom.xml | 3 +- 13 files changed, 1144 insertions(+), 3 deletions(-) create mode 100644 devops/deploy/docker/build/rocky10/Dockerfile create mode 100644 devops/deploy/docker/build/rocky10/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf create mode 100755 devops/deploy/docker/build/rocky10/configs/init_system.sh create mode 100644 devops/deploy/docker/build/rocky10/tests/requirements.txt create mode 100644 devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py create mode 100644 devops/deploy/docker/test/rocky10/Dockerfile create mode 100644 devops/deploy/docker/test/rocky10/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf create mode 100755 devops/deploy/docker/test/rocky10/configs/init_system.sh diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index d42139d0fae..538b4e9b179 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -60,6 +60,7 @@ on: paths: - 'devops/deploy/docker/build/rocky8/**' - 'devops/deploy/docker/build/rocky9/**' + - 'devops/deploy/docker/build/rocky10/**' - 'devops/deploy/docker/build/ubuntu22.04/**' - 'devops/deploy/docker/build/ubuntu24.04/**' pull_request: @@ -81,7 +82,7 @@ jobs: # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 strategy: matrix: - platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] + platform: ['rocky8', 'rocky9', 'rocky10', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code with full history @@ -108,6 +109,8 @@ jobs: - 'devops/deploy/docker/build/rocky8/**' rocky9: - 'devops/deploy/docker/build/rocky9/**' + rocky10: + - 'devops/deploy/docker/build/rocky10/**' ubuntu22.04: - 'devops/deploy/docker/build/ubuntu22.04/**' ubuntu24.04: diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 36d320f0737..4d0fb8def33 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -49,6 +49,7 @@ on: paths: - 'devops/deploy/docker/test/rocky8/**' - 'devops/deploy/docker/test/rocky9/**' + - 'devops/deploy/docker/test/rocky10/**' - 'devops/deploy/docker/test/ubuntu22.04/**' - 'devops/deploy/docker/test/ubuntu24.04/**' pull_request: @@ -68,7 +69,7 @@ jobs: strategy: matrix: # Build for Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 - platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] + platform: ['rocky8', 'rocky9', 'rocky10', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code @@ -92,6 +93,8 @@ jobs: - 'devops/deploy/docker/test/rocky8/**' rocky9: - 'devops/deploy/docker/test/rocky9/**' + rocky10: + - 'devops/deploy/docker/test/rocky10/**' ubuntu22.04: - 'devops/deploy/docker/test/ubuntu22.04/**' ubuntu24.04: diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile new file mode 100644 index 00000000000..5cdaaddf9e6 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -0,0 +1,216 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (Incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Build Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Rocky Linux 10-based container for building +# and developing Apache Cloudberry. It installs necessary system +# utilities, development tools, and configures the environment for SSH +# access and systemd support. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential development tools and libraries installation +# - User configuration for 'gpadmin' with sudo privileges +# +# Usage: +# docker build -t cloudberry-db-env . +# docker run -h cdw -it cloudberry-db-env +# -------------------------------------------------------------------- + +# Base image: Rocky Linux 10 +FROM rockylinux/rockylinux:10 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="America/Los_Angeles" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- +# Install various development tools, system utilities, and libraries +# required for building and running Apache Cloudberry. +# - EPEL repository is enabled for additional packages. +# - Cleanup steps are added to reduce image size after installation. +# -------------------------------------------------------------------- +RUN dnf makecache && \ + dnf install -y \ + epel-release \ + git && \ + dnf makecache && \ + dnf config-manager --disable epel && \ + dnf install -y --enablerepo=epel \ + bat \ + libssh2-devel \ + python3-devel \ + htop && \ + dnf install -y \ + bison \ + cmake3 \ + ed \ + file \ + flex \ + gcc \ + gcc-c++ \ + gdb \ + glibc-langpack-en \ + glibc-locale-source \ + initscripts \ + iproute \ + less \ + lsof \ + m4 \ + net-tools \ + openssh-clients \ + openssh-server \ + perl \ + rpm-build \ + rpmdevtools \ + rsync \ + sudo \ + tar \ + unzip \ + util-linux-ng \ + wget \ + sshpass \ + which && \ + dnf install -y \ + apr-devel \ + bzip2-devel \ + java-21-openjdk \ + java-21-openjdk-devel \ + krb5-devel \ + libcurl-devel \ + libevent-devel \ + libxml2-devel \ + libuuid-devel \ + libzstd-devel \ + lz4 \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + perl-ExtUtils-Embed \ + perl-Test-Simple \ + perl-core \ + python3-setuptools \ + readline-devel \ + zlib-devel && \ + dnf install -y --enablerepo=crb \ + liburing-devel \ + libuv-devel \ + libyaml-devel \ + perl-IPC-Run \ + python3-wheel \ + protobuf-devel && \ + dnf clean all && \ + cd && XERCES_LATEST_RELEASE=3.3.0 && \ + wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - && \ + tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + cd xerces-c-${XERCES_LATEST_RELEASE} && \ + ./configure --prefix=/usr/local/xerces-c && \ + make -j$(nproc) && \ + make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ + rm -rf ~/xerces-c* && \ + cd && GO_VERSION="go1.24.13" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Create and configure the 'gpadmin' user with sudo privileges. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- + +# Copy configuration files from their respective locations +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + sed -i.bak -r 's/^(session\s+required\s+pam_limits.so)/#\1/' /etc/pam.d/* && \ + cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime && \ + chmod 777 /tmp/init_system.sh && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ + setcap cap_net_raw+ep /usr/bin/ping && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/cloudberry-env.sh ]; then\n source /usr/local/cbdb/cloudberry-env.sh\nfi' >> /home/gpadmin/.bashrc && \ + ssh-keygen -A && \ + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" | tee /etc/locale.conf && \ + dnf clean all # Final cleanup to remove unnecessary files + +# Install testinfra via pip +RUN pip3 install pytest-testinfra + +# Copying test files into the container +COPY ./tests /tests + +# -------------------------------------------------------------------- +# Set the Default User and Command +# -------------------------------------------------------------------- +# The default user is set to 'gpadmin', and the container starts by +# running the init_system.sh script. The container also mounts the +# /sys/fs/cgroup volume for systemd compatibility. +# -------------------------------------------------------------------- +USER gpadmin + +VOLUME [ "/sys/fs/cgroup" ] +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits b/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits new file mode 100644 index 00000000000..474957c42f6 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf b/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf new file mode 100644 index 00000000000..d4d312231c5 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky10/configs/init_system.sh b/devops/deploy/docker/build/rocky10/configs/init_system.sh new file mode 100755 index 00000000000..d8c4a00b035 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/init_system.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/devops/deploy/docker/build/rocky10/tests/requirements.txt b/devops/deploy/docker/build/rocky10/tests/requirements.txt new file mode 100644 index 00000000000..b9711eddac5 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/tests/requirements.txt @@ -0,0 +1,3 @@ +testinfra +pytest-testinfra +paramiko diff --git a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 00000000000..445318f5335 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,127 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "epel-release", + "git", + "bat", + "htop", + "bison", + "cmake", + "gcc", + "gcc-c++", + "glibc-langpack-en", + "glibc-locale-source", + "openssh-clients", + "openssh-server", + "sudo", + "rsync", + "wget", + "openssl-devel", + "python3-devel", + "readline-devel", + "zlib-ng-compat-devel", + "libcurl-devel", + "libevent-devel", + "libxml2-devel", + "libuuid-devel", + "libzstd-devel", + "lz4", + "openldap-devel", + "libuv-devel", + "libyaml-devel" + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "wheel" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_locale_configured(host): + """ + Test if the locale is configured correctly. + """ + locale_conf = host.file("/etc/locale.conf") + assert locale_conf.exists + assert locale_conf.contains("LANG=en_US.UTF-8") + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o777 + + +def test_custom_configuration_files(host): + """ + Test if custom configuration files are correctly copied. + """ + config_file = host.file("/tmp/90-cbdb-limits") + assert config_file.exists + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout diff --git a/devops/deploy/docker/test/rocky10/Dockerfile b/devops/deploy/docker/test/rocky10/Dockerfile new file mode 100644 index 00000000000..ec6b268f708 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/Dockerfile @@ -0,0 +1,135 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (Incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Rocky Linux 10-based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry +# Database RPM can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +# Base image: Rocky Linux 10 +FROM rockylinux/rockylinux:10 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="America/Los_Angeles" + +# Environment variables for locale +ENV LANG=en_US.UTF-8 + +# -------------------------------------------------------------------- +# System Update and Installation +# -------------------------------------------------------------------- +# Update the system and install essential system utilities required for +# running and testing Apache Cloudberry. Cleanup the DNF cache afterward +# to reduce the image size. +# -------------------------------------------------------------------- +RUN dnf install -y \ + file \ + gdb \ + glibc-locale-source \ + make \ + openssh \ + openssh-clients \ + openssh-server \ + procps-ng \ + sudo \ + which \ + && \ + dnf clean all # Clean up DNF cache after package installations + +# -------------------------------------------------------------------- +# User Creation and Configuration +# -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + sed -i.bak -r 's/^(session\s+required\s+pam_limits.so)/#\1/' /etc/pam.d/* && \ + cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime && \ + chmod 777 /tmp/init_system.sh && \ + setcap cap_net_raw+ep /usr/bin/ping && \ + ssh-keygen -A && \ + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" | tee /etc/locale.conf + +# -------------------------------------------------------------------- +# Set the Default User and Command +# -------------------------------------------------------------------- +# The default user is set to 'gpadmin', and the container starts by +# running the init_system.sh script. This container serves as a base +# environment, and the Apache Cloudberry RPM can be installed for +# testing and functional verification. +# -------------------------------------------------------------------- +USER gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits b/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits new file mode 100644 index 00000000000..474957c42f6 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf b/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf new file mode 100644 index 00000000000..3dcd5a99365 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf @@ -0,0 +1,87 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# ---------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- diff --git a/devops/deploy/docker/test/rocky10/configs/init_system.sh b/devops/deploy/docker/test/rocky10/configs/init_system.sh new file mode 100755 index 00000000000..3ea7e34b0ff --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/init_system.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if rpm -q apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/pom.xml b/pom.xml index 77ca5e3e3d8..b9915331c0c 100644 --- a/pom.xml +++ b/pom.xml @@ -1749,7 +1749,8 @@ code or new licensing patterns. devops/deploy/docker/build/rocky8/tests/requirements.txt devops/deploy/docker/build/rocky9/tests/requirements.txt - devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt + devops/deploy/docker/build/rocky10/tests/requirements.txt + devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt devops/deploy/docker/build/ubuntu24.04/tests/requirements.txt