fix: [CPCAP-10480] skip readOnlyFileSystem for major upgrade container and fix database deletion in tests

operator/controllers/patroni_core_controller.go

@@ -276,17 +276,9 @@ func (pr *PatroniCoreReconciler) Reconcile(ctx context.Context, request ctrl.Req
 
 	if err := pr.helper.UpdatePatroniConfigMaps(); err != nil {
 		pr.logger.Error("error during update of patroni config maps", zap.Error(err))
-		// will not return err because there is a slight chance, that
-		// update could happen at the same time when patroni will update leader/config info
-		//return reconcile.Result{RequeueAfter: time.Minute}, err
+		return reconcile.Result{RequeueAfter: time.Minute}, err
 	}
 
-	//if err := pr.helper.RevokeGrantOnPublicSchema(pgHost); err != nil {
-	//	pr.logger.Error("Error during revoking grants from public schema", zap.Error(err))
-	//} else {
-	//	pr.logger.Info("REVOKE statement executed successfully from template1")
-	//}
-
 	reconcFunc := func() {
 		cr, _ := helper.GetPatroniHelper().GetPatroniCoreCR()
 		cr.Spec.InstallationTimestamp = strconv.FormatInt(time.Now().Unix(), 10)

operator/pkg/helper/resource_management.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -695,12 +696,16 @@ func (rm *ResourceManager) UpdatePGService() error {
 func (rm *ResourceManager) UpdatePatroniConfigMaps() error {
 	var configMaps = []string{"patroni-leader", "patroni-config"}
 	for _, configMap := range configMaps {
-		cmap, err := rm.GetConfigMap(configMap)
-		if err != nil {
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			cmap, err := rm.GetConfigMap(configMap)
+			if err != nil {
+				return err
+			}
+			cmap.ObjectMeta.OwnerReferences = rm.GetOwnerReferences()
+			_, err = rm.CreateOrUpdateConfigMap(cmap)
 			return err
-		}
-		cmap.OwnerReferences = rm.GetOwnerReferences()
-		if _, err := rm.CreateOrUpdateConfigMap(cmap); err != nil {
+		})
+		if err != nil {
 			logger.Error("error during update of patroni configMap in resource_management.go", zap.Error(err))
 			return err
 		}

operator/pkg/upgrade/upgrade.go

@@ -43,7 +43,7 @@ var (
 	namespace     = opUtil.GetNameSpace()
 	logger        = opUtil.GetLogger()
 	MasterLabel   = map[string]string{"pgtype": "master"}
-	UpgradeLabels = map[string]string{"app": "pg-major-upgrade"}
+	UpgradeLabels = map[string]string{"app": "pg-major-upgrade", "app.kubernetes.io/name": "pg-major-upgrade"}
 	powaUILabels  = map[string]string{"name": "powa"}
 	//noConnectionDatabases = []string{"template0", "template1"}
 )

tests/robot/Lib/pgsLibrary.py

@@ -465,8 +465,7 @@ def delete_test_db(self, *base_names):
             conn.set_isolation_level(0)
             with conn.cursor() as cursor:
                 for base_name in base_names:
-                    cursor.execute("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname = '{}';".format(base_name))
-                    cursor.execute('DROP DATABASE IF EXISTS {}'.format(base_name))
+                    cursor.execute('DROP DATABASE IF EXISTS "{}" WITH (FORCE)'.format(base_name))
 
     @keyword('Get Pod Daemon')
     def get_pod_daemon(self):

tests/robot/check_dbaas_adapter_api/check_dbaas_adapter_api.robot

@@ -88,6 +88,7 @@ Check Updating User By Dbaas Adapter
 
 Check Deleting User By Dbaas Adapter
     [Tags]  full  dbaas
+    ${status_code}=  Set Variable
     Check Database Creating By Dbaas Adapter  ${db_name}
     ${data}=  Set Variable  {"dbName":"${db_name}","password":"qwerty123","role":"admin" }
     ${resp}=  PUT On Session  dbaassession  /api/${api_version}/dbaas/adapter/postgresql/users  data=${data}
@@ -98,5 +99,6 @@ Check Deleting User By Dbaas Adapter
     Should Be True   """${resp_username}""" in """${res}"""   msg=[creating user] Expected user ${resp_username} is not created in pg-${PG_CLUSTER_NAME}: res: ${res}
     ${data}=  Set Variable  [{"kind":"user","name":"${resp_username}"}]
     ${resp}=  POST On Session  dbaassession  /api/${api_version}/dbaas/adapter/postgresql/resources/bulk-drop  data=${data}
-    Should Be Equal As Strings  ${resp.status_code}  200
-    [Teardown]  Delete User And Database  ${db_name}  ${resp_username}
+    ${status_code}=  Set Variable  ${resp.status_code}
+    Should Be Equal As Strings  ${status_code}  200
+    [Teardown]  Teardown Delete User Test  ${status_code}  ${db_name}  ${resp_username}

tests/robot/check_dbaas_adapter_api/keywords.robot

@@ -52,3 +52,11 @@ Delete User And Database
     [Arguments]  ${db_name}  ${user_name}
     Delete Test DB  ${db_name}
     Execute Query  pg-${PG_CLUSTER_NAME}  DROP USER ${user_name}
+
+Teardown Delete User Test
+    [Arguments]  ${status_code}  ${db_name}  ${user_name}
+    IF  '${status_code}' != '200'
+        Delete User And Database  ${db_name}  ${user_name}
+    ELSE
+        Delete Test DB  ${db_name}
+    END

tests/robot/check_granular_api/check_granular_delete_backup_auth_api.robot

@@ -6,8 +6,8 @@ Resource          ../Lib/lib.robot
 
 *** Keywords ***
 Backup Not Exist
-    ${resp}=  Get Request  postgres_backup_daemon  /backup/status/${backup_id}?namespace=${name_space}
-    Should Be True  '${resp.status_code}'!=' ${200}'
+    ${resp}=  GET On Session  postgres_backup_daemon  url=/backup/status/${backup_id}?namespace=${name_space}  expected_status=404
+    Should Be Equal  ${resp.status_code}  ${404}
 
 *** Test Cases ***
 Check Backup Requests Status Endpoint

tests/robot/check_granular_api/check_granular_restore_backup_auth_api.robot

@@ -57,7 +57,7 @@ Check Disabled Auth Regular Backup
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Test DB  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId  ${backup_id}
@@ -110,7 +110,7 @@ Check Enabled Auth Regular Backup
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Test DB  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId  ${backup_id}
@@ -132,7 +132,7 @@ Check Enabled Auth Regular Backup
     ${res}=  Execute Query   pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}   dbname=${db_name}
     Should Be True   """${EXPECTED}""" in """${res}"""   msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup and drop database after test
-    Execute Query   pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Test DB  ${db_name}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
 

tests/robot/check_granular_api/check_granular_restore_backup_with_roles_auth.robot

@@ -47,7 +47,7 @@ Check Disabled Auth With Roles
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Test DB  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}   ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId=${backup_id}
@@ -68,7 +68,7 @@ Check Disabled Auth With Roles
     ${res}=  Execute Query  pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}   dbname=${db_name}
     Should Be True  """${EXPECTED}""" in """${res}"""   msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup and database after test
-    Execute Query  pg-${PG_CLUSTER_NAME}  drop database if exists ${db_name}
+    Delete Test DB  if exists ${db_name}
     Execute Query  pg-${PG_CLUSTER_NAME}  DROP ROLE ${db_role}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}
@@ -104,7 +104,7 @@ Check Enabled Auth With Roles
         Run Keyword If  '${status}' == 'In progress'  Sleep  1s
         Run Keyword If  '${status}' == 'Successful'  Exit For Loop
     END
-    Execute Query  pg-${PG_CLUSTER_NAME}  DROP DATABASE ${db_name}
+    Delete Test DB  ${db_name}
     ${databases}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT datname FROM pg_database
     List Should Not Contain Value  ${databases}  ${db_name}  msg="failed to delete the test database before restore from backup"
     Set To Dictionary  ${data}  backupId=${backup_id}
@@ -125,6 +125,6 @@ Check Enabled Auth With Roles
     ${res}=   Execute Query  pg-${PG_CLUSTER_NAME}  select * from test_insert_robot where id=${RID}  dbname=${db_name}
     Should Be True  """${EXPECTED}""" in """${res}"""  msg=[insert test record] Expected string ${EXPECTED} not found after restore database: ${db_name}. res: ${res}
     #delete backup after test
-    Execute Query  pg-${PG_CLUSTER_NAME}  drop database if exists ${db_name}
+    Delete Test DB  if exists ${db_name}
     ${resp}=  GET On Session  postgres_backup_daemon  url=/delete/${backup_id}?namespace=${name_space}
     Should Be Equal  ${resp.status_code}  ${200}

tests/robot/check_granular_api/check_granular_restore_status_auth_api.robot

@@ -81,7 +81,7 @@ Check Disabled Auth Restore Endpoint
     Create Session    postgres_backup_daemon    ${scheme}://postgres-backup-daemon:9000
     ${resp}=  Get On Session  postgres_backup_daemon  url=/restore/status/${restore_id}
     Should Be Equal  ${resp.status_code}  ${200}
-    execute query   pg-${PG_CLUSTER_NAME}   drop database if exists ${db_name}
+    Delete Test DB  ${db_name}
 
     #delete backup after test
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}
@@ -142,7 +142,7 @@ Check Enabled Auth restore endpoint
     Create Session    postgres_backup_daemon    ${scheme}://postgres-backup-daemon:9000  auth=${auth}
     ${resp}=  Get On Session  postgres_backup_daemon  url=/restore/status/${restore_id}
     Should Be Equal  ${resp.status_code}  ${200}
-    execute query   pg-${PG_CLUSTER_NAME}   drop database if exists ${db_name}
+    Delete Test DB  ${db_name}
 
     #delete backup after test
     ${resp}=  Get On Session  postgres_backup_daemon  url=/delete/${backup_id}

tests/robot/check_granular_api/check_granular_restore_with_owner_auth.robot

@@ -40,7 +40,7 @@ Prepare Database
 
 Teardown Database
     Execute Query  pg-${PG_CLUSTER_NAME}  DROP ROLE ${db_role}
-    Execute Query  pg-${PG_CLUSTER_NAME}  drop database if exists ${db_name}
+    Delete Test DB  ${db_name}
 
 Check That Role Exists
     ${output}=  Execute Query  pg-${PG_CLUSTER_NAME}  SELECT rolname FROM pg_roles where rolname = '${db_role}'

tests/robot/check_installation/check_installation.robot

@@ -30,5 +30,5 @@ Check Backup-daemon Installation Correctness
 
 Test Container Hardening
     [Tags]  backup_basic
-    ${exclusions}=    Create Dictionary    _all=CH12   pg-patroni-node=CH4
+    ${exclusions}=    Create Dictionary  _all=CH12  pg-patroni-node=CH4  pg-major-upgrade=CH4
     Check Container Hardening   exclusions=${exclusions}