Add postgres interpreter for UserStore

changelog.d/2-features/user-pg

@@ -0,0 +1,14 @@
+Allow storing user data in PostgreSQL.
+
+This is currently not the default and is experimental. The migration path from Cassandra is yet to be implemented.
+
+However, new installations can use this by configuring the wire-server Helm chart like this:
+
+```yaml
+galley:
+  config:
+    postgresqlMigration:
+      user: postgresql
+```
+
+(##)

charts/elasticsearch-index/templates/migrate-data.yaml

@@ -71,6 +71,8 @@ spec:
             {{- end }}
             - --pg-settings
             - {{ toJson .Values.postgresql | quote }}
+            - --user-storage-location
+            - {{ .Values.postgresMigration.user }}
           volumeMounts:
           {{- if hasKey .Values.secrets "elasticsearch" }}
             - name: "elasticsearch-index-secrets"

charts/elasticsearch-index/values.yaml

@@ -49,6 +49,9 @@ postgresqlPool:
   agingTimeout: 1d
   idlenessTimeout: 10m
 
+postgresMigration:
+  user: cassandra
+
 galley:
   host: galley
   port: 8080

charts/wire-server/values.yaml

@@ -7,7 +7,7 @@
 
 tags:
   legalhold: false
-  federation: false 
+  federation: false
   backoffice: false
   mlsstats: false
   integration: false
@@ -90,6 +90,7 @@ galley:
       conversationCodes: cassandra
       teamFeatures: cassandra
       domainRegistration: cassandra
+      user: cassandra
     settings:
       httpPoolSize: 128
       maxTeamSize: 10000
@@ -1031,7 +1032,7 @@ brig:
     #   tlsCaSecretRef:
     #     name: <secret-name>
     #     key: <ca-attribute>
-  
+
     elasticsearch:
       scheme: http
       host: elasticsearch-client
@@ -1077,7 +1078,7 @@ brig:
       # tlsCaSecretRef:
       #   name: <secret-name>
       #   key: <ca-attribute>
-  
+
     # Postgres connection settings
     #
     # Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS
@@ -1100,7 +1101,7 @@ brig:
       acquisitionTimeout: 10s
       agingTimeout: 1d
       idlenessTimeout: 10m
-  
+
     emailSMS:
       general:
         templateBranding:
@@ -1195,35 +1196,35 @@ brig:
         maxRateLimitedKeys: 100000 # Estimated memory usage: 4 MB
       # setAuditLogEmailRecipient: security@wire.com
       setEphemeralUserCreationEnabled: true
-  
+
     smtp:
       passwordFile: /etc/wire/brig/secrets/smtp-password.txt
     proxy: {}
     wireServerEnterprise:
       enabled: false
-  
+
   turnStatic:
     v1:
       - turn:localhost:3478
     v2:
       - turn:localhost:3478
       - turn:localhost:3478?transport=tcp
-  
+
   turn:
     serversSource: files # files | dns
     # baseDomain: turn.wire.example # Must be configured if serversSource is dns
     discoveryIntervalSeconds: 10 # Used only if serversSource is dns
-  
+
   serviceAccount:
     # When setting this to 'false', either make sure that a service account named
     # 'brig' exists or change the 'name' field to 'default'
     create: true
     name: brig
     annotations: {}
     automountServiceAccountToken: true
-  
+
   secrets: {}
-  
+
   podSecurityContext:
     allowPrivilegeEscalation: false
     capabilities:
@@ -1237,11 +1238,11 @@ brig:
       {}
       # uploadXml:
       #   baseUrl: s3://bucket/path/
-  
+
     secrets:
       # uploadXmlAwsAccessKeyId: <key-id>
       # uploadXmlAwsSecretAccessKey: <secret>
-  
+
       # These "secrets" are only used in tests and are therefore safe to be stored unencrypted
       providerPrivateKey: |
         -----BEGIN RSA PRIVATE KEY-----
@@ -1303,7 +1304,7 @@ brig:
         hZMuK3BWD3fzkQVfW0yMwz6fWRXB483ZmekGkgndOTDoJQMdJXZxHpI3t2FcxQYj
         T45GXxRd18neXtuYa/OoAw9UQFDN5XfXN0g=
         -----END CERTIFICATE-----
-  
+
       # pgPassword: <postgres-password>
   test:
     elasticsearch:

docs/src/developer/reference/config-options.md

@@ -1893,6 +1893,7 @@ galley:
       conversationCodes: postgresql
       teamFeatures: postgresql
       domainRegistration: postgresql
+      user: postgresql
 background-worker:
   config:
     migrateConversations: false

hack/helm_vars/common.yaml.gotmpl

@@ -18,6 +18,7 @@ conversationStore: {{ $preferredStore }}
 conversationCodesStore: {{ $preferredStore }}
 teamFeaturesStore: {{ $preferredStore }}
 domainRegistration: {{ $preferredStore }}
+userStore: {{ $preferredStore }}
 
 {{- if (eq (env "UPLOAD_XML_S3_BASE_URL") "") }}
 uploadXml: {}

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -305,6 +305,7 @@ galley:
       conversationCodes: {{ .Values.conversationCodesStore }}
       teamFeatures: {{ .Values.teamFeaturesStore }}
       domainRegistration: {{ .Values.domainRegistration }}
+      user: {{ .Values.userStore }}
     settings:
       maxConvAndTeamSize: 16
       maxTeamSize: 32

integration/test/Test/Search.hs

@@ -26,11 +26,14 @@ import qualified API.Common as API
 import API.Galley
 import qualified API.Galley as Galley
 import qualified API.GalleyInternal as GalleyI
+import Control.Monad.Codensity (Codensity (runCodensity))
+import Control.Monad.Reader
 import qualified Data.Set as Set
 import GHC.Stack
 import SetupHelpers
 import Testlib.Assertions
 import Testlib.Prelude
+import Testlib.ResourcePool (acquireResources)
 
 -- * Local Search
 
@@ -563,6 +566,40 @@ testSuspendedUserSearch = do
   BrigI.refreshIndex OwnDomain
   assertCanFind searcher searcheeQid (searchee %. "name") OwnDomain
 
+testReindexAllUsers :: (HasCallStack) => App ()
+testReindexAllUsers = do
+  resourcePool <- asks (.resourcePool)
+  runCodensity (acquireResources 1 resourcePool) $ \[testBackend] -> do
+    let domain = testBackend.berDomain
+
+    (alice, bob, charlie) <- runCodensity (startDynamicBackend testBackend def) $ \_ -> do
+      alice <- randomUser domain def
+      bob <- randomUser domain def
+      charlie <- randomUser domain def
+
+      BrigI.refreshIndex domain
+      assertCanFind alice bob (bob %. "name") domain
+      assertCanFind alice charlie (charlie %. "name") domain
+      pure (alice, bob, charlie)
+
+    -- TODO: Connect with some bogus ES
+    (dan, bobNewName) <- runCodensity (startDynamicBackend testBackend def) $ \_ -> do
+      dan <- randomUser domain def
+
+      BrigI.refreshIndex domain
+      assertCannotFind alice bob (bob %. "name") domain
+      assertCannotFind alice charlie (charlie %. "name") domain
+      assertCanFind alice dan (dan %. "name") domain
+
+      bobNewName <- API.randomName
+      BrigP.putSelf bob (def {BrigP.name = Just bobNewName}) >>= assertSuccess
+      BrigI.refreshIndex domain
+      assertCannotFind alice bob bobNewName domain
+
+      pure (dan, bobNewName)
+
+    undefined
+
 -- * Assertion Helpers
 
 assertCanFind ::

integration/test/Test/User.hs

@@ -185,16 +185,22 @@ testUpdateSelf (MkTagged mode) = do
     TestUpdateEmailAddress -> do
       -- allowed unconditionally *for owner* (this is a bit off-topic: team members can't
       -- change their email addresses themselves under any conditions)
-      someEmail <- (<> "@example.com") . UUID.toString <$> liftIO UUID.nextRandom
-      bindResponse (putUserEmail owner owner someEmail) $ \resp -> do
+      newEmail <- (<> "@example.com") . UUID.toString <$> liftIO UUID.nextRandom
+      bindResponse (putUserEmail owner owner newEmail) $ \resp -> do
         resp.status `shouldMatchInt` 200
+      getSelf owner `bindResponse` \resp -> do
+        resp.status `shouldMatchInt` 200
+        resp.json %. "email_unvalidated" `shouldMatch` newEmail
     TestUpdateLocale -> do
       -- scim maps "User.preferredLanguage" to brig's locale field.  allowed unconditionally.
       -- we try two languages to make sure it doesn't work because it's already the active
       -- locale.
-      forM_ ["uk", "he"] $ \someLocale ->
-        bindResponse (putSelfLocale mem1 someLocale) $ \resp -> do
+      forM_ ["en-GB", "hi", "de-DE", "de", "he"] $ \newLocale -> do
+        bindResponse (putSelfLocale mem1 newLocale) $ \resp -> do
+          resp.status `shouldMatchInt` 200
+        getSelf mem1 `bindResponse` \resp -> do
           resp.status `shouldMatchInt` 200
+          resp.json %. "locale" `shouldMatch` newLocale
 
 data TestUpdateSelfMode
   = TestUpdateDisplayName

libs/wire-api/src/Wire/API/Asset.hs

@@ -94,6 +94,7 @@ import Imports
 import Servant
 import URI.ByteString
 import Wire.API.Error
+import Wire.API.PostgresMarshall
 import Wire.API.Routes.MultiVerb
 import Wire.Arbitrary (Arbitrary (..), GenericUniform (..))
 
@@ -200,6 +201,12 @@ instance C.Cql AssetKey where
   fromCql (C.CqlText txt) = runParser parser . T.encodeUtf8 $ txt
   fromCql _ = Left "AssetKey: Text expected"
 
+instance PostgresMarshall Text AssetKey where
+  postgresMarshall = assetKeyToText
+
+instance PostgresUnmarshall Text AssetKey where
+  postgresUnmarshall = mapLeft (\e -> "failed to parse AssetKey: " <> T.pack e) . runParser parser . T.encodeUtf8
+
 --------------------------------------------------------------------------------
 -- AssetToken
 

libs/wire-api/src/Wire/API/Locale.hs

@@ -48,6 +48,7 @@ import Data.Time.Format
 import Data.Time.LocalTime (TimeZone (..), utc)
 import Imports
 import Test.QuickCheck
+import Wire.API.PostgresMarshall
 import Wire.API.User.Orphans ()
 import Wire.Arbitrary
 
@@ -185,6 +186,14 @@ instance C.Cql Language where
     Nothing -> Left "Language: ISO 639-1 expected."
   fromCql _ = Left "Language: ASCII expected"
 
+instance PostgresMarshall Text Language where
+  postgresMarshall = lan2Text
+
+instance PostgresUnmarshall Text Language where
+  postgresUnmarshall =
+    mapLeft (\e -> "failed to parse Language: " <> Text.pack e)
+      . parseOnly languageParser
+
 languageParser :: Parser Language
 languageParser = codeParser "language" $ fmap Language . checkAndConvert isLower
 
@@ -210,6 +219,14 @@ instance C.Cql Country where
     Nothing -> Left "Country: ISO 3166-1-alpha2 expected."
   fromCql _ = Left "Country: ASCII expected"
 
+instance PostgresMarshall Text Country where
+  postgresMarshall = con2Text
+
+instance PostgresUnmarshall Text Country where
+  postgresUnmarshall =
+    mapLeft (\e -> "failed to parse Country: " <> Text.pack e)
+      . parseOnly countryParser
+
 countryParser :: Parser Country
 countryParser = codeParser "country" $ fmap Country . checkAndConvert isUpper
 

libs/wire-api/src/Wire/API/Password.hs

@@ -74,11 +74,15 @@ instance Cql Password where
   fromCql (CqlBlob lbs) = parsePassword . Text.decodeUtf8 . toStrict $ lbs
   fromCql _ = Left "password: expected blob"
 
-  toCql pw = CqlBlob . fromStrict $ Text.encodeUtf8 encoded
-    where
-      encoded = case pw of
-        Argon2Password argon2pw -> encodeArgon2HashedPassword argon2pw
-        ScryptPassword scryptpw -> encodeScryptPassword scryptpw
+  toCql = CqlBlob . fromStrict . Text.encodeUtf8 . postgresMarshall
+
+instance PostgresMarshall Text Password where
+  postgresMarshall = \case
+    Argon2Password argon2pw -> encodeArgon2HashedPassword argon2pw
+    ScryptPassword scryptpw -> encodeScryptPassword scryptpw
+
+instance PostgresUnmarshall Text Password where
+  postgresUnmarshall = mapLeft Text.pack . parsePassword
 
 -------------------------------------------------------------------------------