diff --git a/Cargo.lock b/Cargo.lock index ad23da4124..65cadfa9c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1795,6 +1795,25 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cached-projection" +version = "0.0.0" +dependencies = [ + "cached-projection-derive", + "postcard", + "serde", + "serde_json", +] + +[[package]] +name = "cached-projection-derive" +version = "0.0.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "cairo-rs" version = "0.18.5" @@ -2133,6 +2152,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.17", +] + [[package]] name = "color-eyre" version = "0.6.5" @@ -3259,6 +3287,18 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -5412,6 +5452,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.9.4", "bytes", + "cached-projection", "censor", "chrono", "clap 4.5.48", @@ -5440,6 +5481,7 @@ dependencies = [ "json-patch 4.1.0", "labrinth", "lettre", + "lz4_flex", "meilisearch-sdk", "modrinth-content-management", "modrinth-util", @@ -5447,6 +5489,7 @@ dependencies = [ "murmur2", "paste", "path-util", + "postcard", "prometheus", "quick-xml 0.38.3", "rand 0.8.5", @@ -7492,6 +7535,18 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.3" diff --git a/Cargo.toml b/Cargo.toml index 301bed803f..e661e6c523 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,8 @@ members = [ "apps/labrinth", "packages/app-lib", "packages/ariadne", + "packages/cached-projection", + "packages/cached-projection-derive", "packages/daedalus", "packages/labrinth-derive", "packages/modrinth-content-management", @@ -54,6 +56,7 @@ bitflags = "2.9.4" bon = "3.9.3" bytemuck = "1.24.0" bytes = "1.10.1" +cached-projection = { path = "packages/cached-projection" } censor = "0.3.0" chardetng = "0.1.17" chrono = "0.4.42" @@ -118,6 +121,12 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +lz4_flex = { version = "0.11.5", default-features = false, features = [ + "checked-decode", + "safe-decode", + "safe-encode", + "std", +] } maxminddb = "0.26.0" meilisearch-sdk = { version = "0.30.0", default-features = false } modrinth-content-management = { path = "packages/modrinth-content-management" } @@ -135,6 +144,7 @@ paste = "1.0.15" path-util = { path = "packages/path-util" } phf = { version = "0.13.1", features = ["macros"] } png = "0.18.0" +postcard = { version = "1.1.3", default-features = false, features = ["alloc"] } proc-macro2 = { version = "1.0" } prometheus = "0.14.0" quartz_nbt = "0.2.9" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 4f702d6bab..e902f41e4c 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -33,6 +33,7 @@ aws-sdk-s3 = { workspace = true } base64 = { workspace = true } bitflags = { workspace = true } bytes = { workspace = true } +cached-projection = { workspace = true } censor = { workspace = true } chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive"] } @@ -73,6 +74,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +lz4_flex = { workspace = true } meilisearch-sdk = { workspace = true, features = ["reqwest"] } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } @@ -80,6 +82,7 @@ muralpay = { workspace = true, features = ["client", "mock", "utoipa"] } murmur2 = { workspace = true } paste = { workspace = true } path-util = { workspace = true } +postcard = { workspace = true } prometheus = { workspace = true } quick-xml = { workspace = true } rand = { workspace = true } diff --git a/apps/labrinth/src/database/models/analytics_event_item.rs b/apps/labrinth/src/database/models/analytics_event_item.rs index bcf89aa344..99de67da7c 100644 --- a/apps/labrinth/src/database/models/analytics_event_item.rs +++ b/apps/labrinth/src/database/models/analytics_event_item.rs @@ -1,3 +1,4 @@ +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use futures::{StreamExt, TryStreamExt}; use sqlx::types::Json; @@ -11,10 +12,10 @@ use crate::{ }; use serde::{Deserialize, Serialize}; -const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events"; +const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v1"; const ANALYTICS_EVENTS_ALL_KEY: &str = "all"; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)] pub struct DBAnalyticsEvent { pub id: DBAnalyticsEventId, pub meta: AnalyticsEventMeta, @@ -88,7 +89,7 @@ impl DBAnalyticsEvent { let mut redis = redis.connect().await?; if let Some(events) = redis - .get_deserialized_from_json( + .get_deserialized( ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY, ) @@ -119,7 +120,7 @@ impl DBAnalyticsEvent { .await?; redis - .set_serialized_to_json( + .set_serialized( ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY, &events, diff --git a/apps/labrinth/src/database/models/categories.rs b/apps/labrinth/src/database/models/categories.rs index 44650d76d4..d376fddc89 100644 --- a/apps/labrinth/src/database/models/categories.rs +++ b/apps/labrinth/src/database/models/categories.rs @@ -4,17 +4,18 @@ use crate::database::redis::RedisPool; use super::DatabaseError; use super::ids::*; +use cached_projection::CachedProjection; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const TAGS_NAMESPACE: &str = "tags"; +const TAGS_NAMESPACE: &str = "tags:v1"; pub struct ProjectType { pub id: ProjectTypeId, pub name: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] pub struct Category { pub id: CategoryId, pub category: String, @@ -28,7 +29,7 @@ pub struct ReportType { pub report_type: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] pub struct LinkPlatform { pub id: LinkPlatformId, pub name: String, @@ -96,9 +97,8 @@ impl Category { { let mut redis = redis.connect().await?; - let res: Option> = redis - .get_deserialized_from_json(TAGS_NAMESPACE, "category") - .await?; + let res: Option> = + redis.get_deserialized(TAGS_NAMESPACE, "category").await?; if let Some(res) = res { return Ok(res); @@ -127,7 +127,7 @@ impl Category { let mut redis = redis.connect().await?; redis - .set_serialized_to_json(TAGS_NAMESPACE, "category", &result, None) + .set_serialized(TAGS_NAMESPACE, "category", &result, None) .await?; Ok(result) @@ -166,7 +166,7 @@ impl LinkPlatform { let mut redis = redis.connect().await?; let res: Option> = redis - .get_deserialized_from_json(TAGS_NAMESPACE, "link_platform") + .get_deserialized(TAGS_NAMESPACE, "link_platform") .await?; if let Some(res) = res { @@ -191,12 +191,7 @@ impl LinkPlatform { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - TAGS_NAMESPACE, - "link_platform", - &result, - None, - ) + .set_serialized(TAGS_NAMESPACE, "link_platform", &result, None) .await?; Ok(result) @@ -235,7 +230,7 @@ impl ReportType { let mut redis = redis.connect().await?; let res: Option> = redis - .get_deserialized_from_json(TAGS_NAMESPACE, "report_type") + .get_deserialized(TAGS_NAMESPACE, "report_type") .await?; if let Some(res) = res { @@ -256,12 +251,7 @@ impl ReportType { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - TAGS_NAMESPACE, - "report_type", - &result, - None, - ) + .set_serialized(TAGS_NAMESPACE, "report_type", &result, None) .await?; Ok(result) @@ -300,7 +290,7 @@ impl ProjectType { let mut redis = redis.connect().await?; let res: Option> = redis - .get_deserialized_from_json(TAGS_NAMESPACE, "project_type") + .get_deserialized(TAGS_NAMESPACE, "project_type") .await?; if let Some(res) = res { @@ -321,12 +311,7 @@ impl ProjectType { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - TAGS_NAMESPACE, - "project_type", - &result, - None, - ) + .set_serialized(TAGS_NAMESPACE, "project_type", &result, None) .await?; Ok(result) diff --git a/apps/labrinth/src/database/models/collection_item.rs b/apps/labrinth/src/database/models/collection_item.rs index 5ca827a834..0ed74a4d94 100644 --- a/apps/labrinth/src/database/models/collection_item.rs +++ b/apps/labrinth/src/database/models/collection_item.rs @@ -3,12 +3,13 @@ use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::database::{PgTransaction, models}; use crate::models::collections::CollectionStatus; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::DashMap; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const COLLECTIONS_NAMESPACE: &str = "collections"; +const COLLECTIONS_NAMESPACE: &str = "collections:v1"; #[derive(Clone)] pub struct CollectionBuilder { @@ -43,7 +44,7 @@ impl CollectionBuilder { Ok(self.collection_id) } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)] pub struct DBCollection { pub id: DBCollectionId, pub user_id: DBUserId, diff --git a/apps/labrinth/src/database/models/flow_item.rs b/apps/labrinth/src/database/models/flow_item.rs index 2bc8dd864d..db0aabda2b 100644 --- a/apps/labrinth/src/database/models/flow_item.rs +++ b/apps/labrinth/src/database/models/flow_item.rs @@ -4,6 +4,7 @@ use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::pats::Scopes; use crate::{auth::AuthProvider, routes::internal::flows::TempUser}; +use cached_projection::CachedProjection; use chrono::Duration; use rand::Rng; use rand::distributions::Alphanumeric; @@ -13,10 +14,9 @@ use serde::{Deserialize, Serialize}; use url::Url; use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration}; -const FLOWS_NAMESPACE: &str = "flows"; +const FLOWS_NAMESPACE: &str = "flows:v2"; -#[derive(Deserialize, Serialize)] -#[serde(tag = "type", rename_all = "snake_case")] +#[derive(Deserialize, Serialize, CachedProjection)] pub enum DBFlow { OAuth { user_id: Option, @@ -61,9 +61,11 @@ pub enum DBFlow { }, RegisterPasskey { user_id: DBUserId, + #[cached_projection(wrap)] state: PasskeyRegistration, }, AuthenticatePasskey { + #[cached_projection(wrap)] state: DiscoverableAuthentication, }, } @@ -78,10 +80,10 @@ impl DBFlow { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( FLOWS_NAMESPACE, &state, - &self, + self, Some(expires.num_seconds()), ) .await?; @@ -109,7 +111,7 @@ impl DBFlow { ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; - redis.get_deserialized_from_json(FLOWS_NAMESPACE, id).await + redis.get_deserialized(FLOWS_NAMESPACE, id).await } /// Gets the flow and removes it from the cache, but only removes if the flow was present and the predicate returned true diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index 2a638326c6..9a96d6d656 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -10,6 +10,7 @@ use crate::models::ids::{ }; use ariadne::ids::base62_impl::to_base62; use ariadne::ids::{UserId, random_base62_rng, random_base62_rng_range}; +use cached_projection::CachedProjection; use censor::Censor; use paste::paste; use rand::SeedableRng; @@ -95,7 +96,7 @@ macro_rules! generate_bulk_ids { macro_rules! impl_db_id_interface { ($id_struct:ident, $db_id_struct:ident, $(, generator: $generator_function:ident @ $db_table:expr, $(bulk_generator: $bulk_generator_function:ident,)?)?) => { - #[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash, utoipa::ToSchema)] + #[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash, utoipa::ToSchema, CachedProjection)] #[sqlx(transparent)] pub struct $db_id_struct(pub i64); @@ -154,6 +155,7 @@ macro_rules! id_type { PartialEq, Hash, utoipa::ToSchema, + CachedProjection, )] #[sqlx(transparent)] pub struct $name(pub $type); diff --git a/apps/labrinth/src/database/models/image_item.rs b/apps/labrinth/src/database/models/image_item.rs index a65736dcf5..cc18af39f9 100644 --- a/apps/labrinth/src/database/models/image_item.rs +++ b/apps/labrinth/src/database/models/image_item.rs @@ -2,13 +2,14 @@ use super::ids::*; use crate::database::PgTransaction; use crate::database::redis::RedisPool; use crate::{database::models::DatabaseError, models::images::ImageContext}; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::DashMap; use serde::{Deserialize, Serialize}; -const IMAGES_NAMESPACE: &str = "images"; +const IMAGES_NAMESPACE: &str = "images:v1"; -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)] pub struct DBImage { pub id: DBImageId, pub url: String, diff --git a/apps/labrinth/src/database/models/loader_fields.rs b/apps/labrinth/src/database/models/loader_fields.rs index e47735910e..3fc29048c7 100644 --- a/apps/labrinth/src/database/models/loader_fields.rs +++ b/apps/labrinth/src/database/models/loader_fields.rs @@ -5,6 +5,7 @@ use super::DatabaseError; use super::ids::*; use crate::database::PgTransaction; use crate::database::redis::RedisPool; +use cached_projection::CachedProjection; use chrono::DateTime; use chrono::Utc; use dashmap::DashMap; @@ -12,15 +13,16 @@ use futures::TryStreamExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; -const GAMES_LIST_NAMESPACE: &str = "games"; -const LOADER_ID: &str = "loader_id"; -const LOADERS_LIST_NAMESPACE: &str = "loaders"; -const LOADER_FIELDS_NAMESPACE: &str = "loader_fields"; -const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all"; -const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums"; -pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values"; +const GAMES_LIST_NAMESPACE: &str = "games:v1"; +const LOADER_ID: &str = "loader_id:v1"; +const LOADERS_LIST_NAMESPACE: &str = "loaders:v2"; +const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v1"; +const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v1"; +const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v1"; +pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = + "loader_field_enum_values:v2"; -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)] pub struct Game { pub id: GameId, pub slug: String, @@ -54,7 +56,7 @@ impl Game { { let mut redis = redis.connect().await?; let cached_games: Option> = redis - .get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games") + .get_deserialized(GAMES_LIST_NAMESPACE, "games") .await?; if let Some(cached_games) = cached_games { return Ok(cached_games); @@ -80,25 +82,21 @@ impl Game { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - GAMES_LIST_NAMESPACE, - "games", - &result, - None, - ) + .set_serialized(GAMES_LIST_NAMESPACE, "games", &result, None) .await?; Ok(result) } } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, CachedProjection)] pub struct Loader { pub id: LoaderId, pub loader: String, pub icon: String, pub supported_project_types: Vec, pub supported_games: Vec, // slugs + #[cached_projection(wrap)] pub metadata: serde_json::Value, } @@ -114,7 +112,7 @@ impl Loader { { let mut redis = redis.connect().await?; let cached_id: Option = - redis.get_deserialized_from_json(LOADER_ID, name).await?; + redis.get_deserialized(LOADER_ID, name).await?; if let Some(cached_id) = cached_id { return Ok(Some(LoaderId(cached_id))); } @@ -134,7 +132,7 @@ impl Loader { if let Some(result) = result { let mut redis = redis.connect().await?; redis - .set_serialized_to_json(LOADER_ID, name, &result.0, None) + .set_serialized(LOADER_ID, name, &result.0, None) .await?; } @@ -151,7 +149,7 @@ impl Loader { { let mut redis = redis.connect().await?; let cached_loaders: Option> = redis - .get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all") + .get_deserialized(LOADERS_LIST_NAMESPACE, "all") .await?; if let Some(cached_loaders) = cached_loaders { return Ok(cached_loaders); @@ -193,19 +191,14 @@ impl Loader { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - LOADERS_LIST_NAMESPACE, - "all", - &result, - None, - ) + .set_serialized(LOADERS_LIST_NAMESPACE, "all", &result, None) .await?; Ok(result) } } -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)] pub struct LoaderField { pub id: LoaderFieldId, pub field: String, @@ -274,7 +267,7 @@ impl LoaderFieldType { } } -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug, CachedProjection)] pub struct LoaderFieldEnum { pub id: LoaderFieldEnumId, pub enum_name: String, @@ -282,14 +275,18 @@ pub struct LoaderFieldEnum { pub hidable: bool, } -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +#[derive( + Clone, Serialize, Deserialize, Debug, PartialEq, Eq, CachedProjection, +)] pub struct LoaderFieldEnumValue { pub id: LoaderFieldEnumValueId, pub enum_id: LoaderFieldEnumId, pub value: String, pub ordering: Option, pub created: DateTime, + #[serde(flatten)] + #[cached_projection(wrap)] pub metadata: serde_json::Value, } @@ -303,22 +300,33 @@ impl std::hash::Hash for LoaderFieldEnumValue { } } -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash, CachedProjection, +)] pub struct VersionField { pub version_id: DBVersionId, pub field_id: LoaderFieldId, pub field_name: String, + #[cached_projection(nested)] pub value: VersionFieldValue, } -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)] +#[derive( + Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash, CachedProjection, +)] pub enum VersionFieldValue { Integer(i32), Text(String), - Enum(LoaderFieldEnumId, LoaderFieldEnumValue), + Enum( + LoaderFieldEnumId, + #[cached_projection(nested)] LoaderFieldEnumValue, + ), Boolean(bool), ArrayInteger(Vec), ArrayText(Vec), - ArrayEnum(LoaderFieldEnumId, Vec), + ArrayEnum( + LoaderFieldEnumId, + #[cached_projection(nested)] Vec, + ), ArrayBoolean(Vec), } @@ -470,10 +478,9 @@ impl LoaderField { { let mut redis = redis.connect().await?; - let cached_fields: Option> = - redis.get(LOADER_FIELDS_NAMESPACE_ALL, "").await?.and_then( - |x| serde_json::from_str::>(&x).ok(), - ); + let cached_fields: Option> = redis + .get_deserialized(LOADER_FIELDS_NAMESPACE_ALL, "") + .await?; if let Some(cached_fields) = cached_fields { return Ok(cached_fields); @@ -506,12 +513,7 @@ impl LoaderField { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - LOADER_FIELDS_NAMESPACE_ALL, - "", - &result, - None, - ) + .set_serialized(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None) .await?; Ok(result) @@ -530,10 +532,7 @@ impl LoaderFieldEnum { let mut redis = redis.connect().await?; let cached_enum = redis - .get_deserialized_from_json( - LOADER_FIELD_ENUMS_ID_NAMESPACE, - enum_name, - ) + .get_deserialized(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name) .await?; if let Some(cached_enum) = cached_enum { return Ok(cached_enum); @@ -561,7 +560,7 @@ impl LoaderFieldEnum { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name, &result, diff --git a/apps/labrinth/src/database/models/mod.rs b/apps/labrinth/src/database/models/mod.rs index a96b96d820..2ca113b42c 100644 --- a/apps/labrinth/src/database/models/mod.rs +++ b/apps/labrinth/src/database/models/mod.rs @@ -76,6 +76,8 @@ pub enum DatabaseError { RedisPool(#[from] deadpool_redis::PoolError), #[error("Error while serializing with the cache: {0}")] SerdeCacheError(#[from] serde_json::Error), + #[error("error while encoding or decoding the cache: {0}")] + PostcardCacheError(#[from] postcard::Error), #[error("Schema error: {0}")] SchemaError(String), #[error( diff --git a/apps/labrinth/src/database/models/moderation_note_item.rs b/apps/labrinth/src/database/models/moderation_note_item.rs index 3df5c8c855..9bdf21dcb5 100644 --- a/apps/labrinth/src/database/models/moderation_note_item.rs +++ b/apps/labrinth/src/database/models/moderation_note_item.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -7,11 +8,11 @@ use crate::database::redis::RedisPool; use super::{DBOrganizationId, DBUserId, DatabaseError}; -const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users"; +const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v1"; const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str = - "moderation_notes_organizations"; + "moderation_notes_organizations:v1"; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)] pub struct DBModerationNote { pub user_id: Option, pub organization_id: Option, @@ -40,7 +41,7 @@ impl DBModerationNote { let cached = { let mut redis = redis.connect().await?; redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( MODERATION_NOTES_USERS_NAMESPACE, &ids, ) @@ -87,7 +88,7 @@ impl DBModerationNote { if let Some(user_id) = note.user_id { redis - .set_serialized_to_json( + .set_serialized( MODERATION_NOTES_USERS_NAMESPACE, user_id.0, ¬e, @@ -130,7 +131,7 @@ impl DBModerationNote { let cached = { let mut redis = redis.connect().await?; redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, &ids, ) @@ -177,7 +178,7 @@ impl DBModerationNote { if let Some(organization_id) = note.organization_id { redis - .set_serialized_to_json( + .set_serialized( MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0, ¬e, diff --git a/apps/labrinth/src/database/models/notification_item.rs b/apps/labrinth/src/database/models/notification_item.rs index dda5b230f7..ed943309fc 100644 --- a/apps/labrinth/src/database/models/notification_item.rs +++ b/apps/labrinth/src/database/models/notification_item.rs @@ -5,20 +5,22 @@ use crate::models::notifications::{ NotificationBody, NotificationChannel, NotificationDeliveryStatus, NotificationType, }; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications"; +const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v2"; pub struct NotificationBuilder { pub body: NotificationBody, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] pub struct DBNotification { pub id: DBNotificationId, pub user_id: DBUserId, + #[cached_projection(nested)] pub body: NotificationBody, pub read: bool, pub created: DateTime, @@ -435,7 +437,7 @@ impl DBNotification { let mut redis = redis.connect().await?; let cached_notifications: Option> = redis - .get_deserialized_from_json( + .get_deserialized( USER_NOTIFICATIONS_NAMESPACE, &user_id.0.to_string(), ) @@ -493,7 +495,7 @@ impl DBNotification { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( USER_NOTIFICATIONS_NAMESPACE, user_id.0, &db_notifications, diff --git a/apps/labrinth/src/database/models/notifications_template_item.rs b/apps/labrinth/src/database/models/notifications_template_item.rs index 607e0d60d0..e3b2712b28 100644 --- a/apps/labrinth/src/database/models/notifications_template_item.rs +++ b/apps/labrinth/src/database/models/notifications_template_item.rs @@ -2,17 +2,19 @@ use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::v3::notifications::{NotificationChannel, NotificationType}; use crate::routes::ApiError; +use cached_projection::CachedProjection; use serde::{Deserialize, Serialize}; -const TEMPLATES_NAMESPACE: &str = "notifications_templates"; -const TEMPLATES_HTML_DATA_NAMESPACE: &str = "notifications_templates_html_data"; +const TEMPLATES_NAMESPACE: &str = "notifications_templates:v1"; +const TEMPLATES_HTML_DATA_NAMESPACE: &str = + "notifications_templates_html_data:v1"; const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str = - "notifications_templates_dynamic_html"; + "notifications_templates_dynamic_html:v1"; const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)] pub struct NotificationTemplate { pub id: i64, pub channel: NotificationChannel, @@ -56,10 +58,7 @@ impl NotificationTemplate { let mut redis = redis.connect().await?; let maybe_cached_templates = redis - .get_deserialized_from_json( - TEMPLATES_NAMESPACE, - channel.as_str(), - ) + .get_deserialized(TEMPLATES_NAMESPACE, channel.as_str()) .await?; if let Some(cached) = maybe_cached_templates { @@ -82,7 +81,7 @@ impl NotificationTemplate { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( TEMPLATES_NAMESPACE, channel.as_str(), &templates, @@ -99,7 +98,7 @@ impl NotificationTemplate { ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; redis - .get_deserialized_from_json( + .get_deserialized( TEMPLATES_HTML_DATA_NAMESPACE, &self.id.to_string(), ) @@ -113,7 +112,7 @@ impl NotificationTemplate { ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( TEMPLATES_HTML_DATA_NAMESPACE, &self.id.to_string(), &data, @@ -131,17 +130,14 @@ pub async fn get_or_set_cached_dynamic_html( where F: Future>, { - #[derive(Debug, Clone, Serialize, Deserialize)] + #[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)] struct HtmlBody { html: String, } let mut redis_conn = redis.connect().await?; if let Some(body) = redis_conn - .get_deserialized_from_json::( - TEMPLATES_DYNAMIC_HTML_NAMESPACE, - key, - ) + .get_deserialized::(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key) .await? { return Ok(body.html); @@ -153,7 +149,7 @@ where let mut redis_conn = redis.connect().await?; redis_conn - .set_serialized_to_json( + .set_serialized( TEMPLATES_DYNAMIC_HTML_NAMESPACE, key, &cached, diff --git a/apps/labrinth/src/database/models/notifications_type_item.rs b/apps/labrinth/src/database/models/notifications_type_item.rs index 0179dbf7d5..f94bb0464c 100644 --- a/apps/labrinth/src/database/models/notifications_type_item.rs +++ b/apps/labrinth/src/database/models/notifications_type_item.rs @@ -1,11 +1,12 @@ use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::v3::notifications::NotificationType; +use cached_projection::CachedProjection; use serde::{Deserialize, Serialize}; -const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types"; +const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v1"; -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] pub struct NotificationTypeItem { pub name: NotificationType, pub delivery_priority: i32, @@ -43,7 +44,7 @@ impl NotificationTypeItem { let mut redis = redis.connect().await?; let cached_types = redis - .get_deserialized_from_json(NOTIFICATION_TYPES_NAMESPACE, "all") + .get_deserialized(NOTIFICATION_TYPES_NAMESPACE, "all") .await?; if let Some(types) = cached_types { @@ -63,12 +64,7 @@ impl NotificationTypeItem { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( - NOTIFICATION_TYPES_NAMESPACE, - "all", - &types, - None, - ) + .set_serialized(NOTIFICATION_TYPES_NAMESPACE, "all", &types, None) .await?; Ok(types) diff --git a/apps/labrinth/src/database/models/organization_item.rs b/apps/labrinth/src/database/models/organization_item.rs index bdef0009bf..2620edaa62 100644 --- a/apps/labrinth/src/database/models/organization_item.rs +++ b/apps/labrinth/src/database/models/organization_item.rs @@ -1,6 +1,7 @@ use crate::database::PgTransaction; use crate::database::redis::RedisPool; use ariadne::ids::base62_impl::parse_base62; +use cached_projection::CachedProjection; use dashmap::DashMap; use futures::TryStreamExt; use std::fmt::{Debug, Display}; @@ -9,10 +10,10 @@ use std::hash::Hash; use super::{DBTeamMember, ids::*}; use serde::{Deserialize, Serialize}; -const ORGANIZATIONS_NAMESPACE: &str = "organizations"; -const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles"; +const ORGANIZATIONS_NAMESPACE: &str = "organizations:v1"; +const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v1"; -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)] /// An organization of users who together control one or more projects and organizations. pub struct DBOrganization { /// The id of the organization diff --git a/apps/labrinth/src/database/models/pat_item.rs b/apps/labrinth/src/database/models/pat_item.rs index 9a9992f5d7..0ed1ca3ebe 100644 --- a/apps/labrinth/src/database/models/pat_item.rs +++ b/apps/labrinth/src/database/models/pat_item.rs @@ -4,6 +4,7 @@ use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::pats::Scopes; use ariadne::ids::base62_impl::parse_base62; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::DashMap; use futures::TryStreamExt; @@ -11,11 +12,11 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const PATS_NAMESPACE: &str = "pats"; -const PATS_TOKENS_NAMESPACE: &str = "pats_tokens"; -const PATS_USERS_NAMESPACE: &str = "pats_users"; +const PATS_NAMESPACE: &str = "pats:v1"; +const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v1"; +const PATS_USERS_NAMESPACE: &str = "pats_users:v1"; -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)] pub struct DBPersonalAccessToken { pub id: DBPatId, pub name: String, @@ -161,7 +162,7 @@ impl DBPersonalAccessToken { let mut redis = redis.connect().await?; let res = redis - .get_deserialized_from_json::>( + .get_deserialized::>( PATS_USERS_NAMESPACE, &user_id.0.to_string(), ) @@ -189,12 +190,7 @@ impl DBPersonalAccessToken { let mut redis = redis.connect().await?; redis - .set( - PATS_USERS_NAMESPACE, - &user_id.0.to_string(), - &serde_json::to_string(&db_pats)?, - None, - ) + .set_serialized(PATS_USERS_NAMESPACE, user_id.0, &db_pats, None) .await?; Ok(db_pats) } diff --git a/apps/labrinth/src/database/models/product_item.rs b/apps/labrinth/src/database/models/product_item.rs index e7d48f810b..763462ccad 100644 --- a/apps/labrinth/src/database/models/product_item.rs +++ b/apps/labrinth/src/database/models/product_item.rs @@ -3,13 +3,14 @@ use crate::database::models::{ }; use crate::database::redis::RedisPool; use crate::models::billing::{Price, ProductMetadata}; +use cached_projection::CachedProjection; use dashmap::DashMap; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use std::convert::TryInto; -const PRODUCTS_NAMESPACE: &str = "products"; +const PRODUCTS_NAMESPACE: &str = "products:v2"; pub struct DBProduct { pub id: DBProductId, @@ -131,13 +132,15 @@ impl DBProduct { } } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, CachedProjection)] pub struct QueryProductWithPrices { pub id: DBProductId, + #[cached_projection(nested)] pub metadata: ProductMetadata, pub unitary: bool, #[serde(skip_serializing_if = "Option::is_none", default)] pub name: Option, + #[cached_projection(nested)] pub prices: Vec, } @@ -153,9 +156,8 @@ impl QueryProductWithPrices { { let mut redis = redis.connect().await?; - let res: Option> = redis - .get_deserialized_from_json(PRODUCTS_NAMESPACE, "all") - .await?; + let res: Option> = + redis.get_deserialized(PRODUCTS_NAMESPACE, "all").await?; if let Some(res) = res { return Ok(res); @@ -196,7 +198,7 @@ impl QueryProductWithPrices { let mut redis = redis.connect().await?; redis - .set_serialized_to_json(PRODUCTS_NAMESPACE, "all", &products, None) + .set_serialized(PRODUCTS_NAMESPACE, "all", &products, None) .await?; Ok(products) @@ -243,10 +245,11 @@ impl QueryProductWithPrices { } } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, CachedProjection)] pub struct DBProductPrice { pub id: DBProductPriceId, pub product_id: DBProductId, + #[cached_projection(nested)] pub prices: Price, pub currency_code: String, } diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index 9fa9091a9b..81c8466ede 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -15,6 +15,7 @@ use crate::models::projects::{ use crate::routes::ApiError; use crate::util::error::Context; use ariadne::ids::base62_impl::parse_base62; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::{DashMap, DashSet}; use futures::TryStreamExt; @@ -23,9 +24,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -pub const PROJECTS_NAMESPACE: &str = "projects"; -pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs"; -const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies"; +pub const PROJECTS_NAMESPACE: &str = "projects:v2"; +pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v1"; +const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v1"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LinkUrl { @@ -276,7 +277,7 @@ impl ProjectBuilder { Ok(self.project_id) } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)] pub struct DBProject { pub id: DBProjectId, pub team_id: DBTeamId, @@ -304,6 +305,7 @@ pub struct DBProject { pub monetization_status: MonetizationStatus, pub side_types_migration_review_status: SideTypesMigrationReviewStatus, pub loaders: Vec, + #[cached_projection(nested)] pub components: exp::ProjectSerial, } @@ -976,7 +978,7 @@ impl DBProject { let mut redis = redis.connect().await?; let dependencies = redis - .get_deserialized_from_json::( + .get_deserialized::( PROJECTS_DEPENDENCIES_NAMESPACE, &id.0.to_string(), ) @@ -1014,7 +1016,7 @@ impl DBProject { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( PROJECTS_DEPENDENCIES_NAMESPACE, id.0, &dependencies, @@ -1050,8 +1052,9 @@ impl DBProject { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, CachedProjection)] pub struct ProjectQueryResult { + #[cached_projection(nested)] pub inner: DBProject, pub categories: Vec, pub additional_categories: Vec, @@ -1061,7 +1064,9 @@ pub struct ProjectQueryResult { pub urls: Vec, pub gallery_items: Vec, pub thread_id: DBThreadId, + #[cached_projection(nested)] pub aggregate_version_fields: Vec, #[serde(flatten)] + #[cached_projection(nested)] pub components: exp::ProjectQuery, } diff --git a/apps/labrinth/src/database/models/session_item.rs b/apps/labrinth/src/database/models/session_item.rs index c3e6025b5b..c2509fab80 100644 --- a/apps/labrinth/src/database/models/session_item.rs +++ b/apps/labrinth/src/database/models/session_item.rs @@ -3,6 +3,7 @@ use crate::database::PgTransaction; use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use ariadne::ids::base62_impl::parse_base62; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::DashMap; use futures_util::TryStreamExt; @@ -10,9 +11,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const SESSIONS_NAMESPACE: &str = "sessions"; -const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids"; -const SESSIONS_USERS_NAMESPACE: &str = "sessions_users"; +const SESSIONS_NAMESPACE: &str = "sessions:v1"; +const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v1"; +const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v1"; pub struct SessionBuilder { pub session: String, @@ -74,7 +75,7 @@ impl SessionBuilder { } } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, CachedProjection)] pub struct DBSession { pub id: DBSessionId, pub session: String, @@ -226,7 +227,7 @@ impl DBSession { let mut redis = redis.connect().await?; let res = redis - .get_deserialized_from_json::>( + .get_deserialized::>( SESSIONS_USERS_NAMESPACE, &user_id.0.to_string(), ) @@ -255,7 +256,7 @@ impl DBSession { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( SESSIONS_USERS_NAMESPACE, user_id.0, &db_sessions, diff --git a/apps/labrinth/src/database/models/team_item.rs b/apps/labrinth/src/database/models/team_item.rs index 12cb44c02c..f0342f39cf 100644 --- a/apps/labrinth/src/database/models/team_item.rs +++ b/apps/labrinth/src/database/models/team_item.rs @@ -3,13 +3,14 @@ use crate::{ database::{PgTransaction, redis::RedisPool}, models::teams::{OrganizationPermissions, ProjectPermissions}, }; +use cached_projection::CachedProjection; use dashmap::DashMap; use futures::TryStreamExt; use itertools::Itertools; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -const TEAMS_NAMESPACE: &str = "teams"; +const TEAMS_NAMESPACE: &str = "teams:v2"; pub struct TeamBuilder { pub members: Vec, @@ -164,7 +165,7 @@ impl DBTeam { } /// A member of a team -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)] pub struct DBTeamMember { pub id: DBTeamMemberId, pub team_id: DBTeamId, @@ -183,6 +184,7 @@ pub struct DBTeamMember { pub organization_permissions: Option, pub accepted: bool, + #[cached_projection(wrap)] pub payouts_split: Decimal, pub ordering: i64, } diff --git a/apps/labrinth/src/database/models/user_item.rs b/apps/labrinth/src/database/models/user_item.rs index bd66a9e4ad..75f7ec9af5 100644 --- a/apps/labrinth/src/database/models/user_item.rs +++ b/apps/labrinth/src/database/models/user_item.rs @@ -9,6 +9,7 @@ use crate::models::billing::ChargeStatus; use crate::models::users::Badges; use crate::util::error::Context; use ariadne::ids::base62_impl::{parse_base62, to_base62}; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::DashMap; use rust_decimal::Decimal; @@ -16,11 +17,11 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const USERS_NAMESPACE: &str = "users"; -const USER_USERNAMES_NAMESPACE: &str = "users_usernames"; -const USERS_PROJECTS_NAMESPACE: &str = "users_projects"; +const USERS_NAMESPACE: &str = "users:v1"; +const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v1"; +const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v1"; -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone, Debug, CachedProjection)] pub struct DBUser { pub id: DBUserId, @@ -391,7 +392,7 @@ impl DBUser { let mut redis = redis.connect().await?; let cached_projects = redis - .get_deserialized_from_json::>( + .get_deserialized::>( USERS_PROJECTS_NAMESPACE, &user_id.0.to_string(), ) @@ -419,7 +420,7 @@ impl DBUser { let mut redis = redis.connect().await?; redis - .set_serialized_to_json( + .set_serialized( USERS_PROJECTS_NAMESPACE, user_id.0, &db_projects, diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index 9740120bda..45485dd7db 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -12,6 +12,7 @@ use crate::models::exp; use crate::models::projects::{FileType, VersionStatus}; use crate::queue::file_scan::scan_file; use crate::routes::internal::delphi::DelphiRunParameters; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use dashmap::{DashMap, DashSet}; use futures::TryStreamExt; @@ -22,8 +23,8 @@ use std::collections::HashMap; use std::iter; use tracing::error; -pub const VERSIONS_NAMESPACE: &str = "versions"; -const VERSION_FILES_NAMESPACE: &str = "versions_files"; +pub const VERSIONS_NAMESPACE: &str = "versions:v2"; +const VERSION_FILES_NAMESPACE: &str = "versions_files:v1"; pub async fn cleanup_unused_attribution_files_and_groups( transaction: &mut PgTransaction<'_>, @@ -375,7 +376,7 @@ impl DBLoaderVersion { } } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, CachedProjection)] pub struct DBVersion { pub id: DBVersionId, pub project_id: DBProjectId, @@ -390,6 +391,7 @@ pub struct DBVersion { pub status: VersionStatus, pub requested_status: Option, pub ordering: Option, + #[cached_projection(nested)] pub components: exp::VersionSerial, } @@ -1090,27 +1092,32 @@ impl DBVersion { } } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, CachedProjection)] pub struct VersionQueryResult { + #[cached_projection(nested)] pub inner: DBVersion, pub files: Vec, + #[cached_projection(nested)] pub version_fields: Vec, pub loaders: Vec, pub project_types: Vec, pub games: Vec, + #[cached_projection(nested)] pub dependencies: Vec, #[serde(flatten)] + #[cached_projection(nested)] pub components: exp::VersionQuery, } -#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Deserialize, Serialize, PartialEq, Eq, CachedProjection)] pub struct DependencyQueryResult { pub id: i32, pub project_id: Option, pub version_id: Option, pub file_name: Option, pub dependency_type: String, + #[cached_projection(nested)] pub attribution: Option, } @@ -1125,7 +1132,7 @@ pub struct FileQueryResult { pub file_type: Option, } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, CachedProjection)] pub struct DBFile { pub id: DBFileId, pub version_id: DBVersionId, diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 1c7a786e17..8708f7cb48 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -2,6 +2,7 @@ use crate::env::ENV; use super::models::DatabaseError; use ariadne::ids::base62_impl::{parse_base62, to_base62}; +use cached_projection::CachedProjection; use chrono::{TimeZone, Utc}; use dashmap::DashMap; use deadpool_redis::{Config, Runtime}; @@ -12,12 +13,15 @@ use prometheus::{IntGauge, Registry}; use redis::ToRedisArgs; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use std::borrow::Cow; use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::future::Future; use std::hash::Hash; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +use thiserror::Error; use tracing::{Instrument, info, info_span}; use util::{cmd, redis_pipe}; @@ -41,8 +45,121 @@ const MGET_CHUNK_SIZE: usize = 32; // BytesMut peak capacity that builds up under steady load. const REDIS_MAX_CONN_AGE: Duration = Duration::from_secs(120); +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Codec { + Raw = 0, + Lz4 = 1, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodingFormat { + Json, + Postcard, +} + +#[derive(Debug, Error)] +#[error("invalid redis codec")] +pub struct InvalidCodec; + +#[derive(Debug, Error)] +#[error("invalid redis encoding format")] +pub struct InvalidEncodingFormat; + +impl TryFrom for Codec { + type Error = InvalidCodec; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Raw), + 1 => Ok(Self::Lz4), + _ => Err(InvalidCodec), + } + } +} + +impl FromStr for Codec { + type Err = InvalidCodec; + + fn from_str(value: &str) -> Result { + match value { + "lz4" => Ok(Self::Lz4), + _ => Err(InvalidCodec), + } + } +} + +impl FromStr for EncodingFormat { + type Err = InvalidEncodingFormat; + + fn from_str(value: &str) -> Result { + match value { + "json" => Ok(Self::Json), + "postcard" => Ok(Self::Postcard), + _ => Err(InvalidEncodingFormat), + } + } +} + +fn encode_value(value: &T) -> Result, DatabaseError> +where + T: CachedProjection + Serialize, +{ + let mut value = match ENV.REDIS_ENCODING_FORMAT { + EncodingFormat::Json => serde_json::to_vec(value)?, + EncodingFormat::Postcard => { + postcard::to_allocvec(&value.project_ref())? + } + }; + + if ENV.REDIS_COMPRESSION_LEVEL > 0 + && ENV.REDIS_COMPRESSION_ALGORITHM == Codec::Lz4 + && value.len() >= ENV.REDIS_COMPRESSION_THRESHOLD_BYTES + { + let compressed = lz4_flex::block::compress_prepend_size(&value); + let savings_ratio = value.len().saturating_sub(compressed.len()) as f64 + / value.len().max(1) as f64 + * 100.0; + + if savings_ratio >= ENV.REDIS_COMPRESSION_MIN_SAVINGS_RATIO { + let mut encoded = Vec::with_capacity(compressed.len() + 1); + encoded.push(Codec::Lz4 as u8); + encoded.extend(compressed); + return Ok(encoded); + } + } + + let mut encoded = Vec::with_capacity(value.len() + 1); + encoded.push(Codec::Raw as u8); + encoded.append(&mut value); + Ok(encoded) +} + +fn decode_value(value: &[u8]) -> Option +where + T: CachedProjection + DeserializeOwned, +{ + let (codec, value) = value.split_first()?; + let value = match Codec::try_from(*codec).ok()? { + Codec::Raw => Cow::Borrowed(value), + Codec::Lz4 => { + Cow::Owned(lz4_flex::block::decompress_size_prepended(value).ok()?) + } + }; + + match ENV.REDIS_ENCODING_FORMAT { + EncodingFormat::Json => serde_json::from_slice(&value).ok(), + EncodingFormat::Postcard => { + let projected = + postcard::from_bytes::(&value).ok()?; + Some(T::from_projected(projected)) + } + } +} + fn cache_expiries(namespace: &str) -> (i64, i64) { - match namespace { + // Namespaces may embed a version suffix like `:v1`, so split it out. + match namespace.split_once(':').map(|t| t.0).unwrap_or(namespace) { "versions" | "versions_files" => { (VERSION_DEFAULT_EXPIRY, VERSION_ACTUAL_EXPIRY) } @@ -188,7 +305,7 @@ impl RedisPool { where F: FnOnce(Vec) -> Fut, Fut: Future, DatabaseError>>, - T: Serialize + DeserializeOwned, + T: CachedProjection + Serialize + DeserializeOwned, K: Display + Hash + Eq @@ -216,7 +333,7 @@ impl RedisPool { where F: FnOnce(Vec) -> Fut, Fut: Future, DatabaseError>>, - T: Serialize + DeserializeOwned, + T: CachedProjection + Serialize + DeserializeOwned, K: Display + Hash + Eq @@ -254,7 +371,7 @@ impl RedisPool { where F: FnOnce(Vec) -> Fut, Fut: Future, T)>, DatabaseError>>, - T: Serialize + DeserializeOwned, + T: CachedProjection + Serialize + DeserializeOwned, I: Display + Hash + Eq + PartialEq + Clone + Debug, K: Display + Hash @@ -291,7 +408,7 @@ impl RedisPool { where F: FnOnce(Vec) -> Fut, Fut: Future, T)>, DatabaseError>>, - T: Serialize + DeserializeOwned, + T: CachedProjection + Serialize + DeserializeOwned, I: Display + Hash + Eq + PartialEq + Clone + Debug, K: Display + Hash @@ -367,12 +484,11 @@ impl RedisPool { for chunk in args.chunks(MGET_CHUNK_SIZE) { let part = cmd("MGET") .arg(chunk) - .query_async::>>(&mut connection) + .query_async::>>>(&mut connection) .await?; cached_values.extend(part.into_iter().filter_map(|x| { x.and_then(|val| { - serde_json::from_str::>(&val) - .ok() + decode_value::>(&val) }) .map(|val| (val.key.clone(), val)) })); @@ -492,7 +608,7 @@ impl RedisPool { "{}_{namespace}:{key}", self.meta_namespace ), - serde_json::to_string(&value)?, + encode_value(&value)?, default_expiry as u64, ); pipe_cmds += 1; @@ -624,47 +740,39 @@ impl<'a> Drop for LockSentinel<'a> { impl RedisConnection { #[tracing::instrument(skip(self))] - pub async fn set( + pub async fn set( &mut self, namespace: &str, id: &str, - data: &str, + data: D, expiry: Option, - ) -> Result<(), DatabaseError> { + ) -> Result<(), DatabaseError> + where + D: ToRedisArgs + Send + Sync + Debug, + { let mut cmd = cmd("SET"); - redis_args( - &mut cmd, - vec![ - format!("{}_{}:{}", self.meta_namespace, namespace, id), - data.to_string(), - "EX".to_string(), - expiry.unwrap_or(DEFAULT_EXPIRY).to_string(), - ] - .as_slice(), - ); + cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)) + .arg(data) + .arg("EX") + .arg(expiry.unwrap_or(DEFAULT_EXPIRY)); redis_execute::<()>(&mut cmd, &mut self.connection).await?; Ok(()) } #[tracing::instrument(skip(self, id, data))] - pub async fn set_serialized_to_json( + pub async fn set_serialized( &mut self, namespace: &str, id: Id, - data: D, + data: &D, expiry: Option, ) -> Result<(), DatabaseError> where Id: Display, - D: serde::Serialize, + D: CachedProjection + serde::Serialize, { - self.set( - namespace, - &id.to_string(), - &serde_json::to_string(&data)?, - expiry, - ) - .await + self.set(namespace, &id.to_string(), encode_value(data)?, expiry) + .await } #[tracing::instrument(skip(self))] @@ -688,7 +796,7 @@ impl RedisConnection { &mut self, namespace: &str, ids: &[String], - ) -> Result>, DatabaseError> { + ) -> Result>>, DatabaseError> { let mut cmd = cmd("MGET"); redis_args( &mut cmd, @@ -702,35 +810,40 @@ impl RedisConnection { } #[tracing::instrument(skip(self))] - pub async fn get_deserialized_from_json( + pub async fn get_deserialized( &mut self, namespace: &str, id: &str, ) -> Result, DatabaseError> where - R: for<'a> serde::Deserialize<'a>, + R: CachedProjection + DeserializeOwned, { - Ok(self - .get(namespace, id) - .await? - .and_then(|x| serde_json::from_str(&x).ok())) + let mut cmd = cmd("GET"); + redis_args( + &mut cmd, + vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)] + .as_slice(), + ); + let value: Option> = + redis_execute(&mut cmd, &mut self.connection).await?; + Ok(value.and_then(|value| decode_value(&value))) } #[tracing::instrument(skip(self))] - pub async fn get_many_deserialized_from_json( + pub async fn get_many_deserialized( &mut self, namespace: &str, ids: &[String], ) -> Result>, DatabaseError> where - R: for<'a> serde::Deserialize<'a>, + R: CachedProjection + DeserializeOwned, { Ok(self .get_many(namespace, ids) .await? .into_iter() - .map(|x| x.and_then(|val| serde_json::from_str::(&val).ok())) - .collect::>()) + .map(|value| value.and_then(|value| decode_value::(&value))) + .collect()) } #[tracing::instrument(skip(self, id))] @@ -799,7 +912,7 @@ impl RedisConnection { namespace: &str, key: &str, timeout: Option, - ) -> Result, DatabaseError> { + ) -> Result; 2]>, DatabaseError> { let key = format!("{}_{namespace}:{key}", self.meta_namespace); // a timeout of 0 is infinite let timeout = timeout.unwrap_or(0.0); @@ -826,15 +939,21 @@ impl RedisConnection { } } -#[derive(Serialize, Deserialize)] +#[derive(CachedProjection, Serialize, Deserialize)] pub struct RedisValue { key: K, - #[serde(skip_serializing_if = "Option::is_none")] alias: Option, iat: i64, + #[cached_projection(nested)] val: T, } +impl RedisValue { + pub fn value(&self) -> &T { + &self.val + } +} + pub fn redis_args(cmd: &mut util::InstrumentedCmd, args: &[String]) { for arg in args { cmd.arg(arg); diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 8c8e32b0b7..e6033acaf1 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -289,6 +289,11 @@ vars! { REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64; REDIS_MAX_CONNECTIONS: u32 = 10000u32; REDIS_MIN_CONNECTIONS: usize = 0usize; + REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Postcard; + REDIS_COMPRESSION_LEVEL: i32 = 0i32; + REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4; + REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize; + REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64; SEARCH_OPERATION_TIMEOUT: u64 = 300000u64; diff --git a/apps/labrinth/src/models/exp/minecraft.rs b/apps/labrinth/src/models/exp/minecraft.rs index 3b8fb5508c..19da37ecd5 100644 --- a/apps/labrinth/src/models/exp/minecraft.rs +++ b/apps/labrinth/src/models/exp/minecraft.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use eyre::Result; use serde::{Deserialize, Serialize}; @@ -109,11 +110,11 @@ pub enum Language { } component::define! { - #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)] + #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)] pub struct ModProject {} /// Listing for a Minecraft server. - #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)] + #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)] pub struct ServerProject { #[base(serde(default))] #[edit(serde( @@ -160,11 +161,11 @@ component::define! { } /// Version of a Minecraft Java server listing. - #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)] + #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)] pub struct JavaServerVersion {} /// Listing for a Minecraft Bedrock server. - #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)] + #[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema, CachedProjection)] pub struct BedrockServerProject { #[base()] #[edit(serde(default))] @@ -200,13 +201,22 @@ impl ProjectComponent for BedrockServerProject { } /// Listing for a Minecraft Java server. -#[derive(Debug, Clone, Serialize, Deserialize, Validate, utoipa::ToSchema)] +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Validate, + utoipa::ToSchema, + CachedProjection, +)] pub struct JavaServerProject { /// Address (IP or domain name) of the Java server, excluding port. #[validate(length(max = 255))] pub address: String, /// What game content this server is using. #[serde(default)] + #[cached_projection(nested)] pub content: ServerContent, } @@ -219,9 +229,12 @@ pub struct JavaServerProjectEdit { pub content: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[derive( + Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection, +)] pub struct JavaServerProjectQuery { pub address: String, + #[cached_projection(nested)] pub content: ServerContentQuery, pub ping: Option, pub verified_plays_2w: Option, @@ -324,7 +337,9 @@ impl ComponentEdit for JavaServerProjectEdit { } /// What game content a [`JavaServerProject`] is using. -#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[derive( + Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection, +)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum ServerContent { /// Server runs modded content with a modpack found on the Modrinth platform. @@ -346,7 +361,9 @@ pub enum ServerContent { } /// What game content a [`JavaServerProject`] is using. -#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[derive( + Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection, +)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum ServerContentQuery { /// Server runs modded content with a modpack found on the Modrinth platform. @@ -399,7 +416,9 @@ pub enum ServerRegion { } /// Recorded ping attempt that Labrinth made to a Minecraft Java server project. -#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[derive( + Debug, Clone, Serialize, Deserialize, utoipa::ToSchema, CachedProjection, +)] pub struct JavaServerPing { /// When the ping was performed. pub when: DateTime, diff --git a/apps/labrinth/src/models/exp/project.rs b/apps/labrinth/src/models/exp/project.rs index 3fdaba6257..ac16417c5a 100644 --- a/apps/labrinth/src/models/exp/project.rs +++ b/apps/labrinth/src/models/exp/project.rs @@ -1,3 +1,4 @@ +use cached_projection::CachedProjection; use eyre::Result; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; @@ -52,11 +53,12 @@ macro_rules! define_project_components { impl ComponentKind for ProjectComponentKind {} - #[derive(Debug, Clone, Default, Serialize, Deserialize, Validate)] + #[derive(Debug, Clone, Default, Serialize, Deserialize, Validate, CachedProjection)] pub struct ProjectSerial { $( #[validate(nested)] #[serde(default, skip_serializing_if = "Option::is_none")] + #[cached_projection(nested)] pub $field_name: Option<$ty>, )* } @@ -113,10 +115,11 @@ macro_rules! define_project_components { } } - #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)] + #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema, CachedProjection)] pub struct ProjectQuery { $( #[serde(skip_serializing_if = "Option::is_none")] + #[cached_projection(nested)] pub $field_name: Option>, )* } @@ -258,7 +261,7 @@ pub async fn fetch_query_context( HashMap::new() } else { redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( server_ping::REDIS_NAMESPACE, &minecraft_java_server_pings .iter() @@ -281,7 +284,7 @@ pub async fn fetch_query_context( HashMap::new() } else { redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( MINECRAFT_SERVER_ANALYTICS, &minecraft_server_analytics .iter() diff --git a/apps/labrinth/src/models/exp/version.rs b/apps/labrinth/src/models/exp/version.rs index 2df4b90954..ab7a6ac5e9 100644 --- a/apps/labrinth/src/models/exp/version.rs +++ b/apps/labrinth/src/models/exp/version.rs @@ -4,6 +4,7 @@ use crate::models::exp::{ minecraft, }; +use cached_projection::CachedProjection; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use validator::Validate; @@ -43,9 +44,10 @@ macro_rules! define_version_components { )* } - #[derive(Debug, Clone, Default, Serialize, Deserialize)] + #[derive(Debug, Clone, Default, Serialize, Deserialize, CachedProjection)] pub struct VersionSerial { $( + #[cached_projection(nested)] pub $field_name: Option<$ty>, )* } @@ -71,9 +73,10 @@ macro_rules! define_version_components { } } - #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)] + #[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema, CachedProjection)] pub struct VersionQuery { $( + #[cached_projection(nested)] pub $field_name: Option>, )* } diff --git a/apps/labrinth/src/models/v3/billing.rs b/apps/labrinth/src/models/v3/billing.rs index c3ef13facb..7f457e2f21 100644 --- a/apps/labrinth/src/models/v3/billing.rs +++ b/apps/labrinth/src/models/v3/billing.rs @@ -2,6 +2,7 @@ use crate::models::ids::{ ChargeId, ProductId, ProductPriceId, UserSubscriptionId, }; use ariadne::ids::UserId; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -14,7 +15,7 @@ pub struct Product { pub unitary: bool, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum ProductMetadata { Midas, @@ -55,7 +56,7 @@ pub struct ProductPrice { pub currency_code: String, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, CachedProjection)] #[serde(tag = "type", rename_all = "kebab-case")] pub enum Price { OneTime { diff --git a/apps/labrinth/src/models/v3/notifications.rs b/apps/labrinth/src/models/v3/notifications.rs index 6a655958f9..1c544b0f92 100644 --- a/apps/labrinth/src/models/v3/notifications.rs +++ b/apps/labrinth/src/models/v3/notifications.rs @@ -10,6 +10,7 @@ use crate::models::ids::{ use crate::models::projects::ProjectStatus; use crate::routes::ApiError; use ariadne::ids::UserId; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -151,7 +152,7 @@ impl NotificationType { } } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, CachedProjection)] #[serde(tag = "type", rename_all = "snake_case")] pub enum NotificationBody { ProjectUpdate { diff --git a/apps/labrinth/src/models/v3/projects.rs b/apps/labrinth/src/models/v3/projects.rs index ac50b33228..a93afa9e5f 100644 --- a/apps/labrinth/src/models/v3/projects.rs +++ b/apps/labrinth/src/models/v3/projects.rs @@ -10,6 +10,7 @@ use crate::models::ids::{ }; use crate::routes::{FileHash, HashAlgorithm}; use ariadne::ids::UserId; +use cached_projection::CachedProjection; use chrono::{DateTime, Utc}; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -676,7 +677,14 @@ pub struct FlameProject { } #[derive( - Debug, Serialize, Deserialize, Clone, PartialEq, Eq, utoipa::ToSchema, + Debug, + Serialize, + Deserialize, + Clone, + PartialEq, + Eq, + utoipa::ToSchema, + CachedProjection, )] #[serde(untagged)] pub enum AttributionLicense { @@ -685,11 +693,19 @@ pub enum AttributionLicense { } #[derive( - Debug, Serialize, Deserialize, Clone, PartialEq, Eq, utoipa::ToSchema, + Debug, + Serialize, + Deserialize, + Clone, + PartialEq, + Eq, + utoipa::ToSchema, + CachedProjection, )] #[serde(tag = "kind", rename_all = "snake_case")] pub enum AttributionResolutionKind { License { + #[cached_projection(nested)] license: AttributionLicense, link_to_work: Url, }, @@ -697,6 +713,7 @@ pub enum AttributionResolutionKind { link_to_work: Url, }, MyProject { + #[cached_projection(nested)] license: AttributionLicense, }, SpecialPermissions { @@ -1004,12 +1021,20 @@ pub struct Dependency { } #[derive( - Serialize, Deserialize, Clone, Debug, PartialEq, Eq, utoipa::ToSchema, + Serialize, + Deserialize, + Clone, + Debug, + PartialEq, + Eq, + utoipa::ToSchema, + CachedProjection, )] pub struct DependencyAttribution { #[serde(skip_serializing_if = "Option::is_none")] pub flame_project: Option, #[serde(skip_serializing_if = "Option::is_none")] + #[cached_projection(nested)] pub resolution: Option, } diff --git a/apps/labrinth/src/queue/analytics/cache.rs b/apps/labrinth/src/queue/analytics/cache.rs index d929173812..87521163e3 100644 --- a/apps/labrinth/src/queue/analytics/cache.rs +++ b/apps/labrinth/src/queue/analytics/cache.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use cached_projection::CachedProjection; use const_format::formatcp; use eyre::{Result, eyre}; use serde::{Deserialize, Serialize}; @@ -13,9 +14,9 @@ use crate::{ util::error::Context, }; -pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics"; +pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v1"; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, CachedProjection)] pub struct MinecraftServerAnalytics { pub verified_plays_2w: u64, pub verified_plays_4w: u64, @@ -117,10 +118,10 @@ pub async fn cache_analytics( debug!("Caching analytics for {project_id}: {analytics:?}"); redis - .set_serialized_to_json( + .set_serialized( MINECRAFT_SERVER_ANALYTICS, project_id.to_string(), - analytics, + &analytics, None, ) .await diff --git a/apps/labrinth/src/queue/analytics/mod.rs b/apps/labrinth/src/queue/analytics/mod.rs index 6d4716c693..79b02b6cd5 100644 --- a/apps/labrinth/src/queue/analytics/mod.rs +++ b/apps/labrinth/src/queue/analytics/mod.rs @@ -13,9 +13,9 @@ use tracing::trace; pub mod cache; -const DOWNLOADS_NAMESPACE: &str = "downloads"; -const VIEWS_NAMESPACE: &str = "views"; -const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays"; +const DOWNLOADS_NAMESPACE: &str = "downloads:v1"; +const VIEWS_NAMESPACE: &str = "views:v1"; +const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v1"; const MINECRAFT_SERVER_PLAYS_EXPIRY: u64 = 86_400; // 24 hours const MINECRAFT_SERVER_PLAYS_LIMIT: u32 = 5; diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index 5b6305961b..46dbd67179 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -26,8 +26,9 @@ pub struct ServerPingQueue { pub incremental_search_queue: IncrementalSearchQueue, } -pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping"; -pub const REDIS_FAILURE_NAMESPACE: &str = "minecraft_java_server_ping_failures"; +pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v1"; +pub const REDIS_FAILURE_NAMESPACE: &str = + "minecraft_java_server_ping_failures:v1"; pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings"; impl ServerPingQueue { @@ -133,12 +134,7 @@ impl ServerPingQueue { // ping succeeded; immediately update its online status in redis redis - .set_serialized_to_json( - REDIS_NAMESPACE, - project_id, - ping, - None, - ) + .set_serialized(REDIS_NAMESPACE, project_id, ping, None) .await .wrap_err("failed to set redis key")?; updated_project = true; @@ -160,7 +156,7 @@ impl ServerPingQueue { && count >= ENV.SERVER_PING_MAX_FAIL_COUNT { redis - .set_serialized_to_json( + .set_serialized( REDIS_NAMESPACE, project_id, ping, @@ -254,7 +250,7 @@ impl ServerPingQueue { .collect::>(); let all_server_last_pings = redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( REDIS_NAMESPACE, &all_project_ids, ) diff --git a/apps/labrinth/src/routes/internal/campaign.rs b/apps/labrinth/src/routes/internal/campaign.rs index e3757b397f..6ba346c3cc 100644 --- a/apps/labrinth/src/routes/internal/campaign.rs +++ b/apps/labrinth/src/routes/internal/campaign.rs @@ -1,5 +1,6 @@ use actix_web::{HttpRequest, get, post, web}; use base64::Engine; +use cached_projection::CachedProjection; use chrono::{DateTime, Duration, Utc}; use eyre::eyre; use hmac::{Hmac, Mac}; @@ -60,15 +61,19 @@ struct TiltifyMeta { subscription_source_type: String, } -#[derive(Clone, Debug, Serialize, Deserialize, utoipa::ToSchema)] +#[derive( + Clone, Debug, Serialize, Deserialize, utoipa::ToSchema, CachedProjection, +)] pub struct CampaignInfo { + #[cached_projection(wrap)] total_donations_usd: Decimal, + #[cached_projection(wrap)] target_usd: Decimal, num_donators: usize, cached_at: DateTime, } -const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info"; +const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v2"; const CAMPAIGN_INFO_CACHE_STALE_SECONDS: i64 = 15 * 60; const CAMPAIGN_INFO_CACHE_TTL_SECONDS: i64 = 24 * 60 * 60; @@ -326,7 +331,7 @@ pub async fn pride_26( .wrap_internal_err("connecting to redis")?; let cached = redis_connection - .get_deserialized_from_json::( + .get_deserialized::( CAMPAIGN_INFO_CACHE_NAMESPACE, campaign_id, ) @@ -382,7 +387,7 @@ pub async fn pride_26( }; redis_connection - .set_serialized_to_json( + .set_serialized( CAMPAIGN_INFO_CACHE_NAMESPACE, campaign_id, &campaign_info, diff --git a/apps/labrinth/src/routes/internal/external_notifications.rs b/apps/labrinth/src/routes/internal/external_notifications.rs index 5d9d2997fa..a013c3028b 100644 --- a/apps/labrinth/src/routes/internal/external_notifications.rs +++ b/apps/labrinth/src/routes/internal/external_notifications.rs @@ -427,7 +427,7 @@ async fn broadcast_notifications( redis, RedisFriendsMessage::Notification { to_user, - notification, + notification_id, }, ) .await diff --git a/apps/labrinth/src/routes/internal/flows.rs b/apps/labrinth/src/routes/internal/flows.rs index 88d4a9bdb2..c81c66a32c 100644 --- a/apps/labrinth/src/routes/internal/flows.rs +++ b/apps/labrinth/src/routes/internal/flows.rs @@ -2126,7 +2126,7 @@ async fn validate_2fa_code( ) .map_err(|_| AuthenticationError::InvalidCredentials)?; - const TOTP_NAMESPACE: &str = "used_totp"; + const TOTP_NAMESPACE: &str = "used_totp:v1"; let mut conn = redis.connect().await?; // Check if TOTP has already been used diff --git a/apps/labrinth/src/routes/internal/gotenberg.rs b/apps/labrinth/src/routes/internal/gotenberg.rs index b5614c3add..71d39dedb4 100644 --- a/apps/labrinth/src/routes/internal/gotenberg.rs +++ b/apps/labrinth/src/routes/internal/gotenberg.rs @@ -70,11 +70,11 @@ pub async fn success_callback( let body = base64::engine::general_purpose::STANDARD.encode(&body); - let redis_msg = serde_json::to_string(&Ok::< + let redis_msg = postcard::to_allocvec(&Ok::< GotenbergDocument, GotenbergError, >(GotenbergDocument { body })) - .wrap_internal_err("failed to serialize document to JSON")?; + .wrap_internal_err("failed to serialize Redis document response")?; redis .lpush( @@ -139,11 +139,11 @@ pub async fn error_callback( .await .wrap_internal_err("failed to get Redis connection")?; - let redis_msg = serde_json::to_string(&Err::< + let redis_msg = postcard::to_allocvec(&Err::< GotenbergDocument, GotenbergError, >(error_body)) - .wrap_internal_err("failed to serialize error to JSON")?; + .wrap_internal_err("failed to serialize Redis error response")?; redis .lpush( diff --git a/apps/labrinth/src/routes/v3/content/mod.rs b/apps/labrinth/src/routes/v3/content/mod.rs index 78e3098c2e..c7f4d97065 100644 --- a/apps/labrinth/src/routes/v3/content/mod.rs +++ b/apps/labrinth/src/routes/v3/content/mod.rs @@ -14,6 +14,7 @@ use crate::queue::session::AuthQueue; use actix_web::{HttpRequest, post, web}; use ariadne::ids::base62_impl::parse_base62; use async_trait::async_trait; +use cached_projection::CachedProjection; use modrinth_content_management::{ ContentMetadataProvider, Error as ResolveError, ResolveContentPlan, ResolveContentRequest, @@ -22,8 +23,8 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; -const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve"; -const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat"; +const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v2"; +const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v1"; const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v1"; const CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS: i64 = 60 * 60 * 24; @@ -90,9 +91,10 @@ struct ResolveContentTrace { project_versions: BTreeMap, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, CachedProjection)] struct CachedResolveContentPlan { trace: ResolveContentTrace, + #[cached_projection(wrap)] plan: ResolveContentPlan, } @@ -340,7 +342,7 @@ async fn get_cached_resolve_content_plan( }; match redis - .get_deserialized_from_json(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key) + .get_deserialized(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key) .await { Ok(cached) => cached, @@ -368,7 +370,7 @@ async fn set_cached_resolve_content_plan( }; if let Err(error) = redis - .set_serialized_to_json( + .set_serialized( CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key, cached, diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index e7b55dd9f3..ac003ed4e5 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -151,7 +151,7 @@ async fn hydrate_search_results( } else { let mut redis = redis_pool.connect().await?; let ping_results = redis - .get_many_deserialized_from_json::( + .get_many_deserialized::( server_ping::REDIS_NAMESPACE, &project_ids .iter() diff --git a/apps/labrinth/src/sync/friends.rs b/apps/labrinth/src/sync/friends.rs index d14699eeeb..770cb1d6c9 100644 --- a/apps/labrinth/src/sync/friends.rs +++ b/apps/labrinth/src/sync/friends.rs @@ -1,4 +1,6 @@ use crate::database::PgPool; +use crate::database::models::notification_item::DBNotification; +use crate::models::ids::NotificationId; use crate::models::notifications::Notification; use crate::queue::socket::ActiveSockets; use crate::routes::internal::statuses::{ @@ -13,10 +15,9 @@ use redis::{RedisWrite, ToRedisArgs}; use serde::{Deserialize, Serialize}; use tokio_stream::StreamExt; -pub const FRIENDS_CHANNEL_NAME: &str = "friends"; +pub const FRIENDS_CHANNEL_NAME: &str = "friends:v1"; #[derive(Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] pub enum RedisFriendsMessage { StatusUpdate { status: UserStatus, @@ -30,7 +31,7 @@ pub enum RedisFriendsMessage { }, Notification { to_user: UserId, - notification: Notification, + notification_id: NotificationId, }, } @@ -39,7 +40,7 @@ impl ToRedisArgs for RedisFriendsMessage { where W: ?Sized + RedisWrite, { - out.write_arg(&serde_json::to_vec(&self).unwrap()) + out.write_arg(&postcard::to_allocvec(&self).unwrap()) } } @@ -54,7 +55,7 @@ pub async fn handle_pubsub( if message.get_channel_name() != FRIENDS_CHANNEL_NAME { continue; } - let payload = serde_json::from_slice(message.get_payload_bytes()); + let payload = postcard::from_bytes(message.get_payload_bytes()); let pool = pool.clone(); let sockets = sockets.clone(); @@ -94,14 +95,18 @@ pub async fn handle_pubsub( Ok(RedisFriendsMessage::Notification { to_user, - notification, + notification_id, }) => { - let _ = send_notification_to_user( - &sockets, - to_user, - ¬ification, - ) - .await; + if let Ok(Some(notification)) = + DBNotification::get(notification_id.into(), &pool).await + { + let _ = send_notification_to_user( + &sockets, + to_user, + &Notification::from(notification), + ) + .await; + } } Err(_) => {} diff --git a/apps/labrinth/src/sync/status.rs b/apps/labrinth/src/sync/status.rs index 58f4a9602e..f16ed3702b 100644 --- a/apps/labrinth/src/sync/status.rs +++ b/apps/labrinth/src/sync/status.rs @@ -17,10 +17,10 @@ pub async fn get_user_status( if let Ok(mut conn) = redis.pool.get().await && let Ok(mut statuses) = - conn.sscan::<_, String>(get_field_name(user)).await - && let Some(status_json) = statuses.next_item().await + conn.sscan::<_, Vec>(get_field_name(user)).await + && let Some(status) = statuses.next_item().await { - return serde_json::from_str::(&status_json).ok(); + return postcard::from_bytes::(&status).ok(); } None @@ -40,11 +40,11 @@ pub async fn replace_user_status( let mut pipe = redis::pipe(); pipe.atomic(); if let Some(status) = old_status { - pipe.srem(&field_name, serde_json::to_string(&status).unwrap()) + pipe.srem(&field_name, postcard::to_allocvec(status).unwrap()) .ignore(); } if let Some(status) = new_status { - pipe.sadd(&field_name, serde_json::to_string(&status).unwrap()) + pipe.sadd(&field_name, postcard::to_allocvec(status).unwrap()) .ignore(); pipe.expire(&field_name, EXPIRY_TIME_SECONDS).ignore(); } @@ -65,5 +65,5 @@ pub async fn push_back_user_expiry( } fn get_field_name(user: UserId) -> String { - format!("user_status:{user}") + format!("user_status:v1:{user}") } diff --git a/apps/labrinth/src/util/gotenberg.rs b/apps/labrinth/src/util/gotenberg.rs index 854c27e124..75e74fe65b 100644 --- a/apps/labrinth/src/util/gotenberg.rs +++ b/apps/labrinth/src/util/gotenberg.rs @@ -14,7 +14,7 @@ pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName = HeaderName::from_static("modrinth-generated-pdf-type"); pub const MODRINTH_PAYMENT_ID: HeaderName = HeaderName::from_static("modrinth-payment-id"); -pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements"; +pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v1"; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PaymentStatement { @@ -203,7 +203,7 @@ impl GotenbergClient { .wrap_internal_err("failed to get document over Redis")? .wrap_internal_err("no document was returned from Redis")?; - let document = serde_json::from_str::< + let document = postcard::from_bytes::< Result, >(&document) .wrap_internal_err("failed to deserialize Redis document response")? diff --git a/apps/labrinth/src/util/ratelimit.rs b/apps/labrinth/src/util/ratelimit.rs index a2534cd205..067af8b498 100644 --- a/apps/labrinth/src/util/ratelimit.rs +++ b/apps/labrinth/src/util/ratelimit.rs @@ -11,7 +11,7 @@ use chrono::Utc; use std::str::FromStr; use std::sync::Arc; -const RATE_LIMIT_NAMESPACE: &str = "rate_limit"; +const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v1"; const RATE_LIMIT_EXPIRY: i64 = 300; // 5 minutes const MINUTE_IN_NANOS: i64 = 60_000_000_000; diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index c337da781c..fd5046cb99 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -18,9 +18,11 @@ use common::environment::{ use common::permissions::{PermissionsTest, PermissionsTestContext}; use futures::StreamExt; use hex::ToHex; +use labrinth::database::models::DBProjectId; use labrinth::database::models::project_item::{ - PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, + PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, ProjectQueryResult, }; +use labrinth::database::redis::RedisValue; use labrinth::models::ids::ProjectId; use labrinth::models::teams::ProjectPermissions; use labrinth::util::actix::{MultipartSegment, MultipartSegmentData}; @@ -67,19 +69,21 @@ async fn test_get_project() { Some(parse_base62(alpha_project_id).unwrap() as i64) ); - let cached_project = redis_pool - .get( + let cached_project: RedisValue< + ProjectQueryResult, + DBProjectId, + String, + > = redis_pool + .get_deserialized( PROJECTS_NAMESPACE, &parse_base62(alpha_project_id).unwrap().to_string(), ) .await .unwrap() .unwrap(); - let cached_project: serde_json::Value = - serde_json::from_str(&cached_project).unwrap(); assert_eq!( - cached_project["val"]["inner"]["slug"], - json!(alpha_project_slug) + cached_project.value().inner.slug.as_ref(), + Some(alpha_project_slug) ); // Make the request again, this time it should be cached diff --git a/apps/labrinth/tests/version.rs b/apps/labrinth/tests/version.rs index 5e5a6ecf97..951b408cd6 100644 --- a/apps/labrinth/tests/version.rs +++ b/apps/labrinth/tests/version.rs @@ -14,7 +14,11 @@ use common::asserts::assert_common_version_ids; use common::database::USER_USER_PAT; use common::environment::{with_test_environment, with_test_environment_all}; use futures::StreamExt; -use labrinth::database::models::version_item::VERSIONS_NAMESPACE; +use labrinth::database::models::DBVersionId; +use labrinth::database::models::version_item::{ + VERSIONS_NAMESPACE, VersionQueryResult, +}; +use labrinth::database::redis::RedisValue; use labrinth::models::ids::VersionId; use labrinth::models::projects::{ Dependency, DependencyType, VersionStatus, VersionType, @@ -47,18 +51,20 @@ async fn test_get_version() { assert_eq!(&version.id.to_string(), alpha_version_id); let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap(); - let cached_project = redis_conn - .get( + let cached_version: RedisValue< + VersionQueryResult, + DBVersionId, + String, + > = redis_conn + .get_deserialized( VERSIONS_NAMESPACE, &parse_base62(alpha_version_id).unwrap().to_string(), ) .await .unwrap() .unwrap(); - let cached_project: serde_json::Value = - serde_json::from_str(&cached_project).unwrap(); assert_eq!( - cached_project["val"]["inner"]["project_id"], + cached_version.value().inner.project_id.0, json!(parse_base62(alpha_project_id).unwrap()) ); diff --git a/packages/cached-projection-derive/Cargo.toml b/packages/cached-projection-derive/Cargo.toml new file mode 100644 index 0000000000..a6e912a4f9 --- /dev/null +++ b/packages/cached-projection-derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "cached-projection-derive" +edition.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true, features = ["full"] } + +[lints] +workspace = true diff --git a/packages/cached-projection-derive/src/lib.rs b/packages/cached-projection-derive/src/lib.rs new file mode 100644 index 0000000000..efaaa53bfe --- /dev/null +++ b/packages/cached-projection-derive/src/lib.rs @@ -0,0 +1,765 @@ +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use quote::{format_ident, quote}; +use syn::punctuated::Punctuated; +use syn::{ + Attribute, Data, DataEnum, DataStruct, DeriveInput, Field, Fields, + GenericParam, Generics, Lifetime, LifetimeParam, Meta, Token, Type, + parse_macro_input, parse_quote, +}; + +#[proc_macro_derive(CachedProjection, attributes(cached_projection))] +pub fn cached_projection(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + derive_cached_projection(&input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} + +#[derive(Clone, Copy)] +enum Projection { + Ordinary, + Nested, + Wrap, +} + +fn derive_cached_projection(input: &DeriveInput) -> syn::Result { + let name = &input.ident; + let projected_name = format_ident!("{name}ProjectedType"); + let projected_ref_name = format_ident!("{name}ProjectedTypeRef"); + let visibility = &input.vis; + let lifetime = Lifetime::new("'__cached_projection", name.span()); + + let fields = all_fields(&input.data); + let ref_lifetime = (!fields.is_empty()).then_some(&lifetime); + let projections = fields + .iter() + .map(|field| field_projection(field)) + .collect::>>()?; + + let mut implementation_generics = input.generics.clone(); + add_projection_bounds( + &mut implementation_generics, + fields.iter().zip(&projections), + ); + let (impl_generics, _, where_clause) = + implementation_generics.split_for_impl(); + let (_, type_generics, _) = input.generics.split_for_impl(); + + let mut ref_generics = implementation_generics.clone(); + if ref_lifetime.is_some() { + ref_generics.params.insert( + 0, + GenericParam::Lifetime(LifetimeParam::new(lifetime.clone())), + ); + add_outlives_bounds(&mut ref_generics, fields.iter(), &lifetime); + } + let ref_type_arguments = generic_arguments(&input.generics, ref_lifetime); + + let owned_definition = projected_definition( + &input.data, + &projected_name, + visibility, + &implementation_generics, + None, + )?; + let ref_definition = projected_definition( + &input.data, + &projected_ref_name, + visibility, + &ref_generics, + ref_lifetime, + )?; + + let project_ref_body = + project_ref_body(&input.data, &projected_ref_name, &projections)?; + let into_projected_body = + into_projected_body(&input.data, &projected_name, &projections)?; + let from_projected_body = + from_projected_body(&input.data, &projected_name, &projections)?; + + Ok(quote! { + #owned_definition + #ref_definition + + impl #impl_generics ::cached_projection::CachedProjection for #name #type_generics #where_clause { + type ProjectedType = #projected_name #type_generics; + type ProjectedTypeRef<#lifetime> = #projected_ref_name #ref_type_arguments + where + Self: #lifetime; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + #project_ref_body + } + + fn into_projected(self) -> Self::ProjectedType { + #into_projected_body + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + #from_projected_body + } + } + }) +} + +fn all_fields(data: &Data) -> Vec<&Field> { + match data { + Data::Struct(data) => data.fields.iter().collect(), + Data::Enum(data) => data + .variants + .iter() + .flat_map(|variant| variant.fields.iter()) + .collect(), + Data::Union(data) => data.fields.named.iter().collect(), + } +} + +fn field_projection(field: &Field) -> syn::Result { + let mut nested = false; + let mut wrap = false; + + for attribute in &field.attrs { + if !attribute.path().is_ident("cached_projection") { + continue; + } + + attribute.parse_nested_meta(|meta| { + if meta.path.is_ident("nested") { + nested = true; + Ok(()) + } else if meta.path.is_ident("wrap") { + wrap = true; + Ok(()) + } else { + Err(meta.error("unsupported cached projection option")) + } + })?; + } + + if nested && wrap { + return Err(syn::Error::new_spanned( + field, + "`nested` cannot be combined with `wrap`", + )); + } + + Ok(if nested { + Projection::Nested + } else if wrap { + Projection::Wrap + } else { + Projection::Ordinary + }) +} + +fn add_projection_bounds<'a>( + generics: &mut Generics, + fields: impl Iterator, +) { + let where_clause = generics.make_where_clause(); + for (field, projection) in fields { + let ty = &field.ty; + let predicate = match projection { + Projection::Nested => { + parse_quote!(#ty: ::cached_projection::CachedProjection) + } + Projection::Ordinary | Projection::Wrap => parse_quote!( + #ty: ::serde::Serialize + ::serde::de::DeserializeOwned + ), + }; + where_clause.predicates.push(predicate); + } +} + +fn add_outlives_bounds<'a>( + generics: &mut Generics, + fields: impl Iterator, + lifetime: &Lifetime, +) { + let where_clause = generics.make_where_clause(); + for field in fields { + let ty = &field.ty; + where_clause.predicates.push(parse_quote!(#ty: #lifetime)); + } +} + +fn generic_arguments( + generics: &Generics, + lifetime: Option<&Lifetime>, +) -> TokenStream2 { + let mut arguments = Vec::new(); + if let Some(lifetime) = lifetime { + arguments.push(quote!(#lifetime)); + } + arguments.extend(generics.params.iter().map(|parameter| match parameter { + GenericParam::Lifetime(parameter) => { + let lifetime = ¶meter.lifetime; + quote!(#lifetime) + } + GenericParam::Type(parameter) => { + let ident = ¶meter.ident; + quote!(#ident) + } + GenericParam::Const(parameter) => { + let ident = ¶meter.ident; + quote!(#ident) + } + })); + + if arguments.is_empty() { + quote!() + } else { + quote!(<#(#arguments),*>) + } +} + +fn projected_definition( + data: &Data, + name: &syn::Ident, + visibility: &syn::Visibility, + generics: &Generics, + lifetime: Option<&Lifetime>, +) -> syn::Result { + let derive = if lifetime.is_some() { + quote!(#[derive(::serde::Serialize)]) + } else { + quote!(#[derive(::serde::Serialize, ::serde::Deserialize)]) + }; + let serde_bound = serde_bound_attribute(data, lifetime); + + match data { + Data::Struct(data) => { + let fields = projected_fields(&data.fields, lifetime)?; + let where_clause = &generics.where_clause; + let declaration = match &data.fields { + Fields::Named(_) => quote!({ #(#fields),* }), + Fields::Unnamed(_) => { + quote!(( #(#fields),* ) #where_clause;) + } + Fields::Unit => quote!(#where_clause;), + }; + let leading_where_clause = match &data.fields { + Fields::Named(_) => quote!(#where_clause), + Fields::Unnamed(_) | Fields::Unit => quote!(), + }; + Ok(quote! { + #[doc(hidden)] + #derive + #serde_bound + #visibility struct #name #generics #leading_where_clause #declaration + }) + } + Data::Enum(data) => { + let where_clause = &generics.where_clause; + let variants = data + .variants + .iter() + .map(|variant| { + let attributes = definition_attributes(&variant.attrs)?; + let ident = &variant.ident; + let fields = projected_fields(&variant.fields, lifetime)?; + let declaration = match &variant.fields { + Fields::Named(_) => quote!({ #(#fields),* }), + Fields::Unnamed(_) => quote!(( #(#fields),* )), + Fields::Unit => quote!(), + }; + Ok(quote!(#(#attributes)* #ident #declaration)) + }) + .collect::>>()?; + Ok(quote! { + #[doc(hidden)] + #derive + #serde_bound + #visibility enum #name #generics #where_clause { + #(#variants),* + } + }) + } + Data::Union(data) => Err(syn::Error::new_spanned( + data.union_token, + "CachedProjection cannot be derived for unions", + )), + } +} + +fn serde_bound_attribute( + data: &Data, + lifetime: Option<&Lifetime>, +) -> TokenStream2 { + let projected_types = all_fields(data) + .into_iter() + .map(|field| { + let projection = field_projection(field) + .expect("field projection was already validated"); + projected_field_type(&field.ty, projection, lifetime) + }) + .collect::>(); + if projected_types.is_empty() { + return quote!(); + } + let serialize_bounds = projected_types + .iter() + .map(|ty| quote!(#ty: ::serde::Serialize)) + .collect::>(); + let serialize_bounds = syn::LitStr::new( + "e!(#(#serialize_bounds),*).to_string(), + proc_macro2::Span::call_site(), + ); + + if lifetime.is_some() { + quote!(#[serde(bound(serialize = #serialize_bounds))]) + } else { + let deserialize_bounds = projected_types + .iter() + .map(|ty| quote!(#ty: ::serde::de::DeserializeOwned)) + .collect::>(); + let deserialize_bounds = syn::LitStr::new( + "e!(#(#deserialize_bounds),*).to_string(), + proc_macro2::Span::call_site(), + ); + quote!(#[serde(bound( + serialize = #serialize_bounds, + deserialize = #deserialize_bounds, + ))]) + } +} + +fn projected_fields( + fields: &Fields, + lifetime: Option<&Lifetime>, +) -> syn::Result> { + let projected = fields + .iter() + .map(|field| { + let attributes = definition_attributes(&field.attrs)?; + let visibility = &field.vis; + let ident = &field.ident; + let projection = field_projection(field)?; + let ty = projected_field_type(&field.ty, projection, lifetime); + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #visibility #ident: #ty) + } else { + quote!(#(#attributes)* #visibility #ty) + }) + }) + .collect::>>()?; + + Ok(projected) +} + +fn projected_field_type( + ty: &Type, + projection: Projection, + lifetime: Option<&Lifetime>, +) -> TokenStream2 { + match (projection, lifetime) { + (Projection::Ordinary, None) => quote!(#ty), + (Projection::Ordinary, Some(lifetime)) => quote!(&#lifetime #ty), + (Projection::Nested, None) => quote!( + <#ty as ::cached_projection::CachedProjection>::ProjectedType + ), + (Projection::Nested, Some(lifetime)) => quote!( + <#ty as ::cached_projection::CachedProjection>::ProjectedTypeRef<#lifetime> + ), + (Projection::Wrap, None) => quote!( + ::cached_projection::DeserializeAnyJsonWrapper<#ty> + ), + (Projection::Wrap, Some(lifetime)) => quote!( + ::cached_projection::DeserializeAnyJsonWrapperRef<#lifetime, #ty> + ), + } +} + +fn definition_attributes( + attributes: &[Attribute], +) -> syn::Result> { + let mut filtered = Vec::new(); + for attribute in attributes { + if attribute.path().is_ident("cfg") { + filtered.push(attribute.clone()); + } else if attribute.path().is_ident("serde") { + if serde_has_skip(attribute)? { + filtered.push(parse_quote!(#[serde(skip)])); + } + } else if attribute.path().is_ident("cfg_attr") + && let Some(attribute) = filter_cfg_attr(attribute, true)? { + filtered.push(attribute); + } + } + Ok(filtered) +} + +fn structural_attributes( + attributes: &[Attribute], +) -> syn::Result> { + let mut filtered = Vec::new(); + for attribute in attributes { + if attribute.path().is_ident("cfg") { + filtered.push(attribute.clone()); + } else if attribute.path().is_ident("cfg_attr") + && let Some(attribute) = filter_cfg_attr(attribute, false)? { + filtered.push(attribute); + } + } + Ok(filtered) +} + +fn serde_has_skip(attribute: &Attribute) -> syn::Result { + let Meta::List(list) = &attribute.meta else { + return Ok(false); + }; + let items = + list.parse_args_with(Punctuated::::parse_terminated)?; + Ok(items + .iter() + .any(|item| matches!(item, Meta::Path(path) if path.is_ident("skip")))) +} + +fn filter_cfg_attr( + attribute: &Attribute, + include_serde_skip: bool, +) -> syn::Result> { + let Meta::List(list) = &attribute.meta else { + return Ok(None); + }; + let mut items = list + .parse_args_with(Punctuated::::parse_terminated)? + .into_iter(); + let Some(condition) = items.next() else { + return Ok(None); + }; + let mut retained = Vec::new(); + for item in items { + if item.path().is_ident("cfg") { + retained.push(item); + } else if include_serde_skip && item.path().is_ident("serde") { + let attribute: Attribute = parse_quote!(#[#item]); + if serde_has_skip(&attribute)? { + retained.push(parse_quote!(serde(skip))); + } + } + } + if retained.is_empty() { + Ok(None) + } else { + Ok(Some(parse_quote!(#[cfg_attr(#condition, #(#retained),*)]))) + } +} + +fn project_ref_body( + data: &Data, + projected_name: &syn::Ident, + projections: &[Projection], +) -> syn::Result { + match data { + Data::Struct(data) => struct_projection_body( + data, + projected_name, + projections, + Conversion::ProjectRef, + ), + Data::Enum(data) => enum_projection_body( + data, + projected_name, + projections, + Conversion::ProjectRef, + ), + Data::Union(_) => unreachable!(), + } +} + +fn into_projected_body( + data: &Data, + projected_name: &syn::Ident, + projections: &[Projection], +) -> syn::Result { + match data { + Data::Struct(data) => struct_projection_body( + data, + projected_name, + projections, + Conversion::IntoProjected, + ), + Data::Enum(data) => enum_projection_body( + data, + projected_name, + projections, + Conversion::IntoProjected, + ), + Data::Union(_) => unreachable!(), + } +} + +fn from_projected_body( + data: &Data, + projected_name: &syn::Ident, + projections: &[Projection], +) -> syn::Result { + match data { + Data::Struct(data) => { + struct_from_projected_body(data, projected_name, projections) + } + Data::Enum(data) => { + enum_from_projected_body(data, projected_name, projections) + } + Data::Union(_) => unreachable!(), + } +} + +#[derive(Clone, Copy)] +enum Conversion { + ProjectRef, + IntoProjected, +} + +fn struct_projection_body( + data: &DataStruct, + projected_name: &syn::Ident, + projections: &[Projection], + conversion: Conversion, +) -> syn::Result { + let bindings = field_bindings(&data.fields); + let pattern = fields_pattern(&data.fields, &bindings, None)?; + let target = parse_quote!(#projected_name); + let values = data + .fields + .iter() + .zip(bindings.iter()) + .zip(projections) + .map(|((field, binding), projection)| { + let attributes = structural_attributes(&field.attrs)?; + let value = conversion_expression(binding, *projection, conversion); + let ident = &field.ident; + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #ident: #value) + } else { + quote!(#(#attributes)* #value) + }) + }) + .collect::>>()?; + let construction = fields_construction(&data.fields, &target, &values); + + Ok(quote! { + let #pattern = self; + #construction + }) +} + +fn struct_from_projected_body( + data: &DataStruct, + projected_name: &syn::Ident, + projections: &[Projection], +) -> syn::Result { + let bindings = field_bindings(&data.fields); + let source = parse_quote!(#projected_name); + let pattern = fields_pattern(&data.fields, &bindings, Some(&source))?; + let values = data + .fields + .iter() + .zip(bindings.iter()) + .zip(projections) + .map(|((field, binding), projection)| { + let attributes = structural_attributes(&field.attrs)?; + let value = from_expression(binding, &field.ty, *projection); + let ident = &field.ident; + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #ident: #value) + } else { + quote!(#(#attributes)* #value) + }) + }) + .collect::>>()?; + let construction = + fields_construction(&data.fields, &parse_quote!(Self), &values); + + Ok(quote! { + let #pattern = projected; + #construction + }) +} + +fn enum_projection_body( + data: &DataEnum, + projected_name: &syn::Ident, + projections: &[Projection], + conversion: Conversion, +) -> syn::Result { + let mut offset = 0; + let arms = data + .variants + .iter() + .map(|variant| { + let count = variant.fields.len(); + let variant_projections = &projections[offset..offset + count]; + offset += count; + let attributes = structural_attributes(&variant.attrs)?; + let bindings = field_bindings(&variant.fields); + let variant_ident = &variant.ident; + let pattern = fields_pattern( + &variant.fields, + &bindings, + Some(&parse_quote!(Self::#variant_ident)), + )?; + let values = variant + .fields + .iter() + .zip(bindings.iter()) + .zip(variant_projections) + .map(|((field, binding), projection)| { + let attributes = structural_attributes(&field.attrs)?; + let value = + conversion_expression(binding, *projection, conversion); + let ident = &field.ident; + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #ident: #value) + } else { + quote!(#(#attributes)* #value) + }) + }) + .collect::>>()?; + let target = parse_quote!(#projected_name::#variant_ident); + let construction = + fields_construction(&variant.fields, &target, &values); + Ok(quote!(#(#attributes)* #pattern => #construction)) + }) + .collect::>>()?; + + Ok(quote!(match self { #(#arms),* })) +} + +fn enum_from_projected_body( + data: &DataEnum, + projected_name: &syn::Ident, + projections: &[Projection], +) -> syn::Result { + let mut offset = 0; + let arms = data + .variants + .iter() + .map(|variant| { + let count = variant.fields.len(); + let variant_projections = &projections[offset..offset + count]; + offset += count; + let attributes = structural_attributes(&variant.attrs)?; + let bindings = field_bindings(&variant.fields); + let variant_ident = &variant.ident; + let source = parse_quote!(#projected_name::#variant_ident); + let pattern = + fields_pattern(&variant.fields, &bindings, Some(&source))?; + let values = variant + .fields + .iter() + .zip(bindings.iter()) + .zip(variant_projections) + .map(|((field, binding), projection)| { + let attributes = structural_attributes(&field.attrs)?; + let value = + from_expression(binding, &field.ty, *projection); + let ident = &field.ident; + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #ident: #value) + } else { + quote!(#(#attributes)* #value) + }) + }) + .collect::>>()?; + let target = parse_quote!(Self::#variant_ident); + let construction = + fields_construction(&variant.fields, &target, &values); + Ok(quote!(#(#attributes)* #pattern => #construction)) + }) + .collect::>>()?; + + Ok(quote!(match projected { #(#arms),* })) +} + +fn field_bindings(fields: &Fields) -> Vec { + fields + .iter() + .enumerate() + .map(|(index, field)| { + field + .ident + .clone() + .unwrap_or_else(|| format_ident!("field_{index}")) + }) + .collect() +} + +fn fields_pattern( + fields: &Fields, + bindings: &[syn::Ident], + prefix: Option<&syn::Path>, +) -> syn::Result { + let prefix = prefix + .map(|prefix| quote!(#prefix)) + .unwrap_or_else(|| quote!(Self)); + let entries = fields + .iter() + .zip(bindings) + .map(|(field, binding)| { + let attributes = structural_attributes(&field.attrs)?; + let ident = &field.ident; + Ok(if let Some(ident) = ident { + quote!(#(#attributes)* #ident) + } else { + quote!(#(#attributes)* #binding) + }) + }) + .collect::>>()?; + + Ok(match fields { + Fields::Named(_) => quote!(#prefix { #(#entries),* }), + Fields::Unnamed(_) => quote!(#prefix( #(#entries),* )), + Fields::Unit => quote!(#prefix), + }) +} + +fn fields_construction( + fields: &Fields, + target: &syn::Path, + values: &[TokenStream2], +) -> TokenStream2 { + match fields { + Fields::Named(_) => quote!(#target { #(#values),* }), + Fields::Unnamed(_) => quote!(#target( #(#values),* )), + Fields::Unit => quote!(#target), + } +} + +fn conversion_expression( + binding: &syn::Ident, + projection: Projection, + conversion: Conversion, +) -> TokenStream2 { + match (projection, conversion) { + (Projection::Ordinary, _) => quote!(#binding), + (Projection::Nested, Conversion::ProjectRef) => quote!( + ::cached_projection::CachedProjection::project_ref(#binding) + ), + (Projection::Nested, Conversion::IntoProjected) => quote!( + ::cached_projection::CachedProjection::into_projected(#binding) + ), + (Projection::Wrap, Conversion::ProjectRef) => quote!( + ::cached_projection::DeserializeAnyJsonWrapperRef(#binding) + ), + (Projection::Wrap, Conversion::IntoProjected) => quote!( + ::cached_projection::DeserializeAnyJsonWrapper(#binding) + ), + } +} + +fn from_expression( + binding: &syn::Ident, + ty: &Type, + projection: Projection, +) -> TokenStream2 { + match projection { + Projection::Ordinary => quote!(#binding), + Projection::Nested => quote!( + <#ty as ::cached_projection::CachedProjection>::from_projected(#binding) + ), + Projection::Wrap => quote!(#binding.0), + } +} diff --git a/packages/cached-projection/Cargo.toml b/packages/cached-projection/Cargo.toml new file mode 100644 index 0000000000..5352e00570 --- /dev/null +++ b/packages/cached-projection/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "cached-projection" +edition.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +cached-projection-derive = { path = "../cached-projection-derive" } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } + +[dev-dependencies] +postcard = { workspace = true } + +[lints] +workspace = true diff --git a/packages/cached-projection/src/lib.rs b/packages/cached-projection/src/lib.rs new file mode 100644 index 0000000000..08574b7c5a --- /dev/null +++ b/packages/cached-projection/src/lib.rs @@ -0,0 +1,429 @@ +// for macro expansion +extern crate self as cached_projection; + +use std::collections::{BTreeMap, HashMap}; +use std::hash::Hash; +use std::sync::Arc; + +pub use cached_projection_derive::CachedProjection; +use serde::de::DeserializeOwned; +use serde::ser::{SerializeMap, SerializeSeq}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +pub trait CachedProjection: Sized { + type ProjectedType: Serialize + DeserializeOwned; + + type ProjectedTypeRef<'a>: Serialize + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_>; + + fn into_projected(self) -> Self::ProjectedType; + + fn from_projected(projected: Self::ProjectedType) -> Self; +} + +pub struct DeserializeAnyJsonWrapper(pub T); + +impl Serialize for DeserializeAnyJsonWrapper +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serde_json::to_string(&self.0) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } +} + +impl<'de, T> Deserialize<'de> for DeserializeAnyJsonWrapper +where + T: DeserializeOwned, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + serde_json::from_str(&value) + .map(Self) + .map_err(serde::de::Error::custom) + } +} + +pub struct DeserializeAnyJsonWrapperRef<'a, T: ?Sized>(pub &'a T); + +impl Serialize for DeserializeAnyJsonWrapperRef<'_, T> +where + T: Serialize + ?Sized, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serde_json::to_string(self.0) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } +} + +macro_rules! identity_projection { + ($($ty:ty),* $(,)?) => { + $( + impl CachedProjection for $ty { + type ProjectedType = Self; + type ProjectedTypeRef<'a> = &'a Self; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + self + } + + fn into_projected(self) -> Self::ProjectedType { + self + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + projected + } + } + )* + }; +} + +identity_projection!( + (), + bool, + char, + String, + i8, + i16, + i32, + i64, + i128, + isize, + u8, + u16, + u32, + u64, + u128, + usize, + f32, + f64, +); + +impl CachedProjection for Option +where + T: CachedProjection, +{ + type ProjectedType = Option; + type ProjectedTypeRef<'a> + = Option> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + self.as_ref().map(CachedProjection::project_ref) + } + + fn into_projected(self) -> Self::ProjectedType { + self.map(CachedProjection::into_projected) + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + projected.map(CachedProjection::from_projected) + } +} + +pub struct ProjectedSliceRef<'a, T>(&'a [T]); + +impl Serialize for ProjectedSliceRef<'_, T> +where + T: CachedProjection, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut sequence = serializer.serialize_seq(Some(self.0.len()))?; + for value in self.0 { + sequence.serialize_element(&value.project_ref())?; + } + sequence.end() + } +} + +impl CachedProjection for Vec +where + T: CachedProjection, +{ + type ProjectedType = Vec; + type ProjectedTypeRef<'a> + = ProjectedSliceRef<'a, T> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + ProjectedSliceRef(self) + } + + fn into_projected(self) -> Self::ProjectedType { + self.into_iter() + .map(CachedProjection::into_projected) + .collect() + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + projected + .into_iter() + .map(CachedProjection::from_projected) + .collect() + } +} + +impl CachedProjection for Box +where + T: CachedProjection, +{ + type ProjectedType = T::ProjectedType; + type ProjectedTypeRef<'a> + = T::ProjectedTypeRef<'a> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + self.as_ref().project_ref() + } + + fn into_projected(self) -> Self::ProjectedType { + (*self).into_projected() + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + Box::new(T::from_projected(projected)) + } +} + +impl CachedProjection for Arc +where + T: CachedProjection + Clone, +{ + type ProjectedType = T::ProjectedType; + type ProjectedTypeRef<'a> + = T::ProjectedTypeRef<'a> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + self.as_ref().project_ref() + } + + fn into_projected(self) -> Self::ProjectedType { + Arc::unwrap_or_clone(self).into_projected() + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + Arc::new(T::from_projected(projected)) + } +} + +pub struct ProjectedHashMapRef<'a, K, V>(&'a HashMap); + +impl Serialize for ProjectedHashMapRef<'_, K, V> +where + K: Serialize, + V: CachedProjection, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(Some(self.0.len()))?; + for (key, value) in self.0 { + map.serialize_entry(key, &value.project_ref())?; + } + map.end() + } +} + +impl CachedProjection for HashMap +where + K: Serialize + DeserializeOwned + Eq + Hash, + V: CachedProjection, +{ + type ProjectedType = HashMap; + type ProjectedTypeRef<'a> + = ProjectedHashMapRef<'a, K, V> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + ProjectedHashMapRef(self) + } + + fn into_projected(self) -> Self::ProjectedType { + self.into_iter() + .map(|(key, value)| (key, value.into_projected())) + .collect() + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + projected + .into_iter() + .map(|(key, value)| (key, V::from_projected(value))) + .collect() + } +} + +pub struct ProjectedBTreeMapRef<'a, K, V>(&'a BTreeMap); + +impl Serialize for ProjectedBTreeMapRef<'_, K, V> +where + K: Serialize, + V: CachedProjection, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(Some(self.0.len()))?; + for (key, value) in self.0 { + map.serialize_entry(key, &value.project_ref())?; + } + map.end() + } +} + +impl CachedProjection for BTreeMap +where + K: Serialize + DeserializeOwned + Ord, + V: CachedProjection, +{ + type ProjectedType = BTreeMap; + type ProjectedTypeRef<'a> + = ProjectedBTreeMapRef<'a, K, V> + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + ProjectedBTreeMapRef(self) + } + + fn into_projected(self) -> Self::ProjectedType { + self.into_iter() + .map(|(key, value)| (key, value.into_projected())) + .collect() + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + projected + .into_iter() + .map(|(key, value)| (key, V::from_projected(value))) + .collect() + } +} + +impl CachedProjection for (A, B) +where + A: CachedProjection, + B: CachedProjection, +{ + type ProjectedType = (A::ProjectedType, B::ProjectedType); + type ProjectedTypeRef<'a> + = (A::ProjectedTypeRef<'a>, B::ProjectedTypeRef<'a>) + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + (self.0.project_ref(), self.1.project_ref()) + } + + fn into_projected(self) -> Self::ProjectedType { + (self.0.into_projected(), self.1.into_projected()) + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + ( + A::from_projected(projected.0), + B::from_projected(projected.1), + ) + } +} + +impl CachedProjection for (A, B, C) +where + A: CachedProjection, + B: CachedProjection, + C: CachedProjection, +{ + type ProjectedType = (A::ProjectedType, B::ProjectedType, C::ProjectedType); + type ProjectedTypeRef<'a> + = ( + A::ProjectedTypeRef<'a>, + B::ProjectedTypeRef<'a>, + C::ProjectedTypeRef<'a>, + ) + where + Self: 'a; + + fn project_ref(&self) -> Self::ProjectedTypeRef<'_> { + ( + self.0.project_ref(), + self.1.project_ref(), + self.2.project_ref(), + ) + } + + fn into_projected(self) -> Self::ProjectedType { + ( + self.0.into_projected(), + self.1.into_projected(), + self.2.into_projected(), + ) + } + + fn from_projected(projected: Self::ProjectedType) -> Self { + ( + A::from_projected(projected.0), + B::from_projected(projected.1), + C::from_projected(projected.2), + ) + } +} + +#[cfg(test)] +mod tests { + use super::CachedProjection; + use serde::{Deserialize, Serialize}; + + #[derive(CachedProjection, Serialize, Deserialize)] + struct Child { + value: String, + } + + #[derive(CachedProjection, Serialize, Deserialize)] + struct Parent { + #[cached_projection(nested)] + child: Child, + #[cached_projection(wrap)] + metadata: serde_json::Value, + } + + #[test] + fn postcard_projection_round_trip() { + let value = Parent { + child: Child { + value: "value".to_owned(), + }, + metadata: serde_json::json!({ "key": [1, 2, 3] }), + }; + + let bytes = postcard::to_allocvec(&value.project_ref()).unwrap(); + let projected = + postcard::from_bytes::(&bytes).unwrap(); + let _value = Parent::from_projected(projected); + } +}