From fc2b1cc68fd90eff8a5282e52ea131250a6ffbdf Mon Sep 17 00:00:00 2001 From: ryoya-hayase Date: Tue, 27 Jan 2026 01:20:18 +0000 Subject: [PATCH 1/5] add cache database update functionality --- configs/app.config.ts | 10 + configs/server.config.toml | 21 ++ nuxt.config.ts | 4 + pyproject.toml | 4 + src/app/components/list/ListRepositories.vue | 106 ++++++++ src/app/composables/cacheGroups.ts | 74 ++++++ src/app/i18n/locales/en.json | 35 ++- src/app/i18n/locales/ja.json | 35 ++- src/app/pages/cache-groups/index.vue | 258 ++++++++++++++++++- src/app/types/cacheGroups.ts | 38 +++ src/app/types/search.ts | 9 +- src/app/utils/search.ts | 15 +- src/server/api/cache_groups.py | 124 +++++++++ src/server/api/router.py | 4 +- src/server/api/schemas.py | 39 +++ src/server/config.py | 54 ++++ src/server/entities/cache.py | 79 ++++++ src/server/ext.py | 2 + src/server/services/cache_groups.py | 255 ++++++++++++++++++ uv.lock | 107 ++++++++ 20 files changed, 1267 insertions(+), 6 deletions(-) create mode 100644 src/app/components/list/ListRepositories.vue create mode 100644 src/app/composables/cacheGroups.ts create mode 100644 src/app/types/cacheGroups.ts create mode 100644 src/server/api/cache_groups.py create mode 100644 src/server/entities/cache.py create mode 100644 src/server/services/cache_groups.py diff --git a/configs/app.config.ts b/configs/app.config.ts index 0ffabd74..8d175a83 100644 --- a/configs/app.config.ts +++ b/configs/app.config.ts @@ -36,6 +36,7 @@ const table = { repositories: [20, 50, 100] as number[], groups: [20, 50, 100] as number[], users: [20, 50, 100] as number[], + cacheGroups: [20, 50, 100] as number[], }, } @@ -123,6 +124,13 @@ const wayf = { ] as { name: string, entityID: string }[], } +const groupCache = { + /** + * Time (in milliseconds) to wait before showing loading indicator when updating cache + */ + loadingWaitTime: 3000 as number, +} + export default { /** Server hostname of this application */ serverName, @@ -134,4 +142,6 @@ export default { table, /** WAYF (Embedded DS) configuration */ wayf, + /** Group cache configuration */ + groupCache, } diff --git a/configs/server.config.toml b/configs/server.config.toml index a3bfe67f..51afeb7a 100644 --- a/configs/server.config.toml +++ b/configs/server.config.toml @@ -159,3 +159,24 @@ port = 26379 [rabbitmq] # Hostname or IP address of the RabbitMQ server for Celery broker. url = "amqp://guest:guest@rabbitmq:5672//" + + +[cache_groups] +# Redis key pattern for caching group data. +cache_redis_key = "{prefix}cache" + +# Redis key pattern for caching GakuNin group data. +gakunin_redis_key = "{fqdn}_gakunin_groups" + +# Map groups API endpoint. +map_groups_api_endpoint = "https://sample.gakunin.jp/api/groups/" + +# If toml_path and either directory_path or fqdn_list_file are defined, toml_path takes precedence. +# Schema for cache database configuration. +toml_path = "cache_db_config.toml" + +# Path to the directory containing institution TLS files. +directory_path = "./cache_db/tls" + +# Path to the file containing FQDN list. +fqdn_list_file = "fqdn_list.toml" diff --git a/nuxt.config.ts b/nuxt.config.ts index 2e34a912..0bfaf079 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -9,6 +9,10 @@ export default defineNuxtConfig({ '@nuxt/test-utils', ], + imports: { + dirs: ['~/types'], + }, + devtools: { enabled: true, }, diff --git a/pyproject.toml b/pyproject.toml index 840b53b0..a8d675d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "pydantic[email]>=2.12.5", "requests>=2.32.5", "sqlalchemy-utils>=0.42.1", + "weko-group-cache-db", ] [dependency-groups] @@ -88,3 +89,6 @@ skip-magic-trailing-comma = false [tool.pyright] extraPaths = ["src/server"] typeCheckingMode = "standard" + +[tool.uv.sources] +weko-group-cache-db = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git", rev = "develop" } diff --git a/src/app/components/list/ListRepositories.vue b/src/app/components/list/ListRepositories.vue new file mode 100644 index 00000000..71f8cd0a --- /dev/null +++ b/src/app/components/list/ListRepositories.vue @@ -0,0 +1,106 @@ + + + diff --git a/src/app/composables/cacheGroups.ts b/src/app/composables/cacheGroups.ts new file mode 100644 index 00000000..0c3eca96 --- /dev/null +++ b/src/app/composables/cacheGroups.ts @@ -0,0 +1,74 @@ +/** + * Composable for managing cache groups. + */ +import type { TableColumn } from '@nuxt/ui' + +const useCacheGroups = () => { + const route = useRoute() + const router = useRouter() + + const { t: $t } = useI18n() + + /** Reactive query object */ + const query = computed(() => normalizeCacheGroupsQuery(route.query)) + /** Update query parameters and push to router */ + const updateQuery = (newQuery: Partial) => { + router.push({ + query: { + ...route.query, + ...newQuery, + }, + }) + } + + const searchTerm = ref(query.value.q) + const filter = ref(query.value.f) + const pageSize = ref(query.value.l) + + const searchIdentityKey = computed(() => { + const { p, l, ...filters } = query.value + return JSON.stringify(filters) + }) + + const selectedMap = useState>( + `selection:${searchIdentityKey.value}`, () => ({}), + ) + + /** Column names with translations */ + const columnNames = { + id: '#', + displayName: $t('cache-groups.table.column.repository-name'), + serviceURL: $t('cache-groups.table.column.repository-url'), + updated: $t('cache-groups.table.column.repository-updated-at'), + } + + type CacheGroupsTableColumn = TableColumn + const columns = computed(() => [ + { + accessorKey: 'name', + header: columnNames.displayName, + }, + { + accessorKey: 'url', + header: columnNames.serviceURL, + }, + { + accessorKey: 'updated', + header: columnNames.updated, + }, + ]) + + return { + query, + updateQuery, + criteria: { + searchTerm, + filter, + pageSize, + }, + selectedMap, + columns, + } +} + +export { useCacheGroups } diff --git a/src/app/i18n/locales/en.json b/src/app/i18n/locales/en.json index f6ccdbc7..3a533512 100644 --- a/src/app/i18n/locales/en.json +++ b/src/app/i18n/locales/en.json @@ -1,12 +1,38 @@ { "button": { + "cancel": "Cancel", "create-new": "Create new", "reload": "Reload", + "update": "Update", "upload-users": "Upload" }, "cache-groups": { + "button": { + "update-all-repositories": "Update all repositories", + "update-selected-repositories": "Update selected repositories" + }, + "confirm-update-all-repositories": "Do you want to update all repositories ({count})?", + "confirm-update-selected-repositories": "Would you like to update the following {count} repositories?", + "count": "{current} / {total} items", "description": "You can manage the cache status of group information used by WEKO3.", - "title": "Caching Groups" + "filter-placeholder": "Cache status", + "search-placeholder": "search...", + "select": { + "cache": "With cash", + "no-cache": "No cache" + }, + "table": { + "column": { + "repository-name": "Repository name", + "repository-status": "Update status", + "repository-updated-at": "Cache update date and time", + "repository-url": "Repository URL" + } + }, + "title": "Caching Groups", + "update-completed": "Update completed ({count} items)", + "update-error": "Cache update processing is in progress.", + "updating": "Updating..." }, "footer": { "privacy-policy": "Privacy policy", @@ -29,6 +55,10 @@ }, "repositories": { "description": "You can check information about the repositories you manage.", + "list": { + "no-repositories-description": "Please change your search criteria and try again.", + "no-repositories-title": "Repository not found" + }, "title": "Repositories" }, "table": { @@ -37,6 +67,9 @@ }, "actions-label": "Action", "display-columns-label": "Columns", + "display-count-label": "Number of items displayed:", + "display-info-text": "Displaying {start}～{end} items (all {total} items)", + "display-info-text-empty": "Displaying 0 items (all 0 items)", "page-size-label": "Page size:" }, "user": { diff --git a/src/app/i18n/locales/ja.json b/src/app/i18n/locales/ja.json index 0b5267cd..219fbcc5 100644 --- a/src/app/i18n/locales/ja.json +++ b/src/app/i18n/locales/ja.json @@ -1,12 +1,38 @@ { "button": { + "cancel": "キャンセル", "create-new": "新規作成", "reload": "再読み込み", + "update": "更新", "upload-users": "アップロード" }, "cache-groups": { + "button": { + "update-all-repositories": "全リポジトリを更新", + "update-selected-repositories": "選択したリポジトリを更新" + }, + "confirm-update-all-repositories": "全リポジトリ（{count}件）の更新を行いますか？", + "confirm-update-selected-repositories": "以下の{count}リポジトリを更新しますか？", + "count": "{current} / {total} 件", "description": "WEKO3が利用するグループ情報のキャッシュ状況を管理できます。", - "title": "グループキャッシュ" + "filter-placeholder": "キャッシュ状態", + "search-placeholder": "検索...", + "select": { + "cache": "キャッシュあり", + "no-cache": "キャッシュなし" + }, + "table": { + "column": { + "repository-name": "リポジトリ名", + "repository-status": "更新状況", + "repository-updated-at": "キャッシュ更新日時", + "repository-url": "リポジトリURL" + } + }, + "title": "グループキャッシュ", + "update-completed": "更新完了（{count}件）", + "update-error": "キャッシュ更新処理が実行中です。", + "updating": "更新中..." }, "footer": { "privacy-policy": "プライバシーポリシー", @@ -29,6 +55,10 @@ }, "repositories": { "description": "あなたが管理しているリポジトリの情報を確認できます。", + "list": { + "no-repositories-description": "検索条件を変更して、再度お試しください。", + "no-repositories-title": "リポジトリが見つかりません" + }, "title": "リポジトリ" }, "table": { @@ -37,6 +67,9 @@ }, "actions-label": "アクション", "display-columns-label": "表示項目", + "display-count-label": "表示件数：", + "display-info-text": "{start}～{end}件を表示（全{total}件）", + "display-info-text-empty": "0件を表示（全0件）", "page-size-label": "表示件数：" }, "user": { diff --git a/src/app/pages/cache-groups/index.vue b/src/app/pages/cache-groups/index.vue index f57787e7..36f3fa62 100644 --- a/src/app/pages/cache-groups/index.vue +++ b/src/app/pages/cache-groups/index.vue @@ -1,11 +1,267 @@ diff --git a/src/app/types/cacheGroups.ts b/src/app/types/cacheGroups.ts new file mode 100644 index 00000000..1b33e0fd --- /dev/null +++ b/src/app/types/cacheGroups.ts @@ -0,0 +1,38 @@ +/** + * Types related to cache groups + */ + +/** Cache group summary information */ +interface CacheGroupsSummary { + id: string + name: string + url: string + updated: string +} + +/** Search result for cache groups */ +interface CacheGroupSearchResult { + total: number + pageSize: number + offset: number + resources: CacheGroupsSummary[] +} + +/** Cache groups update result */ +interface CacheGroupsUpdateResult { + type: string + fqdn: string + status: string + code?: string + repository_cached?: CacheGroupsSummary[] +} + +/** Detail information of a cache groups update task */ +interface TaskDetail { + results: CacheGroupsUpdateResult[] + current: string + done: number + total: number +} + +export type { CacheGroupsSummary, CacheGroupSearchResult, CacheGroupsUpdateResult, TaskDetail } diff --git a/src/app/types/search.ts b/src/app/types/search.ts index 993a8b5b..ed90b006 100644 --- a/src/app/types/search.ts +++ b/src/app/types/search.ts @@ -17,6 +17,13 @@ interface UsersSearchQuery { l?: number } +interface CacheGroupsSearchQuery { + q?: string + f?: string[] + p?: number + l?: number +} + type UsersSortableKeys = 'id' | 'userName' | 'emails' | 'eppns' | 'lastModified' type SortOrder = 'asc' | 'desc' @@ -39,7 +46,7 @@ type GroupsSearchResult = SearchResult type UsersSearchResult = SearchResult export type { - UsersSearchQuery, UsersSortableKeys, + UsersSearchQuery, CacheGroupsSearchQuery, UsersSortableKeys, SortOrder, SearchResult, UsersSearchResult, GroupsSearchResult, RepositoriesSearchResult, } diff --git a/src/app/utils/search.ts b/src/app/utils/search.ts index b9d40e6c..e1b9d3e4 100644 --- a/src/app/utils/search.ts +++ b/src/app/utils/search.ts @@ -37,4 +37,17 @@ const normalizeUsersQuery = (query: LocationQuery): UsersSearchQuery => { } } -export { normalizeUsersQuery } +/** + * Normalize location query to cache groups search query + */ +const normalizeCacheGroupsQuery = (query: LocationQuery): CacheGroupsSearchQuery => { + const { table: { pageSize } } = useAppConfig() + return { + q: query.q ? pickSingle(query.q) : undefined, + f: query.f ? toArray(query.f) : undefined, + p: Number(query.p) || 1, + l: Number(query.l) || pageSize.cacheGroups?.[0], + } +} + +export { normalizeUsersQuery, normalizeCacheGroupsQuery } diff --git a/src/server/api/cache_groups.py b/src/server/api/cache_groups.py new file mode 100644 index 00000000..96d873ce --- /dev/null +++ b/src/server/api/cache_groups.py @@ -0,0 +1,124 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# +"""API router for cache group endpoints.""" + +from ast import literal_eval + +from flask import Blueprint +from flask_login import login_required +from flask_pydantic import validate + +from server.api.helper import roles_required +from server.api.schemas import CacheQuery, CacheRequest, RepositoriesQuery +from server.config import config +from server.datastore import app_cache +from server.entities.cache import CacheResult, RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.services.cache_groups import get_repository_cache, update +from server.services.repositories import search + + +bp = Blueprint("cache-groups", __name__) + + +@bp.get("/") +@login_required +@roles_required("system_admin") +@validate(response_by_alias=True) +def get(query: CacheQuery) -> SearchResult[RepositoryCache]: + """Retrieve repository cache entries based on the provided query. + + Args: + query (CacheQuery): Query parameters for filtering and pagination. + + Returns: + list[RepositoryCache]: List of repository cache entries. + """ + repository_query = RepositoriesQuery( + q=query.q, + k="id", + d="asc", + p=query.p if not query.f else -1, + l=query.l, + ) + + repositories = search(repository_query) + + return get_repository_cache(repositories, query) + + +@bp.post("/") +@login_required +@roles_required("system_admin") +@validate() +def post(body: CacheRequest) -> str: + """Update cache groups for the specified FQDNs. + + Args: + body (CacheRequest): Request body containing FQDNs and operation. + + Returns: + str: Success message. + """ + fqdn_list = body.fqdn_list or [] + update(fqdn_list, body.op) + return "Success" + + +@bp.get("/task") +@login_required +@roles_required("system_admin") +@validate() +def get_task_status() -> TaskDetail: + """Get the status of the cache update task. + + Returns: + TaskDetail: Details of the cache update task. + """ + cache_key = config.CACHE_GROUPS.cache_redis_key.format( + prefix=config.REDIS.key_prefix + ) + task_status = app_cache.hgetall(cache_key) + decode_task_status = {k.decode(): v.decode() for k, v in task_status.items()} # pyright: ignore[reportAttributeAccessIssue] + results = {} + current = "" + done = 0 + total = 0 + for key in decode_task_status: # noqa: PLC0206 + if key == "current": + current = decode_task_status[key] # pyright: ignore[reportIndexIssue] + elif key == "done": + done = int(decode_task_status[key]) # pyright: ignore[reportIndexIssue] + elif key == "total": + total = int(decode_task_status[key]) # pyright: ignore[reportIndexIssue] + else: + result = literal_eval(decode_task_status[key]) # pyright: ignore[reportIndexIssue] + fqdn = key.split("_")[0] + repository_query = RepositoriesQuery( + q=f"/{fqdn}/", + k="id", + d="asc", + p=-1, + l=1, + ) + repository = search(repository_query).resources[0] + results[fqdn] = CacheResult( + type="cache", + fqdn=fqdn, + status=result["status"], + code=result.get("code"), + repository_cached=RepositoryCache( + id=repository.id, + name=repository.display_name, # pyright: ignore[reportArgumentType] + url=str(repository.service_url), + updated=result.get("updated"), + ), + ) + if total > 0 and total == done: + # Clear the task status when the task is completed. + app_cache.delete(cache_key) + + return TaskDetail( + results=list(results.values()), current=current, done=done, total=total + ) diff --git a/src/server/api/router.py b/src/server/api/router.py index c0621df2..31937ea1 100644 --- a/src/server/api/router.py +++ b/src/server/api/router.py @@ -23,7 +23,9 @@ def create_api_blueprint() -> Blueprint: module = import_module(f"{__package__}.{module_name}") if hasattr(module, "bp") and isinstance(module.bp, Blueprint): bp_api.register_blueprint( - module.bp, url_prefix=f"/{module_name}", strict_slashes=False + module.bp, + url_prefix=f"/{module_name.replace('_', '-')}", + strict_slashes=False, ) return bp_api diff --git a/src/server/api/schemas.py b/src/server/api/schemas.py index 729511bd..66c4cc29 100644 --- a/src/server/api/schemas.py +++ b/src/server/api/schemas.py @@ -12,8 +12,16 @@ from datetime import date from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel +camel_case_config = ConfigDict( + validate_assignment=True, + alias_generator=to_camel, + validate_by_name=True, + validate_by_alias=True, +) + ignore_extra_config = ConfigDict( extra="ignore", validate_assignment=True, @@ -167,3 +175,34 @@ class UsersQuery(BaseModel): model_config = ignore_extra_config """Configure to ignore extra fields.""" + + +class CacheQuery(BaseModel): + """Schema for cache query parameters.""" + + q: t.Annotated[str | None, "term"] = None + """Search term for querying cache entries.""" + + f: t.Annotated[list[str] | None, "filter"] = None + """Filter expression for querying cache entries.""" + + p: t.Annotated[int | None, "page"] = None + """Page number for pagination.""" + + l: t.Annotated[int | None, "per"] = None + """Number of items per page for pagination.""" + + +type CacheOperation = t.Literal["all", "id-specified"] + + +class CacheRequest(BaseModel): + """Schema for cache update request.""" + + fqdn_list: list[str] | None = None + """List of fully qualified domain names to update in the cache.""" + + op: CacheOperation + """Operation type: 'all' to update all, 'id-specified' to update specified FQDNs.""" + + model_config = camel_case_config diff --git a/src/server/config.py b/src/server/config.py index c89972c5..eba11a41 100644 --- a/src/server/config.py +++ b/src/server/config.py @@ -81,6 +81,38 @@ class RuntimeConfig(BaseSettings): RABBITMQ: RabbitmqConfig """RabbitMQ configuration values.""" + CACHE_GROUPS: CacheGroupsConfig + """Cache groups task configuration values.""" + + @computed_field + @property + def CACHE_DB(self) -> dict[str, t.Any]: + """Cache database configuration values.""" + return { + "CACHE_KEY_SUFFIX": self.CACHE_GROUPS.gakunin_redis_key, + "MAP_GROUPS_API_ENDPOINT": self.CACHE_GROUPS.map_groups_api_endpoint, + "REDIS_TYPE": "redis" + if self.REDIS.cache_type == "RedisCache" + else "sentinel", + "REDIS_HOST": self.REDIS.single.base_url.replace("redis://", "").split(":")[ + 0 + ] + if self.REDIS.cache_type == "RedisCache" + else "", + "REDIS_PORT": int( + self.REDIS.single.base_url.replace("redis://", "").split(":")[1] + ) + if self.REDIS.cache_type == "RedisCache" + else 0, + "REDIS_DB_INDEX": self.REDIS.database.group_cache, + "REDIS_SENINEL_MASTER: ": self.REDIS.sentinel.master_name + if self.REDIS.cache_type == "RedisSentinelCache" + else "", + "SENTINELS": self.REDIS.sentinel.sentinels + if self.REDIS.cache_type == "RedisSentinelCache" + else [], + } + @computed_field @property def SQLALCHEMY_DATABASE_URI(self) -> URL: @@ -393,6 +425,28 @@ class RabbitmqConfig(BaseModel): """Hostname or IP address of the RabbitMQ server for Celery broker.""" +class CacheGroupsConfig(BaseModel): + """Schema for cache groups configuration.""" + + cache_redis_key: str + """Redis key for cache groups.""" + + gakunin_redis_key: str + """Redis key pattern for caching GakuNin group data.""" + + map_groups_api_endpoint: str + """Map groups API endpoint.""" + + toml_path: str + """Path to the TOML file with cache database configuration.""" + + directory_path: str + """Path to the directory containing institution TLS files.""" + + fqdn_list_file: str + """Path to the file containing FQDN list.""" + + type HasRepoId = t.Annotated[str, StringConstraints(pattern=HAS_REPO_ID_PATTERN)] """Pattern for role-based group IDs. diff --git a/src/server/entities/cache.py b/src/server/entities/cache.py new file mode 100644 index 00000000..849330f1 --- /dev/null +++ b/src/server/entities/cache.py @@ -0,0 +1,79 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Models for updating cache db entities.""" + +import typing as t + +from datetime import datetime + +from pydantic import BaseModel + +from .common import camel_case_config, forbid_extra_config + + +class RepositoryCache(BaseModel): + """Model for repository cache entity.""" + + id: str + """The unique identifier for the repository.""" + + name: str + """The name of the repository.""" + + url: str + """The URL of the repository.""" + + updated: datetime | None = None + """The update timestamp of the repository cache entry.""" + + model_config = camel_case_config | forbid_extra_config + """Configure to use camelCase aliasing and forbid extra fields.""" + + +class TaskDetail(BaseModel): + """Model for task detail entity.""" + + results: list[Result] + """The list of results from the task.""" + + current: str + """Identifier of the object currently being processed.""" + + done: int + """The number of completed items.""" + + total: int + """The total number of items to process.""" + + model_config = camel_case_config | forbid_extra_config + """Configure to use camelCase aliasing and forbid extra fields.""" + + +type TaskStatus = t.Literal["success", "failed"] + + +class CacheResult(BaseModel): + """Model for cache update result.""" + + type: t.Literal["cache"] + """The type of the result, always "cache".""" + + fqdn: str + """The fully qualified domain name of the cached repository.""" + + status: TaskStatus + """The status of the cache update task.""" + + code: str | None = None + """The result code of the cache update task.""" + + repository_cached: RepositoryCache | None = None + """The cached repository information.""" + + model_config = camel_case_config | forbid_extra_config + """Configure to use camelCase aliasing and forbid extra fields.""" + + +Result = CacheResult diff --git a/src/server/ext.py b/src/server/ext.py index 2e86e5fd..94792171 100644 --- a/src/server/ext.py +++ b/src/server/ext.py @@ -6,6 +6,7 @@ import typing as t +from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config from .api.router import create_api_blueprint from .cli.base import register_cli_commands from .config import RuntimeConfig, setup_config @@ -74,6 +75,7 @@ def init_config(self, app: Flask) -> None: app.config.from_mapping(self.config.for_flask) app.config.from_prefixed_env() + setup_weko_group_cache_db_config(self.config.CACHE_DB) def init_db_app(self, app: Flask) -> None: # noqa: PLR6301 """Initialize the database for the this extension. diff --git a/src/server/services/cache_groups.py b/src/server/services/cache_groups.py new file mode 100644 index 00000000..88509366 --- /dev/null +++ b/src/server/services/cache_groups.py @@ -0,0 +1,255 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Service module for managing cache groups.""" + +import typing as t + +from datetime import datetime + +from celery import shared_task +from weko_group_cache_db.groups import fetch_all, fetch_one +from weko_group_cache_db.signal import update_count_signal, update_result_signal + +from server.api.schemas import CacheOperation, CacheQuery +from server.config import config +from server.datastore import app_cache, group_cache +from server.entities.search_request import SearchResult + + +if t.TYPE_CHECKING: + from server.entities.cache import RepositoryCache + from server.entities.summaries import RepositorySummary + + +def get_repository_cache( + repositories: SearchResult[RepositorySummary], query: CacheQuery +) -> SearchResult[RepositoryCache]: + """Retrieve repository cache entries based on the provided query. + + Args: + repositories (SearchResult[RepositorySummary]): List of repository summaries. + query (CacheQuery): Query parameters for filtering and pagination. + + Returns: + SearchResult[RepositoryCache]: List of repository cache entries. + """ + + def _get_repository_redis_key(fqdn: str) -> str: + replaced_fqdn = fqdn.replace("-", "_").replace(".", "_") + return config.CACHE_GROUPS.gakunin_redis_key.format(fqdn=replaced_fqdn) + + repository_cache_list = [] + cursor = 0 + redis_key_list = [] + while True: + cursor, keys = group_cache.scan( + cursor=cursor, + match=config.CACHE_GROUPS.gakunin_redis_key.format(fqdn="*"), + count=100, + ) # pyright: ignore[reportGeneralTypeIssues] + redis_key_list.extend([key.decode("utf-8") for key in keys]) + if cursor == 0: + break + + # filter by cache existence + if query.f: + exist_cache_repository_list = [] + non_exist_cache_repository_list = [] + start_index = (query.p - 1) * query.l if query.p and query.l else 0 + end_index = start_index + query.l if query.l else len(repositories.resources) + + for repository in repositories.resources: + repository_redis_key = _get_repository_redis_key( + repository.service_url.host # pyright: ignore[reportOptionalMemberAccess, reportArgumentType] + ) + repository_updated = None + if repository_redis_key in redis_key_list: + repository_updated = group_cache.hget( + repository_redis_key, "updated_at" + ) + + repo_cache = RepositoryCache( + id=repository.id, + name=repository.display_name, # pyright: ignore[reportArgumentType], + url=str(repository.service_url), + updated=datetime.fromisoformat(repository_updated), # pyright: ignore[reportArgumentType] + ) + exist_cache_repository_list.append(repo_cache) + else: + repo_cache = RepositoryCache( + id=repository.id, + name=repository.display_name, # pyright: ignore[reportArgumentType], + url=str(repository.service_url), + updated=None, + ) + non_exist_cache_repository_list.append(repo_cache) + if query.f == "cache": + target_repositories = exist_cache_repository_list[start_index:end_index] + return SearchResult( + resources=target_repositories, + total=len(exist_cache_repository_list), + page_size=query.l or len(target_repositories), + offset=start_index + 1, + ) + target_repositories = non_exist_cache_repository_list[start_index:end_index] + return SearchResult( + resources=target_repositories, + total=len(non_exist_cache_repository_list), + page_size=query.l or len(target_repositories), + offset=start_index + 1, + ) + repository_cache_list = [] + for repository in repositories.resources: + repository_redis_key = _get_repository_redis_key( + repository.service_url.host # pyright: ignore[reportOptionalMemberAccess, reportArgumentType] + ) + repository_updated = None + if repository_redis_key in redis_key_list: + repository_updated = group_cache.hget(repository_redis_key, "updated_at") + + repo_cache = RepositoryCache( + id=repository.id, + name=repository.display_name, # pyright: ignore[reportArgumentType], + url=str(repository.service_url), + updated=datetime.fromisoformat(repository_updated) # pyright: ignore[reportArgumentType] + if repository_updated + else None, + ) + repository_cache_list.append(repo_cache) + + return SearchResult( + resources=repository_cache_list, + total=repositories.total, + page_size=query.l or repositories.page_size, + offset=(query.p - 1) * query.l + 1 if query.p and query.l else 1, + ) + + +def update(fqdn_list: list[str], op: CacheOperation) -> None: + """Update cache groups based on the operation type. + + Args: + fqdn_list (list[str]): List of fully qualified domain names. + op (CacheOperation): Operation type, either 'all' or 'id-specified'. + + Raises: + RuntimeError: If a cache update task is already running. + """ + if check_updating_cache_is_running(): + err = "A cache update task is already running." + raise RuntimeError(err) + + update_task.apply_async(fqdn_list, op) # pyright: ignore[reportArgumentType] + + +@shared_task() +def update_task(fqdn_list: list[str], op: CacheOperation) -> None: + """Celery task to update cache groups. + + Args: + fqdn_list (list[str]): List of fully qualified domain names. + op (CacheOperation): Operation type, either 'all' or 'id-specified'. + """ + + def _receive_update_count(_: object, **kwargs: dict[str, t.Any]) -> None: + """Receive update count signal and update task status in Redis. + + Args: + _: The sender of the signal. + **kwargs: Additional keyword arguments containing task details. + """ + update_dict = { + "total": kwargs.get("total", 0), + "done": kwargs.get("done", 0), + "current": kwargs.get("current", ""), + } + app_cache.hset(cache_key, mapping=update_dict) + + def _receive_update_result(_: object, **kwargs: dict[str, t.Any]) -> None: + """Receive update result signal and update task result in Redis. + + Args: + _: The sender of the signal. + **kwargs: Additional keyword arguments containing task details. + """ + task_result = { + "status": kwargs.get("status", ""), + "code": kwargs.get("code", ""), + "updated": kwargs.get("updated", ""), + } + update_dict = {str(kwargs.get("task_name", "")): str(task_result)} + app_cache.hset(cache_key, mapping=update_dict) + + cache_key = config.CACHE_GROUPS.cache_redis_key.format( + prefix=config.REDIS.key_prefix + ) + + update_count_signal.connect(_receive_update_count) + update_result_signal.connect(_receive_update_result) + + if op == "all": + update_run_task() + else: + update_one_task(fqdn_list) + + +def update_run_task() -> None: + """Task to update all cache groups.""" + cache_key = config.CACHE_GROUPS.cache_redis_key.format( + prefix=config.REDIS.key_prefix + ) + run_task_init_data = {"total": "", "done": 0, "current": ""} + app_cache.hset(cache_key, mapping=run_task_init_data) + if config.CACHE_GROUPS.toml_path: + fetch_all(toml_path=config.CACHE_GROUPS.toml_path) + else: + fetch_all( + directory_path=config.CACHE_GROUPS.directory_path, + fqdn_list_file=config.CACHE_GROUPS.fqdn_list_file, + ) + + +def update_one_task(fqdn_list: list[str]) -> None: + """Task to update cache groups for specified FQDNs. + + Args: + fqdn_list (list[str]): List of fully qualified domain names to update. + """ + cache_key = config.CACHE_GROUPS.cache_redis_key.format( + prefix=config.REDIS.key_prefix + ) + one_task_init_data = {"total": len(fqdn_list), "done": 0, "current": ""} + app_cache.hset(cache_key, mapping=one_task_init_data) + for fqdn in fqdn_list: + try: + done_str = app_cache.hget(cache_key, "done") + done = int(done_str) if done_str is not None else 0 # pyright: ignore[reportArgumentType] + except TypeError: + done = 0 + if config.CACHE_GROUPS.toml_path: + fetch_one(fqdn, toml_path=config.CACHE_GROUPS.toml_path) + else: + fetch_one( + fqdn, + directory_path=config.CACHE_GROUPS.directory_path, + fqdn_list_file=config.CACHE_GROUPS.fqdn_list_file, + ) + task_result = { + "done": done + 1, + "current": fqdn, + } + app_cache.hset(cache_key, mapping=task_result) + + +def check_updating_cache_is_running() -> bool: + """Check if a cache update task is currently running. + + Returns: + bool: True if a cache update task is running, False otherwise. + """ + cache_key = config.CACHE_GROUPS.cache_redis_key.format( + prefix=config.REDIS.key_prefix + ) + return bool(app_cache.exists(cache_key)) diff --git a/uv.lock b/uv.lock index 9a11971e..64e189f0 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "billiard" version = "4.2.4" @@ -301,6 +310,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] +[[package]] +name = "inflect" +version = "7.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197, upload-time = "2024-12-28T17:11:15.931Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -334,6 +356,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "requests" }, { name = "sqlalchemy-utils" }, + { name = "weko-group-cache-db" }, ] [package.dev-dependencies] @@ -355,6 +378,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.12.0" }, { name = "requests", specifier = ">=2.32.5" }, { name = "sqlalchemy-utils", specifier = ">=0.42.1" }, + { name = "weko-group-cache-db", git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop" }, ] [package.metadata.requires-dev] @@ -396,6 +420,18 @@ redis = [ { name = "redis" }, ] +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -426,6 +462,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -660,6 +714,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rich" +version = "14.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/84/4831f881aa6ff3c976f6d6809b58cdfa350593ffc0dc3c58f5f6586780fb/rich-14.3.1.tar.gz", hash = "sha256:b8c5f568a3a749f9290ec6bddedf835cec33696bfc1e48bcfecb276c7386e4b8", size = 230125, upload-time = "2026-01-24T21:40:44.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/2a/a1810c8627b9ec8c57ec5ec325d306701ae7be50235e8fd81266e002a3cc/rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e", size = 309952, upload-time = "2026-01-24T21:40:42.969Z" }, +] + +[[package]] +name = "rich-click" +version = "1.9.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/50/1497dbc52297d6759451bf5a991e9b2d0a122a5d33ac8cd057f81cb9910a/rich_click-1.9.6.tar.gz", hash = "sha256:463bd3dbef54a812282bfa93dde80c471bce359823fc1301be368eab63391cb2", size = 74777, upload-time = "2026-01-22T02:43:58.374Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/87/508930def644be9fb86fec63520151921061c152289b98798017a498d678/rich_click-1.9.6-py3-none-any.whl", hash = "sha256:e78d71e3f73a55548e573ccfd964e18503936e2e736a4a1f74c6c29479a2a054", size = 71430, upload-time = "2026-01-22T02:43:56.939Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -702,6 +783,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/25/7400c18c3ee97914cc99c90007795c00a4ec5b60c853b49db7ba24d11179/sqlalchemy_utils-0.42.1-py3-none-any.whl", hash = "sha256:243cfe1b3a1dae3c74118ae633f1d1e0ed8c787387bc33e556e37c990594ac80", size = 91761, upload-time = "2025-12-13T03:14:15.014Z" }, ] +[[package]] +name = "typeguard" +version = "4.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/68/71c1a15b5f65f40e91b65da23b8224dad41349894535a97f63a52e462196/typeguard-4.4.4.tar.gz", hash = "sha256:3a7fd2dffb705d4d0efaed4306a704c89b9dee850b688f060a8b1615a79e5f74", size = 75203, upload-time = "2025-06-18T09:56:07.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/a9/e3aee762739c1d7528da1c3e06d518503f8b6c439c35549b53735ba52ead/typeguard-4.4.4-py3-none-any.whl", hash = "sha256:b5f562281b6bfa1f5492470464730ef001646128b180769880468bd84b68b09e", size = 34874, upload-time = "2025-06-18T09:56:05.999Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -771,6 +864,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, ] +[[package]] +name = "weko-group-cache-db" +version = "1.0.0rc4" +source = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop#4b41ff4d972487a6812499f43d10d956b6f72721" } +dependencies = [ + { name = "backoff" }, + { name = "inflect" }, + { name = "pydantic-settings" }, + { name = "redis" }, + { name = "requests" }, + { name = "rich-click" }, + { name = "werkzeug" }, +] + [[package]] name = "werkzeug" version = "3.1.5" From 2808edc0cecd07105993d679538b06a1c76aa5b7 Mon Sep 17 00:00:00 2001 From: ryoya-hayase Date: Fri, 6 Mar 2026 07:43:25 +0000 Subject: [PATCH 2/5] create unit tests and fix bugs --- Dockerfile | 3 + src/server/api/cache_groups.py | 5 +- src/server/config.py | 2 +- src/server/ext.py | 1 + src/server/services/cache_groups.py | 35 +- tests/unit/api/test_cache_groups.py | 243 +++++ tests/unit/api/test_router.py | 113 +++ tests/unit/conftest.py | 94 ++ tests/unit/services/test_cache_groups.py | 1159 ++++++++++++++++++++++ tests/unit/test_ext.py | 22 + 10 files changed, 1651 insertions(+), 26 deletions(-) create mode 100644 tests/unit/api/test_cache_groups.py create mode 100644 tests/unit/api/test_router.py create mode 100644 tests/unit/services/test_cache_groups.py create mode 100644 tests/unit/test_ext.py diff --git a/Dockerfile b/Dockerfile index 372eaaff..429e5561 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ ENV VIRTUAL_ENV="/code/.venv" ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /code +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* RUN pip install -U pip && pip install uv RUN groupadd -g ${GID} ${GROUPNAME} && \ diff --git a/src/server/api/cache_groups.py b/src/server/api/cache_groups.py index 96d873ce..1817c17b 100644 --- a/src/server/api/cache_groups.py +++ b/src/server/api/cache_groups.py @@ -4,6 +4,7 @@ """API router for cache group endpoints.""" from ast import literal_eval +from datetime import datetime from flask import Blueprint from flask_login import login_required @@ -112,7 +113,9 @@ def get_task_status() -> TaskDetail: id=repository.id, name=repository.display_name, # pyright: ignore[reportArgumentType] url=str(repository.service_url), - updated=result.get("updated"), + updated=datetime.strptime( # noqa: DTZ007 + result.get("updated"), "%Y-%m-%dT%H:%M:%SZ" + ), ), ) if total > 0 and total == done: diff --git a/src/server/config.py b/src/server/config.py index eba11a41..9b0fa2f1 100644 --- a/src/server/config.py +++ b/src/server/config.py @@ -105,7 +105,7 @@ def CACHE_DB(self) -> dict[str, t.Any]: if self.REDIS.cache_type == "RedisCache" else 0, "REDIS_DB_INDEX": self.REDIS.database.group_cache, - "REDIS_SENINEL_MASTER: ": self.REDIS.sentinel.master_name + "REDIS_SENTINEL_MASTER": self.REDIS.sentinel.master_name if self.REDIS.cache_type == "RedisSentinelCache" else "", "SENTINELS": self.REDIS.sentinel.sentinels diff --git a/src/server/ext.py b/src/server/ext.py index 94792171..74d28029 100644 --- a/src/server/ext.py +++ b/src/server/ext.py @@ -7,6 +7,7 @@ import typing as t from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config + from .api.router import create_api_blueprint from .cli.base import register_cli_commands from .config import RuntimeConfig, setup_config diff --git a/src/server/services/cache_groups.py b/src/server/services/cache_groups.py index 88509366..949bba67 100644 --- a/src/server/services/cache_groups.py +++ b/src/server/services/cache_groups.py @@ -10,16 +10,16 @@ from celery import shared_task from weko_group_cache_db.groups import fetch_all, fetch_one -from weko_group_cache_db.signal import update_count_signal, update_result_signal +from weko_group_cache_db.signals import update_count_signal, update_result_signal from server.api.schemas import CacheOperation, CacheQuery from server.config import config from server.datastore import app_cache, group_cache +from server.entities.cache import RepositoryCache from server.entities.search_request import SearchResult if t.TYPE_CHECKING: - from server.entities.cache import RepositoryCache from server.entities.summaries import RepositorySummary @@ -55,8 +55,7 @@ def _get_repository_redis_key(fqdn: str) -> str: # filter by cache existence if query.f: - exist_cache_repository_list = [] - non_exist_cache_repository_list = [] + target_repositories = [] start_index = (query.p - 1) * query.l if query.p and query.l else 0 end_index = start_index + query.l if query.l else len(repositories.resources) @@ -76,7 +75,8 @@ def _get_repository_redis_key(fqdn: str) -> str: url=str(repository.service_url), updated=datetime.fromisoformat(repository_updated), # pyright: ignore[reportArgumentType] ) - exist_cache_repository_list.append(repo_cache) + if "cache" in query.f: + target_repositories.append(repo_cache) else: repo_cache = RepositoryCache( id=repository.id, @@ -84,19 +84,11 @@ def _get_repository_redis_key(fqdn: str) -> str: url=str(repository.service_url), updated=None, ) - non_exist_cache_repository_list.append(repo_cache) - if query.f == "cache": - target_repositories = exist_cache_repository_list[start_index:end_index] - return SearchResult( - resources=target_repositories, - total=len(exist_cache_repository_list), - page_size=query.l or len(target_repositories), - offset=start_index + 1, - ) - target_repositories = non_exist_cache_repository_list[start_index:end_index] + if "no_cache" in query.f: + target_repositories.append(repo_cache) return SearchResult( - resources=target_repositories, - total=len(non_exist_cache_repository_list), + resources=target_repositories[start_index:end_index], + total=len(target_repositories), page_size=query.l or len(target_repositories), offset=start_index + 1, ) @@ -222,12 +214,7 @@ def update_one_task(fqdn_list: list[str]) -> None: ) one_task_init_data = {"total": len(fqdn_list), "done": 0, "current": ""} app_cache.hset(cache_key, mapping=one_task_init_data) - for fqdn in fqdn_list: - try: - done_str = app_cache.hget(cache_key, "done") - done = int(done_str) if done_str is not None else 0 # pyright: ignore[reportArgumentType] - except TypeError: - done = 0 + for i, fqdn in enumerate(fqdn_list): if config.CACHE_GROUPS.toml_path: fetch_one(fqdn, toml_path=config.CACHE_GROUPS.toml_path) else: @@ -237,7 +224,7 @@ def update_one_task(fqdn_list: list[str]) -> None: fqdn_list_file=config.CACHE_GROUPS.fqdn_list_file, ) task_result = { - "done": done + 1, + "done": i + 1, "current": fqdn, } app_cache.hset(cache_key, mapping=task_result) diff --git a/tests/unit/api/test_cache_groups.py b/tests/unit/api/test_cache_groups.py new file mode 100644 index 00000000..8aa1fc6b --- /dev/null +++ b/tests/unit/api/test_cache_groups.py @@ -0,0 +1,243 @@ +import typing as t + +from copy import deepcopy +from datetime import datetime +from unittest.mock import call + +from server.api import cache_groups +from server.api.schemas import CacheQuery, CacheRequest, RepositoriesQuery +from server.entities.cache import CacheResult, RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_get(mocker: MockerFixture, repository_summaries, repository_caches, app, unwrap): + repositories = repository_summaries(20) + search_result = SearchResult( + resources=repository_caches(repositories.resources, None, False), + total=20, + page_size=20, + offset=1, + ) + mock_search = mocker.patch( + "server.api.cache_groups.search", + return_value=repositories, + ) + mock_get_repository_cache = mocker.patch( + "server.api.cache_groups.get_repository_cache", + return_value=search_result, + ) + with app.test_request_context(): + result = unwrap(cache_groups.get)(CacheQuery(q=None, p=1, l=20, f=[])) + + assert result == search_result + mock_search.assert_called_once_with(RepositoriesQuery(q=None, k="id", d="asc", p=1, l=20)) + mock_get_repository_cache.assert_called_once_with(repositories, CacheQuery(q=None, p=1, l=20, f=[])) + + +def test_post(mocker: MockerFixture, app, unwrap): + mock_update = mocker.patch("server.api.cache_groups.update") + fqdn_list = ["repo1.example.jp", "repo2.example.jp"] + operation = "all" + with app.test_request_context(): + result = unwrap(cache_groups.post)(body=CacheRequest(fqdn_list=fqdn_list, op=operation)) + + assert result == "Success" + mock_update.assert_called_once_with(fqdn_list, operation) + + +def test_get_task_status_no_task(mocker: MockerFixture, app, unwrap): + mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value={}) + mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") + mock_search = mocker.patch("server.api.cache_groups.search") + + with app.test_request_context(): + result = unwrap(cache_groups.get_task_status)() + + assert result == TaskDetail(results=[], current="", total=0, done=0) + mock_hgetall.assert_called_once_with("jcgroups_cache") + mock_delete.assert_not_called() + mock_search.assert_not_called() + + mocker.stopall() + + +def test_get_task_status_running(mocker: MockerFixture, app, unwrap, repository_summaries): + task_data = { + b"current": b"repo1.example.jp", + b"done": b"5", + b"total": b"10", + b"repo1.example.jp_0": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", + } + mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) + mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") + mock_search = mocker.patch("server.api.cache_groups.search", return_value=repository_summaries(1)) + + expect_result = [ + CacheResult( + type="cache", + fqdn="repo1.example.jp", + status="success", + code=None, + repository_cached=RepositoryCache( + id="repo_1", + name="Repository 1", + url="https://repo1.example.jp/", + updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 + ), + ) + ] + + with app.test_request_context(): + result = unwrap(cache_groups.get_task_status)() + + assert result == TaskDetail(results=expect_result, current="repo1.example.jp", total=10, done=5) + mock_hgetall.assert_called_once_with("jcgroups_cache") + mock_delete.assert_not_called() + mock_search.assert_called_once_with( + RepositoriesQuery( + q="/repo1.example.jp/", + k="id", + d="asc", + p=-1, + l=1, + ) + ) + + mocker.stopall() + + +def test_get_task_status_completed(mocker: MockerFixture, app, unwrap, repository_summaries): + task_data = { + b"current": b"repo1.example.jp", + b"done": b"10", + b"total": b"10", + b"repo1.example.jp_0": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", + } + mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) + mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") + mock_search = mocker.patch("server.api.cache_groups.search", return_value=repository_summaries(1)) + + expect_result = [ + CacheResult( + type="cache", + fqdn="repo1.example.jp", + status="success", + code=None, + repository_cached=RepositoryCache( + id="repo_1", + name="Repository 1", + url="https://repo1.example.jp/", + updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 + ), + ) + ] + + with app.test_request_context(): + result = unwrap(cache_groups.get_task_status)() + + assert result == TaskDetail(results=expect_result, current="repo1.example.jp", total=10, done=10) + mock_hgetall.assert_called_once_with("jcgroups_cache") + mock_delete.assert_called_once_with("jcgroups_cache") + mock_search.assert_called_once_with( + RepositoriesQuery( + q="/repo1.example.jp/", + k="id", + d="asc", + p=-1, + l=1, + ) + ) + + mocker.stopall() + + +def test_get_task_status_multi_try(mocker: MockerFixture, app, unwrap, repository_summaries): + task_data = { + b"current": b"repo2.example.jp", + b"done": b"2", + b"total": b"3", + b"repo1.example.jp_0": b"{'status': 'failed', 'updated': '2026-01-01T00:00:00Z', 'code': 'timeout'}", + b"repo1.example.jp_1": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", + b"repo2.example.jp_0": b"{'status': 'failed', 'updated': '2026-01-02T00:00:00Z', 'code': 'timeout'}", + } + repositories = repository_summaries(2) + first_repo = deepcopy(repositories) + first_repo.resources.pop(1) + second_repo = deepcopy(repositories) + second_repo.resources.pop(0) + + mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) + mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") + mock_search = mocker.patch( + "server.api.cache_groups.search", + side_effect=[first_repo, first_repo, second_repo], + ) + + expect_result = [ + CacheResult( + type="cache", + fqdn="repo1.example.jp", + status="success", + code=None, + repository_cached=RepositoryCache( + id="repo_1", + name="Repository 1", + url="https://repo1.example.jp/", + updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 + ), + ), + CacheResult( + type="cache", + fqdn="repo2.example.jp", + status="failed", + code="timeout", + repository_cached=RepositoryCache( + id="repo_2", + name="Repository 2", + url="https://repo2.example.jp/", + updated=datetime(2026, 1, 2, 0, 0), # noqa: DTZ001 + ), + ), + ] + + with app.test_request_context(): + result = unwrap(cache_groups.get_task_status)() + + assert result == TaskDetail(results=expect_result, current="repo2.example.jp", total=3, done=2) + mock_hgetall.assert_called_once_with("jcgroups_cache") + mock_delete.assert_not_called() + assert mock_search.call_args_list == [ + call( + RepositoriesQuery( + q="/repo1.example.jp/", + k="id", + d="asc", + p=-1, + l=1, + ) + ), + call( + RepositoriesQuery( + q="/repo1.example.jp/", + k="id", + d="asc", + p=-1, + l=1, + ) + ), + call( + RepositoriesQuery( + q="/repo2.example.jp/", + k="id", + d="asc", + p=-1, + l=1, + ) + ), + ] + + mocker.stopall() diff --git a/tests/unit/api/test_router.py b/tests/unit/api/test_router.py new file mode 100644 index 00000000..9289c358 --- /dev/null +++ b/tests/unit/api/test_router.py @@ -0,0 +1,113 @@ +import typing as t + +from flask import Blueprint + +from server.api.router import create_api_blueprint + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_create_api_blueprint(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "test", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_blueprint = mocker.MagicMock(Blueprint) + mock_module = mocker.MagicMock() + mock_module.bp = mock_blueprint + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.test") + mock_register_blueprint.assert_called_once_with( + mock_blueprint, + url_prefix="/test", + strict_slashes=False, + ) + + +def test_create_api_blueprint_multiple_words(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "cache_groups", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_blueprint = mocker.MagicMock(Blueprint) + mock_module = mocker.MagicMock() + mock_module.bp = mock_blueprint + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.cache_groups") + mock_register_blueprint.assert_called_once_with( + mock_blueprint, + url_prefix="/cache-groups", + strict_slashes=False, + ) + + +def test_create_api_blueprint_no_bp(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "no_bp_module", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_module = mocker.MagicMock() + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.no_bp_module") + mock_register_blueprint.assert_not_called() + + +def test_create_api_blueprint_no_modules(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_not_called() + mock_register_blueprint.assert_not_called() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4a2fc3f4..c153c959 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,16 +1,24 @@ +import inspect import typing as t +from datetime import datetime from pathlib import Path import pytest +from pydantic import HttpUrl + from server import const from server.config import RuntimeConfig +from server.entities.cache import RepositoryCache +from server.entities.search_request import SearchResult +from server.entities.summaries import RepositorySummary from server.factory import create_app if t.TYPE_CHECKING: from flask import Flask + from pytest_mock import MockerFixture def is_running_in_docker() -> bool: @@ -59,9 +67,50 @@ def test_config(): } }, "POSTGRES": {"db": "jctest", "host": db_host}, + "LOG": {"level": "DEBUG"}, + "REDIS": { + "single": { + "base_url": "redis://redis-single:6379", + }, + "sentinel": { + "sentinels": [ + { + "host": "", + "port": 26379, + } + ] + }, + }, + "RABBITMQ": { + "url": "amqp://guest:guest@rabbitmq:5672//", + }, + "CACHE_GROUPS": { + "cache_redis_key": "{prefix}cache", + "gakunin_redis_key": "{fqdn}_gakunin_groups", + "map_groups_api_endpoint": "https://sample.gakunin.jp/api/groups/", + "toml_path": "cache_db_config.toml", + "directory_path": "./cache_db/tls", + "fqdn_list_file": "fqdn_list.toml", + }, }) +@pytest.fixture(autouse=True) +def mock_redis(mocker: MockerFixture): + mock_redis = mocker.patch("server.datastore.Redis") + mock_redis_instance = mock_redis.from_url.return_value + mock_redis_instance.ping.return_value = True + return mock_redis_instance + + +@pytest.fixture +def unwrap(): + def _unwrap(f: t.Callable) -> t.Callable: + return inspect.unwrap(f) + + return _unwrap + + @pytest.fixture def base_app(instance_path, test_config): app = create_app(__name__, config=test_config) @@ -75,3 +124,48 @@ def base_app(instance_path, test_config): def app(base_app: Flask): with base_app.app_context(): yield base_app + + +@pytest.fixture +def repository_summaries(): + def _data(num: int) -> SearchResult[RepositorySummary]: + return SearchResult( + resources=[ + RepositorySummary( + id=f"repo_{i}", + display_name=f"Repository {i}", + service_url=HttpUrl(f"https://repo{i}.example.jp"), + sp_connector_id=f"jc_repo_{i}_sp", + ) + for i in range(1, num + 1) + ], + total=num, + page_size=20, + offset=1, + ) + + return _data + + +@pytest.fixture +def cache_redis_key(): + def _keys(fqdn_list: list[str]) -> list[bytes]: + return [f"{fqdn.replace('-', '_').replace('.', '_')}_gakunin_groups".encode() for fqdn in fqdn_list] + + return _keys + + +@pytest.fixture +def repository_caches(): + def _data(repositories: list[RepositorySummary], now: datetime, every_other: bool) -> list[RepositoryCache]: + return [ + RepositoryCache( + id=repositories[i].id, + name=repositories[i].display_name, # pyright: ignore[reportArgumentType], + url=str(repositories[i].service_url), + updated=now if not every_other or i % 2 == 0 else None, + ) + for i in range(len(repositories)) + ] + + return _data diff --git a/tests/unit/services/test_cache_groups.py b/tests/unit/services/test_cache_groups.py new file mode 100644 index 00000000..09e7d534 --- /dev/null +++ b/tests/unit/services/test_cache_groups.py @@ -0,0 +1,1159 @@ +import typing as t + +from datetime import UTC, datetime +from unittest.mock import MagicMock, call + +from server.api.schemas import CacheQuery +from server.entities.search_request import SearchResult +from server.services.cache_groups import ( + check_updating_cache_is_running, + get_repository_cache, + update, + update_count_signal, + update_one_task, + update_result_signal, + update_run_task, + update_task, +) + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_get_repository_cache(mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches): + with app.app_context(): + query = CacheQuery(l=20, p=1) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources, now, True) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=20, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 10 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_multi_scan( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(l=20, p=1) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources, now, True) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (1, keys[:10]), + (0, keys[10:]), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=20, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 10 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_all_cache_not_exceeding_page_size( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["cache"], l=20, p=1) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources, now, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=20, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 20 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["cache"], l=20, p=1) + repositories = repository_summaries(30) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[:20], now, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 30 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["cache"], l=20, p=2) + repositories = repository_summaries(30) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[20:30], now, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=21, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 30 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_empty_cache(mocker: MockerFixture, app, repository_summaries): + with app.app_context(): + query = CacheQuery(f=["cache"], l=20, p=1) + repositories = repository_summaries(20) + now = datetime.now(UTC) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, []), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + mock_hget.assert_not_called() + + mocker.stopall() + + +def test_get_repository_cache_half_cache( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["cache"], l=20, p=1) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[::2], now, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 10 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_no_cache_not_exceeding_page_size( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["no_cache"], l=20, p=1) + repositories = repository_summaries(20) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources, None, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, []), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=20, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + mock_hget.assert_not_called() + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["no_cache"], l=20, p=1) + repositories = repository_summaries(30) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[:20], None, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, []), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + mock_hget.assert_not_called() + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["no_cache"], l=20, p=2) + repositories = repository_summaries(30) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[20:30], None, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, []), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=21, + ) + assert result == expect + mock_scan.assert_called() + mock_hget.assert_not_called() + + mocker.stopall() + + +def test_get_repository_cache_no_cache_all_cache(mocker: MockerFixture, app, repository_summaries, cache_redis_key): + with app.app_context(): + query = CacheQuery(f=["no_cache"], l=20, p=1) + now = datetime.now(UTC) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 20 # noqa: PLR2004 + + mocker.stopall() + + +def test_get_repository_cache_no_cache_half_cache( + mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches +): + with app.app_context(): + query = CacheQuery(f=["no_cache"], l=20, p=1) + repositories = repository_summaries(20) + keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = repository_caches(repositories.resources[1::2], None, False) + mock_scan = mocker.patch( + "server.services.cache_groups.group_cache.scan", + side_effect=[ + (0, keys), + ], + ) + mock_hget = mocker.patch( + "server.services.cache_groups.group_cache.hget", + return_value=now.isoformat(), + ) + + result = get_repository_cache(repositories, query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_scan.assert_called() + assert mock_hget.call_count == 10 # noqa: PLR2004 + + mocker.stopall() + + +def test_update(mocker: MockerFixture, app): + with app.app_context(): + mock_check = mocker.patch( + "server.services.cache_groups.check_updating_cache_is_running", + return_value=False, + ) + mock_update_task = mocker.patch( + "server.services.cache_groups.update_task.apply_async", + ) + + fqdn_list = ["example.com"] + op = "all" + update(fqdn_list, op) + + mock_check.assert_called_once() + mock_update_task.assert_called_once_with(fqdn_list, op) + + mocker.stopall() + + +def test_update_raises_runtime_error_when_task_running(mocker: MockerFixture, app): + with app.app_context(): + mock_check = mocker.patch( + "server.services.cache_groups.check_updating_cache_is_running", + return_value=True, + ) + mock_update_task = mocker.patch( + "server.services.cache_groups.update_task.apply_async", + ) + fqdn_list = ["example.com"] + op = "all" + + try: + update(fqdn_list, op) + except RuntimeError as e: + assert str(e) == "A cache update task is already running." + + mock_check.assert_called_once() + mock_update_task.assert_not_called() + + mocker.stopall() + + +def test_update_task_all(mocker: MockerFixture, app): + with app.app_context(): + mock_update_all_caches = mocker.patch( + "server.services.cache_groups.update_run_task", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "all" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + mock_update_all_caches.assert_called_once() + mock_hset.assert_not_called() + + mocker.stopall() + + +def test_update_task_all_count_signal_once(mocker: MockerFixture, app): + with app.app_context(): + mock_update_all_caches = mocker.patch( + "server.services.cache_groups.update_run_task", + side_effect=lambda: update_count_signal.send( + None, + total=100, + done=50, + current="example.com", + ), + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "all" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count signal + + mock_update_all_caches.assert_called_once() + mock_hset.assert_called_with( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50, + "current": "example.com", + }, + ) + + mocker.stopall() + + +def test_update_task_all_result_signal_once(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + mock_update_all_caches = mocker.patch( + "server.services.cache_groups.update_run_task", + side_effect=lambda: update_result_signal.send( + None, + task_name="example_com_0", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ), + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "all" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update result signal + + mock_update_all_caches.assert_called_once() + mock_hset.assert_called_with( + "jcgroups_cache", + mapping={ + "example_com_0": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ) + + mocker.stopall() + + +def test_update_task_all_signals_once(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + + def side_effect(): + update_count_signal.send( + None, + total=100, + done=50, + current="example.com", + ) + update_result_signal.send( + None, + task_name="example_com_0", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + mock_update_all_caches = mocker.patch( + "server.services.cache_groups.update_run_task", + side_effect=side_effect, + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "all" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count and result signals + + mock_update_all_caches.assert_called_once() + expected_calls = [ + call( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50, + "current": "example.com", + }, + ), + call( + "jcgroups_cache", + mapping={ + "example_com_0": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ), + ] + mock_hset.assert_has_calls(expected_calls, any_order=False) + + mocker.stopall() + + +def test_update_task_all_signal_multiple(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + + def side_effect(): + for i in range(3): + update_count_signal.send( + None, + total=100, + done=50 + i, + current=f"example{i}.com", + ) + update_result_signal.send( + None, + task_name=f"example_com_{i}", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + mock_update_all_caches = mocker.patch( + "server.services.cache_groups.update_run_task", + side_effect=side_effect, + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "all" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count and result signals multiple times + + mock_update_all_caches.assert_called_once() + expected_calls = [ + call( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50 + i, + "current": f"example{i}.com", + }, + ) + for i in range(3) + ] + [ + call( + "jcgroups_cache", + mapping={ + f"example_com_{i}": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ) + for i in range(3) + ] + mock_hset.assert_has_calls(expected_calls, any_order=True) + + mocker.stopall() + + +def test_update_task_specified(mocker: MockerFixture, app): + with app.app_context(): + mock_update_specified_caches = mocker.patch( + "server.services.cache_groups.update_one_task", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "id-specified" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + mock_update_specified_caches.assert_called_once_with(fqdn_list) + mock_hset.assert_not_called() + + mocker.stopall() + + +def test_update_task_specified_count_signal_once(mocker: MockerFixture, app): + with app.app_context(): + mock_update_specified_caches = mocker.patch( + "server.services.cache_groups.update_one_task", + side_effect=lambda *_, **__: update_count_signal.send( + None, + total=100, + done=50, + current="example.com", + ), + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "id-specified" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count signal + + mock_update_specified_caches.assert_called_once_with(fqdn_list) + mock_hset.assert_called_with( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50, + "current": "example.com", + }, + ) + + mocker.stopall() + + +def test_update_task_specified_result_signal_once(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + mock_update_specified_caches = mocker.patch( + "server.services.cache_groups.update_one_task", + side_effect=lambda *_, **__: update_result_signal.send( + None, + task_name="example_com_0", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ), + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "id-specified" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update result signal + + mock_update_specified_caches.assert_called_once_with(fqdn_list) + mock_hset.assert_called_with( + "jcgroups_cache", + mapping={ + "example_com_0": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ) + + mocker.stopall() + + +def test_update_task_specified_signals_once(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + + def side_effect(*_, **__): + update_count_signal.send( + None, + total=100, + done=50, + current="example.com", + ) + update_result_signal.send( + None, + task_name="example_com_0", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + mock_update_specified_caches = mocker.patch( + "server.services.cache_groups.update_one_task", + side_effect=side_effect, + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "id-specified" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count and result signals + + mock_update_specified_caches.assert_called_once_with(fqdn_list) + expected_calls = [ + call( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50, + "current": "example.com", + }, + ), + call( + "jcgroups_cache", + mapping={ + "example_com_0": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ), + ] + mock_hset.assert_has_calls(expected_calls, any_order=False) + + mocker.stopall() + + +def test_update_task_specified_signal_multiple(mocker: MockerFixture, app): + with app.app_context(): + now = datetime.now(UTC) + + def side_effect(*_, **__): + for i in range(3): + update_count_signal.send( + None, + total=100, + done=50 + i, + current=f"example{i}.com", + ) + update_result_signal.send( + None, + task_name=f"example_com_{i}", + status="success", + code="", + updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + mock_update_specified_caches = mocker.patch( + "server.services.cache_groups.update_one_task", + side_effect=side_effect, + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + fqdn_list = ["example.com"] + op = "id-specified" + + update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] + + # Simulate sending the update count and result signals multiple times + + mock_update_specified_caches.assert_called_once_with(fqdn_list) + expected_calls = [ + call( + "jcgroups_cache", + mapping={ + "total": 100, + "done": 50 + i, + "current": f"example{i}.com", + }, + ) + for i in range(3) + ] + [ + call( + "jcgroups_cache", + mapping={ + f"example_com_{i}": "{'status': 'success', 'code': '', 'updated': '" + + now.strftime("%Y-%m-%dT%H:%M:%SZ") + + "'}" + }, + ) + for i in range(3) + ] + mock_hset.assert_has_calls(expected_calls, any_order=True) + + mocker.stopall() + + +def test_update_run_task_toml_path(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_all = mocker.patch( + "server.services.cache_groups.fetch_all", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="/path/to/toml", + directory_path="", + fqdn_list_file="", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + + update_run_task() + + mock_fetch_all.assert_called_once_with(toml_path="/path/to/toml") + mock_hset.assert_called_once_with( + "jcgroups_cache", + mapping={"total": "", "done": 0, "current": ""}, + ) + + mocker.stopall() + + +def test_update_run_task_not_toml_path(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_all = mocker.patch( + "server.services.cache_groups.fetch_all", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + + update_run_task() + + mock_fetch_all.assert_called_once_with( + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ) + mock_hset.assert_called_once_with( + "jcgroups_cache", + mapping={"total": "", "done": 0, "current": ""}, + ) + + mocker.stopall() + + +def test_update_run_task_all_settings(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_all = mocker.patch( + "server.services.cache_groups.fetch_all", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="/path/to/toml", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + + update_run_task() + + mock_fetch_all.assert_called_once_with( + toml_path="/path/to/toml", + ) + mock_hset.assert_called_once_with( + "jcgroups_cache", + mapping={"total": "", "done": 0, "current": ""}, + ) + + mocker.stopall() + + +def test_update_one_task_toml_path(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_one = mocker.patch( + "server.services.cache_groups.fetch_one", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="/path/to/toml", + directory_path="", + fqdn_list_file="", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + fqdn_list = ["example.com"] + + update_one_task(fqdn_list) + + mock_fetch_one.assert_called_once_with("example.com", toml_path="/path/to/toml") + assert mock_hset.call_args_list == [ + call( + "jcgroups_cache", + mapping={"total": 1, "done": 0, "current": ""}, + ), + call( + "jcgroups_cache", + mapping={"done": 1, "current": "example.com"}, + ), + ] + + mocker.stopall() + + +def test_update_one_task_not_toml_path(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_one = mocker.patch( + "server.services.cache_groups.fetch_one", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + fqdn_list = ["example.com"] + + update_one_task(fqdn_list) + + mock_fetch_one.assert_called_once_with( + "example.com", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ) + assert mock_hset.call_args_list == [ + call( + "jcgroups_cache", + mapping={"total": 1, "done": 0, "current": ""}, + ), + call( + "jcgroups_cache", + mapping={"done": 1, "current": "example.com"}, + ), + ] + + mocker.stopall() + + +def test_update_one_task_all_settings(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_one = mocker.patch( + "server.services.cache_groups.fetch_one", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="/path/to/toml", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + fqdn_list = ["example.com"] + + update_one_task(fqdn_list) + + mock_fetch_one.assert_called_once_with("example.com", toml_path="/path/to/toml") + assert mock_hset.call_args_list == [ + call( + "jcgroups_cache", + mapping={"total": 1, "done": 0, "current": ""}, + ), + call( + "jcgroups_cache", + mapping={"done": 1, "current": "example.com"}, + ), + ] + + mocker.stopall() + + +def test_update_one_task_multi_repositories(mocker: MockerFixture, app): + with app.app_context(): + mock_fetch_one = mocker.patch( + "server.services.cache_groups.fetch_one", + ) + mock_hset = mocker.patch( + "server.services.cache_groups.app_cache.hset", + ) + mocker.patch( + "server.services.cache_groups.config", + CACHE_GROUPS=MagicMock( + cache_redis_key="{prefix}cache", + toml_path="/path/to/toml", + directory_path="./cache_db/tls", + fqdn_list_file="fqdn_list.toml", + ), + REDIS=MagicMock(key_prefix="jcgroups_"), + ) + fqdn_list = ["example1.com", "example2.com", "example3.com"] + + update_one_task(fqdn_list) + + assert mock_fetch_one.call_count == 3 + expected_calls = [ + call( + "jcgroups_cache", + mapping={"total": 3, "done": 0, "current": ""}, + ), + call( + "jcgroups_cache", + mapping={"done": 1, "current": "example1.com"}, + ), + call( + "jcgroups_cache", + mapping={"done": 2, "current": "example2.com"}, + ), + call( + "jcgroups_cache", + mapping={"done": 3, "current": "example3.com"}, + ), + ] + mock_hset.assert_has_calls(expected_calls, any_order=False) + + mocker.stopall() + + +def test_check_updating_cache_is_running_exists(mocker: MockerFixture, app): + with app.app_context(): + mock_get = mocker.patch( + "server.services.cache_groups.app_cache.exists", + return_value=True, + ) + + result = check_updating_cache_is_running() + assert result is True + mock_get.assert_called_once_with("jcgroups_cache") + + mocker.stopall() + + +def test_check_updating_cache_is_running_not_exists(mocker: MockerFixture, app): + with app.app_context(): + mock_get = mocker.patch( + "server.services.cache_groups.app_cache.exists", + return_value=False, + ) + + result = check_updating_cache_is_running() + assert result is False + mock_get.assert_called_once_with("jcgroups_cache") + + mocker.stopall() diff --git a/tests/unit/test_ext.py b/tests/unit/test_ext.py new file mode 100644 index 00000000..5adfbae3 --- /dev/null +++ b/tests/unit/test_ext.py @@ -0,0 +1,22 @@ +import typing as t + +from server.ext import JAIROCloudGroupsManager + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_init_config(mocker: MockerFixture): + mock_setup_config = mocker.patch( + "server.ext.setup_config", + ) + mock_cache_db_setup_config = mocker.patch( + "server.ext.setup_weko_group_cache_db_config", + ) + mock_app = mocker.MagicMock() + + ext = JAIROCloudGroupsManager() + ext.init_config(mock_app) + mock_setup_config.assert_called_once() + mock_cache_db_setup_config.assert_called_once_with(ext.config.CACHE_DB) From 68dd8a15e73e7cf5c844cfdf148c42b819c5bfd6 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Tue, 10 Mar 2026 23:52:05 +0000 Subject: [PATCH 3/5] refactor: add error handling and logic streamlining --- configs/server.config.toml | 39 +- src/server/api/cache_groups.py | 127 -- src/server/api/group_caches.py | 92 ++ src/server/api/router.py | 3 +- src/server/api/schemas.py | 12 +- src/server/config.py | 76 +- src/server/const.py | 3 + src/server/datastore.py | 2 +- src/server/entities/cache.py | 49 +- src/server/exc.py | 9 +- src/server/messages/error.py | 28 +- src/server/messages/info.py | 6 + src/server/messages/warning.py | 12 + src/server/services/cache_groups.py | 242 ---- src/server/services/group_caches.py | 319 +++++ src/server/services/utils/search_queries.py | 29 + tests/unit/api/test_cache_groups.py | 243 ---- tests/unit/api/test_group_caches.py | 117 ++ tests/unit/conftest.py | 57 +- tests/unit/services/test_cache_groups.py | 1159 ------------------- tests/unit/services/test_group_caches.py | 641 ++++++++++ 21 files changed, 1361 insertions(+), 1904 deletions(-) delete mode 100644 src/server/api/cache_groups.py create mode 100644 src/server/api/group_caches.py delete mode 100644 src/server/services/cache_groups.py create mode 100644 src/server/services/group_caches.py delete mode 100644 tests/unit/api/test_cache_groups.py create mode 100644 tests/unit/api/test_group_caches.py delete mode 100644 tests/unit/services/test_cache_groups.py create mode 100644 tests/unit/services/test_group_caches.py diff --git a/configs/server.config.toml b/configs/server.config.toml index df16df20..fc7d437b 100644 --- a/configs/server.config.toml +++ b/configs/server.config.toml @@ -222,24 +222,37 @@ url = "amqp://guest:guest@rabbitmq:5672//" [cache_groups] -# Redis key pattern for caching group data. -cache_redis_key = "{prefix}cache" - -# Redis key pattern for caching GakuNin group data. -gakunin_redis_key = "{fqdn}_gakunin_groups" +# Cache key suffix of group information in Redis. +cache_key_suffix = "_gakunin_groups" # Map groups API endpoint. -map_groups_api_endpoint = "https://sample.gakunin.jp/api/groups/" +api_endpoint = "/api/groups" -# If toml_path and either directory_path or fqdn_list_file are defined, toml_path takes precedence. -# Schema for cache database configuration. -toml_path = "cache_db_config.toml" +# Cache time-to-live of group information in Redis. +# if it specified less than 0, it will be considered as no expiration. +cache_ttl = 86400 -# Path to the directory containing institution TLS files. -directory_path = "./cache_db/tls" +# Request timeout (in seconds) when connecting to mAP API. +request_timeout = 20 + +# Request interval (in seconds) between mAP API requests. +request_interval = 3 + +# Request retries when failed to fetch groups from mAP API. +request_retries = 3 + +# Base time (in seconds) for exponential backoff during request retries. +request_retry_base = 4 -# Path to the file containing FQDN list. -fqdn_list_file = "fqdn_list.toml" +# Factor (in seconds) for exponential backoff during request retries. +request_retry_factor = 5 + +# Maximum time (in seconds) for exponential backoff during request retries. +request_retry_max = 90 + + +# Path to the directory containing institution TLS files. +directory_path = "/var/mnt" # [develop] diff --git a/src/server/api/cache_groups.py b/src/server/api/cache_groups.py deleted file mode 100644 index 1817c17b..00000000 --- a/src/server/api/cache_groups.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright (C) 2025 National Institute of Informatics. -# -"""API router for cache group endpoints.""" - -from ast import literal_eval -from datetime import datetime - -from flask import Blueprint -from flask_login import login_required -from flask_pydantic import validate - -from server.api.helper import roles_required -from server.api.schemas import CacheQuery, CacheRequest, RepositoriesQuery -from server.config import config -from server.datastore import app_cache -from server.entities.cache import CacheResult, RepositoryCache, TaskDetail -from server.entities.search_request import SearchResult -from server.services.cache_groups import get_repository_cache, update -from server.services.repositories import search - - -bp = Blueprint("cache-groups", __name__) - - -@bp.get("/") -@login_required -@roles_required("system_admin") -@validate(response_by_alias=True) -def get(query: CacheQuery) -> SearchResult[RepositoryCache]: - """Retrieve repository cache entries based on the provided query. - - Args: - query (CacheQuery): Query parameters for filtering and pagination. - - Returns: - list[RepositoryCache]: List of repository cache entries. - """ - repository_query = RepositoriesQuery( - q=query.q, - k="id", - d="asc", - p=query.p if not query.f else -1, - l=query.l, - ) - - repositories = search(repository_query) - - return get_repository_cache(repositories, query) - - -@bp.post("/") -@login_required -@roles_required("system_admin") -@validate() -def post(body: CacheRequest) -> str: - """Update cache groups for the specified FQDNs. - - Args: - body (CacheRequest): Request body containing FQDNs and operation. - - Returns: - str: Success message. - """ - fqdn_list = body.fqdn_list or [] - update(fqdn_list, body.op) - return "Success" - - -@bp.get("/task") -@login_required -@roles_required("system_admin") -@validate() -def get_task_status() -> TaskDetail: - """Get the status of the cache update task. - - Returns: - TaskDetail: Details of the cache update task. - """ - cache_key = config.CACHE_GROUPS.cache_redis_key.format( - prefix=config.REDIS.key_prefix - ) - task_status = app_cache.hgetall(cache_key) - decode_task_status = {k.decode(): v.decode() for k, v in task_status.items()} # pyright: ignore[reportAttributeAccessIssue] - results = {} - current = "" - done = 0 - total = 0 - for key in decode_task_status: # noqa: PLC0206 - if key == "current": - current = decode_task_status[key] # pyright: ignore[reportIndexIssue] - elif key == "done": - done = int(decode_task_status[key]) # pyright: ignore[reportIndexIssue] - elif key == "total": - total = int(decode_task_status[key]) # pyright: ignore[reportIndexIssue] - else: - result = literal_eval(decode_task_status[key]) # pyright: ignore[reportIndexIssue] - fqdn = key.split("_")[0] - repository_query = RepositoriesQuery( - q=f"/{fqdn}/", - k="id", - d="asc", - p=-1, - l=1, - ) - repository = search(repository_query).resources[0] - results[fqdn] = CacheResult( - type="cache", - fqdn=fqdn, - status=result["status"], - code=result.get("code"), - repository_cached=RepositoryCache( - id=repository.id, - name=repository.display_name, # pyright: ignore[reportArgumentType] - url=str(repository.service_url), - updated=datetime.strptime( # noqa: DTZ007 - result.get("updated"), "%Y-%m-%dT%H:%M:%SZ" - ), - ), - ) - if total > 0 and total == done: - # Clear the task status when the task is completed. - app_cache.delete(cache_key) - - return TaskDetail( - results=list(results.values()), current=current, done=done, total=total - ) diff --git a/src/server/api/group_caches.py b/src/server/api/group_caches.py new file mode 100644 index 00000000..cadc874d --- /dev/null +++ b/src/server/api/group_caches.py @@ -0,0 +1,92 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# +"""API router for cache group endpoints.""" + +import traceback +import typing as t + +from flask import Blueprint, current_app +from flask_login import login_required +from flask_pydantic import validate + +from server.api.helpers import roles_required +from server.api.schemas import ( + CacheQuery, + CacheRequest, + ErrorResponse, +) +from server.const import USER_ROLES +from server.entities.cache import TaskDetail +from server.entities.search_request import SearchResult +from server.exc import InvalidQueryError, RequestConflict +from server.messages import E +from server.services import group_caches + + +bp = Blueprint("group-caches", __name__) + + +@bp.get("/", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate(response_by_alias=True) +def get(query: CacheQuery) -> tuple[SearchResult, int] | tuple[ErrorResponse, int]: + """Retrieve repository cache entries based on the provided query. + + Args: + query (CacheQuery): Query parameters for filtering and pagination. + + Returns: + - If succeeded in getting repository cache, search result and status code 200 + - If query is invalid, error message and status code 400 + """ + try: + cache_result = group_caches.get_repository_cache(query) + except InvalidQueryError as exc: + traceback.print_exc() + return ErrorResponse(message=exc.message), 400 + + return cache_result, 200 + + +@bp.post("/", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate() +def post(body: CacheRequest) -> tuple[t.Literal[""], int] | tuple[ErrorResponse, int]: + """Update cache groups for the specified repositories. + + Args: + body (CacheRequest): Request body containing repositories and operation. + + Returns: + - If the update task is successfully started, empty response and status code 202 + - If there is a conflict in starting the task, error message and status code 409 + """ + try: + group_caches.update(body.op, body.ids) + except RequestConflict as exc: + traceback.print_exc() + return ErrorResponse(message=exc.message), 409 + + return "", 202 + + +@bp.get("/status", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate() +def status() -> tuple[TaskDetail | ErrorResponse, int]: + """Get the status of the cache update task. + + Returns: + - If a task is running, details of the task and status code 200 + - If no task is running, error message and status code 400 + """ + task_status = group_caches.get_task_status() + if task_status is None: + current_app.logger.error(E.UPDATE_TASK_NOT_RUNNING) + return ErrorResponse(message=E.UPDATE_TASK_NOT_RUNNING), 400 + + return task_status, 200 diff --git a/src/server/api/router.py b/src/server/api/router.py index bf62bdef..c0e1fc39 100644 --- a/src/server/api/router.py +++ b/src/server/api/router.py @@ -36,8 +36,9 @@ def create_api_blueprint() -> Blueprint: for _, module_name, _ in iter_modules([str(Path(__file__).parent)]): module = import_module(f"{__package__}.{module_name}") + url_prefix = f"/{module_name}".replace("_", "-") if hasattr(module, "bp") and isinstance(module.bp, Blueprint): - bp_api.register_blueprint(module.bp, url_prefix=f"/{module_name}") + bp_api.register_blueprint(module.bp, url_prefix=url_prefix) @bp_api.errorhandler(JAIROCloudGroupsManagerError) @validate() diff --git a/src/server/api/schemas.py b/src/server/api/schemas.py index e0495753..b1c9b6f8 100644 --- a/src/server/api/schemas.py +++ b/src/server/api/schemas.py @@ -362,13 +362,13 @@ class CacheQuery(BaseModel): q: t.Annotated[str | None, "term"] = None """Search term for querying cache entries.""" - f: t.Annotated[list[str] | None, "filter"] = None + f: t.Annotated[list[t.Literal["e", "n"]] | None, "filter"] = None """Filter expression for querying cache entries.""" p: t.Annotated[int | None, "page"] = None """Page number for pagination.""" - l: t.Annotated[int | None, "per"] = None + l: t.Annotated[int | None, "per"] = None # noqa: E741 """Number of items per page for pagination.""" @@ -378,10 +378,12 @@ class CacheQuery(BaseModel): class CacheRequest(BaseModel): """Schema for cache update request.""" - fqdn_list: list[str] | None = None - """List of fully qualified domain names to update in the cache.""" + ids: list[str] | None = None + """List of repository IDs to update cache for. + Required if operation is 'id-specified'. + """ op: CacheOperation - """Operation type: 'all' to update all, 'id-specified' to update specified FQDNs.""" + """Operation type: 'all' to update all, 'id-specified' to update specified IDs.""" model_config = camel_case_config diff --git a/src/server/config.py b/src/server/config.py index 5e3a6afc..27e1f5fb 100644 --- a/src/server/config.py +++ b/src/server/config.py @@ -17,6 +17,7 @@ from flask import current_app from pydantic import ( + AnyUrl, BaseModel, Field, StringConstraints, @@ -30,6 +31,10 @@ TomlConfigSettingsSource, ) from sqlalchemy.engine import URL, make_url +from weko_group_cache_db.config import ( + Sentinel, + Settings as CacheDbSettings, +) from werkzeug.local import LocalProxy from .const import ( @@ -102,32 +107,27 @@ class RuntimeConfig(BaseSettings): @computed_field @property - def CACHE_DB(self) -> dict[str, t.Any]: + def CACHE_DB(self) -> CacheDbSettings: """Cache database configuration values.""" - return { - "CACHE_KEY_SUFFIX": self.CACHE_GROUPS.gakunin_redis_key, - "MAP_GROUPS_API_ENDPOINT": self.CACHE_GROUPS.map_groups_api_endpoint, - "REDIS_TYPE": "redis" - if self.REDIS.cache_type == "RedisCache" - else "sentinel", - "REDIS_HOST": self.REDIS.single.base_url.replace("redis://", "").split(":")[ - 0 - ] - if self.REDIS.cache_type == "RedisCache" - else "", - "REDIS_PORT": int( - self.REDIS.single.base_url.replace("redis://", "").split(":")[1] - ) - if self.REDIS.cache_type == "RedisCache" - else 0, - "REDIS_DB_INDEX": self.REDIS.database.group_cache, - "REDIS_SENTINEL_MASTER": self.REDIS.sentinel.master_name - if self.REDIS.cache_type == "RedisSentinelCache" - else "", - "SENTINELS": self.REDIS.sentinel.nodes - if self.REDIS.cache_type == "RedisSentinelCache" - else [], - } + endpoint = ( + self.MAP_CORE.base_url.rstrip("/") + + "/" + + self.CACHE_GROUPS.api_endpoint.lstrip("/") + ) + + return self.CACHE_GROUPS.model_copy( + update={ + "LOG_LEVEL": self.LOG.level, + "SP_CONNECTOR_ID_PREFIX": self.SP.connector_id, + "MAP_GROUPS_API_ENDPOINT": endpoint, + "REDIS_TYPE": self.REDIS.cache_type, + "REDIS_HOST": self.REDIS.single.base_url.host, + "REDIS_PORT": self.REDIS.single.base_url.port, + "REDIS_DB_INDEX": self.REDIS.database.group_cache, + "REDIS_SENTINEL_MASTER": self.REDIS.sentinel.master_name, + "SENTINELS": t.cast("list[Sentinel]", self.REDIS.sentinel.nodes), + } + ) @computed_field @property @@ -148,10 +148,10 @@ def CELERY(self) -> dict[str, t.Any]: """ cache_type = self.REDIS.cache_type database = self.REDIS.database.result_backend - config: dict[str, t.Any] = {"broker_url": self.RABBITMQ.url} + config: dict[str, t.Any] = {"broker_url": str(self.RABBITMQ.url)} if cache_type == "RedisCache" and self.REDIS.single: - base_url = self.REDIS.single.base_url.rstrip("/") + base_url = self.REDIS.single.base_url config["result_backend"] = f"{base_url}/{database}" elif cache_type == "RedisSentinelCache" and self.REDIS.sentinel: @@ -490,7 +490,7 @@ class RedisConfig(BaseModel): cache_timeout: t.Annotated[int, "seconds"] = 300 """Default timeout (in seconds) for cached items.""" - key_prefix: str = "jcgroups_" + key_prefix: str = "jcgroups-" """Prefix for cache keys used by the application.""" database: RedisDatabaseConfig = Field( @@ -530,7 +530,7 @@ class RedisDatabaseConfig(BaseModel): class RedisSingleConfig(BaseModel): """Schema for single Redis server configuration.""" - base_url: str = "redis://localhost:6379" + base_url: AnyUrl = AnyUrl("redis://localhost:6379") class RedisSentinelCacheConfig(BaseModel): @@ -555,31 +555,19 @@ class SentinelNodeConfig(BaseModel): class RabbitmqConfig(BaseModel): """Schema for RabbitMQ configuration.""" - url: str = "amqp://guest:guest@localhost:5672//" + url: AnyUrl = AnyUrl("amqp://guest:guest@localhost:5672//") """Hostname or IP address of the RabbitMQ server for Celery broker.""" -class CacheGroupsConfig(BaseModel): +class CacheGroupsConfig(CacheDbSettings): """Schema for cache groups configuration.""" - cache_redis_key: str - """Redis key for cache groups.""" - - gakunin_redis_key: str - """Redis key pattern for caching GakuNin group data.""" - - map_groups_api_endpoint: str + api_endpoint: str """Map groups API endpoint.""" - toml_path: str - """Path to the TOML file with cache database configuration.""" - directory_path: str """Path to the directory containing institution TLS files.""" - fqdn_list_file: str - """Path to the file containing FQDN list.""" - type HasRepoId = t.Annotated[str, StringConstraints(pattern=HAS_REPO_ID_PATTERN)] """Pattern for role-based group IDs. diff --git a/src/server/const.py b/src/server/const.py index 2b387641..6a618a35 100644 --- a/src/server/const.py +++ b/src/server/const.py @@ -191,6 +191,9 @@ class USER_ROLES(StrEnum): - URLs ending with "/admin" are excluded from matching. """ +GROUP_CACHE_KEY_PATTERN: Final = "weko-group-cache-db" +"""Regular expression pattern to identify cache keys in Redis.""" + class ValidationEntity: """Constants for validation entities.""" diff --git a/src/server/datastore.py b/src/server/datastore.py index d6d1a75c..e907e97a 100644 --- a/src/server/datastore.py +++ b/src/server/datastore.py @@ -59,7 +59,7 @@ def connection( timeout = config.REDIS.socket_timeout try: if config.REDIS.cache_type == "RedisCache": - base_url = config.REDIS.single.base_url.rstrip("/") + base_url = config.REDIS.single.base_url store = Redis.from_url(f"{base_url}/{db}") else: sentinels = sentinel.Sentinel( diff --git a/src/server/entities/cache.py b/src/server/entities/cache.py index 849330f1..e05d9927 100644 --- a/src/server/entities/cache.py +++ b/src/server/entities/cache.py @@ -11,35 +11,30 @@ from pydantic import BaseModel from .common import camel_case_config, forbid_extra_config +from .summaries import RepositorySummary -class RepositoryCache(BaseModel): +class RepositoryCache(RepositorySummary): """Model for repository cache entity.""" - id: str - """The unique identifier for the repository.""" - - name: str - """The name of the repository.""" - - url: str - """The URL of the repository.""" - updated: datetime | None = None """The update timestamp of the repository cache entry.""" - model_config = camel_case_config | forbid_extra_config - """Configure to use camelCase aliasing and forbid extra fields.""" + status: RepositoryStatus | None = None + """The status of the cache update task.""" class TaskDetail(BaseModel): """Model for task detail entity.""" - results: list[Result] + results: list[RepositoryCache] """The list of results from the task.""" + status: TaskStatus | None = None + """The status of the task.""" + current: str - """Identifier of the object currently being processed.""" + """The repository id currently being processed.""" done: int """The number of completed items.""" @@ -51,29 +46,7 @@ class TaskDetail(BaseModel): """Configure to use camelCase aliasing and forbid extra fields.""" -type TaskStatus = t.Literal["success", "failed"] - - -class CacheResult(BaseModel): - """Model for cache update result.""" - - type: t.Literal["cache"] - """The type of the result, always "cache".""" - - fqdn: str - """The fully qualified domain name of the cached repository.""" - - status: TaskStatus - """The status of the cache update task.""" - - code: str | None = None - """The result code of the cache update task.""" - - repository_cached: RepositoryCache | None = None - """The cached repository information.""" - - model_config = camel_case_config | forbid_extra_config - """Configure to use camelCase aliasing and forbid extra fields.""" +type RepositoryStatus = t.Literal["success", "failed"] -Result = CacheResult +type TaskStatus = t.Literal["pending", "started", "in_progress", "completed"] diff --git a/src/server/exc.py b/src/server/exc.py index 0be07137..a973cdf0 100644 --- a/src/server/exc.py +++ b/src/server/exc.py @@ -97,7 +97,7 @@ class DatastoreError(InfrastructureError): """ -class TaskExcutionError(DatastoreError): +class TaskExecutionError(DatastoreError): """Exception for task execution errors. Errors caused by issues during task execution. @@ -193,3 +193,10 @@ class FileValidationError(BulkOperationError): Errors caused by validation failures during bulk operations. """ + + +class GroupCacheError(JAIROCloudGroupsManagerError): + """Exception for group cache errors. + + Errors caused by issues in group cache operations. + """ diff --git a/src/server/messages/error.py b/src/server/messages/error.py index 885c53ed..5b55f9b0 100644 --- a/src/server/messages/error.py +++ b/src/server/messages/error.py @@ -20,7 +20,7 @@ UNSUPPORTED_EXPRESSION = LogMessage( "E002", "Unsupported expression in server configuration; " - "supported: 1. int / flaot literal, 2. literal str for len, " + "supported: 1. int / float literal, 2. literal str for len, " "3. +, -, *, / operators, 4. len, max, min functions.", ) @@ -577,6 +577,32 @@ ) +GROUP_CACHE_UPDATE_CONFLICT = LogMessage( + "E800", + "The cache update operation is already in progress.", +) + +FAILED_ENQUEUE_CACHE_UPDATE_TASK = LogMessage( + "E801", + "Failed to enqueue cache update task.", +) + +FAILED_FETCH_UPDATE_TASK_STATUS = LogMessage( + "E802", + "Failed to fetch cache update task status.", +) + +FAILED_PARSE_UPDATE_TASK_STATUS = LogMessage( + "E803", + "Failed to parse cache update task status.", +) + +UPDATE_TASK_NOT_RUNNING = LogMessage( + "E804", + "No cache update task is currently running.", +) + + UNNECESSARY_CONTRIB = LogMessage( "E999", "Contrib utilities can only be used in development mode." ) diff --git a/src/server/messages/info.py b/src/server/messages/info.py index 8c5e825d..2c8636d5 100644 --- a/src/server/messages/info.py +++ b/src/server/messages/info.py @@ -177,3 +177,9 @@ "I321", "Successfully updated affiliations of User resource (id: %(id)s, ePPN: %(eppn)s).", ) + + +GROUP_CACHE_UPDATE_STARTED = LogMessage( + "I800", + "Group cache update task started (operation: %(op)s, task: %(task_id)s).", +) diff --git a/src/server/messages/warning.py b/src/server/messages/warning.py index 45a0dd9f..25a50200 100644 --- a/src/server/messages/warning.py +++ b/src/server/messages/warning.py @@ -65,3 +65,15 @@ "W083", "Failed to delete cache (func %(func)s, id: %(id)s).", ) + + +FAILED_UPDATE_TASK_PROGRESS = LogMessage( + "W800", + "Failed to update current task progress (done: %(done)s, total: %(total)s).", +) + +FAILED_UPDATE_TASK_EXECUT_STATUS = LogMessage( + "W801", + "Failed to update current task execution status " + "(repository: %(rid)s, status: %(status)s, retries: %(retries)s).", +) diff --git a/src/server/services/cache_groups.py b/src/server/services/cache_groups.py deleted file mode 100644 index 949bba67..00000000 --- a/src/server/services/cache_groups.py +++ /dev/null @@ -1,242 +0,0 @@ -# -# Copyright (C) 2025 National Institute of Informatics. -# - -"""Service module for managing cache groups.""" - -import typing as t - -from datetime import datetime - -from celery import shared_task -from weko_group_cache_db.groups import fetch_all, fetch_one -from weko_group_cache_db.signals import update_count_signal, update_result_signal - -from server.api.schemas import CacheOperation, CacheQuery -from server.config import config -from server.datastore import app_cache, group_cache -from server.entities.cache import RepositoryCache -from server.entities.search_request import SearchResult - - -if t.TYPE_CHECKING: - from server.entities.summaries import RepositorySummary - - -def get_repository_cache( - repositories: SearchResult[RepositorySummary], query: CacheQuery -) -> SearchResult[RepositoryCache]: - """Retrieve repository cache entries based on the provided query. - - Args: - repositories (SearchResult[RepositorySummary]): List of repository summaries. - query (CacheQuery): Query parameters for filtering and pagination. - - Returns: - SearchResult[RepositoryCache]: List of repository cache entries. - """ - - def _get_repository_redis_key(fqdn: str) -> str: - replaced_fqdn = fqdn.replace("-", "_").replace(".", "_") - return config.CACHE_GROUPS.gakunin_redis_key.format(fqdn=replaced_fqdn) - - repository_cache_list = [] - cursor = 0 - redis_key_list = [] - while True: - cursor, keys = group_cache.scan( - cursor=cursor, - match=config.CACHE_GROUPS.gakunin_redis_key.format(fqdn="*"), - count=100, - ) # pyright: ignore[reportGeneralTypeIssues] - redis_key_list.extend([key.decode("utf-8") for key in keys]) - if cursor == 0: - break - - # filter by cache existence - if query.f: - target_repositories = [] - start_index = (query.p - 1) * query.l if query.p and query.l else 0 - end_index = start_index + query.l if query.l else len(repositories.resources) - - for repository in repositories.resources: - repository_redis_key = _get_repository_redis_key( - repository.service_url.host # pyright: ignore[reportOptionalMemberAccess, reportArgumentType] - ) - repository_updated = None - if repository_redis_key in redis_key_list: - repository_updated = group_cache.hget( - repository_redis_key, "updated_at" - ) - - repo_cache = RepositoryCache( - id=repository.id, - name=repository.display_name, # pyright: ignore[reportArgumentType], - url=str(repository.service_url), - updated=datetime.fromisoformat(repository_updated), # pyright: ignore[reportArgumentType] - ) - if "cache" in query.f: - target_repositories.append(repo_cache) - else: - repo_cache = RepositoryCache( - id=repository.id, - name=repository.display_name, # pyright: ignore[reportArgumentType], - url=str(repository.service_url), - updated=None, - ) - if "no_cache" in query.f: - target_repositories.append(repo_cache) - return SearchResult( - resources=target_repositories[start_index:end_index], - total=len(target_repositories), - page_size=query.l or len(target_repositories), - offset=start_index + 1, - ) - repository_cache_list = [] - for repository in repositories.resources: - repository_redis_key = _get_repository_redis_key( - repository.service_url.host # pyright: ignore[reportOptionalMemberAccess, reportArgumentType] - ) - repository_updated = None - if repository_redis_key in redis_key_list: - repository_updated = group_cache.hget(repository_redis_key, "updated_at") - - repo_cache = RepositoryCache( - id=repository.id, - name=repository.display_name, # pyright: ignore[reportArgumentType], - url=str(repository.service_url), - updated=datetime.fromisoformat(repository_updated) # pyright: ignore[reportArgumentType] - if repository_updated - else None, - ) - repository_cache_list.append(repo_cache) - - return SearchResult( - resources=repository_cache_list, - total=repositories.total, - page_size=query.l or repositories.page_size, - offset=(query.p - 1) * query.l + 1 if query.p and query.l else 1, - ) - - -def update(fqdn_list: list[str], op: CacheOperation) -> None: - """Update cache groups based on the operation type. - - Args: - fqdn_list (list[str]): List of fully qualified domain names. - op (CacheOperation): Operation type, either 'all' or 'id-specified'. - - Raises: - RuntimeError: If a cache update task is already running. - """ - if check_updating_cache_is_running(): - err = "A cache update task is already running." - raise RuntimeError(err) - - update_task.apply_async(fqdn_list, op) # pyright: ignore[reportArgumentType] - - -@shared_task() -def update_task(fqdn_list: list[str], op: CacheOperation) -> None: - """Celery task to update cache groups. - - Args: - fqdn_list (list[str]): List of fully qualified domain names. - op (CacheOperation): Operation type, either 'all' or 'id-specified'. - """ - - def _receive_update_count(_: object, **kwargs: dict[str, t.Any]) -> None: - """Receive update count signal and update task status in Redis. - - Args: - _: The sender of the signal. - **kwargs: Additional keyword arguments containing task details. - """ - update_dict = { - "total": kwargs.get("total", 0), - "done": kwargs.get("done", 0), - "current": kwargs.get("current", ""), - } - app_cache.hset(cache_key, mapping=update_dict) - - def _receive_update_result(_: object, **kwargs: dict[str, t.Any]) -> None: - """Receive update result signal and update task result in Redis. - - Args: - _: The sender of the signal. - **kwargs: Additional keyword arguments containing task details. - """ - task_result = { - "status": kwargs.get("status", ""), - "code": kwargs.get("code", ""), - "updated": kwargs.get("updated", ""), - } - update_dict = {str(kwargs.get("task_name", "")): str(task_result)} - app_cache.hset(cache_key, mapping=update_dict) - - cache_key = config.CACHE_GROUPS.cache_redis_key.format( - prefix=config.REDIS.key_prefix - ) - - update_count_signal.connect(_receive_update_count) - update_result_signal.connect(_receive_update_result) - - if op == "all": - update_run_task() - else: - update_one_task(fqdn_list) - - -def update_run_task() -> None: - """Task to update all cache groups.""" - cache_key = config.CACHE_GROUPS.cache_redis_key.format( - prefix=config.REDIS.key_prefix - ) - run_task_init_data = {"total": "", "done": 0, "current": ""} - app_cache.hset(cache_key, mapping=run_task_init_data) - if config.CACHE_GROUPS.toml_path: - fetch_all(toml_path=config.CACHE_GROUPS.toml_path) - else: - fetch_all( - directory_path=config.CACHE_GROUPS.directory_path, - fqdn_list_file=config.CACHE_GROUPS.fqdn_list_file, - ) - - -def update_one_task(fqdn_list: list[str]) -> None: - """Task to update cache groups for specified FQDNs. - - Args: - fqdn_list (list[str]): List of fully qualified domain names to update. - """ - cache_key = config.CACHE_GROUPS.cache_redis_key.format( - prefix=config.REDIS.key_prefix - ) - one_task_init_data = {"total": len(fqdn_list), "done": 0, "current": ""} - app_cache.hset(cache_key, mapping=one_task_init_data) - for i, fqdn in enumerate(fqdn_list): - if config.CACHE_GROUPS.toml_path: - fetch_one(fqdn, toml_path=config.CACHE_GROUPS.toml_path) - else: - fetch_one( - fqdn, - directory_path=config.CACHE_GROUPS.directory_path, - fqdn_list_file=config.CACHE_GROUPS.fqdn_list_file, - ) - task_result = { - "done": i + 1, - "current": fqdn, - } - app_cache.hset(cache_key, mapping=task_result) - - -def check_updating_cache_is_running() -> bool: - """Check if a cache update task is currently running. - - Returns: - bool: True if a cache update task is running, False otherwise. - """ - cache_key = config.CACHE_GROUPS.cache_redis_key.format( - prefix=config.REDIS.key_prefix - ) - return bool(app_cache.exists(cache_key)) diff --git a/src/server/services/group_caches.py b/src/server/services/group_caches.py new file mode 100644 index 00000000..f87d7972 --- /dev/null +++ b/src/server/services/group_caches.py @@ -0,0 +1,319 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Service module for managing cache groups.""" + +import traceback +import typing as t + +from datetime import datetime +from functools import cache + +from celery import shared_task +from flask import current_app +from pydantic_core import PydanticSerializationError, ValidationError +from redis import RedisError +from weko_group_cache_db import groups as wgcd +from weko_group_cache_db.signals import ( + ExecutedData, + ProgressData as ProgressDataBase, + executed_signal, + progress_signal, +) + +from server.config import config +from server.const import GROUP_CACHE_KEY_PATTERN +from server.datastore import app_cache, group_cache +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.exc import ( + DatastoreError, + GroupCacheError, + RequestConflict, + TaskExecutionError, +) +from server.messages import E, I, W +from server.services import repositories + +from .utils import make_criteria_object, resolve_repository_id + + +if t.TYPE_CHECKING: + from server.entities.summaries import RepositorySummary + + from .utils.search_queries import ( + GroupCacheCriteria, + GroupCacheFilter, + GroupCacheOperation, + ) + + +def get_repository_cache(query: GroupCacheCriteria) -> SearchResult[RepositoryCache]: + """Retrieve repository cache entries based on the provided query. + + Args: + query (GroupCacheCriteria): Query parameters for filtering and pagination. + + Returns: + SearchResult[RepositoryCache]: List of repository cache entries. + """ + repository_query = make_criteria_object( + "repositories", + q=query.q, + k="id", + d="asc", + # when filtering by cache status, + # get all repositories to apply pagination in this app. + p=query.p if not query.f else -1, + l=query.l, + ) + searched = repositories.search(repository_query) + + page_size = searched.page_size + start = (query.p - 1) * page_size if query.p else 0 + end = min(start + page_size, len(searched.resources)) + + results = check_cache_exists( + repositories=searched.resources, + status_filter=query.f, + ) + + resources = results + total = searched.total + if query.f: + # If filtering by status, apply pagination in this app. + resources = results[start:end] + total = len(results) + + return SearchResult( + resources=resources, + total=total, + page_size=page_size, + offset=start + 1, + ) + + +def check_cache_exists( + repositories: list[RepositorySummary], + status_filter: list[GroupCacheFilter] | None = None, +) -> list[RepositoryCache]: + """Check if cache exists for the given list of repositories. + + Args: + repositories (list[RepositorySummary]): List of repository summaries. + status_filter (list | None): List of status filters, e.g., ["e", "n"]. + + Returns: + list[RepositoryCache]: List of repository caches that exist. + """ + result_repositories: list[RepositoryCache] = [] + for repository in repositories: + if not repository.service_url or not repository.service_name: + # service URL and name should exist. + continue # pragma: no cover + + fqdn = t.cast("str", repository.service_url.host) + cache_key = wgcd.cache_key(fqdn) + # when cache exists, `updated_at` is always present. + updated: str | None = group_cache.hget(cache_key, "updated_at") # pyright: ignore[reportAssignmentType] + + repo_cache = RepositoryCache( + id=repository.id, + service_name=repository.service_name, + service_url=repository.service_url, + updated=datetime.fromisoformat(updated) if updated else None, + ) + + if ( + not status_filter + or (updated and "e" in status_filter) + or (not updated and "n" in status_filter) + ): + result_repositories.append(repo_cache) + + return result_repositories + + +@cache +def _unique_progress_key() -> str: + """Generate a unique key for tracking progress in Redis. + + Returns: + str: A unique key for progress tracking. + """ + return config.REDIS.key_prefix + GROUP_CACHE_KEY_PATTERN + + +def update(op: GroupCacheOperation, repository_ids: list[str] | None = None) -> None: + """Update cache groups based on the operation type. + + Args: + op (str): Operation type, either 'all' or 'id-specified'. + repository_ids (list[str]): List of repository IDs. + + Raises: + RequestConflict: If the cache update task is already running. + TaskExecutionError: If there is an error connecting to Redis. + """ + if is_update_task_running(): + raise RequestConflict(E.GROUP_CACHE_UPDATE_CONFLICT) + + repository_ids = repository_ids if op == "id-specified" else [] + query = make_criteria_object( + "repositories", i=repository_ids, l=-1, k="id", d="asc" + ) + repositories_result = repositories.search(query) + fqdn_list = [ + t.cast("str", repo.service_url.host) + for repo in repositories_result.resources + if repo.service_url + ] + + cache_key = _unique_progress_key() + try: + app_cache.delete(cache_key) + app_cache.hset(cache_key, mapping={"status": "pending"}) + task = update_task.apply_async((fqdn_list,)) + except RedisError as exc: + error = E.FAILED_ENQUEUE_CACHE_UPDATE_TASK + raise TaskExecutionError(error) from exc + + current_app.logger.info( + I.GROUP_CACHE_UPDATE_STARTED, {"op": op, "task_id": task.id} + ) + + +@shared_task() +def update_task(fqdn_list: list[str]) -> None: + """Celery task to update cache groups. + + Args: + fqdn_list (list[str]): List of fully qualified domain names. + """ + wgcd.fetch_all( + directory_path=config.CACHE_GROUPS.directory_path, fqdn_list=fqdn_list + ) + + +def is_update_task_running() -> bool: + """Check if a cache update task is currently running. + + Returns: + bool: True if a cache update task is running, False otherwise. + """ + cache_key = _unique_progress_key() + progress_status: str | None = app_cache.hget(cache_key, "status") # pyright: ignore[reportAssignmentType] + + return progress_status in {"pending", "started", "in_progress"} + + +@progress_signal.connect +def handle_progress(_: object, data: ProgressDataBase, **kwargs: object) -> None: # noqa: ARG001 + """Receive progress update signal and update task progress in Redis. + + Args: + _: The sender of the signal. + data (ProgressData): Data containing progress information. + **kwargs: Additional keyword arguments containing task details. + """ + cache_key = _unique_progress_key() + try: + update_dict = data.model_dump(mode="json") + app_cache.hset(cache_key, mapping=update_dict) + except RedisError, PydanticSerializationError: + current_app.logger.warning( + W.FAILED_UPDATE_TASK_PROGRESS, {"done": data.done, "total": data.total} + ) + traceback.print_exc() + + +@executed_signal.connect +def handle_excuted(_: object, data: ExecutedData, **kwargs: object) -> None: # noqa: ARG001 + """Receive executed signal and update task execution status in Redis. + + Args: + _: The sender of the signal. + data (ExecutedData): Data containing executed information. + **kwargs: Additional keyword arguments containing task details. + """ + cache_key = _unique_progress_key() + repository_id = resolve_repository_id(fqdn=data.fqdn) + field_name = f"{repository_id}_{data.retries}" + try: + app_cache.hset(cache_key, mapping={field_name: data.model_dump_json()}) + except RedisError, PydanticSerializationError: + current_app.logger.warning( + W.FAILED_UPDATE_TASK_EXECUT_STATUS, + {"rid": repository_id, "status": data.status, "retries": data.retries}, + ) + traceback.print_exc() + + +def get_task_status() -> TaskDetail | None: + """Get the status of the cache update task. + + Returns: + TaskDetail: + Details of the cache update task. if no task is running, returns None. + + Raises: + GroupCacheError: If there is an error connecting to Redis. + DatastoreError: If there is an error parsing task status data. + """ + cache_key = _unique_progress_key() + try: + raw = app_cache.hgetall(cache_key) + if not raw: + return None + except RedisError as exc: + raise DatastoreError(E.FAILED_FETCH_UPDATE_TASK_STATUS) from exc + + task_data = { + k.decode("utf-8"): v.decode("utf-8") + for k, v in t.cast("dict[bytes, bytes]", raw).items() + } + + try: + progress = ProgressData.model_validate(task_data, extra="ignore") + + results: list[ExecutedData] = [ + ExecutedData.model_validate_json(value) + for key, value in task_data.items() + if key not in {"status", "current", "done", "total"} + ] + repository_ids = [resolve_repository_id(fqdn=result.fqdn) for result in results] + repository_query = make_criteria_object( + "repositories", i=repository_ids, l=len(repository_ids) + ) + searchd = repositories.search(repository_query) + repository_map = {repo.id: repo for repo in searchd.resources} + detail_results = [ + RepositoryCache( + id=r.id, + service_name=r.service_name, + updated=result.updated_at, + status=result.status, + ) + for result in results + if (r := repository_map.get(resolve_repository_id(fqdn=result.fqdn))) + ] + + task_status = TaskDetail( + results=detail_results, + status=progress.status, + current=resolve_repository_id(fqdn=progress.current), + done=progress.done, + total=progress.total, + ) + except ValidationError as exc: + raise GroupCacheError(E.FAILED_PARSE_UPDATE_TASK_STATUS) from exc + + return task_status + + +class ProgressData(ProgressDataBase): + """Model for progress data entity.""" + + status: t.Literal["pending", "started", "in_progress", "completed"] # pyright: ignore[reportIncompatibleVariableOverride] + """The status of the cache update task.""" diff --git a/src/server/services/utils/search_queries.py b/src/server/services/utils/search_queries.py index e502dd65..7572afbd 100644 --- a/src/server/services/utils/search_queries.py +++ b/src/server/services/utils/search_queries.py @@ -912,3 +912,32 @@ def make_criteria_object(resource_type: str, **kwargs: t.Any) -> Criteria: # py attrs[key] = value return t.cast("Criteria", SimpleNamespace(**attrs)) + + +class GroupCacheCriteria(t.Protocol): + """Schema for cache query parameters.""" + + q: t.Annotated[str | None, "term"] = None + """Search term for querying cache entries.""" + + f: t.Annotated[list[GroupCacheFilter] | None, "filter"] = None + """Filter expression for querying cache entries.""" + + p: t.Annotated[int | None, "page"] = None + """Page number for pagination.""" + + l: t.Annotated[int | None, "per"] = None # noqa: E741 + """Number of items per page for pagination.""" + + +type GroupCacheFilter = t.Literal["e", "n"] +"""Group cache filter options: + - “e”: Filter existing cache entries. + - “n”: Filter non-existent cache entries. +""" + +type GroupCacheOperation = t.Literal["all", "id-specified"] +"""Group cache operation options: + - “all”: Update all cache entries. + - “id-specified”: Update cache entries by specified IDs. +""" diff --git a/tests/unit/api/test_cache_groups.py b/tests/unit/api/test_cache_groups.py deleted file mode 100644 index 8aa1fc6b..00000000 --- a/tests/unit/api/test_cache_groups.py +++ /dev/null @@ -1,243 +0,0 @@ -import typing as t - -from copy import deepcopy -from datetime import datetime -from unittest.mock import call - -from server.api import cache_groups -from server.api.schemas import CacheQuery, CacheRequest, RepositoriesQuery -from server.entities.cache import CacheResult, RepositoryCache, TaskDetail -from server.entities.search_request import SearchResult - - -if t.TYPE_CHECKING: - from pytest_mock import MockerFixture - - -def test_get(mocker: MockerFixture, repository_summaries, repository_caches, app, unwrap): - repositories = repository_summaries(20) - search_result = SearchResult( - resources=repository_caches(repositories.resources, None, False), - total=20, - page_size=20, - offset=1, - ) - mock_search = mocker.patch( - "server.api.cache_groups.search", - return_value=repositories, - ) - mock_get_repository_cache = mocker.patch( - "server.api.cache_groups.get_repository_cache", - return_value=search_result, - ) - with app.test_request_context(): - result = unwrap(cache_groups.get)(CacheQuery(q=None, p=1, l=20, f=[])) - - assert result == search_result - mock_search.assert_called_once_with(RepositoriesQuery(q=None, k="id", d="asc", p=1, l=20)) - mock_get_repository_cache.assert_called_once_with(repositories, CacheQuery(q=None, p=1, l=20, f=[])) - - -def test_post(mocker: MockerFixture, app, unwrap): - mock_update = mocker.patch("server.api.cache_groups.update") - fqdn_list = ["repo1.example.jp", "repo2.example.jp"] - operation = "all" - with app.test_request_context(): - result = unwrap(cache_groups.post)(body=CacheRequest(fqdn_list=fqdn_list, op=operation)) - - assert result == "Success" - mock_update.assert_called_once_with(fqdn_list, operation) - - -def test_get_task_status_no_task(mocker: MockerFixture, app, unwrap): - mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value={}) - mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") - mock_search = mocker.patch("server.api.cache_groups.search") - - with app.test_request_context(): - result = unwrap(cache_groups.get_task_status)() - - assert result == TaskDetail(results=[], current="", total=0, done=0) - mock_hgetall.assert_called_once_with("jcgroups_cache") - mock_delete.assert_not_called() - mock_search.assert_not_called() - - mocker.stopall() - - -def test_get_task_status_running(mocker: MockerFixture, app, unwrap, repository_summaries): - task_data = { - b"current": b"repo1.example.jp", - b"done": b"5", - b"total": b"10", - b"repo1.example.jp_0": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", - } - mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) - mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") - mock_search = mocker.patch("server.api.cache_groups.search", return_value=repository_summaries(1)) - - expect_result = [ - CacheResult( - type="cache", - fqdn="repo1.example.jp", - status="success", - code=None, - repository_cached=RepositoryCache( - id="repo_1", - name="Repository 1", - url="https://repo1.example.jp/", - updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 - ), - ) - ] - - with app.test_request_context(): - result = unwrap(cache_groups.get_task_status)() - - assert result == TaskDetail(results=expect_result, current="repo1.example.jp", total=10, done=5) - mock_hgetall.assert_called_once_with("jcgroups_cache") - mock_delete.assert_not_called() - mock_search.assert_called_once_with( - RepositoriesQuery( - q="/repo1.example.jp/", - k="id", - d="asc", - p=-1, - l=1, - ) - ) - - mocker.stopall() - - -def test_get_task_status_completed(mocker: MockerFixture, app, unwrap, repository_summaries): - task_data = { - b"current": b"repo1.example.jp", - b"done": b"10", - b"total": b"10", - b"repo1.example.jp_0": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", - } - mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) - mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") - mock_search = mocker.patch("server.api.cache_groups.search", return_value=repository_summaries(1)) - - expect_result = [ - CacheResult( - type="cache", - fqdn="repo1.example.jp", - status="success", - code=None, - repository_cached=RepositoryCache( - id="repo_1", - name="Repository 1", - url="https://repo1.example.jp/", - updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 - ), - ) - ] - - with app.test_request_context(): - result = unwrap(cache_groups.get_task_status)() - - assert result == TaskDetail(results=expect_result, current="repo1.example.jp", total=10, done=10) - mock_hgetall.assert_called_once_with("jcgroups_cache") - mock_delete.assert_called_once_with("jcgroups_cache") - mock_search.assert_called_once_with( - RepositoriesQuery( - q="/repo1.example.jp/", - k="id", - d="asc", - p=-1, - l=1, - ) - ) - - mocker.stopall() - - -def test_get_task_status_multi_try(mocker: MockerFixture, app, unwrap, repository_summaries): - task_data = { - b"current": b"repo2.example.jp", - b"done": b"2", - b"total": b"3", - b"repo1.example.jp_0": b"{'status': 'failed', 'updated': '2026-01-01T00:00:00Z', 'code': 'timeout'}", - b"repo1.example.jp_1": b"{'status': 'success', 'updated': '2026-01-01T00:00:00Z'}", - b"repo2.example.jp_0": b"{'status': 'failed', 'updated': '2026-01-02T00:00:00Z', 'code': 'timeout'}", - } - repositories = repository_summaries(2) - first_repo = deepcopy(repositories) - first_repo.resources.pop(1) - second_repo = deepcopy(repositories) - second_repo.resources.pop(0) - - mock_hgetall = mocker.patch("server.api.cache_groups.app_cache.hgetall", return_value=task_data) - mock_delete = mocker.patch("server.api.cache_groups.app_cache.delete") - mock_search = mocker.patch( - "server.api.cache_groups.search", - side_effect=[first_repo, first_repo, second_repo], - ) - - expect_result = [ - CacheResult( - type="cache", - fqdn="repo1.example.jp", - status="success", - code=None, - repository_cached=RepositoryCache( - id="repo_1", - name="Repository 1", - url="https://repo1.example.jp/", - updated=datetime(2026, 1, 1, 0, 0), # noqa: DTZ001 - ), - ), - CacheResult( - type="cache", - fqdn="repo2.example.jp", - status="failed", - code="timeout", - repository_cached=RepositoryCache( - id="repo_2", - name="Repository 2", - url="https://repo2.example.jp/", - updated=datetime(2026, 1, 2, 0, 0), # noqa: DTZ001 - ), - ), - ] - - with app.test_request_context(): - result = unwrap(cache_groups.get_task_status)() - - assert result == TaskDetail(results=expect_result, current="repo2.example.jp", total=3, done=2) - mock_hgetall.assert_called_once_with("jcgroups_cache") - mock_delete.assert_not_called() - assert mock_search.call_args_list == [ - call( - RepositoriesQuery( - q="/repo1.example.jp/", - k="id", - d="asc", - p=-1, - l=1, - ) - ), - call( - RepositoriesQuery( - q="/repo1.example.jp/", - k="id", - d="asc", - p=-1, - l=1, - ) - ), - call( - RepositoriesQuery( - q="/repo2.example.jp/", - k="id", - d="asc", - p=-1, - l=1, - ) - ), - ] - - mocker.stopall() diff --git a/tests/unit/api/test_group_caches.py b/tests/unit/api/test_group_caches.py new file mode 100644 index 00000000..484ebf1f --- /dev/null +++ b/tests/unit/api/test_group_caches.py @@ -0,0 +1,117 @@ +import typing as t + +from flask import Flask + +from server.api import group_caches +from server.api.schemas import CacheQuery, CacheRequest, ErrorResponse +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.exc import InvalidQueryError +from server.messages import E + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_get(app, mocker: MockerFixture, gen_summaries, cached_data, unwrap): + query = CacheQuery(q=None, p=1, l=20, f=[]) + repositories: SearchResult = gen_summaries(20) + search_result = SearchResult( + resources=cached_data(repositories.resources, None, every_other=False), + total=20, + page_size=20, + offset=1, + ) + mock_get_cache = mocker.patch("server.api.group_caches.group_caches.get_repository_cache") + mock_get_cache.return_value = search_result + success = 200 + + result, status = unwrap(group_caches.get)(query) + + assert result == search_result + assert status == success + mock_get_cache.assert_called_once_with(query) + + +def test_get_invalid_query(mocker: MockerFixture, unwrap): + query = CacheQuery(q=None, p=1, l=20, f=[]) + messege = "Invalid query" + mock_search = mocker.patch("server.api.group_caches.group_caches.get_repository_cache") + mock_search.side_effect = InvalidQueryError(messege) + bad_request = 400 + + result, status = unwrap(group_caches.get)(query) + + assert isinstance(result, ErrorResponse) + assert result.message == messege + assert status == bad_request + mock_search.assert_called_once_with(query) + + +def test_post(app: Flask, mocker: MockerFixture, unwrap): + mock_update = mocker.patch("server.api.group_caches.group_caches.update") + ids = ["repo1_example_jp", "repo2_example_jp"] + operation = "all" + accepted = 202 + + result, status = unwrap(group_caches.post)(body=CacheRequest(ids=ids, op=operation)) + + assert not result + assert status == accepted + mock_update.assert_called_once_with(operation, ids) + + +def test_post_conflict(app: Flask, mocker: MockerFixture, unwrap): + mock_update = mocker.patch("server.api.group_caches.group_caches.update") + mock_update.side_effect = group_caches.RequestConflict(E.GROUP_CACHE_UPDATE_CONFLICT) + ids = ["repo1_example_jp", "repo2_example_jp"] + operation = "all" + conflict = 409 + + result, status = unwrap(group_caches.post)(body=CacheRequest(ids=ids, op=operation)) + + assert isinstance(result, ErrorResponse) + assert result.message in str(E.GROUP_CACHE_UPDATE_CONFLICT) + assert status == conflict + mock_update.assert_called_once_with(operation, ids) + + +def test_status(app: Flask, mocker: MockerFixture, unwrap, gen_summaries): + repository = gen_summaries(1).resources[0] + repository_cache = RepositoryCache( + id=repository.id, + service_name=repository.service_name, # pyright: ignore[reportArgumentType], + service_url=repository.service_url, + updated=None, + ) + task_detail = TaskDetail( + results=[repository_cache], + status="in_progress", + current="repo1_example_jp", + done=10, + total=20, + ) + + mock_status = mocker.patch("server.api.group_caches.group_caches.get_task_status") + mock_status.return_value = task_detail + success = 200 + + result, status = unwrap(group_caches.status)() + + assert result == task_detail + assert status == success + mock_status.assert_called_once_with() + + +def test_status_no_task(app: Flask, mocker: MockerFixture, unwrap): + mock_status = mocker.patch("server.api.group_caches.group_caches.get_task_status") + mock_status.return_value = None + bad_request = 400 + + result, status = unwrap(group_caches.status)() + + assert isinstance(result, ErrorResponse) + assert result.message in str(E.UPDATE_TASK_NOT_RUNNING) + assert status == bad_request + mock_status.assert_called_once_with() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3aae5502..7e49cef8 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,7 +1,6 @@ import inspect import typing as t -from datetime import datetime from pathlib import Path import pytest @@ -17,6 +16,8 @@ if t.TYPE_CHECKING: + from datetime import datetime + from flask import Flask from pytest_mock import MockerFixture @@ -86,19 +87,16 @@ def test_config(): "single": {"base_url": f"redis://{redis_host}:6379/0"}, "sentinel": { "nodes": [ - {"host", "sentinel-1", "port": 26379}, - {"host", "sentinel-2", "port": 26379}, + {"host": "sentinel-1", "port": 26379}, + {"host": "sentinel-2", "port": 26379}, ], }, }, "RABBITMQ": {"url": f"amqp://guest:guest@{amqp_host}:5672//"}, "CACHE_GROUPS": { - "cache_redis_key": "{prefix}cache", - "gakunin_redis_key": "{fqdn}_gakunin_groups", - "map_groups_api_endpoint": "https://sample.gakunin.jp/api/groups/", - "toml_path": "cache_db_config.toml", - "directory_path": "./cache_db/tls", - "fqdn_list_file": "fqdn_list.toml", + "cache_key_suffix": "_gakunin_groups", + "api_endpoint": "https://sample.gakunin.jp/api/groups/", + "directory_path": "/var/mnt", }, }) @@ -158,28 +156,24 @@ def app(base_app: Flask): @pytest.fixture -def repository_summaries(): +def gen_summaries(): def _data(num: int) -> SearchResult[RepositorySummary]: - return SearchResult( - resources=[ - RepositorySummary( - id=f"repo_{i}", - display_name=f"Repository {i}", - service_url=HttpUrl(f"https://repo{i}.example.jp"), - sp_connector_id=f"jc_repo_{i}_sp", - ) - for i in range(1, num + 1) - ], - total=num, - page_size=20, - offset=1, - ) + resources = [ + RepositorySummary( + id=f"repo_{i}", + service_name=f"Repository {i}", + service_url=HttpUrl(f"https://repo{i}.example.jp"), + service_id=f"jc_repo_{i}_sp", + ) + for i in range(1, num + 1) + ] + return SearchResult(resources=resources, total=num, page_size=20, offset=1) return _data @pytest.fixture -def cache_redis_key(): +def cache_keys(): def _keys(fqdn_list: list[str]) -> list[bytes]: return [f"{fqdn.replace('-', '_').replace('.', '_')}_gakunin_groups".encode() for fqdn in fqdn_list] @@ -187,13 +181,18 @@ def _keys(fqdn_list: list[str]) -> list[bytes]: @pytest.fixture -def repository_caches(): - def _data(repositories: list[RepositorySummary], now: datetime, every_other: bool) -> list[RepositoryCache]: +def cached_data(): + def _data( + repositories: list[RepositorySummary], + now: datetime, + *, + every_other: bool, + ) -> list[RepositoryCache]: return [ RepositoryCache( id=repositories[i].id, - name=repositories[i].display_name, # pyright: ignore[reportArgumentType], - url=str(repositories[i].service_url), + service_name=repositories[i].service_name, # pyright: ignore[reportArgumentType], + service_url=repositories[i].service_url, updated=now if not every_other or i % 2 == 0 else None, ) for i in range(len(repositories)) diff --git a/tests/unit/services/test_cache_groups.py b/tests/unit/services/test_cache_groups.py deleted file mode 100644 index 09e7d534..00000000 --- a/tests/unit/services/test_cache_groups.py +++ /dev/null @@ -1,1159 +0,0 @@ -import typing as t - -from datetime import UTC, datetime -from unittest.mock import MagicMock, call - -from server.api.schemas import CacheQuery -from server.entities.search_request import SearchResult -from server.services.cache_groups import ( - check_updating_cache_is_running, - get_repository_cache, - update, - update_count_signal, - update_one_task, - update_result_signal, - update_run_task, - update_task, -) - - -if t.TYPE_CHECKING: - from pytest_mock import MockerFixture - - -def test_get_repository_cache(mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches): - with app.app_context(): - query = CacheQuery(l=20, p=1) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources, now, True) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=20, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 10 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_multi_scan( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(l=20, p=1) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources, now, True) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (1, keys[:10]), - (0, keys[10:]), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=20, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 10 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_all_cache_not_exceeding_page_size( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["cache"], l=20, p=1) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources, now, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=20, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 20 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_all_cache_exceeding_page_size( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["cache"], l=20, p=1) - repositories = repository_summaries(30) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[:20], now, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=30, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 30 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_all_cache_exceeding_page_size_next_page( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["cache"], l=20, p=2) - repositories = repository_summaries(30) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[20:30], now, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=30, - resources=caches, - page_size=20, - offset=21, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 30 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_empty_cache(mocker: MockerFixture, app, repository_summaries): - with app.app_context(): - query = CacheQuery(f=["cache"], l=20, p=1) - repositories = repository_summaries(20) - now = datetime.now(UTC) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, []), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=0, - resources=[], - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - mock_hget.assert_not_called() - - mocker.stopall() - - -def test_get_repository_cache_half_cache( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["cache"], l=20, p=1) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[::2], now, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=10, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 10 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_no_cache_not_exceeding_page_size( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["no_cache"], l=20, p=1) - repositories = repository_summaries(20) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources, None, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, []), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=20, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - mock_hget.assert_not_called() - - mocker.stopall() - - -def test_get_repository_cache_no_cache_exceeding_page_size( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["no_cache"], l=20, p=1) - repositories = repository_summaries(30) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[:20], None, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, []), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=30, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - mock_hget.assert_not_called() - - mocker.stopall() - - -def test_get_repository_cache_no_cache_exceeding_page_size_next_page( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["no_cache"], l=20, p=2) - repositories = repository_summaries(30) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[20:30], None, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, []), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=30, - resources=caches, - page_size=20, - offset=21, - ) - assert result == expect - mock_scan.assert_called() - mock_hget.assert_not_called() - - mocker.stopall() - - -def test_get_repository_cache_no_cache_all_cache(mocker: MockerFixture, app, repository_summaries, cache_redis_key): - with app.app_context(): - query = CacheQuery(f=["no_cache"], l=20, p=1) - now = datetime.now(UTC) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources]) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=0, - resources=[], - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 20 # noqa: PLR2004 - - mocker.stopall() - - -def test_get_repository_cache_no_cache_half_cache( - mocker: MockerFixture, app, repository_summaries, cache_redis_key, repository_caches -): - with app.app_context(): - query = CacheQuery(f=["no_cache"], l=20, p=1) - repositories = repository_summaries(20) - keys = cache_redis_key([repo.service_url.host for repo in repositories.resources[::2]]) - now = datetime.now(UTC) - caches = repository_caches(repositories.resources[1::2], None, False) - mock_scan = mocker.patch( - "server.services.cache_groups.group_cache.scan", - side_effect=[ - (0, keys), - ], - ) - mock_hget = mocker.patch( - "server.services.cache_groups.group_cache.hget", - return_value=now.isoformat(), - ) - - result = get_repository_cache(repositories, query) - expect = SearchResult( - total=10, - resources=caches, - page_size=20, - offset=1, - ) - assert result == expect - mock_scan.assert_called() - assert mock_hget.call_count == 10 # noqa: PLR2004 - - mocker.stopall() - - -def test_update(mocker: MockerFixture, app): - with app.app_context(): - mock_check = mocker.patch( - "server.services.cache_groups.check_updating_cache_is_running", - return_value=False, - ) - mock_update_task = mocker.patch( - "server.services.cache_groups.update_task.apply_async", - ) - - fqdn_list = ["example.com"] - op = "all" - update(fqdn_list, op) - - mock_check.assert_called_once() - mock_update_task.assert_called_once_with(fqdn_list, op) - - mocker.stopall() - - -def test_update_raises_runtime_error_when_task_running(mocker: MockerFixture, app): - with app.app_context(): - mock_check = mocker.patch( - "server.services.cache_groups.check_updating_cache_is_running", - return_value=True, - ) - mock_update_task = mocker.patch( - "server.services.cache_groups.update_task.apply_async", - ) - fqdn_list = ["example.com"] - op = "all" - - try: - update(fqdn_list, op) - except RuntimeError as e: - assert str(e) == "A cache update task is already running." - - mock_check.assert_called_once() - mock_update_task.assert_not_called() - - mocker.stopall() - - -def test_update_task_all(mocker: MockerFixture, app): - with app.app_context(): - mock_update_all_caches = mocker.patch( - "server.services.cache_groups.update_run_task", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "all" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - mock_update_all_caches.assert_called_once() - mock_hset.assert_not_called() - - mocker.stopall() - - -def test_update_task_all_count_signal_once(mocker: MockerFixture, app): - with app.app_context(): - mock_update_all_caches = mocker.patch( - "server.services.cache_groups.update_run_task", - side_effect=lambda: update_count_signal.send( - None, - total=100, - done=50, - current="example.com", - ), - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "all" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count signal - - mock_update_all_caches.assert_called_once() - mock_hset.assert_called_with( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50, - "current": "example.com", - }, - ) - - mocker.stopall() - - -def test_update_task_all_result_signal_once(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - mock_update_all_caches = mocker.patch( - "server.services.cache_groups.update_run_task", - side_effect=lambda: update_result_signal.send( - None, - task_name="example_com_0", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ), - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "all" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update result signal - - mock_update_all_caches.assert_called_once() - mock_hset.assert_called_with( - "jcgroups_cache", - mapping={ - "example_com_0": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ) - - mocker.stopall() - - -def test_update_task_all_signals_once(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - - def side_effect(): - update_count_signal.send( - None, - total=100, - done=50, - current="example.com", - ) - update_result_signal.send( - None, - task_name="example_com_0", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - mock_update_all_caches = mocker.patch( - "server.services.cache_groups.update_run_task", - side_effect=side_effect, - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "all" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count and result signals - - mock_update_all_caches.assert_called_once() - expected_calls = [ - call( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50, - "current": "example.com", - }, - ), - call( - "jcgroups_cache", - mapping={ - "example_com_0": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ), - ] - mock_hset.assert_has_calls(expected_calls, any_order=False) - - mocker.stopall() - - -def test_update_task_all_signal_multiple(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - - def side_effect(): - for i in range(3): - update_count_signal.send( - None, - total=100, - done=50 + i, - current=f"example{i}.com", - ) - update_result_signal.send( - None, - task_name=f"example_com_{i}", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - mock_update_all_caches = mocker.patch( - "server.services.cache_groups.update_run_task", - side_effect=side_effect, - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "all" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count and result signals multiple times - - mock_update_all_caches.assert_called_once() - expected_calls = [ - call( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50 + i, - "current": f"example{i}.com", - }, - ) - for i in range(3) - ] + [ - call( - "jcgroups_cache", - mapping={ - f"example_com_{i}": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ) - for i in range(3) - ] - mock_hset.assert_has_calls(expected_calls, any_order=True) - - mocker.stopall() - - -def test_update_task_specified(mocker: MockerFixture, app): - with app.app_context(): - mock_update_specified_caches = mocker.patch( - "server.services.cache_groups.update_one_task", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "id-specified" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - mock_update_specified_caches.assert_called_once_with(fqdn_list) - mock_hset.assert_not_called() - - mocker.stopall() - - -def test_update_task_specified_count_signal_once(mocker: MockerFixture, app): - with app.app_context(): - mock_update_specified_caches = mocker.patch( - "server.services.cache_groups.update_one_task", - side_effect=lambda *_, **__: update_count_signal.send( - None, - total=100, - done=50, - current="example.com", - ), - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "id-specified" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count signal - - mock_update_specified_caches.assert_called_once_with(fqdn_list) - mock_hset.assert_called_with( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50, - "current": "example.com", - }, - ) - - mocker.stopall() - - -def test_update_task_specified_result_signal_once(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - mock_update_specified_caches = mocker.patch( - "server.services.cache_groups.update_one_task", - side_effect=lambda *_, **__: update_result_signal.send( - None, - task_name="example_com_0", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ), - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "id-specified" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update result signal - - mock_update_specified_caches.assert_called_once_with(fqdn_list) - mock_hset.assert_called_with( - "jcgroups_cache", - mapping={ - "example_com_0": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ) - - mocker.stopall() - - -def test_update_task_specified_signals_once(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - - def side_effect(*_, **__): - update_count_signal.send( - None, - total=100, - done=50, - current="example.com", - ) - update_result_signal.send( - None, - task_name="example_com_0", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - mock_update_specified_caches = mocker.patch( - "server.services.cache_groups.update_one_task", - side_effect=side_effect, - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "id-specified" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count and result signals - - mock_update_specified_caches.assert_called_once_with(fqdn_list) - expected_calls = [ - call( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50, - "current": "example.com", - }, - ), - call( - "jcgroups_cache", - mapping={ - "example_com_0": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ), - ] - mock_hset.assert_has_calls(expected_calls, any_order=False) - - mocker.stopall() - - -def test_update_task_specified_signal_multiple(mocker: MockerFixture, app): - with app.app_context(): - now = datetime.now(UTC) - - def side_effect(*_, **__): - for i in range(3): - update_count_signal.send( - None, - total=100, - done=50 + i, - current=f"example{i}.com", - ) - update_result_signal.send( - None, - task_name=f"example_com_{i}", - status="success", - code="", - updated=now.strftime("%Y-%m-%dT%H:%M:%SZ"), - ) - - mock_update_specified_caches = mocker.patch( - "server.services.cache_groups.update_one_task", - side_effect=side_effect, - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - fqdn_list = ["example.com"] - op = "id-specified" - - update_task(fqdn_list, op) # pyright: ignore[reportArgumentType] - - # Simulate sending the update count and result signals multiple times - - mock_update_specified_caches.assert_called_once_with(fqdn_list) - expected_calls = [ - call( - "jcgroups_cache", - mapping={ - "total": 100, - "done": 50 + i, - "current": f"example{i}.com", - }, - ) - for i in range(3) - ] + [ - call( - "jcgroups_cache", - mapping={ - f"example_com_{i}": "{'status': 'success', 'code': '', 'updated': '" - + now.strftime("%Y-%m-%dT%H:%M:%SZ") - + "'}" - }, - ) - for i in range(3) - ] - mock_hset.assert_has_calls(expected_calls, any_order=True) - - mocker.stopall() - - -def test_update_run_task_toml_path(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_all = mocker.patch( - "server.services.cache_groups.fetch_all", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="/path/to/toml", - directory_path="", - fqdn_list_file="", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - - update_run_task() - - mock_fetch_all.assert_called_once_with(toml_path="/path/to/toml") - mock_hset.assert_called_once_with( - "jcgroups_cache", - mapping={"total": "", "done": 0, "current": ""}, - ) - - mocker.stopall() - - -def test_update_run_task_not_toml_path(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_all = mocker.patch( - "server.services.cache_groups.fetch_all", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - - update_run_task() - - mock_fetch_all.assert_called_once_with( - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ) - mock_hset.assert_called_once_with( - "jcgroups_cache", - mapping={"total": "", "done": 0, "current": ""}, - ) - - mocker.stopall() - - -def test_update_run_task_all_settings(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_all = mocker.patch( - "server.services.cache_groups.fetch_all", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="/path/to/toml", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - - update_run_task() - - mock_fetch_all.assert_called_once_with( - toml_path="/path/to/toml", - ) - mock_hset.assert_called_once_with( - "jcgroups_cache", - mapping={"total": "", "done": 0, "current": ""}, - ) - - mocker.stopall() - - -def test_update_one_task_toml_path(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_one = mocker.patch( - "server.services.cache_groups.fetch_one", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="/path/to/toml", - directory_path="", - fqdn_list_file="", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - fqdn_list = ["example.com"] - - update_one_task(fqdn_list) - - mock_fetch_one.assert_called_once_with("example.com", toml_path="/path/to/toml") - assert mock_hset.call_args_list == [ - call( - "jcgroups_cache", - mapping={"total": 1, "done": 0, "current": ""}, - ), - call( - "jcgroups_cache", - mapping={"done": 1, "current": "example.com"}, - ), - ] - - mocker.stopall() - - -def test_update_one_task_not_toml_path(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_one = mocker.patch( - "server.services.cache_groups.fetch_one", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - fqdn_list = ["example.com"] - - update_one_task(fqdn_list) - - mock_fetch_one.assert_called_once_with( - "example.com", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ) - assert mock_hset.call_args_list == [ - call( - "jcgroups_cache", - mapping={"total": 1, "done": 0, "current": ""}, - ), - call( - "jcgroups_cache", - mapping={"done": 1, "current": "example.com"}, - ), - ] - - mocker.stopall() - - -def test_update_one_task_all_settings(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_one = mocker.patch( - "server.services.cache_groups.fetch_one", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="/path/to/toml", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - fqdn_list = ["example.com"] - - update_one_task(fqdn_list) - - mock_fetch_one.assert_called_once_with("example.com", toml_path="/path/to/toml") - assert mock_hset.call_args_list == [ - call( - "jcgroups_cache", - mapping={"total": 1, "done": 0, "current": ""}, - ), - call( - "jcgroups_cache", - mapping={"done": 1, "current": "example.com"}, - ), - ] - - mocker.stopall() - - -def test_update_one_task_multi_repositories(mocker: MockerFixture, app): - with app.app_context(): - mock_fetch_one = mocker.patch( - "server.services.cache_groups.fetch_one", - ) - mock_hset = mocker.patch( - "server.services.cache_groups.app_cache.hset", - ) - mocker.patch( - "server.services.cache_groups.config", - CACHE_GROUPS=MagicMock( - cache_redis_key="{prefix}cache", - toml_path="/path/to/toml", - directory_path="./cache_db/tls", - fqdn_list_file="fqdn_list.toml", - ), - REDIS=MagicMock(key_prefix="jcgroups_"), - ) - fqdn_list = ["example1.com", "example2.com", "example3.com"] - - update_one_task(fqdn_list) - - assert mock_fetch_one.call_count == 3 - expected_calls = [ - call( - "jcgroups_cache", - mapping={"total": 3, "done": 0, "current": ""}, - ), - call( - "jcgroups_cache", - mapping={"done": 1, "current": "example1.com"}, - ), - call( - "jcgroups_cache", - mapping={"done": 2, "current": "example2.com"}, - ), - call( - "jcgroups_cache", - mapping={"done": 3, "current": "example3.com"}, - ), - ] - mock_hset.assert_has_calls(expected_calls, any_order=False) - - mocker.stopall() - - -def test_check_updating_cache_is_running_exists(mocker: MockerFixture, app): - with app.app_context(): - mock_get = mocker.patch( - "server.services.cache_groups.app_cache.exists", - return_value=True, - ) - - result = check_updating_cache_is_running() - assert result is True - mock_get.assert_called_once_with("jcgroups_cache") - - mocker.stopall() - - -def test_check_updating_cache_is_running_not_exists(mocker: MockerFixture, app): - with app.app_context(): - mock_get = mocker.patch( - "server.services.cache_groups.app_cache.exists", - return_value=False, - ) - - result = check_updating_cache_is_running() - assert result is False - mock_get.assert_called_once_with("jcgroups_cache") - - mocker.stopall() diff --git a/tests/unit/services/test_group_caches.py b/tests/unit/services/test_group_caches.py new file mode 100644 index 00000000..13375f94 --- /dev/null +++ b/tests/unit/services/test_group_caches.py @@ -0,0 +1,641 @@ +import typing as t + +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from redis import RedisError +from weko_group_cache_db.signals import ExecutedData, ProgressData + +from server.api.schemas import CacheQuery +from server.config import config +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.entities.summaries import RepositorySummary +from server.exc import DatastoreError, GroupCacheError, RequestConflict +from server.messages import E, W +from server.services.group_caches import ( + get_repository_cache, + get_task_status, + handle_excuted, + handle_progress, + is_update_task_running, + update, + update_task, +) + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_get_repository_cache(app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=True) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + expect = SearchResult(total=20, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_multi_scan(mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=True) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + expect = SearchResult(total=20, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_all_cache_not_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories: SearchResult[RepositorySummary] = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + result = get_repository_cache(query) + expect = SearchResult( + total=num, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num = 30 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + now = datetime.now(UTC) + caches = cached_data(repositories.resources[:20], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + expect = SearchResult(total=num, resources=caches, page_size=20, offset=1) + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num = 30 + query = CacheQuery(f=["e"], l=20, p=2) + repositories = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[20:30], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + result = get_repository_cache(query) + expect = SearchResult(total=30, resources=caches, page_size=20, offset=21) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_empty_cache(app, mocker: MockerFixture, gen_summaries, datastore): + num_repo = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(20) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + _, _, group_cache = datastore + group_cache.hget.return_value = None + + result = get_repository_cache(query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_half_cache(app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[::2], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + result = get_repository_cache(query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + mock_search.assert_called_once() + assert result == expect + + assert group_cache.hget.call_count == num_repo + mocker.stopall() + + +def test_get_repository_cache_no_cache_not_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources, None, every_other=False) + _, _, group_cache = datastore + group_cache.hget.return_value = None + + expect = SearchResult(total=num_repo, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 30 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources[:20], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = None + result = get_repository_cache(query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 30 + query = CacheQuery(f=["n"], l=20, p=2) + repositories = gen_summaries(30) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources[20:30], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = None + + result = get_repository_cache(query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=21, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_all_cache(mocker: MockerFixture, app, gen_summaries, cache_keys, datastore): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + now = datetime.now(UTC) + repositories: SearchResult = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + result = get_repository_cache(query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_half_cache( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[1::2], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + result = get_repository_cache(query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_update_all(app, mocker: MockerFixture, datastore, gen_summaries): + + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + query = SimpleNamespace(q=None, i=[], p=None, l=-1, k="id", d="asc") + + app_cache, _, _ = datastore + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "all" + + update(op, ids) + + mock_check.assert_called_once() + mock_search.assert_called_once_with(query) + mock_update_task.assert_called_once_with((fqdn_list,)) + app_cache.delete.assert_called_once_with("jcgroups-weko-group-cache-db") + app_cache.hset.assert_called_once_with("jcgroups-weko-group-cache-db", mapping={"status": "pending"}) + mocker.stopall() + + +def test_update_id_specified(app, mocker: MockerFixture, datastore, gen_summaries): + + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "id-specified" + query = SimpleNamespace(q=None, i=ids, p=None, l=-1, k="id", d="asc") + + app_cache, _, _ = datastore + + update(op, ids) + + mock_check.assert_called_once() + mock_search.assert_called_once_with(query) + mock_update_task.assert_called_once_with((fqdn_list,)) + app_cache.delete.assert_called_once_with("jcgroups-weko-group-cache-db") + app_cache.hset.assert_called_once_with("jcgroups-weko-group-cache-db", mapping={"status": "pending"}) + mocker.stopall() + + +def test_update_raises_task_running(app, mocker: MockerFixture): + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = True + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + fqdn_list = ["example.com"] + op = "all" + + with pytest.raises(RequestConflict, match=str(E.GROUP_CACHE_UPDATE_CONFLICT)): + update(op, fqdn_list) + + mock_check.assert_called_once() + mock_update_task.assert_not_called() + + mocker.stopall() + + +def test_update_raises_failed_task_running(app, mocker: MockerFixture, datastore, gen_summaries): + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + mock_update_task.side_effect = RedisError("Failed to connect to Redis.") + + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + + app_cache, _, _ = datastore + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "all" + + with pytest.raises(DatastoreError, match=str(E.FAILED_ENQUEUE_CACHE_UPDATE_TASK)): + update(op, ids) + + mock_check.assert_called_once() + app_cache.hset.assert_called_once_with("jcgroups-weko-group-cache-db", mapping={"status": "pending"}) + + mock_update_task.assert_called_once_with((fqdn_list,)) + + mocker.stopall() + + +def test_update_task_all(app, mocker: MockerFixture, gen_summaries, unwrap): + repositories = gen_summaries(1).resources + mock_fetch_all = mocker.patch("server.services.group_caches.wgcd.fetch_all") + + fqdn_list = [repositories[0].service_url.host] + + unwrap(update_task)(fqdn_list) + mock_fetch_all.assert_called_once_with( + directory_path=config.CACHE_GROUPS.directory_path, + fqdn_list=fqdn_list, + ) + mocker.stopall() + + +def test_is_update_task_running_pending(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "pending" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-weko-group-cache-db", "status") + + +def test_is_update_task_running_started(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "started" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-weko-group-cache-db", "status") + + +def test_is_update_task_running_in_progress(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "in_progress" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-weko-group-cache-db", "status") + + +def test_is_update_task_running_completed(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "completed" + + result = is_update_task_running() + assert result is False + app_cache.hget.assert_called_once_with("jcgroups-weko-group-cache-db", "status") + + +def test_is_update_task_running_not_exists(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = None + + result = is_update_task_running() + assert result is False + app_cache.hget.assert_called_once_with("jcgroups-weko-group-cache-db", "status") + + +def test_handle_progress(app, mocker: MockerFixture, unwrap, datastore): + data = ProgressData(status="in_progress", total=10, done=5, current="example.com") + app_cache, _, _ = datastore + + unwrap(handle_progress)(None, data) + + cache_key = "jcgroups-weko-group-cache-db" + app_cache.hset.assert_called_once_with(cache_key, mapping=data.model_dump(mode="json")) + + +def test_handle_progress_redis_error(app, mocker: MockerFixture, unwrap, datastore, caplog): + data = ProgressData(status="in_progress", total=10, done=5, current="example.com") + app_cache, _, _ = datastore + app_cache.hset.side_effect = RedisError("Redis error") + + unwrap(handle_progress)(None, data) + + cache_key = "jcgroups-weko-group-cache-db" + app_cache.hset.assert_called_once_with(cache_key, mapping=data.model_dump(mode="json")) + + assert str(W.FAILED_UPDATE_TASK_PROGRESS % {"done": 5, "total": 10}) in caplog.text + + +def test_handle_excuted(app, mocker: MockerFixture, unwrap, datastore): + data = ExecutedData( + fqdn="example.com", + status="success", + retries=0, + error_type=None, + error_message=None, + updated_at=datetime.now(UTC), + ) + app_cache, _, _ = datastore + + unwrap(handle_excuted)(None, data) + + cache_key = "jcgroups-weko-group-cache-db" + field_name = "example_com_0" + app_cache.hset.assert_called_once_with(cache_key, mapping={field_name: data.model_dump_json()}) + + +def test_handle_excuted_redis_error(app, mocker: MockerFixture, unwrap, datastore, caplog): + data = ExecutedData( + fqdn="example.com", + status="success", + retries=0, + error_type=None, + error_message=None, + updated_at=datetime.now(UTC), + ) + app_cache, _, _ = datastore + app_cache.hset.side_effect = RedisError("Redis error") + + unwrap(handle_excuted)(None, data) + + cache_key = "jcgroups-weko-group-cache-db" + field_name = "example_com_0" + app_cache.hset.assert_called_once_with(cache_key, mapping={field_name: data.model_dump_json()}) + + assert ( + str(W.FAILED_UPDATE_TASK_EXECUT_STATUS % {"rid": "example_com", "status": "success", "retries": 0}) + in caplog.text + ) + + +def test_get_task_status(app, mocker: MockerFixture, unwrap, datastore): + task_data = { + b"current": b"example.com", + b"status": b"in_progress", + b"done": b"5", + b"total": b"10", + b"example_com_0": b'{"fqdn": "example.com", "status": "success", "updated_at": "2026-01-01T00:00:00Z"}', + } + ids = ["example_com"] + + app_cache, _, _ = datastore + app_cache.hgetall.return_value = task_data + mocker.patch("server.services.group_caches.make_criteria_object") + mock_query = MagicMock() + mock_query.i = ids + mock_search = mocker.patch("server.services.group_caches.repositories.search") + mock_search.return_value = SearchResult( + resources=[RepositorySummary(id="example_com", service_name="Example Repository")], + total=1, + page_size=1, + offset=1, + ) + + result = unwrap(get_task_status)() + + expect_result = [ + RepositoryCache( + id="example_com", + service_name="Example Repository", + updated=datetime.fromisoformat("2026-01-01T00:00:00+00:00"), + status="success", + ) + ] + + assert result == TaskDetail(results=expect_result, status="in_progress", current="example_com", total=10, done=5) + + app_cache.hgetall.assert_called_once_with("jcgroups-weko-group-cache-db") + app_cache.delete.assert_not_called() + + +def test_get_task_status_not_running(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.return_value = {} + app_cache.hget.return_value = "completed" + + result = unwrap(get_task_status)() + + assert result is None + + +def test_get_task_status_no_task(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.return_value = {} + + result = unwrap(get_task_status)() + + assert result is None + + +def test_get_task_status_redis_error(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.side_effect = RedisError("Redis error") + + with pytest.raises(DatastoreError, match=str(E.FAILED_FETCH_UPDATE_TASK_STATUS)): + unwrap(get_task_status)() + + +def test_get_task_status_parse_error(app, unwrap, datastore): + app_cache, _, _ = datastore + + cache_data = { + b"current": b"example.com", + b"done": b"5", + b"total": b"10", + b"example_com_0": b"invalid_json", + } + app_cache.hgetall.return_value = cache_data + + error = str(E.FAILED_PARSE_UPDATE_TASK_STATUS) + with pytest.raises(GroupCacheError, match=error): + unwrap(get_task_status)() From 4854d7795a9d6f819bd7d19f68174d31d9e8a608 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Wed, 11 Mar 2026 05:36:43 +0000 Subject: [PATCH 4/5] refactor: improve error handling and unify implemention --- configs/app.config.ts | 9 - nuxt.config.ts | 4 - src/app/components/list/ListRepositories.vue | 106 ------- src/app/composables/cacheGroups.ts | 74 ----- src/app/composables/groupCaches.ts | 246 +++++++++++++++ src/app/composables/useMenu.ts | 4 +- src/app/i18n/locales/en.json | 74 +++-- src/app/i18n/locales/ja.json | 56 +++- src/app/pages/cache-groups/index.vue | 267 ---------------- src/app/pages/group-caches/index.vue | 301 +++++++++++++++++++ src/app/types/cacheGroups.ts | 38 --- src/app/types/groupCaches.ts | 28 ++ src/app/types/search.ts | 6 +- src/app/utils/search.ts | 2 +- src/server/api/group_caches.py | 21 +- src/server/ext.py | 2 - src/server/services/group_caches.py | 1 + 17 files changed, 684 insertions(+), 555 deletions(-) delete mode 100644 src/app/components/list/ListRepositories.vue delete mode 100644 src/app/composables/cacheGroups.ts create mode 100644 src/app/composables/groupCaches.ts delete mode 100644 src/app/pages/cache-groups/index.vue create mode 100644 src/app/pages/group-caches/index.vue delete mode 100644 src/app/types/cacheGroups.ts create mode 100644 src/app/types/groupCaches.ts diff --git a/configs/app.config.ts b/configs/app.config.ts index 00ccc0fc..740bc493 100644 --- a/configs/app.config.ts +++ b/configs/app.config.ts @@ -176,13 +176,6 @@ const wayf = { ] as { name: string, entityID: string }[], } -const groupCache = { - /** - * Time (in milliseconds) to wait before showing loading indicator when updating cache - */ - loadingWaitTime: 3000 as number, -} - export default { /** Server hostname of this application */ serverName, @@ -200,6 +193,4 @@ export default { polling, /** WAYF (Embedded DS) configuration */ wayf, - /** Group cache configuration */ - groupCache, } diff --git a/nuxt.config.ts b/nuxt.config.ts index 0b75156c..3982d426 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -15,10 +15,6 @@ export default defineNuxtConfig({ scan: true, }, - imports: { - dirs: ['~/types'], - }, - devtools: { enabled: true, }, diff --git a/src/app/components/list/ListRepositories.vue b/src/app/components/list/ListRepositories.vue deleted file mode 100644 index 71f8cd0a..00000000 --- a/src/app/components/list/ListRepositories.vue +++ /dev/null @@ -1,106 +0,0 @@ - - - diff --git a/src/app/composables/cacheGroups.ts b/src/app/composables/cacheGroups.ts deleted file mode 100644 index 0c3eca96..00000000 --- a/src/app/composables/cacheGroups.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Composable for managing cache groups. - */ -import type { TableColumn } from '@nuxt/ui' - -const useCacheGroups = () => { - const route = useRoute() - const router = useRouter() - - const { t: $t } = useI18n() - - /** Reactive query object */ - const query = computed(() => normalizeCacheGroupsQuery(route.query)) - /** Update query parameters and push to router */ - const updateQuery = (newQuery: Partial) => { - router.push({ - query: { - ...route.query, - ...newQuery, - }, - }) - } - - const searchTerm = ref(query.value.q) - const filter = ref(query.value.f) - const pageSize = ref(query.value.l) - - const searchIdentityKey = computed(() => { - const { p, l, ...filters } = query.value - return JSON.stringify(filters) - }) - - const selectedMap = useState>( - `selection:${searchIdentityKey.value}`, () => ({}), - ) - - /** Column names with translations */ - const columnNames = { - id: '#', - displayName: $t('cache-groups.table.column.repository-name'), - serviceURL: $t('cache-groups.table.column.repository-url'), - updated: $t('cache-groups.table.column.repository-updated-at'), - } - - type CacheGroupsTableColumn = TableColumn - const columns = computed(() => [ - { - accessorKey: 'name', - header: columnNames.displayName, - }, - { - accessorKey: 'url', - header: columnNames.serviceURL, - }, - { - accessorKey: 'updated', - header: columnNames.updated, - }, - ]) - - return { - query, - updateQuery, - criteria: { - searchTerm, - filter, - pageSize, - }, - selectedMap, - columns, - } -} - -export { useCacheGroups } diff --git a/src/app/composables/groupCaches.ts b/src/app/composables/groupCaches.ts new file mode 100644 index 00000000..e739331f --- /dev/null +++ b/src/app/composables/groupCaches.ts @@ -0,0 +1,246 @@ +/** + * Composable for managing cache groups. + */ +import { UCheckbox, ULink } from '#components' + +import type { Row, Table } from '@tanstack/table-core' +import type { DropdownMenuItem, SelectItem, TableColumn } from '@nuxt/ui' + +const useCacheGroups = () => { + const route = useRoute() + const router = useRouter() + + const { t: $t } = useI18n() + + /** Reactive query object */ + const query = computed(() => normalizeCacheGroupsQuery(route.query)) + /** Update query parameters and push to router */ + const updateQuery = (newQuery: Partial) => { + router.push({ + query: { + ...route.query, + ...newQuery, + }, + }) + } + + const searchTerm = ref(query.value.q) + const filter = ref(query.value.f) + const pageNumber = ref(query.value.p) + const pageSize = ref(query.value.l) + + const searchIdentityKey = computed(() => { + const { p, l, ...filters } = query.value + return JSON.stringify(filters) + }) + + const selectedMap = useState>( + `selection-group-caches:${searchIdentityKey.value}`, () => ({}), + ) + + const selectedCount = computed(() => { + return Object.values(selectedMap.value).filter(value => value !== undefined).length + }) + const toggleSelection = (event: Event | undefined, row: Row) => { + selectedMap.value[row.original.id] + = selectedMap.value[row.original.id] ? undefined : row.original + } + const toggleAllPageRows = (table: Table) => { + const pageRows = table.getRowModel().rows + const allSelected = pageRows.every(row => selectedMap.value[row.original.id] !== undefined) + + if (allSelected) { + for (const row of pageRows) { + selectedMap.value[row.original.id] = undefined + } + } + else { + for (const row of pageRows) { + selectedMap.value[row.original.id] = row.original + } + } + } + const isAllPageRowsSelected = (table: Table) => { + const pageRows = table.getRowModel().rows + return pageRows.length > 0 && pageRows.every( + row => selectedMap.value[row.original.id] !== undefined, + ) + } + const isSomePageRowsSelected = (table: Table) => { + const pageRows = table.getRowModel().rows + const selectedRows = pageRows.filter(row => selectedMap.value[row.original.id] !== undefined) + return selectedRows.length > 0 && selectedRows.length < pageRows.length + } + + const getSelected = (): { id: string, serviceName: string, serviceUrl: string }[] => { + return Object.entries(selectedMap.value) + .filter(([_, service]) => service !== undefined) + .map(([id, service]) => ({ + id, serviceName: service!.serviceName, serviceUrl: service!.serviceUrl, + })) + } + const clearSelection = () => { + selectedMap.value = {} + } + + const modals = reactive>({ + 'all': false, + 'id-specified': false, + }) + const isUpdating = ref(false) + + /** Column names with translations */ + const columnNames = computed(() => ({ + id: '#', + serviceName: $t('group-caches.table.column.repository-name'), + serviceUrl: $t('group-caches.table.column.repository-url'), + updated: $t('group-caches.table.column.repository-updated-at'), + })) + + const filterItems = computed(() => [ + { + label: $t('group-caches.status.cached'), + value: 'e' as GroupCacheStatus, + }, + { + label: $t('group-caches.status.no-cached'), + value: 'n' as GroupCacheStatus, + }, + ]) + + const selectedRepositoriesAction = computed<[DropdownMenuItem, ...DropdownMenuItem[]]>(() => [ + { + type: 'label' as const, + label: $t('repositories.all-repositories-actions'), + }, + { + icon: 'i-lucide-refresh-cw', + label: $t('group-caches.button.update-all-repositories'), + onSelect: () => modals.all = true, + }, + { + type: 'separator' as const, + }, + { + type: 'label' as const, + label: $t('repositories.selected-repositories-actions'), + }, + { + icon: 'i-lucide-refresh-cw', + label: $t('group-caches.button.update-selected-repositories'), + onSelect: () => modals['id-specified'] = true, + disabled: selectedCount.value === 0, + }, + ]) + + type CacheGroupsTableColumn = TableColumn + const columns = computed(() => [ + { + id: 'select', + header: ({ table }) => + h(UCheckbox, { + 'modelValue': isSomePageRowsSelected(table) + ? 'indeterminate' + : isAllPageRowsSelected(table), + 'onUpdate:modelValue': () => toggleAllPageRows(table), + 'ui': { root: 'py-0.5' }, + 'disabled': isUpdating.value, + 'aria-label': 'Select all', + }), + cell: ({ row }) => + h(UCheckbox, { + 'modelValue': selectedMap.value[row.original.id] !== undefined, + 'onUpdate:modelValue': () => toggleSelection(undefined, row), + 'disabled': isUpdating.value, + 'aria-label': 'Select row', + }), + enableHiding: false, + }, + { + accessorKey: 'serviceName', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.serviceName, + ), + cell: ({ row }) => h( + ULink, { + to: `/repositories/${row.original.id}`, + class: 'font-bold hover:underline inline-flex items-center', + }, () => [ + h('span', row.original.serviceName), + ], + + ), + }, + { + accessorKey: 'url', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.serviceUrl, + ), + }, + { + accessorKey: 'updated', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.updated, + ), + cell: ({ row }) => + row.original.updated + ? datetimeFormatter.format(new Date(row.original.updated)) + : $t('group-caches.status.no-cached'), + }, + ]) + + const makePageInfo = (result: Ref) => { + return computed(() => { + const start = result.value?.offset ?? 1 + const total = result.value?.total ?? 0 + const end = Math.min(start + pageSize.value!, total) + const count = selectedCount.value + + if (count > 0) + return `${start} - ${end} / ${total} (${$t('table.selected')} ${count})` + return `${start} - ${end} / ${total}` + }) + } + + return { + /** Computed reference for the current query */ + query, + /** Update query parameters and push to router */ + updateQuery, + /** Criteria for filtering and sorting repositories */ + criteria: { + /** Reactive object for the search term */ + searchTerm, + /** Reactive object for the filter */ + filter, + /** Reactive object for the current page number */ + pageNumber, + /** Reactive object for the page size */ + pageSize, + }, + /** Flag indicating if the data is being updated */ + isUpdating, + /** Reactive object for the selected repositories */ + selectedMap, + /** Computed reference for the count of selected repositories */ + selectedCount, + /** Toggle the selection of a repository */ + toggleSelection, + /** Get the selected repositories */ + getSelected, + /** Clear all selection */ + clearSelection, + /** Dropdown items of actions for the selected repositories */ + selectedRepositoriesAction, + /** Items for filtering the repositories */ + filterItems, + /** Column definitions for the table with translations */ + columns, + /** Make indicator for the page information */ + makePageInfo, + /** Reactive object for the state of modals */ + modals, + } +} + +export { useCacheGroups } diff --git a/src/app/composables/useMenu.ts b/src/app/composables/useMenu.ts index ab50e192..c34b2e00 100644 --- a/src/app/composables/useMenu.ts +++ b/src/app/composables/useMenu.ts @@ -45,8 +45,8 @@ export function useMenu() { }, { - label: $t('cache-groups.title'), - to: '/cache-groups', + label: $t('group-caches.title'), + to: '/group-caches', icon: 'i-lucide-database', requiredSystemAdmin: true, }, diff --git a/src/app/i18n/locales/en.json b/src/app/i18n/locales/en.json index 7148fbef..ddcee4d6 100644 --- a/src/app/i18n/locales/en.json +++ b/src/app/i18n/locales/en.json @@ -88,48 +88,27 @@ }, "cache-groups": { "button": { - "update-all-repositories": "Update all repositories", - "update-selected-repositories": "Update selected repositories" }, - "confirm-update-all-repositories": "Do you want to update all repositories ({count})?", "confirm-update-selected-repositories": "Would you like to update the following {count} repositories?", "count": "{current} / {total} items", - "description": "You can manage the cache status of group information used by WEKO3.", "filter-placeholder": "Cache status", - "search-placeholder": "search...", "select": { - "cache": "With cash", - "no-cache": "No cache" }, "table": { "column": { - "repository-name": "Repository name", - "repository-status": "Update status", - "repository-updated-at": "Cache update date and time", - "repository-url": "Repository URL" + "repository-status": "Update status" } }, - "title": "Caching Groups", "update-completed": "Update completed ({count} items)", - "update-error": "Cache update processing is in progress.", - "updating": "Updating..." + "update-error": "Cache update processing is in progress." }, "common": { "loading": "Loading..." }, - "error": { - "conflict": { - }, - "server": { - }, - "unexpected": { - }, - "validation": { - } - }, "error-page": { "forbidden": { "group-access": "You do not have permission to access this group.", + "group-caches": "You do not have permission to manage group caches.", "group-create": "You do not have permission to create groups in the selected repository.", "group-delete": "You do not have permission to delete this group.", "group-edit": "You do not have permission to change information for this group.", @@ -203,6 +182,29 @@ } } }, + "group-caches": { + "button": { + "update-all": "Update all repositories", + "update-all-repositories": "Update all cache", + "update-selected-repositories": "Update cache" + }, + "description": "You can manage the cache status of group information used by WEKO3.", + "search-placeholder": "search...", + "status": { + "cached": "Cached", + "no-cached": "Not cached" + }, + "table": { + "column": { + "repository-name": "Repository name", + "repository-updated-at": "Cache update date and time", + "repository-url": "Repository URL" + }, + "no-cache": "" + }, + "title": "Caching Groups", + "updating": "Updating..." + }, "groups": { "actions": { "copy-id": "Copy ID to clipboard" @@ -211,7 +213,7 @@ }, "delete-selected-button": "Delete groups", "description": "You can manage groups in your repository.", - "selected-groups-actions": "Selected groups", + "selected-groups-actions": "Selected Groups", "table": { "cell": { "public": { @@ -339,18 +341,24 @@ "remove-users-from-group": { "selection": "Please select the group to remove.", "title": "Remove the following users from the group." + }, + "update-all-repositories-cache": { + "alert": "This process will take a long time to complete. It cannot be stopped at this time.", + "title": "Are you sure to update the cache for all repositories?" } }, "repositories": { "actions": { "copy-sp-connector-id": "Copy SP Connecter ID to clipboard\t" }, + "all-repositories-actions": "All Repositories", "description": "You can check information about the repositories you manage.", + "entity-id-label": "Entity ID: ", "list": { "no-repositories-description": "Please change your search criteria and try again.", "no-repositories-title": "Repository not found" }, - "entity-id-label": "Entity ID: ", + "selected-repositories-actions": "Selected Repositories", "table": { "column": { "entity-ids": "Entity ID", @@ -439,7 +447,6 @@ "display-count-label": "Number of items displayed:", "display-info-text": "Displaying {start}～{end} items (all {total} items)", "display-info-text-empty": "Displaying 0 items (all 0 items)", - "page-size-label": "Page size:" "filter-button-label": "Filter", "page-size-label": "Page size:", "selected": "selected", @@ -456,6 +463,9 @@ "description": "The server marked the request as invalid.", "title": "Bad Request" }, + "cache-update-in-progress": { + "description": "The group cache update operation is already in progress." + }, "conflict": { "description": "There was a conflict in the input content.", "title": "Conflict Error" @@ -470,6 +480,9 @@ "invalid-search-query": { "description": "Contains invalid search criteria." }, + "no-cache-update-task": { + "description": "The cache update process is not currently running." + }, "not-found": { "description": "Request destination not found.", "title": "Not Found" @@ -526,6 +539,9 @@ "deleted": { "title": "Delete successful" }, + "group-cache-update-started": { + "description": "The group cache update operation has started." + }, "group-created": { "description": "Successfully created group." }, @@ -630,7 +646,7 @@ } }, "users": { - "all-users-actions": "All users", + "all-users-actions": "All Users", "button": { "all-users-download": "Download all users", "selected-users-add-to-group": "Add to group", @@ -647,7 +663,7 @@ "repository-admin": "Repository Administrator", "system-admin": "System Administrator" }, - "selected-users-actions": "Selected users", + "selected-users-actions": "Selected Users", "table": { "column": { "emails": "Email", diff --git a/src/app/i18n/locales/ja.json b/src/app/i18n/locales/ja.json index 69abeffd..f9ab08ea 100644 --- a/src/app/i18n/locales/ja.json +++ b/src/app/i18n/locales/ja.json @@ -88,31 +88,19 @@ }, "cache-groups": { "button": { - "update-all-repositories": "全リポジトリを更新", - "update-selected-repositories": "選択したリポジトリを更新" }, - "confirm-update-all-repositories": "全リポジトリ（{count}件）の更新を行いますか？", "confirm-update-selected-repositories": "以下の{count}リポジトリを更新しますか？", "count": "{current} / {total} 件", - "description": "WEKO3が利用するグループ情報のキャッシュ状況を管理できます。", "filter-placeholder": "キャッシュ状態", - "search-placeholder": "検索...", "select": { - "cache": "キャッシュあり", - "no-cache": "キャッシュなし" }, "table": { "column": { - "repository-name": "リポジトリ名", - "repository-status": "更新状況", - "repository-updated-at": "キャッシュ更新日時", - "repository-url": "リポジトリURL" + "repository-status": "更新状況" } }, - "title": "グループキャッシュ", "update-completed": "更新完了（{count}件）", - "update-error": "キャッシュ更新処理が実行中です。", - "updating": "更新中..." + "update-error": "キャッシュ更新処理が実行中です。" }, "common": { "loading": "読み込み中..." @@ -120,6 +108,7 @@ "error-page": { "forbidden": { "group-access": "このグループにアクセスする権限がありません。", + "group-caches": "グループキャッシュを管理する権限がありません。", "group-create": "選択したリポジトリにグループを作成する権限がありません。", "group-delete": "このグループを削除する権限がありません。", "group-edit": "このグループの情報を変更する権限がありません。", @@ -189,6 +178,28 @@ } } }, + "group-caches": { + "button": { + "update-all": "全リポジトリを更新", + "update-all-repositories": "キャッシュを全更新", + "update-selected-repositories": "キャッシュを更新" + }, + "description": "WEKO3 が利用するグループ情報のキャッシュ状況を管理できます。", + "search-placeholder": "検索...", + "status": { + "cached": "キャッシュあり", + "no-cached": "キャッシュなし" + }, + "table": { + "column": { + "repository-name": "リポジトリ名", + "repository-updated-at": "キャッシュ更新日時", + "repository-url": "リポジトリURL" + } + }, + "title": "グループキャッシュ", + "updating": "更新中..." + }, "groups": { "actions": { "copy-id": "グループ ID をコピー" @@ -324,18 +335,24 @@ "remove-users-from-group": { "selection": "除外元のグループを選択してください。", "title": "以下のユーザーをグループから除外します。" + }, + "update-all-repositories-cache": { + "alert": "すべての処理は完了までに長い時間がかかります。現時点ではこの処理を途中で停止できません。", + "title": "全リポジトリのキャッシュの更新を実行しますか。" } }, "repositories": { "actions": { "copy-sp-connector-id": "SP コネクタ ID をコピー" }, + "all-repositories-actions": "すべてのリポジトリ", "description": "あなたが管理しているリポジトリの情報を確認できます。", + "entity-id-label": "Entity ID：", "list": { "no-repositories-description": "検索条件を変更して、再度お試しください。", "no-repositories-title": "リポジトリが見つかりません" }, - "entity-id-label": "Entity ID：", + "selected-repositories-actions": "選択したリポジトリ", "table": { "column": { "entity-ids": "Entity ID", @@ -428,6 +445,9 @@ "description": "サーバーがリクエストを不正とみなしました。", "title": "不正なリクエスト" }, + "cache-update-in-progress": { + "description": "すでにグループキャッシュの更新処理が進行中です。" + }, "conflict": { "description": "入力内容に競合がありました。", "title": "競合エラー" @@ -442,6 +462,9 @@ "invalid-search-query": { "description": "不正な検索条件が含まれます。" }, + "no-cache-update-task": { + "description": "キャッシュの更新処理は実行中ではありません。" + }, "not-found": { "description": "リクエスト先が見つかりません。", "title": "未検出" @@ -498,6 +521,9 @@ "deleted": { "title": "削除成功" }, + "group-cache-update-started": { + "description": "グループキャッシュの更新処理が開始しました。" + }, "group-created": { "description": "グループの作成に成功しました。" }, diff --git a/src/app/pages/cache-groups/index.vue b/src/app/pages/cache-groups/index.vue deleted file mode 100644 index 36f3fa62..00000000 --- a/src/app/pages/cache-groups/index.vue +++ /dev/null @@ -1,267 +0,0 @@ - - - diff --git a/src/app/pages/group-caches/index.vue b/src/app/pages/group-caches/index.vue new file mode 100644 index 00000000..dcc66a99 --- /dev/null +++ b/src/app/pages/group-caches/index.vue @@ -0,0 +1,301 @@ + + + diff --git a/src/app/types/cacheGroups.ts b/src/app/types/cacheGroups.ts deleted file mode 100644 index 1b33e0fd..00000000 --- a/src/app/types/cacheGroups.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Types related to cache groups - */ - -/** Cache group summary information */ -interface CacheGroupsSummary { - id: string - name: string - url: string - updated: string -} - -/** Search result for cache groups */ -interface CacheGroupSearchResult { - total: number - pageSize: number - offset: number - resources: CacheGroupsSummary[] -} - -/** Cache groups update result */ -interface CacheGroupsUpdateResult { - type: string - fqdn: string - status: string - code?: string - repository_cached?: CacheGroupsSummary[] -} - -/** Detail information of a cache groups update task */ -interface TaskDetail { - results: CacheGroupsUpdateResult[] - current: string - done: number - total: number -} - -export type { CacheGroupsSummary, CacheGroupSearchResult, CacheGroupsUpdateResult, TaskDetail } diff --git a/src/app/types/groupCaches.ts b/src/app/types/groupCaches.ts new file mode 100644 index 00000000..5a7490a6 --- /dev/null +++ b/src/app/types/groupCaches.ts @@ -0,0 +1,28 @@ +/** + * Types related to cache groups + */ + +/** Cache group summary information */ +interface RepositoryCache extends RepositorySummary { + updated?: string + status?: 'success' | 'failed' +} + +/** Detail information of a cache groups update task */ +interface TaskDetail { + results: RepositoryCache[] + current: string + done: number + total: number +} + +/** Group cache status for filtering */ +type GroupCacheStatus = 'e' | 'n' + +/** Group cache update action */ +type GroupCacheUpdateAction = 'all' | 'id-specified' + +export type { + RepositoryCache, TaskDetail, + GroupCacheStatus, GroupCacheUpdateAction, +} diff --git a/src/app/types/search.ts b/src/app/types/search.ts index 19027ca1..cad3282a 100644 --- a/src/app/types/search.ts +++ b/src/app/types/search.ts @@ -52,7 +52,7 @@ interface UsersSearchQuery { interface CacheGroupsSearchQuery { q?: string - f?: string[] + f?: GroupCacheStatus[] p?: number l?: number } @@ -89,6 +89,8 @@ type GlobalSearchResults = ( | UsersSearchResult & { type: 'users' } )[] +type GroupCachesSearchResult = SearchResult + export type { FilterOption, RepositoriesSearchQuery, RepositoriesSortableKeys, @@ -97,5 +99,5 @@ export type { SortOrder, SearchResult, UsersSearchResult, GroupsSearchResult, RepositoriesSearchResult, GlobalSearchResults, - CacheGroupsSearchQuery, + CacheGroupsSearchQuery, GroupCachesSearchResult, } diff --git a/src/app/utils/search.ts b/src/app/utils/search.ts index 73728fda..d2a88ddb 100644 --- a/src/app/utils/search.ts +++ b/src/app/utils/search.ts @@ -76,7 +76,7 @@ const normalizeCacheGroupsQuery = (query: LocationQuery): CacheGroupsSearchQuery const { table: { pageSize } } = useAppConfig() return { q: query.q ? pickSingle(query.q) : undefined, - f: query.f ? toArray(query.f) : undefined, + f: query.f ? toArray(query.f) as GroupCacheStatus[] : undefined, p: Number(query.p) || 1, l: Number(query.l) || pageSize.cacheGroups?.[0], } diff --git a/src/server/api/group_caches.py b/src/server/api/group_caches.py index cadc874d..2708750e 100644 --- a/src/server/api/group_caches.py +++ b/src/server/api/group_caches.py @@ -9,13 +9,9 @@ from flask import Blueprint, current_app from flask_login import login_required from flask_pydantic import validate +from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config -from server.api.helpers import roles_required -from server.api.schemas import ( - CacheQuery, - CacheRequest, - ErrorResponse, -) +from server.config import config from server.const import USER_ROLES from server.entities.cache import TaskDetail from server.entities.search_request import SearchResult @@ -23,10 +19,23 @@ from server.messages import E from server.services import group_caches +from .helpers import roles_required +from .schemas import ( + CacheQuery, + CacheRequest, + ErrorResponse, +) + bp = Blueprint("group-caches", __name__) +@bp.before_request +def init_settings() -> None: + """Initialize settings for the request.""" + setup_weko_group_cache_db_config(config.CACHE_DB) + + @bp.get("/", strict_slashes=False) @login_required @roles_required(USER_ROLES.SYSTEM_ADMIN) diff --git a/src/server/ext.py b/src/server/ext.py index 63097ac0..d0388801 100644 --- a/src/server/ext.py +++ b/src/server/ext.py @@ -9,7 +9,6 @@ from pathlib import Path from sqlalchemy_utils import database_exists -from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config from .api.router import create_api_blueprint from .auth import login_manager @@ -83,7 +82,6 @@ def init_config(self, app: Flask) -> None: app.config.from_mapping(self.config.for_flask) app.config.from_prefixed_env() - setup_weko_group_cache_db_config(self.config.CACHE_DB) def init_db_app(self, app: Flask) -> None: # noqa: PLR6301 """Initialize the database for the this extension. diff --git a/src/server/services/group_caches.py b/src/server/services/group_caches.py index f87d7972..f1007f66 100644 --- a/src/server/services/group_caches.py +++ b/src/server/services/group_caches.py @@ -292,6 +292,7 @@ def get_task_status() -> TaskDetail | None: RepositoryCache( id=r.id, service_name=r.service_name, + service_url=r.service_url, updated=result.updated_at, status=result.status, ) From 4796ed4d6e7c0108cb88e1c671f632945cc49629 Mon Sep 17 00:00:00 2001 From: ivis-kuroda Date: Wed, 11 Mar 2026 05:47:46 +0000 Subject: [PATCH 5/5] chore: update dependencies --- pyproject.toml | 2 +- uv.lock | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f44b85ee..44a73298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ select = ["ALL"] "__init__.py" = ["F401"] "**/api/**.py" = ["TC001", "TC002", "TC003"] "**/entities/**.py" = ["TC001", "TC002", "TC003"] -"*.pyi" = ["CPY001"] +"*.pyi" = ["CPY001", "E501"] [tool.ruff.lint.isort] # refer to https://docs.astral.sh/ruff/settings/#lintisort diff --git a/uv.lock b/uv.lock index ff72284e..2a376dcb 100644 --- a/uv.lock +++ b/uv.lock @@ -870,18 +870,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/25/7400c18c3ee97914cc99c90007795c00a4ec5b60c853b49db7ba24d11179/sqlalchemy_utils-0.42.1-py3-none-any.whl", hash = "sha256:243cfe1b3a1dae3c74118ae633f1d1e0ed8c787387bc33e556e37c990594ac80", size = 91761, upload-time = "2025-12-13T03:14:15.014Z" }, ] -[[package]] -name = "typeguard" -version = "4.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c7/68/71c1a15b5f65f40e91b65da23b8224dad41349894535a97f63a52e462196/typeguard-4.4.4.tar.gz", hash = "sha256:3a7fd2dffb705d4d0efaed4306a704c89b9dee850b688f060a8b1615a79e5f74", size = 75203, upload-time = "2025-06-18T09:56:07.624Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/a9/e3aee762739c1d7528da1c3e06d518503f8b6c439c35549b53735ba52ead/typeguard-4.4.4-py3-none-any.whl", hash = "sha256:b5f562281b6bfa1f5492470464730ef001646128b180769880468bd84b68b09e", size = 34874, upload-time = "2025-06-18T09:56:05.999Z" }, -] - [[package]] name = "taskipy" version = "1.14.1" @@ -924,6 +912,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] +[[package]] +name = "typeguard" +version = "4.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/68/71c1a15b5f65f40e91b65da23b8224dad41349894535a97f63a52e462196/typeguard-4.4.4.tar.gz", hash = "sha256:3a7fd2dffb705d4d0efaed4306a704c89b9dee850b688f060a8b1615a79e5f74", size = 75203, upload-time = "2025-06-18T09:56:07.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/a9/e3aee762739c1d7528da1c3e06d518503f8b6c439c35549b53735ba52ead/typeguard-4.4.4-py3-none-any.whl", hash = "sha256:b5f562281b6bfa1f5492470464730ef001646128b180769880468bd84b68b09e", size = 34874, upload-time = "2025-06-18T09:56:05.999Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -995,10 +995,11 @@ wheels = [ [[package]] name = "weko-group-cache-db" -version = "1.0.0rc4" -source = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop#4b41ff4d972487a6812499f43d10d956b6f72721" } +version = "1.0.0rc6" +source = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop#a8d12f62b486e8805a7b9dab059d8be4f9ce1a10" } dependencies = [ { name = "backoff" }, + { name = "blinker" }, { name = "inflect" }, { name = "pydantic-settings" }, { name = "redis" },