diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c63f74db..b581ba62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,6 @@ jobs: - name: Run Task run: | export PYTHONPATH=$(pwd) - FLASK_ENV=development uv run flask -A server.app --debug app version uv run ${{ matrix.task.command }} - name: Minimize uv cache diff --git a/Dockerfile b/Dockerfile index fb91fbb5..7dbc218a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ ENV VIRTUAL_ENV="/code/.venv" ENV PATH="$VIRTUAL_ENV/bin:$PATH" WORKDIR /code +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* RUN pip install -U pip && pip install uv RUN groupadd -g ${GID} ${GROUPNAME} && \ diff --git a/configs/app.config.ts b/configs/app.config.ts index f4609ebb..8fe71043 100644 --- a/configs/app.config.ts +++ b/configs/app.config.ts @@ -56,6 +56,7 @@ const table = { users: [20, 50, 100] as [number, ...number[]], history: [20, 50, 100] as [number, ...number[]], bulks: [20, 50, 100] as [number, ...number[]], + cacheGroups: [20, 50, 100] as number[], }, } diff --git a/configs/server.config.toml b/configs/server.config.toml index b9baef58..27730a90 100644 --- a/configs/server.config.toml +++ b/configs/server.config.toml @@ -233,6 +233,40 @@ port = 26379 url = "amqp://guest:guest@rabbitmq:5672//" +[cache_groups] +# Cache key suffix of group information in Redis. +cache_key_suffix = "_gakunin_groups" + +# Map groups API endpoint. +api_endpoint = "/api/groups" + +# Cache time-to-live of group information in Redis. +# if it specified less than 0, it will be considered as no expiration. +cache_ttl = 86400 + +# Request timeout (in seconds) when connecting to mAP API. +request_timeout = 20 + +# Request interval (in seconds) between mAP API requests. +request_interval = 3 + +# Request retries when failed to fetch groups from mAP API. +request_retries = 3 + +# Base time (in seconds) for exponential backoff during request retries. +request_retry_base = 4 + +# Factor (in seconds) for exponential backoff during request retries. +request_retry_factor = 5 + +# Maximum time (in seconds) for exponential backoff during request retries. +request_retry_max = 90 + + +# Path to the directory containing institution TLS files. +directory_path = "/var/mnt" + + # [develop] # Enable or disable developer login feature. # developer_login = false diff --git a/pyproject.toml b/pyproject.toml index 18255b99..44a73298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "pydantic[email]>=2.12.5", "requests>=2.32.5", "sqlalchemy-utils>=0.42.1", + "weko-group-cache-db", ] [dependency-groups] @@ -77,7 +78,7 @@ select = ["ALL"] "__init__.py" = ["F401"] "**/api/**.py" = ["TC001", "TC002", "TC003"] "**/entities/**.py" = ["TC001", "TC002", "TC003"] -"*.pyi" = ["CPY001"] +"*.pyi" = ["CPY001", "E501"] [tool.ruff.lint.isort] # refer to https://docs.astral.sh/ruff/settings/#lintisort @@ -105,3 +106,6 @@ skip-magic-trailing-comma = false include = ["src"] extraPaths = ["src"] typeCheckingMode = "standard" + +[tool.uv.sources] +weko-group-cache-db = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git", rev = "develop" } diff --git a/src/app/composables/groupCaches.ts b/src/app/composables/groupCaches.ts new file mode 100644 index 00000000..e739331f --- /dev/null +++ b/src/app/composables/groupCaches.ts @@ -0,0 +1,246 @@ +/** + * Composable for managing cache groups. + */ +import { UCheckbox, ULink } from '#components' + +import type { Row, Table } from '@tanstack/table-core' +import type { DropdownMenuItem, SelectItem, TableColumn } from '@nuxt/ui' + +const useCacheGroups = () => { + const route = useRoute() + const router = useRouter() + + const { t: $t } = useI18n() + + /** Reactive query object */ + const query = computed(() => normalizeCacheGroupsQuery(route.query)) + /** Update query parameters and push to router */ + const updateQuery = (newQuery: Partial) => { + router.push({ + query: { + ...route.query, + ...newQuery, + }, + }) + } + + const searchTerm = ref(query.value.q) + const filter = ref(query.value.f) + const pageNumber = ref(query.value.p) + const pageSize = ref(query.value.l) + + const searchIdentityKey = computed(() => { + const { p, l, ...filters } = query.value + return JSON.stringify(filters) + }) + + const selectedMap = useState>( + `selection-group-caches:${searchIdentityKey.value}`, () => ({}), + ) + + const selectedCount = computed(() => { + return Object.values(selectedMap.value).filter(value => value !== undefined).length + }) + const toggleSelection = (event: Event | undefined, row: Row) => { + selectedMap.value[row.original.id] + = selectedMap.value[row.original.id] ? undefined : row.original + } + const toggleAllPageRows = (table: Table) => { + const pageRows = table.getRowModel().rows + const allSelected = pageRows.every(row => selectedMap.value[row.original.id] !== undefined) + + if (allSelected) { + for (const row of pageRows) { + selectedMap.value[row.original.id] = undefined + } + } + else { + for (const row of pageRows) { + selectedMap.value[row.original.id] = row.original + } + } + } + const isAllPageRowsSelected = (table: Table) => { + const pageRows = table.getRowModel().rows + return pageRows.length > 0 && pageRows.every( + row => selectedMap.value[row.original.id] !== undefined, + ) + } + const isSomePageRowsSelected = (table: Table) => { + const pageRows = table.getRowModel().rows + const selectedRows = pageRows.filter(row => selectedMap.value[row.original.id] !== undefined) + return selectedRows.length > 0 && selectedRows.length < pageRows.length + } + + const getSelected = (): { id: string, serviceName: string, serviceUrl: string }[] => { + return Object.entries(selectedMap.value) + .filter(([_, service]) => service !== undefined) + .map(([id, service]) => ({ + id, serviceName: service!.serviceName, serviceUrl: service!.serviceUrl, + })) + } + const clearSelection = () => { + selectedMap.value = {} + } + + const modals = reactive>({ + 'all': false, + 'id-specified': false, + }) + const isUpdating = ref(false) + + /** Column names with translations */ + const columnNames = computed(() => ({ + id: '#', + serviceName: $t('group-caches.table.column.repository-name'), + serviceUrl: $t('group-caches.table.column.repository-url'), + updated: $t('group-caches.table.column.repository-updated-at'), + })) + + const filterItems = computed(() => [ + { + label: $t('group-caches.status.cached'), + value: 'e' as GroupCacheStatus, + }, + { + label: $t('group-caches.status.no-cached'), + value: 'n' as GroupCacheStatus, + }, + ]) + + const selectedRepositoriesAction = computed<[DropdownMenuItem, ...DropdownMenuItem[]]>(() => [ + { + type: 'label' as const, + label: $t('repositories.all-repositories-actions'), + }, + { + icon: 'i-lucide-refresh-cw', + label: $t('group-caches.button.update-all-repositories'), + onSelect: () => modals.all = true, + }, + { + type: 'separator' as const, + }, + { + type: 'label' as const, + label: $t('repositories.selected-repositories-actions'), + }, + { + icon: 'i-lucide-refresh-cw', + label: $t('group-caches.button.update-selected-repositories'), + onSelect: () => modals['id-specified'] = true, + disabled: selectedCount.value === 0, + }, + ]) + + type CacheGroupsTableColumn = TableColumn + const columns = computed(() => [ + { + id: 'select', + header: ({ table }) => + h(UCheckbox, { + 'modelValue': isSomePageRowsSelected(table) + ? 'indeterminate' + : isAllPageRowsSelected(table), + 'onUpdate:modelValue': () => toggleAllPageRows(table), + 'ui': { root: 'py-0.5' }, + 'disabled': isUpdating.value, + 'aria-label': 'Select all', + }), + cell: ({ row }) => + h(UCheckbox, { + 'modelValue': selectedMap.value[row.original.id] !== undefined, + 'onUpdate:modelValue': () => toggleSelection(undefined, row), + 'disabled': isUpdating.value, + 'aria-label': 'Select row', + }), + enableHiding: false, + }, + { + accessorKey: 'serviceName', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.serviceName, + ), + cell: ({ row }) => h( + ULink, { + to: `/repositories/${row.original.id}`, + class: 'font-bold hover:underline inline-flex items-center', + }, () => [ + h('span', row.original.serviceName), + ], + + ), + }, + { + accessorKey: 'url', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.serviceUrl, + ), + }, + { + accessorKey: 'updated', + header: () => h( + 'span', { class: 'text-xs text-default font-medium' }, columnNames.value.updated, + ), + cell: ({ row }) => + row.original.updated + ? datetimeFormatter.format(new Date(row.original.updated)) + : $t('group-caches.status.no-cached'), + }, + ]) + + const makePageInfo = (result: Ref) => { + return computed(() => { + const start = result.value?.offset ?? 1 + const total = result.value?.total ?? 0 + const end = Math.min(start + pageSize.value!, total) + const count = selectedCount.value + + if (count > 0) + return `${start} - ${end} / ${total} (${$t('table.selected')} ${count})` + return `${start} - ${end} / ${total}` + }) + } + + return { + /** Computed reference for the current query */ + query, + /** Update query parameters and push to router */ + updateQuery, + /** Criteria for filtering and sorting repositories */ + criteria: { + /** Reactive object for the search term */ + searchTerm, + /** Reactive object for the filter */ + filter, + /** Reactive object for the current page number */ + pageNumber, + /** Reactive object for the page size */ + pageSize, + }, + /** Flag indicating if the data is being updated */ + isUpdating, + /** Reactive object for the selected repositories */ + selectedMap, + /** Computed reference for the count of selected repositories */ + selectedCount, + /** Toggle the selection of a repository */ + toggleSelection, + /** Get the selected repositories */ + getSelected, + /** Clear all selection */ + clearSelection, + /** Dropdown items of actions for the selected repositories */ + selectedRepositoriesAction, + /** Items for filtering the repositories */ + filterItems, + /** Column definitions for the table with translations */ + columns, + /** Make indicator for the page information */ + makePageInfo, + /** Reactive object for the state of modals */ + modals, + } +} + +export { useCacheGroups } diff --git a/src/app/composables/useMenu.ts b/src/app/composables/useMenu.ts index ab50e192..c34b2e00 100644 --- a/src/app/composables/useMenu.ts +++ b/src/app/composables/useMenu.ts @@ -45,8 +45,8 @@ export function useMenu() { }, { - label: $t('cache-groups.title'), - to: '/cache-groups', + label: $t('group-caches.title'), + to: '/group-caches', icon: 'i-lucide-database', requiredSystemAdmin: true, }, diff --git a/src/app/i18n/locales/en.json b/src/app/i18n/locales/en.json index 49a51db7..a7ae2fbd 100644 --- a/src/app/i18n/locales/en.json +++ b/src/app/i18n/locales/en.json @@ -85,21 +85,23 @@ "upload": "Upload" }, "cache-groups": { - "description": "You can manage the cache status of group information used by WEKO3.", - "title": "Caching Groups" - }, - "common": { - "loading": "Loading..." - }, - "error": { - "conflict": { + "button": { }, - "server": { + "confirm-update-selected-repositories": "Would you like to update the following {count} repositories?", + "count": "{current} / {total} items", + "filter-placeholder": "Cache status", + "select": { }, - "unexpected": { + "table": { + "column": { + "repository-status": "Update status" + } }, - "validation": { - } + "update-completed": "Update completed ({count} items)", + "update-error": "Cache update processing is in progress." + }, + "common": { + "loading": "Loading..." }, "error-page": { "failed": { @@ -109,6 +111,7 @@ "bulk-edit": "You do not have permission to work with users of this repository.", "bulk-result": "You do not have permission to view upload history.", "group-access": "You do not have permission to access this group.", + "group-caches": "You do not have permission to manage group caches.", "group-create": "You do not have permission to create groups in the selected repository.", "group-delete": "You do not have permission to delete this group.", "group-edit": "You do not have permission to change information for this group.", @@ -184,6 +187,29 @@ } } }, + "group-caches": { + "button": { + "update-all": "Update all repositories", + "update-all-repositories": "Update all cache", + "update-selected-repositories": "Update cache" + }, + "description": "You can manage the cache status of group information used by WEKO3.", + "search-placeholder": "search...", + "status": { + "cached": "Cached", + "no-cached": "Not cached" + }, + "table": { + "column": { + "repository-name": "Repository name", + "repository-updated-at": "Cache update date and time", + "repository-url": "Repository URL" + }, + "no-cache": "" + }, + "title": "Caching Groups", + "updating": "Updating..." + }, "groups": { "actions": { "copy-id": "Copy ID to clipboard" @@ -192,7 +218,7 @@ }, "delete-selected-button": "Delete groups", "description": "You can manage groups in your repository.", - "selected-groups-actions": "Selected groups", + "selected-groups-actions": "Selected Groups", "table": { "cell": { "public": { @@ -298,14 +324,24 @@ "remove-users-from-group": { "selection": "Please select the group to remove.", "title": "Remove the following users from the group." + }, + "update-all-repositories-cache": { + "alert": "This process will take a long time to complete. It cannot be stopped at this time.", + "title": "Are you sure to update the cache for all repositories?" } }, "repositories": { "actions": { "copy-sp-connector-id": "Copy SP Connecter ID to clipboard\t" }, + "all-repositories-actions": "All Repositories", "description": "You can check information about the repositories you manage.", "entity-id-label": "Entity ID: ", + "list": { + "no-repositories-description": "Please change your search criteria and try again.", + "no-repositories-title": "Repository not found" + }, + "selected-repositories-actions": "Selected Repositories", "table": { "column": { "entity-ids": "Entity ID", @@ -393,6 +429,9 @@ }, "actions-label": "Action", "display-columns-label": "Columns", + "display-count-label": "Number of items displayed:", + "display-info-text": "Displaying {start}～{end} items (all {total} items)", + "display-info-text-empty": "Displaying 0 items (all 0 items)", "filter-button-label": "Filter", "page-size-label": "Page size:", "selected": "selected", @@ -409,6 +448,9 @@ "description": "The server marked the request as invalid.", "title": "Bad Request" }, + "cache-update-in-progress": { + "description": "The group cache update operation is already in progress." + }, "conflict": { "description": "There was a conflict in the input content.", "title": "Conflict Error" @@ -423,6 +465,9 @@ "invalid-search-query": { "description": "Contains invalid search criteria." }, + "no-cache-update-task": { + "description": "The cache update process is not currently running." + }, "not-found": { "description": "Request destination not found.", "title": "Not Found" @@ -479,6 +524,9 @@ "deleted": { "title": "Delete successful" }, + "group-cache-update-started": { + "description": "The group cache update operation has started." + }, "group-created": { "description": "Successfully created group." }, @@ -583,11 +631,11 @@ } }, "users": { + "all-users-actions": "All Users", "alert": { "filter-by-both-role-group-disabled": "Currently, due to limited functionality, it is not possible to select role and group filters at the same time.", "filter-by-last-modified-disabled": "Due to limited functionality, filtering by last updated time is currently not available." }, - "all-users-actions": "All users", "button": { "all-users-download": "Download all users", "selected-users-add-to-group": "Add to group", @@ -604,7 +652,7 @@ "repository-admin": "Repository Administrator", "system-admin": "System Administrator" }, - "selected-users-actions": "Selected users", + "selected-users-actions": "Selected Users", "table": { "column": { "emails": "Email", diff --git a/src/app/i18n/locales/ja.json b/src/app/i18n/locales/ja.json index c49dcb16..1a7eb5fc 100644 --- a/src/app/i18n/locales/ja.json +++ b/src/app/i18n/locales/ja.json @@ -85,8 +85,20 @@ "upload": "アップロード" }, "cache-groups": { - "description": "WEKO3が利用するグループ情報のキャッシュ状況を管理できます。", - "title": "グループキャッシュ" + "button": { + }, + "confirm-update-selected-repositories": "以下の{count}リポジトリを更新しますか？", + "count": "{current} / {total} 件", + "filter-placeholder": "キャッシュ状態", + "select": { + }, + "table": { + "column": { + "repository-status": "更新状況" + } + }, + "update-completed": "更新完了（{count}件）", + "update-error": "キャッシュ更新処理が実行中です。" }, "common": { "loading": "読み込み中..." @@ -99,6 +111,7 @@ "bulk-edit": "このリポジトリのユーザーを操作する権限がありません。", "bulk-result": "アップロード履歴を閲覧する権限がありません。", "group-access": "このグループにアクセスする権限がありません。", + "group-caches": "グループキャッシュを管理する権限がありません。", "group-create": "選択したリポジトリにグループを作成する権限がありません。", "group-delete": "このグループを削除する権限がありません。", "group-edit": "このグループの情報を変更する権限がありません。", @@ -170,6 +183,28 @@ } } }, + "group-caches": { + "button": { + "update-all": "全リポジトリを更新", + "update-all-repositories": "キャッシュを全更新", + "update-selected-repositories": "キャッシュを更新" + }, + "description": "WEKO3 が利用するグループ情報のキャッシュ状況を管理できます。", + "search-placeholder": "検索...", + "status": { + "cached": "キャッシュあり", + "no-cached": "キャッシュなし" + }, + "table": { + "column": { + "repository-name": "リポジトリ名", + "repository-updated-at": "キャッシュ更新日時", + "repository-url": "リポジトリURL" + } + }, + "title": "グループキャッシュ", + "updating": "更新中..." + }, "groups": { "actions": { "copy-id": "グループ ID をコピー" @@ -284,14 +319,24 @@ "remove-users-from-group": { "selection": "除外元のグループを選択してください。", "title": "以下のユーザーをグループから除外します。" + }, + "update-all-repositories-cache": { + "alert": "すべての処理は完了までに長い時間がかかります。現時点ではこの処理を途中で停止できません。", + "title": "全リポジトリのキャッシュの更新を実行しますか。" } }, "repositories": { "actions": { "copy-sp-connector-id": "SP コネクタ ID をコピー" }, + "all-repositories-actions": "すべてのリポジトリ", "description": "あなたが管理しているリポジトリの情報を確認できます。", "entity-id-label": "Entity ID：", + "list": { + "no-repositories-description": "検索条件を変更して、再度お試しください。", + "no-repositories-title": "リポジトリが見つかりません" + }, + "selected-repositories-actions": "選択したリポジトリ", "table": { "column": { "entity-ids": "Entity ID", @@ -367,6 +412,9 @@ }, "actions-label": "アクション", "display-columns-label": "表示項目", + "display-count-label": "表示件数：", + "display-info-text": "{start}～{end}件を表示（全{total}件）", + "display-info-text-empty": "0件を表示（全0件）", "filter-button-label": "フィルター", "page-size-label": "表示件数：", "selected": "選択", @@ -383,6 +431,9 @@ "description": "サーバーがリクエストを不正とみなしました。", "title": "不正なリクエスト" }, + "cache-update-in-progress": { + "description": "すでにグループキャッシュの更新処理が進行中です。" + }, "conflict": { "description": "入力内容に競合がありました。", "title": "競合エラー" @@ -397,6 +448,9 @@ "invalid-search-query": { "description": "不正な検索条件が含まれます。" }, + "no-cache-update-task": { + "description": "キャッシュの更新処理は実行中ではありません。" + }, "not-found": { "description": "リクエスト先が見つかりません。", "title": "未検出" @@ -453,6 +507,9 @@ "deleted": { "title": "削除成功" }, + "group-cache-update-started": { + "description": "グループキャッシュの更新処理が開始しました。" + }, "group-created": { "description": "グループの作成に成功しました。" }, diff --git a/src/app/pages/cache-groups/index.vue b/src/app/pages/cache-groups/index.vue deleted file mode 100644 index f57787e7..00000000 --- a/src/app/pages/cache-groups/index.vue +++ /dev/null @@ -1,11 +0,0 @@ - - - diff --git a/src/app/pages/group-caches/index.vue b/src/app/pages/group-caches/index.vue new file mode 100644 index 00000000..dcc66a99 --- /dev/null +++ b/src/app/pages/group-caches/index.vue @@ -0,0 +1,301 @@ + + + diff --git a/src/app/types/groupCaches.ts b/src/app/types/groupCaches.ts new file mode 100644 index 00000000..5a7490a6 --- /dev/null +++ b/src/app/types/groupCaches.ts @@ -0,0 +1,28 @@ +/** + * Types related to cache groups + */ + +/** Cache group summary information */ +interface RepositoryCache extends RepositorySummary { + updated?: string + status?: 'success' | 'failed' +} + +/** Detail information of a cache groups update task */ +interface TaskDetail { + results: RepositoryCache[] + current: string + done: number + total: number +} + +/** Group cache status for filtering */ +type GroupCacheStatus = 'e' | 'n' + +/** Group cache update action */ +type GroupCacheUpdateAction = 'all' | 'id-specified' + +export type { + RepositoryCache, TaskDetail, + GroupCacheStatus, GroupCacheUpdateAction, +} diff --git a/src/app/types/search.ts b/src/app/types/search.ts index 3dff368c..cad3282a 100644 --- a/src/app/types/search.ts +++ b/src/app/types/search.ts @@ -50,6 +50,13 @@ interface UsersSearchQuery { l?: number } +interface CacheGroupsSearchQuery { + q?: string + f?: GroupCacheStatus[] + p?: number + l?: number +} + type RepositoriesSortableKeys = 'id' | 'serviceName' | 'serviceUrl' | 'entityIds' type GroupsSortableKeys = 'id' | 'displayName' | 'public' | 'memberListVisibility' @@ -82,6 +89,8 @@ type GlobalSearchResults = ( | UsersSearchResult & { type: 'users' } )[] +type GroupCachesSearchResult = SearchResult + export type { FilterOption, RepositoriesSearchQuery, RepositoriesSortableKeys, @@ -90,4 +99,5 @@ export type { SortOrder, SearchResult, UsersSearchResult, GroupsSearchResult, RepositoriesSearchResult, GlobalSearchResults, + CacheGroupsSearchQuery, GroupCachesSearchResult, } diff --git a/src/app/utils/search.ts b/src/app/utils/search.ts index 922c3201..d2a88ddb 100644 --- a/src/app/utils/search.ts +++ b/src/app/utils/search.ts @@ -69,6 +69,19 @@ const normalizeUsersQuery = (query: LocationQuery): UsersSearchQuery => { } } +/** + * Normalize location query to cache groups search query + */ +const normalizeCacheGroupsQuery = (query: LocationQuery): CacheGroupsSearchQuery => { + const { table: { pageSize } } = useAppConfig() + return { + q: query.q ? pickSingle(query.q) : undefined, + f: query.f ? toArray(query.f) as GroupCacheStatus[] : undefined, + p: Number(query.p) || 1, + l: Number(query.l) || pageSize.cacheGroups?.[0], + } +} + /** * Normalize location query to history */ @@ -101,4 +114,5 @@ const normalizeUploadQuery = (query: LocationQuery): UploadQuery => { export { normalizeRepositoriesQuery, normalizeGroupsQuery, normalizeUsersQuery, normalizeHistoryQuery, normalizeUploadQuery, + normalizeCacheGroupsQuery, } diff --git a/src/server/api/bulk.py b/src/server/api/bulk.py index d5caf463..d486fbb4 100644 --- a/src/server/api/bulk.py +++ b/src/server/api/bulk.py @@ -21,7 +21,7 @@ FileNotFound, FileValidationError, RecordNotFound, - TaskExcutionError, + TaskExecutionError, ) from server.messages import E from server.services import bulks, history_table, repositories @@ -101,7 +101,7 @@ def validate_status(task_id: str) -> tuple[BulkBody | ErrorResponse, int]: """ try: res = bulks.get_validate_task_result(task_id) - except TaskExcutionError as exc: + except TaskExecutionError as exc: return ErrorResponse(message=exc.message), 404 return BulkBody(status=res.state), 200 @@ -148,7 +148,7 @@ def validate_result( result = bulks.get_validate_result( history_id=history_id, status_filter=status_filter, offset=offset, size=size ) - except (RecordNotFound, TaskExcutionError) as exc: + except (RecordNotFound, TaskExecutionError) as exc: return ErrorResponse(message=exc.message), 404 return result, 200 @@ -207,7 +207,7 @@ def execute_status(task_id: str) -> tuple[BulkBody | ErrorResponse, int]: """ try: res = bulks.get_execute_task_result(task_id) - except TaskExcutionError as exc: + except TaskExecutionError as exc: return ErrorResponse(message=exc.message), 404 return BulkBody(status=res.state), 200 diff --git a/src/server/api/group_caches.py b/src/server/api/group_caches.py new file mode 100644 index 00000000..2708750e --- /dev/null +++ b/src/server/api/group_caches.py @@ -0,0 +1,101 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# +"""API router for cache group endpoints.""" + +import traceback +import typing as t + +from flask import Blueprint, current_app +from flask_login import login_required +from flask_pydantic import validate +from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config + +from server.config import config +from server.const import USER_ROLES +from server.entities.cache import TaskDetail +from server.entities.search_request import SearchResult +from server.exc import InvalidQueryError, RequestConflict +from server.messages import E +from server.services import group_caches + +from .helpers import roles_required +from .schemas import ( + CacheQuery, + CacheRequest, + ErrorResponse, +) + + +bp = Blueprint("group-caches", __name__) + + +@bp.before_request +def init_settings() -> None: + """Initialize settings for the request.""" + setup_weko_group_cache_db_config(config.CACHE_DB) + + +@bp.get("/", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate(response_by_alias=True) +def get(query: CacheQuery) -> tuple[SearchResult, int] | tuple[ErrorResponse, int]: + """Retrieve repository cache entries based on the provided query. + + Args: + query (CacheQuery): Query parameters for filtering and pagination. + + Returns: + - If succeeded in getting repository cache, search result and status code 200 + - If query is invalid, error message and status code 400 + """ + try: + cache_result = group_caches.get_repository_cache(query) + except InvalidQueryError as exc: + traceback.print_exc() + return ErrorResponse(message=exc.message), 400 + + return cache_result, 200 + + +@bp.post("/", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate() +def post(body: CacheRequest) -> tuple[t.Literal[""], int] | tuple[ErrorResponse, int]: + """Update cache groups for the specified repositories. + + Args: + body (CacheRequest): Request body containing repositories and operation. + + Returns: + - If the update task is successfully started, empty response and status code 202 + - If there is a conflict in starting the task, error message and status code 409 + """ + try: + group_caches.update(body.op, body.ids) + except RequestConflict as exc: + traceback.print_exc() + return ErrorResponse(message=exc.message), 409 + + return "", 202 + + +@bp.get("/status", strict_slashes=False) +@login_required +@roles_required(USER_ROLES.SYSTEM_ADMIN) +@validate() +def status() -> tuple[TaskDetail | ErrorResponse, int]: + """Get the status of the cache update task. + + Returns: + - If a task is running, details of the task and status code 200 + - If no task is running, error message and status code 400 + """ + task_status = group_caches.get_task_status() + if task_status is None: + current_app.logger.error(E.UPDATE_TASK_NOT_RUNNING) + return ErrorResponse(message=E.UPDATE_TASK_NOT_RUNNING), 400 + + return task_status, 200 diff --git a/src/server/api/router.py b/src/server/api/router.py index bf62bdef..c0e1fc39 100644 --- a/src/server/api/router.py +++ b/src/server/api/router.py @@ -36,8 +36,9 @@ def create_api_blueprint() -> Blueprint: for _, module_name, _ in iter_modules([str(Path(__file__).parent)]): module = import_module(f"{__package__}.{module_name}") + url_prefix = f"/{module_name}".replace("_", "-") if hasattr(module, "bp") and isinstance(module.bp, Blueprint): - bp_api.register_blueprint(module.bp, url_prefix=f"/{module_name}") + bp_api.register_blueprint(module.bp, url_prefix=url_prefix) @bp_api.errorhandler(JAIROCloudGroupsManagerError) @validate() diff --git a/src/server/api/schemas.py b/src/server/api/schemas.py index 0331b4a6..b8b4d6db 100644 --- a/src/server/api/schemas.py +++ b/src/server/api/schemas.py @@ -364,3 +364,37 @@ class FileQuery(UsersQuery): model_config = ignore_extra_config """Configure to ignore extra fields.""" + + +class CacheQuery(BaseModel): + """Schema for cache query parameters.""" + + q: t.Annotated[str | None, "term"] = None + """Search term for querying cache entries.""" + + f: t.Annotated[list[t.Literal["e", "n"]] | None, "filter"] = None + """Filter expression for querying cache entries.""" + + p: t.Annotated[int | None, "page"] = None + """Page number for pagination.""" + + l: t.Annotated[int | None, "per"] = None # noqa: E741 + """Number of items per page for pagination.""" + + +type CacheOperation = t.Literal["all", "id-specified"] + + +class CacheRequest(BaseModel): + """Schema for cache update request.""" + + ids: list[str] | None = None + """List of repository IDs to update cache for. + Required if operation is 'id-specified'. + """ + + op: CacheOperation + """Operation type: 'all' to update all, 'id-specified' to update specified IDs.""" + + model_config = camel_case_config + """Configure to use camelCase aliasing.""" diff --git a/src/server/config.py b/src/server/config.py index 18ef59ec..32952604 100644 --- a/src/server/config.py +++ b/src/server/config.py @@ -17,6 +17,7 @@ from flask import current_app from pydantic import ( + AnyUrl, BaseModel, Field, StringConstraints, @@ -30,6 +31,10 @@ TomlConfigSettingsSource, ) from sqlalchemy.engine import URL, make_url +from weko_group_cache_db.config import ( + Sentinel, + Settings as CacheDbSettings, +) from werkzeug.local import LocalProxy from .const import ( @@ -98,6 +103,9 @@ class RuntimeConfig(BaseSettings): RABBITMQ: RabbitmqConfig """RabbitMQ configuration values.""" + CACHE_GROUPS: CacheGroupsConfig + """Cache groups task configuration values.""" + DEVELOP: DevelopConfig | None = None FEATURES: FeaturesConfig @@ -106,6 +114,30 @@ class RuntimeConfig(BaseSettings): These are due to temporary constraints in the future, all features will be enabled and the settings will be deleted.""" + @computed_field + @property + def CACHE_DB(self) -> CacheDbSettings: + """Cache database configuration values.""" + endpoint = ( + self.MAP_CORE.base_url.rstrip("/") + + "/" + + self.CACHE_GROUPS.api_endpoint.lstrip("/") + ) + + return self.CACHE_GROUPS.model_copy( + update={ + "LOG_LEVEL": self.LOG.level, + "SP_CONNECTOR_ID_PREFIX": self.SP.connector_id, + "MAP_GROUPS_API_ENDPOINT": endpoint, + "REDIS_TYPE": self.REDIS.cache_type, + "REDIS_HOST": self.REDIS.single.base_url.host, + "REDIS_PORT": self.REDIS.single.base_url.port, + "REDIS_DB_INDEX": self.REDIS.database.group_cache, + "REDIS_SENTINEL_MASTER": self.REDIS.sentinel.master_name, + "SENTINELS": t.cast("list[Sentinel]", self.REDIS.sentinel.nodes), + } + ) + @computed_field @property def SQLALCHEMY_DATABASE_URI(self) -> URL: @@ -125,10 +157,10 @@ def CELERY(self) -> dict[str, t.Any]: """ cache_type = self.REDIS.cache_type database = self.REDIS.database.result_backend - config: dict[str, t.Any] = {"broker_url": self.RABBITMQ.url} + config: dict[str, t.Any] = {"broker_url": str(self.RABBITMQ.url)} if cache_type == "RedisCache" and self.REDIS.single: - base_url = self.REDIS.single.base_url.rstrip("/") + base_url = self.REDIS.single.base_url config["result_backend"] = f"{base_url}/{database}" elif cache_type == "RedisSentinelCache" and self.REDIS.sentinel: @@ -483,7 +515,7 @@ class RedisConfig(BaseModel): cache_timeout: t.Annotated[int, "seconds"] = 300 """Default timeout (in seconds) for cached items.""" - key_prefix: str = "jcgroups_" + key_prefix: str = "jcgroups-" """Prefix for cache keys used by the application.""" database: RedisDatabaseConfig = Field( @@ -523,7 +555,7 @@ class RedisDatabaseConfig(BaseModel): class RedisSingleConfig(BaseModel): """Schema for single Redis server configuration.""" - base_url: str = "redis://localhost:6379" + base_url: AnyUrl = AnyUrl("redis://localhost:6379") class RedisSentinelCacheConfig(BaseModel): @@ -548,10 +580,20 @@ class SentinelNodeConfig(BaseModel): class RabbitmqConfig(BaseModel): """Schema for RabbitMQ configuration.""" - url: str = "amqp://guest:guest@localhost:5672//" + url: AnyUrl = AnyUrl("amqp://guest:guest@localhost:5672//") """Hostname or IP address of the RabbitMQ server for Celery broker.""" +class CacheGroupsConfig(CacheDbSettings): + """Schema for cache groups configuration.""" + + api_endpoint: str + """Map groups API endpoint.""" + + directory_path: str + """Path to the directory containing institution TLS files.""" + + type HasRepoId = t.Annotated[str, StringConstraints(pattern=HAS_REPO_ID_PATTERN)] """Pattern for role-based group IDs. diff --git a/src/server/const.py b/src/server/const.py index 2b387641..6a618a35 100644 --- a/src/server/const.py +++ b/src/server/const.py @@ -191,6 +191,9 @@ class USER_ROLES(StrEnum): - URLs ending with "/admin" are excluded from matching. """ +GROUP_CACHE_KEY_PATTERN: Final = "weko-group-cache-db" +"""Regular expression pattern to identify cache keys in Redis.""" + class ValidationEntity: """Constants for validation entities.""" diff --git a/src/server/datastore.py b/src/server/datastore.py index d6d1a75c..e907e97a 100644 --- a/src/server/datastore.py +++ b/src/server/datastore.py @@ -59,7 +59,7 @@ def connection( timeout = config.REDIS.socket_timeout try: if config.REDIS.cache_type == "RedisCache": - base_url = config.REDIS.single.base_url.rstrip("/") + base_url = config.REDIS.single.base_url store = Redis.from_url(f"{base_url}/{db}") else: sentinels = sentinel.Sentinel( diff --git a/src/server/entities/cache.py b/src/server/entities/cache.py new file mode 100644 index 00000000..e05d9927 --- /dev/null +++ b/src/server/entities/cache.py @@ -0,0 +1,52 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Models for updating cache db entities.""" + +import typing as t + +from datetime import datetime + +from pydantic import BaseModel + +from .common import camel_case_config, forbid_extra_config +from .summaries import RepositorySummary + + +class RepositoryCache(RepositorySummary): + """Model for repository cache entity.""" + + updated: datetime | None = None + """The update timestamp of the repository cache entry.""" + + status: RepositoryStatus | None = None + """The status of the cache update task.""" + + +class TaskDetail(BaseModel): + """Model for task detail entity.""" + + results: list[RepositoryCache] + """The list of results from the task.""" + + status: TaskStatus | None = None + """The status of the task.""" + + current: str + """The repository id currently being processed.""" + + done: int + """The number of completed items.""" + + total: int + """The total number of items to process.""" + + model_config = camel_case_config | forbid_extra_config + """Configure to use camelCase aliasing and forbid extra fields.""" + + +type RepositoryStatus = t.Literal["success", "failed"] + + +type TaskStatus = t.Literal["pending", "started", "in_progress", "completed"] diff --git a/src/server/exc.py b/src/server/exc.py index ea1d7b8f..f861e4f0 100644 --- a/src/server/exc.py +++ b/src/server/exc.py @@ -97,7 +97,7 @@ class DatastoreError(InfrastructureError): """ -class TaskExcutionError(DatastoreError): +class TaskExecutionError(DatastoreError): """Exception for task execution errors. Errors caused by issues during task execution. @@ -221,3 +221,10 @@ class InvalidExportError(BulkOperationError): Errors caused by issues during export operations. """ + + +class GroupCacheError(JAIROCloudGroupsManagerError): + """Exception for group cache errors. + + Errors caused by issues in group cache operations. + """ diff --git a/src/server/ext.py b/src/server/ext.py index d0388801..67e3da8d 100644 --- a/src/server/ext.py +++ b/src/server/ext.py @@ -9,6 +9,7 @@ from pathlib import Path from sqlalchemy_utils import database_exists +from weko_group_cache_db.config import setup_config as setup_weko_group_cache_db_config from .api.router import create_api_blueprint from .auth import login_manager @@ -83,6 +84,8 @@ def init_config(self, app: Flask) -> None: app.config.from_mapping(self.config.for_flask) app.config.from_prefixed_env() + setup_weko_group_cache_db_config(self.config.CACHE_DB) + def init_db_app(self, app: Flask) -> None: # noqa: PLR6301 """Initialize the database for the this extension. diff --git a/src/server/messages/error.py b/src/server/messages/error.py index fbb17e09..fab10355 100644 --- a/src/server/messages/error.py +++ b/src/server/messages/error.py @@ -20,7 +20,7 @@ UNSUPPORTED_EXPRESSION = LogMessage( "E002", "Unsupported expression in server configuration; " - "supported: 1. int / flaot literal, 2. literal str for len, " + "supported: 1. int / float literal, 2. literal str for len, " "3. +, -, *, / operators, 4. len, max, min functions.", ) @@ -720,6 +720,32 @@ "Failed to get file path for file (id: %(file_id)s) from database.", ) +GROUP_CACHE_UPDATE_CONFLICT = LogMessage( + "E800", + "The cache update operation is already in progress.", +) + +FAILED_ENQUEUE_CACHE_UPDATE_TASK = LogMessage( + "E801", + "Failed to enqueue cache update task.", +) + +FAILED_FETCH_UPDATE_TASK_STATUS = LogMessage( + "E802", + "Failed to fetch cache update task status.", +) + +FAILED_PARSE_UPDATE_TASK_STATUS = LogMessage( + "E803", + "Failed to parse cache update task status.", +) + +UPDATE_TASK_NOT_RUNNING = LogMessage( + "E804", + "No cache update task is currently running.", +) + + UNNECESSARY_CONTRIB = LogMessage( "E999", "Contrib utilities can only be used in development mode." ) diff --git a/src/server/messages/info.py b/src/server/messages/info.py index a65a805c..97d9c630 100644 --- a/src/server/messages/info.py +++ b/src/server/messages/info.py @@ -204,8 +204,15 @@ "successfully retrieved bulk operation result for history record: %(history_id)s", ) + SUCCESS_UPDATE_PUBLIC_STATUS = LogMessage( "I700", "successfully update public status of history record (id: %(history_id)s) in" " database.", ) + + +GROUP_CACHE_UPDATE_STARTED = LogMessage( + "I800", + "Group cache update task started (operation: %(op)s, task: %(task_id)s).", +) diff --git a/src/server/messages/warning.py b/src/server/messages/warning.py index 45a0dd9f..25a50200 100644 --- a/src/server/messages/warning.py +++ b/src/server/messages/warning.py @@ -65,3 +65,15 @@ "W083", "Failed to delete cache (func %(func)s, id: %(id)s).", ) + + +FAILED_UPDATE_TASK_PROGRESS = LogMessage( + "W800", + "Failed to update current task progress (done: %(done)s, total: %(total)s).", +) + +FAILED_UPDATE_TASK_EXECUT_STATUS = LogMessage( + "W801", + "Failed to update current task execution status " + "(repository: %(rid)s, status: %(status)s, retries: %(retries)s).", +) diff --git a/src/server/services/bulks.py b/src/server/services/bulks.py index 6c530c51..2d892608 100644 --- a/src/server/services/bulks.py +++ b/src/server/services/bulks.py @@ -51,7 +51,7 @@ InvalidFormError, OAuthTokenError, RecordNotFound, - TaskExcutionError, + TaskExecutionError, UnexpectedResponseError, ) from server.messages import E, I @@ -607,7 +607,7 @@ def get_validate_task_result(task_id: str) -> AsyncResult[UUID]: Raises: DatastoreError: If there is an error connecting to the datastore. - TaskExcutionError: If the task with the given ID does not exist. + TaskExecutionError: If the task with the given ID does not exist. """ try: res = validate_upload_data.AsyncResult(task_id) @@ -616,7 +616,7 @@ def get_validate_task_result(task_id: str) -> AsyncResult[UUID]: raise DatastoreError(E.FAILED_CONNECT_REDIS % {"error": str(exc)}) from exc if not res: current_app.logger.error(E.TASK_NOT_FOUND, {"task_id": task_id}) - raise TaskExcutionError(E.TASK_NOT_FOUND % {"task_id": task_id}) + raise TaskExecutionError(E.TASK_NOT_FOUND % {"task_id": task_id}) return res @@ -962,7 +962,7 @@ def get_execute_task_result(task_id: str) -> AsyncResult[UUID]: Raises: DatastoreError: If there is an error connecting to the datastore. - TaskExcutionError: If the task with the given ID does not exist. + TaskExecutionError: If the task with the given ID does not exist. """ try: res = update_users.AsyncResult(task_id) @@ -971,7 +971,7 @@ def get_execute_task_result(task_id: str) -> AsyncResult[UUID]: raise DatastoreError(E.FAILED_CONNECT_REDIS % {"error": str(exc)}) from exc if not res: current_app.logger.error(E.TASK_NOT_FOUND, {"task_id": task_id}) - raise TaskExcutionError(E.TASK_NOT_FOUND % {"task_id": task_id}) + raise TaskExecutionError(E.TASK_NOT_FOUND % {"task_id": task_id}) return res diff --git a/src/server/services/group_caches.py b/src/server/services/group_caches.py new file mode 100644 index 00000000..f1007f66 --- /dev/null +++ b/src/server/services/group_caches.py @@ -0,0 +1,320 @@ +# +# Copyright (C) 2025 National Institute of Informatics. +# + +"""Service module for managing cache groups.""" + +import traceback +import typing as t + +from datetime import datetime +from functools import cache + +from celery import shared_task +from flask import current_app +from pydantic_core import PydanticSerializationError, ValidationError +from redis import RedisError +from weko_group_cache_db import groups as wgcd +from weko_group_cache_db.signals import ( + ExecutedData, + ProgressData as ProgressDataBase, + executed_signal, + progress_signal, +) + +from server.config import config +from server.const import GROUP_CACHE_KEY_PATTERN +from server.datastore import app_cache, group_cache +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.exc import ( + DatastoreError, + GroupCacheError, + RequestConflict, + TaskExecutionError, +) +from server.messages import E, I, W +from server.services import repositories + +from .utils import make_criteria_object, resolve_repository_id + + +if t.TYPE_CHECKING: + from server.entities.summaries import RepositorySummary + + from .utils.search_queries import ( + GroupCacheCriteria, + GroupCacheFilter, + GroupCacheOperation, + ) + + +def get_repository_cache(query: GroupCacheCriteria) -> SearchResult[RepositoryCache]: + """Retrieve repository cache entries based on the provided query. + + Args: + query (GroupCacheCriteria): Query parameters for filtering and pagination. + + Returns: + SearchResult[RepositoryCache]: List of repository cache entries. + """ + repository_query = make_criteria_object( + "repositories", + q=query.q, + k="id", + d="asc", + # when filtering by cache status, + # get all repositories to apply pagination in this app. + p=query.p if not query.f else -1, + l=query.l, + ) + searched = repositories.search(repository_query) + + page_size = searched.page_size + start = (query.p - 1) * page_size if query.p else 0 + end = min(start + page_size, len(searched.resources)) + + results = check_cache_exists( + repositories=searched.resources, + status_filter=query.f, + ) + + resources = results + total = searched.total + if query.f: + # If filtering by status, apply pagination in this app. + resources = results[start:end] + total = len(results) + + return SearchResult( + resources=resources, + total=total, + page_size=page_size, + offset=start + 1, + ) + + +def check_cache_exists( + repositories: list[RepositorySummary], + status_filter: list[GroupCacheFilter] | None = None, +) -> list[RepositoryCache]: + """Check if cache exists for the given list of repositories. + + Args: + repositories (list[RepositorySummary]): List of repository summaries. + status_filter (list | None): List of status filters, e.g., ["e", "n"]. + + Returns: + list[RepositoryCache]: List of repository caches that exist. + """ + result_repositories: list[RepositoryCache] = [] + for repository in repositories: + if not repository.service_url or not repository.service_name: + # service URL and name should exist. + continue # pragma: no cover + + fqdn = t.cast("str", repository.service_url.host) + cache_key = wgcd.cache_key(fqdn) + # when cache exists, `updated_at` is always present. + updated: str | None = group_cache.hget(cache_key, "updated_at") # pyright: ignore[reportAssignmentType] + + repo_cache = RepositoryCache( + id=repository.id, + service_name=repository.service_name, + service_url=repository.service_url, + updated=datetime.fromisoformat(updated) if updated else None, + ) + + if ( + not status_filter + or (updated and "e" in status_filter) + or (not updated and "n" in status_filter) + ): + result_repositories.append(repo_cache) + + return result_repositories + + +@cache +def _unique_progress_key() -> str: + """Generate a unique key for tracking progress in Redis. + + Returns: + str: A unique key for progress tracking. + """ + return config.REDIS.key_prefix + GROUP_CACHE_KEY_PATTERN + + +def update(op: GroupCacheOperation, repository_ids: list[str] | None = None) -> None: + """Update cache groups based on the operation type. + + Args: + op (str): Operation type, either 'all' or 'id-specified'. + repository_ids (list[str]): List of repository IDs. + + Raises: + RequestConflict: If the cache update task is already running. + TaskExecutionError: If there is an error connecting to Redis. + """ + if is_update_task_running(): + raise RequestConflict(E.GROUP_CACHE_UPDATE_CONFLICT) + + repository_ids = repository_ids if op == "id-specified" else [] + query = make_criteria_object( + "repositories", i=repository_ids, l=-1, k="id", d="asc" + ) + repositories_result = repositories.search(query) + fqdn_list = [ + t.cast("str", repo.service_url.host) + for repo in repositories_result.resources + if repo.service_url + ] + + cache_key = _unique_progress_key() + try: + app_cache.delete(cache_key) + app_cache.hset(cache_key, mapping={"status": "pending"}) + task = update_task.apply_async((fqdn_list,)) + except RedisError as exc: + error = E.FAILED_ENQUEUE_CACHE_UPDATE_TASK + raise TaskExecutionError(error) from exc + + current_app.logger.info( + I.GROUP_CACHE_UPDATE_STARTED, {"op": op, "task_id": task.id} + ) + + +@shared_task() +def update_task(fqdn_list: list[str]) -> None: + """Celery task to update cache groups. + + Args: + fqdn_list (list[str]): List of fully qualified domain names. + """ + wgcd.fetch_all( + directory_path=config.CACHE_GROUPS.directory_path, fqdn_list=fqdn_list + ) + + +def is_update_task_running() -> bool: + """Check if a cache update task is currently running. + + Returns: + bool: True if a cache update task is running, False otherwise. + """ + cache_key = _unique_progress_key() + progress_status: str | None = app_cache.hget(cache_key, "status") # pyright: ignore[reportAssignmentType] + + return progress_status in {"pending", "started", "in_progress"} + + +@progress_signal.connect +def handle_progress(_: object, data: ProgressDataBase, **kwargs: object) -> None: # noqa: ARG001 + """Receive progress update signal and update task progress in Redis. + + Args: + _: The sender of the signal. + data (ProgressData): Data containing progress information. + **kwargs: Additional keyword arguments containing task details. + """ + cache_key = _unique_progress_key() + try: + update_dict = data.model_dump(mode="json") + app_cache.hset(cache_key, mapping=update_dict) + except RedisError, PydanticSerializationError: + current_app.logger.warning( + W.FAILED_UPDATE_TASK_PROGRESS, {"done": data.done, "total": data.total} + ) + traceback.print_exc() + + +@executed_signal.connect +def handle_excuted(_: object, data: ExecutedData, **kwargs: object) -> None: # noqa: ARG001 + """Receive executed signal and update task execution status in Redis. + + Args: + _: The sender of the signal. + data (ExecutedData): Data containing executed information. + **kwargs: Additional keyword arguments containing task details. + """ + cache_key = _unique_progress_key() + repository_id = resolve_repository_id(fqdn=data.fqdn) + field_name = f"{repository_id}_{data.retries}" + try: + app_cache.hset(cache_key, mapping={field_name: data.model_dump_json()}) + except RedisError, PydanticSerializationError: + current_app.logger.warning( + W.FAILED_UPDATE_TASK_EXECUT_STATUS, + {"rid": repository_id, "status": data.status, "retries": data.retries}, + ) + traceback.print_exc() + + +def get_task_status() -> TaskDetail | None: + """Get the status of the cache update task. + + Returns: + TaskDetail: + Details of the cache update task. if no task is running, returns None. + + Raises: + GroupCacheError: If there is an error connecting to Redis. + DatastoreError: If there is an error parsing task status data. + """ + cache_key = _unique_progress_key() + try: + raw = app_cache.hgetall(cache_key) + if not raw: + return None + except RedisError as exc: + raise DatastoreError(E.FAILED_FETCH_UPDATE_TASK_STATUS) from exc + + task_data = { + k.decode("utf-8"): v.decode("utf-8") + for k, v in t.cast("dict[bytes, bytes]", raw).items() + } + + try: + progress = ProgressData.model_validate(task_data, extra="ignore") + + results: list[ExecutedData] = [ + ExecutedData.model_validate_json(value) + for key, value in task_data.items() + if key not in {"status", "current", "done", "total"} + ] + repository_ids = [resolve_repository_id(fqdn=result.fqdn) for result in results] + repository_query = make_criteria_object( + "repositories", i=repository_ids, l=len(repository_ids) + ) + searchd = repositories.search(repository_query) + repository_map = {repo.id: repo for repo in searchd.resources} + detail_results = [ + RepositoryCache( + id=r.id, + service_name=r.service_name, + service_url=r.service_url, + updated=result.updated_at, + status=result.status, + ) + for result in results + if (r := repository_map.get(resolve_repository_id(fqdn=result.fqdn))) + ] + + task_status = TaskDetail( + results=detail_results, + status=progress.status, + current=resolve_repository_id(fqdn=progress.current), + done=progress.done, + total=progress.total, + ) + except ValidationError as exc: + raise GroupCacheError(E.FAILED_PARSE_UPDATE_TASK_STATUS) from exc + + return task_status + + +class ProgressData(ProgressDataBase): + """Model for progress data entity.""" + + status: t.Literal["pending", "started", "in_progress", "completed"] # pyright: ignore[reportIncompatibleVariableOverride] + """The status of the cache update task.""" diff --git a/src/server/services/utils/search_queries.py b/src/server/services/utils/search_queries.py index aa2fab43..98d3558b 100644 --- a/src/server/services/utils/search_queries.py +++ b/src/server/services/utils/search_queries.py @@ -947,3 +947,32 @@ def make_criteria_object(resource_type: str, **kwargs: t.Any) -> Criteria: # py attrs[key] = value return t.cast("Criteria", SimpleNamespace(**attrs)) + + +class GroupCacheCriteria(t.Protocol): + """Schema for cache query parameters.""" + + q: t.Annotated[str | None, "term"] = None + """Search term for querying cache entries.""" + + f: t.Annotated[list[GroupCacheFilter] | None, "filter"] = None + """Filter expression for querying cache entries.""" + + p: t.Annotated[int | None, "page"] = None + """Page number for pagination.""" + + l: t.Annotated[int | None, "per"] = None # noqa: E741 + """Number of items per page for pagination.""" + + +type GroupCacheFilter = t.Literal["e", "n"] +"""Group cache filter options: + - “e”: Filter existing cache entries. + - “n”: Filter non-existent cache entries. +""" + +type GroupCacheOperation = t.Literal["all", "id-specified"] +"""Group cache operation options: + - “all”: Update all cache entries. + - “id-specified”: Update cache entries by specified IDs. +""" diff --git a/tests/unit/api/test_bulk.py b/tests/unit/api/test_bulk.py index b6ae24e4..cc7c462c 100644 --- a/tests/unit/api/test_bulk.py +++ b/tests/unit/api/test_bulk.py @@ -11,7 +11,7 @@ from server.api.schemas import BulkBody, ErrorResponse, ExcuteRequest, TargetRepositoryForm, UploadQuery from server.entities.bulk import ExecuteResults, ResultSummary, ValidateResults from server.entities.login_user import LoginUser -from server.exc import FileNotFound, FileValidationError, RecordNotFound, TaskExcutionError +from server.exc import FileNotFound, FileValidationError, RecordNotFound, TaskExecutionError from server.messages import E @@ -91,7 +91,7 @@ def test_validate_status_not_found(app, mocker: MockerFixture): task_id = "task_id" test_func = inspect.unwrap(bulk.validate_status) expected_message = E.TASK_NOT_FOUND % {"task_id": task_id} - mocker.patch("server.services.bulks.get_validate_task_result", side_effect=TaskExcutionError(expected_message)) + mocker.patch("server.services.bulks.get_validate_task_result", side_effect=TaskExecutionError(expected_message)) expected = ErrorResponse(message=expected_message) with app.test_request_context(): result = test_func(task_id) @@ -269,7 +269,7 @@ def test_execute_status_not_found(app, mocker: MockerFixture): test_func = inspect.unwrap(bulk.execute_status) task_id = "task_id" expected_message = E.TASK_NOT_FOUND % {"task_id": task_id} - mocker.patch("server.services.bulks.get_execute_task_result", side_effect=TaskExcutionError(expected_message)) + mocker.patch("server.services.bulks.get_execute_task_result", side_effect=TaskExecutionError(expected_message)) with app.test_request_context(): result = test_func(task_id) assert result[0] == ErrorResponse(message=expected_message) diff --git a/tests/unit/api/test_group_caches.py b/tests/unit/api/test_group_caches.py new file mode 100644 index 00000000..484ebf1f --- /dev/null +++ b/tests/unit/api/test_group_caches.py @@ -0,0 +1,117 @@ +import typing as t + +from flask import Flask + +from server.api import group_caches +from server.api.schemas import CacheQuery, CacheRequest, ErrorResponse +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.exc import InvalidQueryError +from server.messages import E + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_get(app, mocker: MockerFixture, gen_summaries, cached_data, unwrap): + query = CacheQuery(q=None, p=1, l=20, f=[]) + repositories: SearchResult = gen_summaries(20) + search_result = SearchResult( + resources=cached_data(repositories.resources, None, every_other=False), + total=20, + page_size=20, + offset=1, + ) + mock_get_cache = mocker.patch("server.api.group_caches.group_caches.get_repository_cache") + mock_get_cache.return_value = search_result + success = 200 + + result, status = unwrap(group_caches.get)(query) + + assert result == search_result + assert status == success + mock_get_cache.assert_called_once_with(query) + + +def test_get_invalid_query(mocker: MockerFixture, unwrap): + query = CacheQuery(q=None, p=1, l=20, f=[]) + messege = "Invalid query" + mock_search = mocker.patch("server.api.group_caches.group_caches.get_repository_cache") + mock_search.side_effect = InvalidQueryError(messege) + bad_request = 400 + + result, status = unwrap(group_caches.get)(query) + + assert isinstance(result, ErrorResponse) + assert result.message == messege + assert status == bad_request + mock_search.assert_called_once_with(query) + + +def test_post(app: Flask, mocker: MockerFixture, unwrap): + mock_update = mocker.patch("server.api.group_caches.group_caches.update") + ids = ["repo1_example_jp", "repo2_example_jp"] + operation = "all" + accepted = 202 + + result, status = unwrap(group_caches.post)(body=CacheRequest(ids=ids, op=operation)) + + assert not result + assert status == accepted + mock_update.assert_called_once_with(operation, ids) + + +def test_post_conflict(app: Flask, mocker: MockerFixture, unwrap): + mock_update = mocker.patch("server.api.group_caches.group_caches.update") + mock_update.side_effect = group_caches.RequestConflict(E.GROUP_CACHE_UPDATE_CONFLICT) + ids = ["repo1_example_jp", "repo2_example_jp"] + operation = "all" + conflict = 409 + + result, status = unwrap(group_caches.post)(body=CacheRequest(ids=ids, op=operation)) + + assert isinstance(result, ErrorResponse) + assert result.message in str(E.GROUP_CACHE_UPDATE_CONFLICT) + assert status == conflict + mock_update.assert_called_once_with(operation, ids) + + +def test_status(app: Flask, mocker: MockerFixture, unwrap, gen_summaries): + repository = gen_summaries(1).resources[0] + repository_cache = RepositoryCache( + id=repository.id, + service_name=repository.service_name, # pyright: ignore[reportArgumentType], + service_url=repository.service_url, + updated=None, + ) + task_detail = TaskDetail( + results=[repository_cache], + status="in_progress", + current="repo1_example_jp", + done=10, + total=20, + ) + + mock_status = mocker.patch("server.api.group_caches.group_caches.get_task_status") + mock_status.return_value = task_detail + success = 200 + + result, status = unwrap(group_caches.status)() + + assert result == task_detail + assert status == success + mock_status.assert_called_once_with() + + +def test_status_no_task(app: Flask, mocker: MockerFixture, unwrap): + mock_status = mocker.patch("server.api.group_caches.group_caches.get_task_status") + mock_status.return_value = None + bad_request = 400 + + result, status = unwrap(group_caches.status)() + + assert isinstance(result, ErrorResponse) + assert result.message in str(E.UPDATE_TASK_NOT_RUNNING) + assert status == bad_request + mock_status.assert_called_once_with() diff --git a/tests/unit/api/test_router.py b/tests/unit/api/test_router.py new file mode 100644 index 00000000..bf6bc673 --- /dev/null +++ b/tests/unit/api/test_router.py @@ -0,0 +1,111 @@ +import typing as t + +from flask import Blueprint + +from server.api.router import create_api_blueprint + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_create_api_blueprint(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "test", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_blueprint = mocker.MagicMock(Blueprint) + mock_module = mocker.MagicMock() + mock_module.bp = mock_blueprint + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.test") + mock_register_blueprint.assert_called_once_with( + mock_blueprint, + url_prefix="/test", + ) + + +def test_create_api_blueprint_multiple_words(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "cache_groups", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_blueprint = mocker.MagicMock(Blueprint) + mock_module = mocker.MagicMock() + mock_module.bp = mock_blueprint + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.cache_groups") + mock_register_blueprint.assert_called_once_with( + mock_blueprint, + url_prefix="/cache-groups", + ) + + +def test_create_api_blueprint_no_bp(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[ + (None, "no_bp_module", None), + ], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + mock_module = mocker.MagicMock() + mock_import_module.return_value = mock_module + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_called_once_with("server.api.no_bp_module") + mock_register_blueprint.assert_not_called() + + +def test_create_api_blueprint_no_modules(mocker: MockerFixture): + mock_iter_modules = mocker.patch( + "server.api.router.iter_modules", + return_value=[], + ) + mock_import_module = mocker.patch( + "server.api.router.import_module", + ) + mock_register_blueprint = mocker.patch( + "server.api.router.Blueprint.register_blueprint", + ) + + bp = create_api_blueprint() + + assert bp.name == "api" + mock_iter_modules.assert_called_once() + mock_import_module.assert_not_called() + mock_register_blueprint.assert_not_called() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 0c29ad5a..e7434fe1 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,15 +1,23 @@ +import inspect import typing as t from pathlib import Path import pytest +from pydantic import HttpUrl + from server import const from server.config import RuntimeConfig +from server.entities.cache import RepositoryCache +from server.entities.search_request import SearchResult +from server.entities.summaries import RepositorySummary from server.factory import create_app if t.TYPE_CHECKING: + from datetime import datetime + from flask import Flask from pytest_mock import MockerFixture @@ -35,66 +43,94 @@ def test_config(): db_host = "postgres" if is_running_in_docker() else "localhost" redis_host = "redis" if is_running_in_docker() else "localhost" amqp_host = "rabbitmq" if is_running_in_docker() else "localhost" - return RuntimeConfig.model_validate({ - "SECRET_KEY": "test_secret_key", - "LOG": { - "level": "INFO", - }, - "SP": { - "connector_id": "jairocloud-groups-manager_test", - "entity_id": "https://test/shibboleth-sp", - "crt": "/test/server.crt", - "key": "/test/server.key", - }, - "MAP_CORE": { - "base_url": "https://mapcore.test.jp", - "timeout": 3, - }, - "REPOSITORIES": { - "id_patterns": { - "sp_connector": "jc_{repository_id}_test", + return RuntimeConfig.model_validate( + { + "SECRET_KEY": "test_secret_key", + "LOG": { + "level": "DEBUG", }, - }, - "GROUPS": { - "id_patterns": { - "system_admin": "jc_roles_sysadm_test", - "repository_admin": "jc_{repository_id}_ro_radm_test", - "community_admin": "jc_{repository_id}_ro_cadm_test", - "contributor": "jc_{repository_id}_ro_cont_test", - "general_user": "jc_{repository_id}_ro_user_test", - "user_defined": "jc_{repository_id}_gr_{user_defined_id}_test", + "SP": { + "connector_id": "jairocloud-groups-manager_test", + "entity_id": "https://test/shibboleth-sp", + "crt": "/test/server.crt", + "key": "/test/server.key", }, - "name_patterns": { - "system_admin": "ジャイロクラウドシステム管理者_テスト", - "repository_admin": "{repository_name}管理者_テスト", - "community_admin": "{repository_name}コミュニティ管理者_テスト", - "contributor": "{repository_name}投稿ユーザー_テスト", - "general_user": "{repository_name}一般ユーザー_テスト", + "MAP_CORE": { + "base_url": "https://mapcore.test.jp", + "timeout": 3, }, - "max_id_length": "50 - len('jc_') - len('_gr_')", - }, - "POSTGRES": {"db": "jctest", "host": db_host}, - "USERS": { - "export_fields": [ - "id", - "user_name", - "groups[].id", - "groups[].name", - "role", - "edu_person_principal_names[]", - "preferred_language", - "emails[]", - ] - }, - "REDIS": { - "cache_type": "RedisCache", - "single": {"base_url": f"redis://{redis_host}:6379/0"}, - "key_prefix": "jcgroups-test", + "REPOSITORIES": { + "id_patterns": { + "sp_connector": "jc_{repository_id}_test", + }, + }, + "GROUPS": { + "id_patterns": { + "system_admin": "jc_roles_sysadm_test", + "repository_admin": "jc_{repository_id}_ro_radm_test", + "community_admin": "jc_{repository_id}_ro_cadm_test", + "contributor": "jc_{repository_id}_ro_cont_test", + "general_user": "jc_{repository_id}_ro_user_test", + "user_defined": "jc_{repository_id}_gr_{user_defined_id}_test", + }, + "name_patterns": { + "system_admin": "ジャイロクラウドシステム管理者_テスト", + "repository_admin": "{repository_name}管理者_テスト", + "community_admin": "{repository_name}コミュニティ管理者_テスト", + "contributor": "{repository_name}投稿ユーザー_テスト", + "general_user": "{repository_name}一般ユーザー_テスト", + }, + "max_id_length": "50 - len('jc_') - len('_gr_')", + }, + "POSTGRES": {"db": "jctest", "host": db_host}, + "USERS": { + "export_fields": [ + "id", + "user_name", + "groups[].id", + "groups[].name", + "role", + "edu_person_principal_names[]", + "preferred_language", + "emails[]", + ] + }, + "REDIS": { + "cache_type": "RedisCache", + "key_prefix": "jcgroups-test-", + "single": {"base_url": f"redis://{redis_host}:6379/0"}, + "sentinel": { + "nodes": [ + {"host": "sentinel-1", "port": 26379}, + {"host": "sentinel-2", "port": 26379}, + ], + }, + }, + "RABBITMQ": {"url": f"amqp://guest:guest@{amqp_host}:5672//"}, + "STORAGE": {"local": {"temporary": "/var/tmp/jcgroups"}}, # noqa: S108 + "CACHE_GROUPS": { + "cache_key_suffix": "_gakunin_groups", + "api_endpoint": "https://sample.gakunin.jp/api/groups/", + "directory_path": "/var/mnt", + }, + "FEATURES": {"search_only_username": False, "enable_bulk_operation": True}, }, - "RABBITMQ": {"url": f"amqp://guest:guest@{amqp_host}:5672//"}, - "STORAGE": {"local": {"temporary": "/var/tmp/jcgroups"}}, # noqa: S108 - "FEATURES": {"search_only_username": False, "enable_bulk_operation": True}, - }) + ) + + +def mock_redis(mocker: MockerFixture): + mock_redis = mocker.patch("server.datastore.Redis") + mock_redis_instance = mock_redis.from_url.return_value + mock_redis_instance.ping.return_value = True + return mock_redis_instance + + +@pytest.fixture +def unwrap(): + def _unwrap(f: t.Callable) -> t.Callable: + return inspect.unwrap(f) + + return _unwrap @pytest.fixture(autouse=True) @@ -134,3 +170,49 @@ def base_app(instance_path, test_config): def app(base_app: Flask): with base_app.app_context(): yield base_app + + +@pytest.fixture +def gen_summaries(): + def _data(num: int) -> SearchResult[RepositorySummary]: + resources = [ + RepositorySummary( + id=f"repo_{i}", + service_name=f"Repository {i}", + service_url=HttpUrl(f"https://repo{i}.example.jp"), + service_id=f"jc_repo_{i}_sp", + ) + for i in range(1, num + 1) + ] + return SearchResult(resources=resources, total=num, page_size=20, offset=1) + + return _data + + +@pytest.fixture +def cache_keys(): + def _keys(fqdn_list: list[str]) -> list[bytes]: + return [f"{fqdn.replace('-', '_').replace('.', '_')}_gakunin_groups".encode() for fqdn in fqdn_list] + + return _keys + + +@pytest.fixture +def cached_data(): + def _data( + repositories: list[RepositorySummary], + now: datetime, + *, + every_other: bool, + ) -> list[RepositoryCache]: + return [ + RepositoryCache( + id=repositories[i].id, + service_name=repositories[i].service_name, # pyright: ignore[reportArgumentType], + service_url=repositories[i].service_url, + updated=now if not every_other or i % 2 == 0 else None, + ) + for i in range(len(repositories)) + ] + + return _data diff --git a/tests/unit/services/test_bulk.py b/tests/unit/services/test_bulk.py index 3d090ab4..3752d48e 100644 --- a/tests/unit/services/test_bulk.py +++ b/tests/unit/services/test_bulk.py @@ -35,7 +35,7 @@ InvalidFormError, OAuthTokenError, RecordNotFound, - TaskExcutionError, + TaskExecutionError, UnexpectedResponseError, ) from server.messages import E @@ -773,7 +773,7 @@ def test_get_validate_task_result_none(app, mocker: MockerFixture): task_id = "test_task_id" mocker.patch("server.services.bulks.validate_upload_data.AsyncResult", return_value=None) expected = E.TASK_NOT_FOUND % {"task_id": "test_task_id"} - with pytest.raises(TaskExcutionError) as exc: + with pytest.raises(TaskExecutionError) as exc: bulks.get_validate_task_result(task_id) assert str(exc.value) == str(expected) @@ -1254,7 +1254,7 @@ def test_get_execute_task_result_none(app, mocker: MockerFixture): task_id = "test_task_id" mocker.patch("server.services.bulks.update_users.AsyncResult", return_value=None) expected = E.TASK_NOT_FOUND % {"task_id": "test_task_id"} - with pytest.raises(TaskExcutionError) as exc: + with pytest.raises(TaskExecutionError) as exc: bulks.get_execute_task_result(task_id) assert str(exc.value) == str(expected) diff --git a/tests/unit/services/test_group_caches.py b/tests/unit/services/test_group_caches.py new file mode 100644 index 00000000..30230601 --- /dev/null +++ b/tests/unit/services/test_group_caches.py @@ -0,0 +1,647 @@ +import typing as t + +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from redis import RedisError +from weko_group_cache_db.config import setup_config as setup_wgcd_config +from weko_group_cache_db.signals import ExecutedData, ProgressData + +from server.api.schemas import CacheQuery +from server.config import config +from server.entities.cache import RepositoryCache, TaskDetail +from server.entities.search_request import SearchResult +from server.entities.summaries import RepositorySummary +from server.exc import DatastoreError, GroupCacheError, RequestConflict +from server.messages import E, W +from server.services.group_caches import ( + get_repository_cache, + get_task_status, + handle_excuted, + handle_progress, + is_update_task_running, + update, + update_task, +) + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +@pytest.fixture(autouse=True) +def setup_config(app): + setup_wgcd_config(config.CACHE_GROUPS) + + +def test_get_repository_cache(app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=True) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + expect = SearchResult(total=20, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_multi_scan(mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=True) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + expect = SearchResult(total=20, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_all_cache_not_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories: SearchResult[RepositorySummary] = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + now = datetime.now(UTC) + caches = cached_data(repositories.resources, now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + result = get_repository_cache(query) + expect = SearchResult( + total=num, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num = 30 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + now = datetime.now(UTC) + caches = cached_data(repositories.resources[:20], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + expect = SearchResult(total=num, resources=caches, page_size=20, offset=1) + result = get_repository_cache(query) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_all_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num = 30 + query = CacheQuery(f=["e"], l=20, p=2) + repositories = gen_summaries(num) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[20:30], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + result = get_repository_cache(query) + expect = SearchResult(total=30, resources=caches, page_size=20, offset=21) + + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num + + mocker.stopall() + + +def test_get_repository_cache_empty_cache(app, mocker: MockerFixture, gen_summaries, datastore): + num_repo = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(20) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + _, _, group_cache = datastore + group_cache.hget.return_value = None + + result = get_repository_cache(query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_half_cache(app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore): + num_repo = 20 + query = CacheQuery(f=["e"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[::2], now, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + result = get_repository_cache(query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + mock_search.assert_called_once() + assert result == expect + + assert group_cache.hget.call_count == num_repo + mocker.stopall() + + +def test_get_repository_cache_no_cache_not_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources, None, every_other=False) + _, _, group_cache = datastore + group_cache.hget.return_value = None + + expect = SearchResult(total=num_repo, resources=caches, page_size=20, offset=1) + + result = get_repository_cache(query) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size( + app, mocker: MockerFixture, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 30 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources[:20], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = None + result = get_repository_cache(query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_exceeding_page_size_next_page( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 30 + query = CacheQuery(f=["n"], l=20, p=2) + repositories = gen_summaries(30) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + caches = cached_data(repositories.resources[20:30], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.return_value = None + + result = get_repository_cache(query) + expect = SearchResult( + total=30, + resources=caches, + page_size=20, + offset=21, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_all_cache(mocker: MockerFixture, app, gen_summaries, cache_keys, datastore): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + now = datetime.now(UTC) + repositories: SearchResult = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + + _, _, group_cache = datastore + group_cache.hget.return_value = now.isoformat() + + result = get_repository_cache(query) + expect = SearchResult( + total=0, + resources=[], + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_get_repository_cache_no_cache_half_cache( + mocker: MockerFixture, app, gen_summaries, cache_keys, cached_data, datastore +): + num_repo = 20 + query = CacheQuery(f=["n"], l=20, p=1) + repositories = gen_summaries(num_repo) + mock_search = mocker.patch("server.services.group_caches.repositories.search", return_value=repositories) + keys = cache_keys([repo.service_url.host for repo in repositories.resources[::2]]) + now = datetime.now(UTC) + caches = cached_data(repositories.resources[1::2], None, every_other=False) + + _, _, group_cache = datastore + group_cache.hget.side_effect = lambda key, _: now.isoformat() if key.encode() in keys else None + + result = get_repository_cache(query) + expect = SearchResult( + total=10, + resources=caches, + page_size=20, + offset=1, + ) + assert result == expect + mock_search.assert_called_once() + assert group_cache.hget.call_count == num_repo + + mocker.stopall() + + +def test_update_all(app, mocker: MockerFixture, datastore, gen_summaries): + + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + query = SimpleNamespace(q=None, i=[], p=None, l=-1, k="id", d="asc") + + app_cache, _, _ = datastore + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "all" + + update(op, ids) + + mock_check.assert_called_once() + mock_search.assert_called_once_with(query) + mock_update_task.assert_called_once_with((fqdn_list,)) + app_cache.delete.assert_called_once_with("jcgroups-test-weko-group-cache-db") + app_cache.hset.assert_called_once_with("jcgroups-test-weko-group-cache-db", mapping={"status": "pending"}) + mocker.stopall() + + +def test_update_id_specified(app, mocker: MockerFixture, datastore, gen_summaries): + + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "id-specified" + query = SimpleNamespace(q=None, i=ids, p=None, l=-1, k="id", d="asc") + + app_cache, _, _ = datastore + + update(op, ids) + + mock_check.assert_called_once() + mock_search.assert_called_once_with(query) + mock_update_task.assert_called_once_with((fqdn_list,)) + app_cache.delete.assert_called_once_with("jcgroups-test-weko-group-cache-db") + app_cache.hset.assert_called_once_with("jcgroups-test-weko-group-cache-db", mapping={"status": "pending"}) + mocker.stopall() + + +def test_update_raises_task_running(app, mocker: MockerFixture): + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = True + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + + fqdn_list = ["example.com"] + op = "all" + + with pytest.raises(RequestConflict, match=str(E.GROUP_CACHE_UPDATE_CONFLICT)): + update(op, fqdn_list) + + mock_check.assert_called_once() + mock_update_task.assert_not_called() + + mocker.stopall() + + +def test_update_raises_failed_task_running(app, mocker: MockerFixture, datastore, gen_summaries): + mock_check = mocker.patch("server.services.group_caches.is_update_task_running") + mock_check.return_value = False + mock_update_task = mocker.patch("server.services.group_caches.update_task.apply_async") + mock_update_task.side_effect = RedisError("Failed to connect to Redis.") + + mock_search = mocker.patch("server.services.group_caches.repositories.search") + repositories = SearchResult( + resources=[gen_summaries(1).resources[0]], + total=1, + page_size=1, + offset=1, + ) + mock_search.return_value = repositories + + app_cache, _, _ = datastore + + ids = [repositories.resources[0].id] + fqdn_list = [repositories.resources[0].service_url.host] + op = "all" + + with pytest.raises(DatastoreError, match=str(E.FAILED_ENQUEUE_CACHE_UPDATE_TASK)): + update(op, ids) + + mock_check.assert_called_once() + app_cache.hset.assert_called_once_with("jcgroups-test-weko-group-cache-db", mapping={"status": "pending"}) + + mock_update_task.assert_called_once_with((fqdn_list,)) + + mocker.stopall() + + +def test_update_task_all(app, mocker: MockerFixture, gen_summaries, unwrap): + repositories = gen_summaries(1).resources + mock_fetch_all = mocker.patch("server.services.group_caches.wgcd.fetch_all") + + fqdn_list = [repositories[0].service_url.host] + + unwrap(update_task)(fqdn_list) + mock_fetch_all.assert_called_once_with( + directory_path=config.CACHE_GROUPS.directory_path, + fqdn_list=fqdn_list, + ) + mocker.stopall() + + +def test_is_update_task_running_pending(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "pending" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-test-weko-group-cache-db", "status") + + +def test_is_update_task_running_started(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "started" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-test-weko-group-cache-db", "status") + + +def test_is_update_task_running_in_progress(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "in_progress" + + result = is_update_task_running() + assert result is True + app_cache.hget.assert_called_once_with("jcgroups-test-weko-group-cache-db", "status") + + +def test_is_update_task_running_completed(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = "completed" + + result = is_update_task_running() + assert result is False + app_cache.hget.assert_called_once_with("jcgroups-test-weko-group-cache-db", "status") + + +def test_is_update_task_running_not_exists(app, datastore): + app_cache, _, _ = datastore + app_cache.hget.return_value = None + + result = is_update_task_running() + assert result is False + app_cache.hget.assert_called_once_with("jcgroups-test-weko-group-cache-db", "status") + + +def test_handle_progress(app, mocker: MockerFixture, unwrap, datastore): + data = ProgressData(status="in_progress", total=10, done=5, current="example.com") + app_cache, _, _ = datastore + + unwrap(handle_progress)(None, data) + + cache_key = "jcgroups-test-weko-group-cache-db" + app_cache.hset.assert_called_once_with(cache_key, mapping=data.model_dump(mode="json")) + + +def test_handle_progress_redis_error(app, mocker: MockerFixture, unwrap, datastore, caplog): + data = ProgressData(status="in_progress", total=10, done=5, current="example.com") + app_cache, _, _ = datastore + app_cache.hset.side_effect = RedisError("Redis error") + + unwrap(handle_progress)(None, data) + + cache_key = "jcgroups-test-weko-group-cache-db" + app_cache.hset.assert_called_once_with(cache_key, mapping=data.model_dump(mode="json")) + + assert str(W.FAILED_UPDATE_TASK_PROGRESS % {"done": 5, "total": 10}) in caplog.text + + +def test_handle_excuted(app, mocker: MockerFixture, unwrap, datastore): + data = ExecutedData( + fqdn="example.com", + status="success", + retries=0, + error_type=None, + error_message=None, + updated_at=datetime.now(UTC), + ) + app_cache, _, _ = datastore + + unwrap(handle_excuted)(None, data) + + cache_key = "jcgroups-test-weko-group-cache-db" + field_name = "example_com_0" + app_cache.hset.assert_called_once_with(cache_key, mapping={field_name: data.model_dump_json()}) + + +def test_handle_excuted_redis_error(app, mocker: MockerFixture, unwrap, datastore, caplog): + data = ExecutedData( + fqdn="example.com", + status="success", + retries=0, + error_type=None, + error_message=None, + updated_at=datetime.now(UTC), + ) + app_cache, _, _ = datastore + app_cache.hset.side_effect = RedisError("Redis error") + + unwrap(handle_excuted)(None, data) + + cache_key = "jcgroups-test-weko-group-cache-db" + field_name = "example_com_0" + app_cache.hset.assert_called_once_with(cache_key, mapping={field_name: data.model_dump_json()}) + + assert ( + str(W.FAILED_UPDATE_TASK_EXECUT_STATUS % {"rid": "example_com", "status": "success", "retries": 0}) + in caplog.text + ) + + +def test_get_task_status(app, mocker: MockerFixture, unwrap, datastore): + task_data = { + b"current": b"example.com", + b"status": b"in_progress", + b"done": b"5", + b"total": b"10", + b"example_com_0": b'{"fqdn": "example.com", "status": "success", "updated_at": "2026-01-01T00:00:00Z"}', + } + ids = ["example_com"] + + app_cache, _, _ = datastore + app_cache.hgetall.return_value = task_data + mocker.patch("server.services.group_caches.make_criteria_object") + mock_query = MagicMock() + mock_query.i = ids + mock_search = mocker.patch("server.services.group_caches.repositories.search") + mock_search.return_value = SearchResult( + resources=[RepositorySummary(id="example_com", service_name="Example Repository")], + total=1, + page_size=1, + offset=1, + ) + + result = unwrap(get_task_status)() + + expect_result = [ + RepositoryCache( + id="example_com", + service_name="Example Repository", + updated=datetime.fromisoformat("2026-01-01T00:00:00+00:00"), + status="success", + ) + ] + + assert result == TaskDetail(results=expect_result, status="in_progress", current="example_com", total=10, done=5) + + app_cache.hgetall.assert_called_once_with("jcgroups-test-weko-group-cache-db") + app_cache.delete.assert_not_called() + + +def test_get_task_status_not_running(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.return_value = {} + app_cache.hget.return_value = "completed" + + result = unwrap(get_task_status)() + + assert result is None + + +def test_get_task_status_no_task(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.return_value = {} + + result = unwrap(get_task_status)() + + assert result is None + + +def test_get_task_status_redis_error(app, unwrap, datastore): + app_cache, _, _ = datastore + app_cache.hgetall.side_effect = RedisError("Redis error") + + with pytest.raises(DatastoreError, match=str(E.FAILED_FETCH_UPDATE_TASK_STATUS)): + unwrap(get_task_status)() + + +def test_get_task_status_parse_error(app, unwrap, datastore): + app_cache, _, _ = datastore + + cache_data = { + b"current": b"example.com", + b"done": b"5", + b"total": b"10", + b"example_com_0": b"invalid_json", + } + app_cache.hgetall.return_value = cache_data + + error = str(E.FAILED_PARSE_UPDATE_TASK_STATUS) + with pytest.raises(GroupCacheError, match=error): + unwrap(get_task_status)() diff --git a/tests/unit/test_ext.py b/tests/unit/test_ext.py new file mode 100644 index 00000000..5adfbae3 --- /dev/null +++ b/tests/unit/test_ext.py @@ -0,0 +1,22 @@ +import typing as t + +from server.ext import JAIROCloudGroupsManager + + +if t.TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_init_config(mocker: MockerFixture): + mock_setup_config = mocker.patch( + "server.ext.setup_config", + ) + mock_cache_db_setup_config = mocker.patch( + "server.ext.setup_weko_group_cache_db_config", + ) + mock_app = mocker.MagicMock() + + ext = JAIROCloudGroupsManager() + ext.init_config(mock_app) + mock_setup_config.assert_called_once() + mock_cache_db_setup_config.assert_called_once_with(ext.config.CACHE_DB) diff --git a/uv.lock b/uv.lock index 2c4d4e96..2a376dcb 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "billiard" version = "4.2.4" @@ -322,6 +331,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] +[[package]] +name = "inflect" +version = "7.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197, upload-time = "2024-12-28T17:11:15.931Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -356,6 +378,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "requests" }, { name = "sqlalchemy-utils" }, + { name = "weko-group-cache-db" }, ] [package.dev-dependencies] @@ -381,6 +404,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.13.1" }, { name = "requests", specifier = ">=2.32.5" }, { name = "sqlalchemy-utils", specifier = ">=0.42.1" }, + { name = "weko-group-cache-db", git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop" }, ] [package.metadata.requires-dev] @@ -425,6 +449,18 @@ redis = [ { name = "redis" }, ] +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -455,6 +491,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + [[package]] name = "mslex" version = "1.3.0" @@ -747,6 +801,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rich" +version = "14.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/84/4831f881aa6ff3c976f6d6809b58cdfa350593ffc0dc3c58f5f6586780fb/rich-14.3.1.tar.gz", hash = "sha256:b8c5f568a3a749f9290ec6bddedf835cec33696bfc1e48bcfecb276c7386e4b8", size = 230125, upload-time = "2026-01-24T21:40:44.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/2a/a1810c8627b9ec8c57ec5ec325d306701ae7be50235e8fd81266e002a3cc/rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e", size = 309952, upload-time = "2026-01-24T21:40:42.969Z" }, +] + +[[package]] +name = "rich-click" +version = "1.9.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/50/1497dbc52297d6759451bf5a991e9b2d0a122a5d33ac8cd057f81cb9910a/rich_click-1.9.6.tar.gz", hash = "sha256:463bd3dbef54a812282bfa93dde80c471bce359823fc1301be368eab63391cb2", size = 74777, upload-time = "2026-01-22T02:43:58.374Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/87/508930def644be9fb86fec63520151921061c152289b98798017a498d678/rich_click-1.9.6-py3-none-any.whl", hash = "sha256:e78d71e3f73a55548e573ccfd964e18503936e2e736a4a1f74c6c29479a2a054", size = 71430, upload-time = "2026-01-22T02:43:56.939Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -831,6 +912,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] +[[package]] +name = "typeguard" +version = "4.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/68/71c1a15b5f65f40e91b65da23b8224dad41349894535a97f63a52e462196/typeguard-4.4.4.tar.gz", hash = "sha256:3a7fd2dffb705d4d0efaed4306a704c89b9dee850b688f060a8b1615a79e5f74", size = 75203, upload-time = "2025-06-18T09:56:07.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/a9/e3aee762739c1d7528da1c3e06d518503f8b6c439c35549b53735ba52ead/typeguard-4.4.4-py3-none-any.whl", hash = "sha256:b5f562281b6bfa1f5492470464730ef001646128b180769880468bd84b68b09e", size = 34874, upload-time = "2025-06-18T09:56:05.999Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -900,6 +993,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, ] +[[package]] +name = "weko-group-cache-db" +version = "1.0.0rc6" +source = { git = "https://github.com/ivis-weko3-dev/weko-group-cache-db.git?rev=develop#a8d12f62b486e8805a7b9dab059d8be4f9ce1a10" } +dependencies = [ + { name = "backoff" }, + { name = "blinker" }, + { name = "inflect" }, + { name = "pydantic-settings" }, + { name = "redis" }, + { name = "requests" }, + { name = "rich-click" }, + { name = "werkzeug" }, +] + [[package]] name = "werkzeug" version = "3.1.5"