From e9cfd945cb76f729f271a7285e7b27369b07486b Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 30 Sep 2025 10:14:06 +1000 Subject: [PATCH 1/3] temp --- src/aaz_dev/command/api/_cmds.py | 143 +++++++++++++++++++++++++++++-- 1 file changed, 134 insertions(+), 9 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index a1947d74..c243642f 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -8,10 +8,15 @@ from flask import Blueprint from command.controller.specs_manager import AAZSpecsManager -from command.templates import get_templates -from swagger.utils.tools import swagger_resource_path_to_resource_id +from command.controller.workspace_manager import WorkspaceManager +from command.model.configuration import CMDHelp +from swagger.controller.specs_manager import SwaggerSpecsManager +from swagger.model.specs import TypeSpecResourceProvider from swagger.utils.source import SourceTypeEnum +from swagger.utils.tools import swagger_resource_path_to_resource_id + from utils.config import Config +from utils.exceptions import InvalidAPIUsage logger = logging.getLogger('aaz') @@ -80,13 +85,6 @@ def resource_id_type(value): help="The path to export the workspace for modification." ) def generate_command_models_from_swagger(swagger_tag, workspace_path=None): - from swagger.controller.specs_manager import SwaggerSpecsManager - from command.controller.specs_manager import AAZSpecsManager - from command.controller.workspace_manager import WorkspaceManager - from utils.config import Config - from utils.exceptions import InvalidAPIUsage - from command.model.configuration import CMDHelp - try: swagger_specs = SwaggerSpecsManager() aaz_specs = AAZSpecsManager() @@ -162,6 +160,133 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): sys.exit(1) +@bp.cli.command("generate-all", short_help="Fully generate data model from swagger, mainly used in https://github.com/magodo/aaz-metadata.") +@click.option( + "--swagger-path", '-s', + type=click.Path(file_okay=False, dir_okay=True, readable=True, resolve_path=True), + default=Config.SWAGGER_PATH, + callback=Config.validate_and_setup_swagger_path, + expose_value=False, + help="The local path of azure-rest-api-specs repo. Official repo is https://github.com/Azure/azure-rest-api-specs" +) +@click.option( + "--aaz-path", '-a', + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + default=Config.AAZ_PATH, + required=not Config.AAZ_PATH, + callback=Config.validate_and_setup_aaz_path, + expose_value=False, + help="The local path of aaz repo." +) +@click.option( + "--module", '-m', + default=Config.DEFAULT_SWAGGER_MODULE, + callback=Config.validate_and_setup_default_swagger_module, + expose_value=False, + help="The name of swagger module." +) +def generate_all(): + def to_aaz(module_name): + try: + swagger_specs = SwaggerSpecsManager() + module_manager = swagger_specs.get_module_manager(Config.DEFAULT_PLANE, module_name) + + rps = module_manager.get_resource_providers() + for r in rps: + if isinstance(r, TypeSpecResourceProvider): + continue + + rp = module_manager.get_openapi_resource_provider(r.name) + + for k in r.tags: + resource_map = rp.get_resource_map_by_tag(k) + if not resource_map: + raise InvalidAPIUsage(f"Tag `{k}` is not exist") + + version_resource_map = {} + for resource_id, version_map in resource_map.items(): + v_list = [v for v in version_map] + if len(v_list) > 1: + raise InvalidAPIUsage( + f"Tag `{k}` contains multiple api versions of one resource", + payload={ + "Resource": resource_id, + "versions": v_list, + } + ) + + v = v_list[0] + resource = version_map[v] + has_patch = "patch" in resource.operations.values() + has_generic_update = "get" in resource.operations.values() and "put" in resource.operations.values() + + update_by = "PatchOnly" if has_patch else ("None" if has_generic_update else None) + + if v not in version_resource_map: + version_resource_map[v] = [] + + new_resource = {"id": resource_id} + if update_by: + new_resource["options"] = {"update_by": update_by} + version_resource_map[v].append(new_resource) + + mod_names = module_name + ws = WorkspaceManager.new( + name=module_name, + plane=Config.DEFAULT_PLANE, + folder=WorkspaceManager.IN_MEMORY, + mod_names=mod_names, + resource_provider=rp.name, + swagger_manager=swagger_specs, + aaz_manager=AAZSpecsManager(), + source=SourceTypeEnum.OpenAPI, + ) + for version, resources in version_resource_map.items(): + ws.add_new_resources_by_swagger( + mod_names=mod_names, version=version, resources=resources + ) + + # provide default short summary + for node in ws.iter_command_tree_nodes(): + if not node.help: + node.help = CMDHelp() + if not node.help.short: + node.help.short = f"Manage {node.names[-1]}" + + for leaf in ws.iter_command_tree_leaves(): + if not leaf.help: + leaf.help = CMDHelp() + if not leaf.help.short: + n = leaf.names[-1] + n = n[0].upper() + n[1:] + leaf.help.short = f"{n} {leaf.names[-2]}" + + if not ws.is_in_memory: + ws.save() + + ws.generate_to_aaz() + + except InvalidAPIUsage as err: + logger.error(err, exc_info=True) + sys.exit(1) + + except ValueError as err: + logger.error(err, exc_info=True) + sys.exit(1) + + if Config.DEFAULT_SWAGGER_MODULE: + to_aaz(Config.DEFAULT_SWAGGER_MODULE) + + else: + spec_path = os.path.join(Config.SWAGGER_PATH, "specification") + for item in os.listdir(spec_path): + curr_path = os.path.join(spec_path, item) + if os.path.isfile(curr_path): + continue + + to_aaz(item) + + @bp.cli.command("verify", short_help="Verify data consistency within `aaz` repository.") @click.option( "--aaz-path", "-a", From fcfc25587afdc94a193cb1f404d3716efe79d6ee Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 2 Oct 2025 14:50:25 +1000 Subject: [PATCH 2/3] feat: add generate_all --- src/aaz_dev/command/api/_cmds.py | 168 ++++++++++++++----------------- 1 file changed, 74 insertions(+), 94 deletions(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index c243642f..e6616262 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -186,105 +186,85 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): help="The name of swagger module." ) def generate_all(): + def update_strategy(resource): + has_patch = "patch" in resource.operations.values() + # has_generic_update = {"get", "put"}.issubset(resource.operations.values()) + if has_patch: + return "PatchOnly" + # if has_generic_update: # fallback to get + put + # return "None" + return None + + def normalize_resource_map(resource_map): + version_resource_map = defaultdict(list) + for resource_id, version_map in resource_map.items(): + for version in sorted(version_map.keys())[-4:]: # only care about the latest 4 versions + resource = {"id": resource_id} + update_by = update_strategy(version_map[version]) + if update_by: + resource["options"] = {"update_by": update_by} + + version_resource_map[version].append(resource) + + return version_resource_map + def to_aaz(module_name): - try: - swagger_specs = SwaggerSpecsManager() - module_manager = swagger_specs.get_module_manager(Config.DEFAULT_PLANE, module_name) + module_manager = SwaggerSpecsManager().get_module_manager(Config.DEFAULT_PLANE, module_name) + rps = module_manager.get_resource_providers() - rps = module_manager.get_resource_providers() - for r in rps: - if isinstance(r, TypeSpecResourceProvider): + for r in rps: + if isinstance(r, TypeSpecResourceProvider): + continue + + rp = module_manager.get_openapi_resource_provider(r.name) + version_resource_map = normalize_resource_map(rp.get_resource_map()) + + for version, resources in version_resource_map.items(): + ws = WorkspaceManager.new( + name=module_name, + plane=Config.DEFAULT_PLANE, + folder=WorkspaceManager.IN_MEMORY, + mod_names=module_name, + resource_provider=rp.name, + swagger_manager=SwaggerSpecsManager(), + aaz_manager=AAZSpecsManager(), + source=SourceTypeEnum.OpenAPI, + ) + ws.add_new_resources_by_swagger(mod_names=module_name, version=version, resources=resources) + + # provide default short summary + for node in ws.iter_command_tree_nodes(): + if not node.help: + node.help = CMDHelp() + if not node.help.short: + node.help.short = f"Manage {node.names[-1]}." + + for leaf in ws.iter_command_tree_leaves(): + if not leaf.help: + leaf.help = CMDHelp() + if not leaf.help.short: + n = leaf.names[-1] + n = n[0].upper() + n[1:] + leaf.help.short = f"{n} {leaf.names[-2]}." + + ws.generate_to_aaz() + + try: + if Config.DEFAULT_SWAGGER_MODULE: + to_aaz(Config.DEFAULT_SWAGGER_MODULE) + + else: + spec_path = os.path.join(Config.SWAGGER_PATH, "specification") + for item in os.listdir(spec_path): + curr_path = os.path.join(spec_path, item) + if os.path.isfile(curr_path): continue - rp = module_manager.get_openapi_resource_provider(r.name) - - for k in r.tags: - resource_map = rp.get_resource_map_by_tag(k) - if not resource_map: - raise InvalidAPIUsage(f"Tag `{k}` is not exist") - - version_resource_map = {} - for resource_id, version_map in resource_map.items(): - v_list = [v for v in version_map] - if len(v_list) > 1: - raise InvalidAPIUsage( - f"Tag `{k}` contains multiple api versions of one resource", - payload={ - "Resource": resource_id, - "versions": v_list, - } - ) - - v = v_list[0] - resource = version_map[v] - has_patch = "patch" in resource.operations.values() - has_generic_update = "get" in resource.operations.values() and "put" in resource.operations.values() - - update_by = "PatchOnly" if has_patch else ("None" if has_generic_update else None) - - if v not in version_resource_map: - version_resource_map[v] = [] - - new_resource = {"id": resource_id} - if update_by: - new_resource["options"] = {"update_by": update_by} - version_resource_map[v].append(new_resource) - - mod_names = module_name - ws = WorkspaceManager.new( - name=module_name, - plane=Config.DEFAULT_PLANE, - folder=WorkspaceManager.IN_MEMORY, - mod_names=mod_names, - resource_provider=rp.name, - swagger_manager=swagger_specs, - aaz_manager=AAZSpecsManager(), - source=SourceTypeEnum.OpenAPI, - ) - for version, resources in version_resource_map.items(): - ws.add_new_resources_by_swagger( - mod_names=mod_names, version=version, resources=resources - ) - - # provide default short summary - for node in ws.iter_command_tree_nodes(): - if not node.help: - node.help = CMDHelp() - if not node.help.short: - node.help.short = f"Manage {node.names[-1]}" - - for leaf in ws.iter_command_tree_leaves(): - if not leaf.help: - leaf.help = CMDHelp() - if not leaf.help.short: - n = leaf.names[-1] - n = n[0].upper() + n[1:] - leaf.help.short = f"{n} {leaf.names[-2]}" - - if not ws.is_in_memory: - ws.save() - - ws.generate_to_aaz() - - except InvalidAPIUsage as err: - logger.error(err, exc_info=True) - sys.exit(1) - - except ValueError as err: - logger.error(err, exc_info=True) - sys.exit(1) - - if Config.DEFAULT_SWAGGER_MODULE: - to_aaz(Config.DEFAULT_SWAGGER_MODULE) - - else: - spec_path = os.path.join(Config.SWAGGER_PATH, "specification") - for item in os.listdir(spec_path): - curr_path = os.path.join(spec_path, item) - if os.path.isfile(curr_path): - continue + to_aaz(item) - to_aaz(item) + except (InvalidAPIUsage, ValueError) as err: + logger.error(err, exc_info=True) + raise sys.exit(1) @bp.cli.command("verify", short_help="Verify data consistency within `aaz` repository.") From 5b25325f1721df07377b649ac63a56e43968d655 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 2 Oct 2025 14:55:51 +1000 Subject: [PATCH 3/3] chore: refine help message --- src/aaz_dev/command/api/_cmds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/command/api/_cmds.py b/src/aaz_dev/command/api/_cmds.py index e6616262..278c49c0 100644 --- a/src/aaz_dev/command/api/_cmds.py +++ b/src/aaz_dev/command/api/_cmds.py @@ -160,7 +160,7 @@ def generate_command_models_from_swagger(swagger_tag, workspace_path=None): sys.exit(1) -@bp.cli.command("generate-all", short_help="Fully generate data model from swagger, mainly used in https://github.com/magodo/aaz-metadata.") +@bp.cli.command("generate-all", short_help="Fully generate data model from OpenAPI specification, mainly for use in https://github.com/magodo/az-rs.") @click.option( "--swagger-path", '-s', type=click.Path(file_okay=False, dir_okay=True, readable=True, resolve_path=True),