From ae9b3c371947836bac47dd7cc64bbc146115b5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 17:35:28 +0100 Subject: [PATCH 01/89] Close every spec schema object and reject removed keys through schema validation Validate the raw spec, including the ui block, before any transform. Drop filter, writeFields, long-form role, top-level roles, SAML issuer/audience/trust, 0.3 root input and auth.bindings, string imports in 0.3, options, security redaction, 0.1 OIDC discovery/claim mapping/auto-provision, indexes, store and expand, together with their key-specific checks and IR fields. Add a guard test for schema keys without a consumer. --- examples/v02/api-with-admin.yaml | 11 +- examples/v02/api-with-ui.yaml | 11 +- examples/v03/access-bindings/spec.yaml | 1 - examples/v03/crud-pagination/spec.yaml | 1 - examples/v03/files/spec.yaml | 7 +- examples/v03/mongo-migrations/spec.yaml | 1 - examples/v03/oidc-pkce-dpop/spec.yaml | 1 - examples/v03/password-role/spec.yaml | 1 - examples/v03/secure-session/spec.yaml | 3 +- examples/v03/where-authorization.yaml | 1 - examples/valid/access-control-advanced.yaml | 5 - examples/valid/deploy-systemd.yaml | 3 - examples/valid/mongodb-all-commands.yaml | 8 +- examples/valid/path-parameters.yaml | 57 - examples/valid/role-hierarchies.yaml | 10 - examples/valid/v02-advanced-mongodb.yaml | 8 +- .../backend/builders/file_builder.py | 4 - .../generators/backend/compile_context.py | 1 - .../backend/fastapi/templates/routes.py.jinja | 6 +- src/yaml_api_generator/ir/api.py | 13 +- src/yaml_api_generator/ir/ui.py | 1 - .../parsers/schemas/ui.schema.json | 91 ++ .../schemas/yaml-spec-v0.1.schema.json | 445 ++----- .../schemas/yaml-spec-v0.2.schema.json | 1076 +++++++++++++---- .../schemas/yaml-spec-v0.3.schema.json | 1068 +++++++++------- .../parsers/transformers/access_validation.py | 17 +- .../parsers/transformers/shape_validation.py | 37 - .../parsers/transformers/trust_validation.py | 101 +- .../transformers/v01/route_transformer.py | 18 +- .../parsers/transformers/v02/__init__.py | 10 - .../transformers/v02/access_transformer.py | 42 +- .../transformers/v02/identity_transformer.py | 34 - .../transformers/v02/route_transformer.py | 17 +- .../parsers/transformers/v03/__init__.py | 2 +- .../transformers/v03/vendored_imports.py | 9 - src/yaml_api_generator/parsers/ui_parser.py | 12 +- src/yaml_api_generator/parsers/yaml_parser.py | 56 +- tests/fixtures/valid/roles-access.yaml | 10 +- tests/fixtures/valid/ui-block.yaml | 15 +- tests/fixtures/valid/v03-service-input.yaml | 1 - .../test_generated_where_authorization.py | 1 - tests/unit/test_access_reason.py | 1 - tests/unit/test_compiler_context_builder.py | 1 - tests/unit/test_controller_bind.py | 13 +- tests/unit/test_controller_role.py | 193 +-- tests/unit/test_dpop.py | 2 +- tests/unit/test_fastapi_authz_generation.py | 2 +- tests/unit/test_generators.py | 2 +- tests/unit/test_parser.py | 10 +- tests/unit/test_pkce.py | 2 +- tests/unit/test_removed_access_filter.py | 132 -- .../test_repository_operation_validation.py | 2 +- tests/unit/test_runtime_context_builder.py | 2 - tests/unit/test_schema_consumers.py | 114 ++ tests/unit/test_schema_rejection.py | 139 +++ tests/unit/test_service_input.py | 3 +- tests/unit/test_shape_validation.py | 69 -- tests/unit/test_trust_validation.py | 115 +- tests/unit/test_v02_transformer.py | 19 +- tests/unit/test_v03_spec_version.py | 22 +- tests/unit/test_vendored_imports.py | 2 +- 61 files changed, 2098 insertions(+), 1963 deletions(-) create mode 100644 src/yaml_api_generator/parsers/schemas/ui.schema.json delete mode 100644 src/yaml_api_generator/parsers/transformers/shape_validation.py delete mode 100644 tests/unit/test_removed_access_filter.py create mode 100644 tests/unit/test_schema_consumers.py create mode 100644 tests/unit/test_schema_rejection.py delete mode 100644 tests/unit/test_shape_validation.py diff --git a/examples/v02/api-with-admin.yaml b/examples/v02/api-with-admin.yaml index cd47de0a..ed0ee97c 100644 --- a/examples/v02/api-with-admin.yaml +++ b/examples/v02/api-with-admin.yaml @@ -29,6 +29,12 @@ entities: - provider: password identifier: email roles: role + roles: + values: [admin, moderator, user] + default: user + hierarchy: + admin: [moderator] + moderator: [user] fields: id: { type: id } email: { type: string, required: true, unique: true } @@ -49,11 +55,6 @@ entities: values: [draft, published, archived] default: draft -roles: - user: - admin: [moderator] - moderator: [user] - repositories: users: entity: user diff --git a/examples/v02/api-with-ui.yaml b/examples/v02/api-with-ui.yaml index 90356134..32cdbe81 100644 --- a/examples/v02/api-with-ui.yaml +++ b/examples/v02/api-with-ui.yaml @@ -30,6 +30,12 @@ entities: - provider: password identifier: email roles: role + roles: + values: [admin, moderator, user] + default: user + hierarchy: + admin: [moderator] + moderator: [user] fields: id: { type: id } email: { type: string, required: true, unique: true } @@ -50,11 +56,6 @@ entities: values: [draft, published, archived] default: draft -roles: - user: - admin: [moderator] - moderator: [user] - repositories: users: entity: user diff --git a/examples/v03/access-bindings/spec.yaml b/examples/v03/access-bindings/spec.yaml index d04f431a..6eaa50ef 100644 --- a/examples/v03/access-bindings/spec.yaml +++ b/examples/v03/access-bindings/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-access-bindings basePath: /api mongo: diff --git a/examples/v03/crud-pagination/spec.yaml b/examples/v03/crud-pagination/spec.yaml index ec1577fb..3c2ac2ea 100644 --- a/examples/v03/crud-pagination/spec.yaml +++ b/examples/v03/crud-pagination/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-crud-pagination basePath: /api operations: defaults: diff --git a/examples/v03/files/spec.yaml b/examples/v03/files/spec.yaml index ce818aac..ccc272a1 100644 --- a/examples/v03/files/spec.yaml +++ b/examples/v03/files/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-files basePath: /api files: maxSize: 3KB @@ -96,20 +95,18 @@ controllers: path: /{id} access: - given: { principal: "identity:user", document: documents } - role: editor when: document.ownerId equals principal.sub - then: deny - reason: Only an editor who owns the document may change it. + reason: Only the owner may change the document. delete: service: documents method: DELETE path: /{id} access: - given: { principal: "identity:user", document: documents } - role: editor when: document.ownerId equals principal.sub - then: deny - reason: Only an editor who owns the document may delete it. + reason: Only the owner may delete the document. assets: base_path: /assets routes: diff --git a/examples/v03/mongo-migrations/spec.yaml b/examples/v03/mongo-migrations/spec.yaml index bc34a30a..7e8d0c69 100644 --- a/examples/v03/mongo-migrations/spec.yaml +++ b/examples/v03/mongo-migrations/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-mongo-migrations basePath: /api mongo: diff --git a/examples/v03/oidc-pkce-dpop/spec.yaml b/examples/v03/oidc-pkce-dpop/spec.yaml index 2e7ec296..01ae3f4b 100644 --- a/examples/v03/oidc-pkce-dpop/spec.yaml +++ b/examples/v03/oidc-pkce-dpop/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-oidc-pkce-dpop basePath: /api providers: diff --git a/examples/v03/password-role/spec.yaml b/examples/v03/password-role/spec.yaml index f7f4eb9a..b0473d27 100644 --- a/examples/v03/password-role/spec.yaml +++ b/examples/v03/password-role/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-password-role basePath: /api version: v1 operations: diff --git a/examples/v03/secure-session/spec.yaml b/examples/v03/secure-session/spec.yaml index 6608898e..eede1bc2 100644 --- a/examples/v03/secure-session/spec.yaml +++ b/examples/v03/secure-session/spec.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: v03-secure-session basePath: /api session: @@ -41,7 +40,7 @@ controllers: operations: list: true get: true - create: public + create: true update: true replace: false delete: true diff --git a/examples/v03/where-authorization.yaml b/examples/v03/where-authorization.yaml index b11c2621..22e64a7d 100644 --- a/examples/v03/where-authorization.yaml +++ b/examples/v03/where-authorization.yaml @@ -1,7 +1,6 @@ version: "0.3" api: - name: organization-reports basePath: /api version: v1 diff --git a/examples/valid/access-control-advanced.yaml b/examples/valid/access-control-advanced.yaml index a8070433..69a0f313 100644 --- a/examples/valid/access-control-advanced.yaml +++ b/examples/valid/access-control-advanced.yaml @@ -52,11 +52,6 @@ entities: confidential_info: { type: string } owner_id: { type: string, required: true } -roles: - user: - admin: [editor] - editor: [viewer] - repositories: users: entity: user diff --git a/examples/valid/deploy-systemd.yaml b/examples/valid/deploy-systemd.yaml index 2ab197c5..8fec1605 100644 --- a/examples/valid/deploy-systemd.yaml +++ b/examples/valid/deploy-systemd.yaml @@ -48,12 +48,10 @@ repositories: query: command: find filter: { level: "{{level}}" } - options: { sort: { timestamp: -1 }, limit: 100 } recent: query: command: find filter: { timestamp: { $gte: "{{start_time}}" } } - options: { sort: { timestamp: -1 } } metrics: entity: metric @@ -63,7 +61,6 @@ repositories: query: command: find filter: { name: "{{name}}" } - options: { sort: { timestamp: -1 }, limit: 50 } aggregate_by_name: query: command: aggregate diff --git a/examples/valid/mongodb-all-commands.yaml b/examples/valid/mongodb-all-commands.yaml index bfbbeb07..2893602a 100644 --- a/examples/valid/mongodb-all-commands.yaml +++ b/examples/valid/mongodb-all-commands.yaml @@ -45,7 +45,6 @@ repositories: query: command: find filter: { category: "{{category}}" } - options: { sort: { price: 1 } } find_one_by_name: query: command: findOne @@ -57,7 +56,7 @@ repositories: distinct_categories: query: command: distinct - field: category + key: category filter: {} aggregate_stats: query: @@ -101,13 +100,11 @@ repositories: command: findOneAndUpdate filter: { id: "{{id}}" } update: { $inc: { stock: "{{amount}}" } } - options: { returnDocument: "after" } find_and_replace: query: command: findOneAndReplace filter: { id: "{{id}}" } replacement: "{{product}}" - options: { returnDocument: "after" } find_and_delete: query: command: findOneAndDelete @@ -125,7 +122,6 @@ repositories: query: command: find filter: { customer_id: "{{customer_id}}" } - options: { sort: { created_at: -1 } } by_status: query: command: find @@ -137,7 +133,7 @@ repositories: distinct_customers: query: command: distinct - field: customer_id + key: customer_id filter: {} customer_totals: query: diff --git a/examples/valid/path-parameters.yaml b/examples/valid/path-parameters.yaml index d47fd77e..408105de 100644 --- a/examples/valid/path-parameters.yaml +++ b/examples/valid/path-parameters.yaml @@ -95,37 +95,6 @@ controllers: base_path: /posts service: posts access: public - routes: - list: - method: GET - path: "" - service: posts - access: public - create: - method: POST - path: "" - service: posts - access: public - get_by_id: - method: GET - path: /{id} - service: posts - access: public - get_by_slug: - method: GET - path: /slug/{slug} - service: posts - access: public - update: - method: PUT - path: /{id} - service: posts - access: public - delete: - method: DELETE - path: /{id} - service: posts - access: public comments: base_path: /posts @@ -160,29 +129,3 @@ controllers: base_path: /categories service: categories access: public - routes: - list: - method: GET - path: "" - service: categories - access: public - get_by_slug: - method: GET - path: /{slug} - service: categories - access: public - create: - method: POST - path: "" - service: categories - access: public - update: - method: PUT - path: /{slug} - service: categories - access: public - delete: - method: DELETE - path: /{slug} - service: categories - access: public diff --git a/examples/valid/role-hierarchies.yaml b/examples/valid/role-hierarchies.yaml index 84cc4631..a5e9551d 100644 --- a/examples/valid/role-hierarchies.yaml +++ b/examples/valid/role-hierarchies.yaml @@ -63,16 +63,6 @@ entities: default: public created_by: { type: string, required: true } -roles: - admin_user: - super_admin: [admin, moderator] - admin: [moderator] - moderator: [support] - regular_user: - premium: [standard] - standard: [trial] - trial: [guest] - repositories: admins: entity: admin_user diff --git a/examples/valid/v02-advanced-mongodb.yaml b/examples/valid/v02-advanced-mongodb.yaml index 21f02fb4..79a4b806 100644 --- a/examples/valid/v02-advanced-mongodb.yaml +++ b/examples/valid/v02-advanced-mongodb.yaml @@ -37,7 +37,6 @@ repositories: query: command: find filter: { category: "{{category}}" } - options: { sort: { price: 1 } } price_range: query: command: find @@ -47,7 +46,6 @@ repositories: query: command: find filter: { stock: { $lt: 10 } } - options: { sort: { stock: 1 } } search_name: query: command: find @@ -76,7 +74,6 @@ repositories: query: command: find filter: { customer_id: "{{customer_id}}" } - options: { sort: { created_at: -1 } } by_status: query: command: find @@ -85,12 +82,10 @@ repositories: query: command: find filter: { created_at: { $gte: "{{start_date}}" } } - options: { sort: { created_at: -1 }, limit: 100 } high_value: query: command: find filter: { total: { $gte: 1000 } } - options: { sort: { total: -1 } } customer_total: query: command: aggregate @@ -115,7 +110,6 @@ repositories: query: command: find filter: { user_id: "{{user_id}}" } - options: { sort: { timestamp: -1 }, limit: 100 } count_events: query: command: countDocuments @@ -130,7 +124,7 @@ repositories: distinct_users: query: command: distinct - field: user_id + key: user_id filter: { event_name: "{{event}}" } services: diff --git a/src/yaml_api_generator/generators/backend/builders/file_builder.py b/src/yaml_api_generator/generators/backend/builders/file_builder.py index f0ff9660..999caa75 100644 --- a/src/yaml_api_generator/generators/backend/builders/file_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/file_builder.py @@ -65,8 +65,4 @@ def _file_route( field=document.collections[route.collection].files[route.field], item=route.item, source=source, - writable=not any( - rule.write_fields and route.field not in rule.write_fields - for rule in route.source.access_rules or () - ), ) diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index 1b7dc15c..f0634b49 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -300,7 +300,6 @@ class FileRouteDescriptor: field: IRFileField item: bool source: RouteDescriptor - writable: bool = True @property def handler_name(self) -> str: diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index b404436d..0365f2c8 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -18,7 +18,7 @@ from typing import Any {% set is_mongo = datastore_type not in ["postgres", "postgresql"] %} {% set needs_request = custom_routes or repo_op_routes or (crud_routes | selectattr("needs_request") | list) or (file_routes | selectattr("action", "equalto", "replace") | list) %} -from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes or (file_routes | rejectattr("writable") | list) %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} +from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} {% if ns_service_input.enabled or ns_where.collections %} from dependencies.service_inputs import {% if ns_service_input.enabled %}prepare_service_inputs{% endif %}{% if ns_service_input.enabled and (ns_service_input.access_bind or ns_where.collections) %}, {% endif %}{% if ns_service_input.access_bind or ns_where.collections %}resolve_access_bind{% endif +%} @@ -331,15 +331,11 @@ async def delete_{{ collection_name }}({{ route.record_id_param }}: str{{ extra_ {% set file_id = ", fileId" if file_route.item else "" %} @router.{{ file_route.method | lower }}("{{ file_route.relative_path }}"{{ openapi_extra(source, none if file_route.action != "replace" else "file_upload_body(_files.fields[" ~ field_name ~ "], item=" ~ (file_route.item | py_bool) ~ ")") }}{% if file_route.action == "download" %}, responses=file_download_responses(){% endif %}) async def {{ file_route.handler_name }}({{ source.record_id_param }}: str{{ extra_path_params(source) }}{% if file_route.item %}, fileId: str{% endif %}{% if file_route.action == "replace" or source.needs_request %}, request: Request{% endif %}{{ auth_param(source) }}, {{ svc_dep() }}{{ where_repo_params(source) }}) -> Any: -{% if not file_route.writable %} - raise HTTPException(status_code=403, detail="Forbidden") -{% else %} {{ request_locals(source, "None") }} {{ pre_fetch_guard(source) }} existing = await {{ record_call(source, "get") }} {{ post_fetch_guard(source, "existing") }} {{ file_actions[file_route.action](source, field_name, file_id) }} -{% endif %} {% endfor %} diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 54d57cc7..652cd674 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -22,7 +22,7 @@ from collections.abc import Iterable from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any +from typing import Any, ClassVar from urllib.parse import urlparse from .when import WhenExpr @@ -51,15 +51,9 @@ def is_idp(self) -> bool: class IRSAMLProvider: name: str type: str = "saml" - trust: str = "app" idp_metadata_url: str | None = None sp_entity_id: str | None = None - issuer: str | None = None - audience: str | None = None - - @property - def is_idp(self) -> bool: - return self.trust == "idp" + is_idp: ClassVar[bool] = False IRProvider = IROIDCProvider | IRSAMLProvider @@ -328,8 +322,6 @@ class IRAccessRule: request_symbols: dict[str, IRRequestBinding] = field(default_factory=dict) when: WhenExpr | None = None effect: str = "allow" - filter: str | None = None - write_fields: list[str] = field(default_factory=list) reason: str | None = None bind: dict[str, WhereRhs] = field(default_factory=dict) @@ -471,7 +463,6 @@ class IRDocument: identityCollections: dict[str, IRIdentityCollection] dtos: dict[str, IRDto] = field(default_factory=dict) services: dict[str, IRService] = field(default_factory=dict) - role_hierarchies: dict[str, dict[str, list[str]]] = field(default_factory=dict) providers: dict[str, IRProvider] = field(default_factory=dict) mongo: IRMongo = field(default_factory=IRMongo) datastores: dict[str, IRDatastore] = field(default_factory=dict) diff --git a/src/yaml_api_generator/ir/ui.py b/src/yaml_api_generator/ir/ui.py index c24648cf..72be515a 100644 --- a/src/yaml_api_generator/ir/ui.py +++ b/src/yaml_api_generator/ir/ui.py @@ -106,5 +106,4 @@ class GeneratorContext: pack_config: dict[str, Any] entities: dict[str, EntityInfo] identity_entities: list[str] - role_hierarchies: dict[str, dict[str, list[str]]] session_delivery: str = "token-url" diff --git a/src/yaml_api_generator/parsers/schemas/ui.schema.json b/src/yaml_api_generator/parsers/schemas/ui.schema.json new file mode 100644 index 00000000..d25e5265 --- /dev/null +++ b/src/yaml_api_generator/parsers/schemas/ui.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "ui.schema.json", + "title": "YAML API Generator UI block", + "type": "object", + "additionalProperties": false, + "required": ["frontend"], + "properties": { + "frontend": { + "type": "object", + "additionalProperties": false, + "required": ["pack"], + "properties": { + "pack": { "type": "string", "minLength": 1 }, + "output": { "type": "string" }, + "overwrite": { "enum": ["never", "always"], "default": "never" } + } + }, + "config": { + "type": "object", + "description": "Pack-defined settings, passed to the pack templates as pack_config." + }, + "collections": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["fields"], + "properties": { + "fields": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["from"], + "properties": { + "from": { "type": "string", "minLength": 1 }, + "format": { "type": "string" } + } + } + } + } + } + }, + "components": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["render"], + "properties": { + "props": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "required": { "type": "boolean" } + } + } + }, + "render": { "$ref": "#/$defs/ComponentNode" } + } + } + }, + "views": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["path", "component"], + "properties": { + "path": { "type": "string" }, + "component": { "type": "string" }, + "props": { "type": "object" } + } + } + } + }, + "$defs": { + "ComponentNode": { + "type": "object", + "additionalProperties": false, + "required": ["component"], + "properties": { + "component": { "type": "string", "minLength": 1 }, + "props": { "type": "object" } + } + } + } +} diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json index bb757399..a86ce97e 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json @@ -4,134 +4,95 @@ "title": "YAML API Generator Structure Spec v0.1", "type": "object", "required": ["version", "collections"], + "additionalProperties": false, "properties": { "version": { - "type": "string" + "type": "string", + "const": "0.1" }, "api": { "type": "object", + "additionalProperties": false, "properties": { - "basePath": { - "type": "string" - }, - "version": { - "type": "string" - }, + "basePath": { "type": "string" }, + "version": { "type": "string" }, "pagination": { "type": "object", + "additionalProperties": false, "properties": { - "defaultPageSize": { - "type": "integer" - }, - "maxPageSize": { - "type": "integer" - } + "defaultPageSize": { "type": "integer" }, + "maxPageSize": { "type": "integer" } } }, "operations": { "type": "object", + "additionalProperties": false, "properties": { - "defaults": { - "$ref": "#/$defs/OperationFlags" - } + "defaults": { "$ref": "#/$defs/OperationFlags" } } } } }, "security": { "type": "object", + "additionalProperties": false, "properties": { "cors": { "type": "object", + "additionalProperties": false, "properties": { - "allowedOrigins": { - "type": "array", - "items": { - "type": "string" - } - }, - "allowCredentials": { - "type": "boolean" - }, - "allowedMethods": { - "type": "array", - "items": { - "type": "string" - } - } + "allowedOrigins": { "$ref": "#/$defs/StringList" }, + "allowCredentials": { "type": "boolean" }, + "allowedMethods": { "$ref": "#/$defs/StringList" } } }, "headers": { "type": "object", + "additionalProperties": false, "properties": { - "enableStandard": { - "type": "boolean" - } - } - }, - "redaction": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - }, - "keys": { - "type": "array", - "items": { - "type": "string" - } - } + "enableStandard": { "type": "boolean" } } } } }, "collections": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/CollectionConfig" - } + "additionalProperties": { "$ref": "#/$defs/CollectionConfig" } }, "providers": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/ProviderConfig" - } + "additionalProperties": { "$ref": "#/$defs/ProviderConfig" } }, - "options": { + "mongo": { "type": "object", + "additionalProperties": false, "properties": { - "strict": { - "type": "boolean" - } + "uri": { "type": "string" }, + "database": { "type": "string" } } - } + }, + "ui": { "$ref": "ui.schema.json" } }, "$defs": { + "StringList": { + "type": "array", + "items": { "type": "string" } + }, "OperationFlags": { "type": "object", + "additionalProperties": false, "properties": { - "list": { - "type": "boolean" - }, - "get": { - "type": "boolean" - }, - "create": { - "type": "boolean" - }, - "update": { - "type": "boolean" - }, - "replace": { - "type": "boolean" - }, - "delete": { - "type": "boolean" - } + "list": { "type": "boolean" }, + "get": { "type": "boolean" }, + "create": { "type": "boolean" }, + "update": { "type": "boolean" }, + "replace": { "type": "boolean" }, + "delete": { "type": "boolean" } } }, "CollectionConfig": { "type": "object", + "additionalProperties": false, "required": ["fields"], "properties": { "type": { @@ -141,239 +102,131 @@ }, "fields": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/FieldConfig" - } - }, - "auth": { - "$ref": "#/$defs/AuthCollectionConfig" - }, - "roles": { - "$ref": "#/$defs/RolesConfig" - }, - "operations": { - "$ref": "#/$defs/OperationFlags" + "additionalProperties": { "$ref": "#/$defs/FieldConfig" } }, + "auth": { "$ref": "#/$defs/AuthCollectionConfig" }, + "roles": { "$ref": "#/$defs/RolesConfig" }, + "operations": { "$ref": "#/$defs/OperationFlags" }, "access": { "type": "object", + "additionalProperties": false, "properties": { - "list": { - "$ref": "#/$defs/AccessOperationConfig" - }, - "get": { - "$ref": "#/$defs/AccessOperationConfig" - }, - "create": { - "$ref": "#/$defs/AccessOperationConfig" - }, - "update": { - "$ref": "#/$defs/AccessOperationConfig" - }, - "replace": { - "$ref": "#/$defs/AccessOperationConfig" - }, - "delete": { - "$ref": "#/$defs/AccessOperationConfig" - } + "list": { "$ref": "#/$defs/AccessOperationConfig" }, + "get": { "$ref": "#/$defs/AccessOperationConfig" }, + "create": { "$ref": "#/$defs/AccessOperationConfig" }, + "update": { "$ref": "#/$defs/AccessOperationConfig" }, + "replace": { "$ref": "#/$defs/AccessOperationConfig" }, + "delete": { "$ref": "#/$defs/AccessOperationConfig" } } }, - "softDelete": { - "type": "boolean" - }, - "indexes": { - "type": "array", - "items": { - "$ref": "#/$defs/IndexConfig" - } - } + "softDelete": { "type": "boolean" } }, "allOf": [ { "if": { - "properties": { - "type": { - "const": "auth" - } - }, + "properties": { "type": { "const": "auth" } }, "required": ["type"] }, - "then": { - "required": ["auth"] - } + "then": { "required": ["auth"] } } ] }, "AuthCollectionConfig": { "type": "object", + "additionalProperties": false, "required": ["identifier", "via"], "properties": { - "identifier": { - "type": "string" - }, + "identifier": { "type": "string" }, "via": { "oneOf": [ { "type": "string" }, - { - "type": "array", - "items": { "type": "string" } - } + { "$ref": "#/$defs/StringList" } ] - }, - "claim_mapping": { - "type": "object", - "additionalProperties": { "type": "string" } - }, - "auto_provision": { "type": "boolean" } + } } }, "ProviderConfig": { - "oneOf": [ - { "$ref": "#/$defs/OIDCProviderConfig" }, - { "$ref": "#/$defs/SAMLProviderConfig" } + "type": "object", + "required": ["type"], + "properties": { + "type": { "enum": ["oidc", "saml"] } + }, + "allOf": [ + { + "if": { "properties": { "type": { "const": "oidc" } } }, + "then": { "$ref": "#/$defs/OIDCProviderConfig" } + }, + { + "if": { "properties": { "type": { "const": "saml" } } }, + "then": { "$ref": "#/$defs/SAMLProviderConfig" } + } ] }, "OIDCProviderConfig": { "type": "object", + "additionalProperties": false, "required": ["type", "client_id", "client_secret"], "properties": { - "type": { "type": "string", "const": "oidc" }, + "type": { "const": "oidc" }, "client_id": { "type": "string" }, - "client_secret": { "type": "string" }, - "discovery_url": { "type": "string" } - }, - "additionalProperties": false + "client_secret": { "type": "string" } + } }, "SAMLProviderConfig": { "type": "object", + "additionalProperties": false, "required": ["type", "idp_metadata_url", "sp_entity_id"], "properties": { - "type": { "type": "string", "const": "saml" }, + "type": { "const": "saml" }, "idp_metadata_url": { "type": "string" }, "sp_entity_id": { "type": "string" } - }, - "additionalProperties": false + } }, "FieldConfig": { "type": "object", + "additionalProperties": false, "required": ["type"], "properties": { "type": { "enum": ["string", "number", "boolean", "datetime", "enum", "array", "object", "ref", "id"] }, - "required": { - "type": "boolean" - }, + "required": { "type": "boolean" }, "default": {}, - "unique": { - "type": "boolean" - }, - "min": { - "type": "number" - }, - "max": { - "type": "number" - }, - "minLength": { - "type": "integer" - }, - "maxLength": { - "type": "integer" - }, + "unique": { "type": "boolean" }, + "min": { "type": "number" }, + "max": { "type": "number" }, + "minLength": { "type": "integer" }, + "maxLength": { "type": "integer" }, "format": { "enum": ["email", "e164", "uri", "uuidv7", "uuidv4"] }, - "values": { - "type": "array", - "items": { - "type": "string" - } - }, - "items": { - "$ref": "#/$defs/FieldConfigLite" - }, + "values": { "$ref": "#/$defs/StringList" }, + "items": { "$ref": "#/$defs/FieldConfig" }, "properties": { "type": "object", - "additionalProperties": { - "$ref": "#/$defs/FieldConfig" - } - }, - "additionalProperties": { - "type": "boolean" - }, - "ref": { - "type": "string" - }, - "store": { - "enum": ["id", "embed"] + "additionalProperties": { "$ref": "#/$defs/FieldConfig" } }, - "expand": { - "type": "object", - "properties": { - "allowed": { - "type": "boolean" - }, - "always": { - "type": "boolean" - }, - "fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "maxDepth": { - "type": "integer" - } - } - } + "additionalProperties": { "type": "boolean" }, + "ref": { "type": "string" } }, "allOf": [ { - "if": { - "properties": { - "type": { - "const": "enum" - } - }, - "required": ["type"] - }, - "then": { - "required": ["values"] - } + "if": { "properties": { "type": { "const": "enum" } }, "required": ["type"] }, + "then": { "required": ["values"] } }, { - "if": { - "properties": { - "type": { - "const": "array" - } - }, - "required": ["type"] - }, - "then": { - "required": ["items"] - } + "if": { "properties": { "type": { "const": "array" } }, "required": ["type"] }, + "then": { "required": ["items"] } }, { - "if": { - "properties": { - "type": { - "const": "ref" - } - }, - "required": ["type"] - }, - "then": { - "required": ["ref"] - } + "if": { "properties": { "type": { "const": "ref" } }, "required": ["type"] }, + "then": { "required": ["ref"] } } ] }, - "FieldConfigLite": { - "$ref": "#/$defs/FieldConfig" - }, "RolesConfig": { "type": "object", + "additionalProperties": false, "required": ["values", "default"], "properties": { "values": { @@ -384,111 +237,35 @@ "default": { "type": "string" }, "hierarchy": { "type": "object", - "additionalProperties": { - "type": "array", - "items": { "type": "string" } - } + "additionalProperties": { "$ref": "#/$defs/StringList" } } - }, - "additionalProperties": false + } + }, + "Principal": { + "anyOf": [ + { "type": "string", "enum": ["public", "*"] }, + { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*:(?:[A-Za-z_][A-Za-z0-9_]*|\\*)$" }, + { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" } + ] }, "AccessRule": { "type": "object", + "additionalProperties": false, "required": ["principal"], "properties": { - "principal": { - "anyOf": [ - { - "type": "string", - "enum": ["public", "*"] - }, - { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*:(?:[A-Za-z_][A-Za-z0-9_]*|\\*)$" - }, - { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - } - ] - }, - "when": { - "type": "string" - }, - "filter": { - "type": "string" - }, - "writeFields": { - "type": "array", - "items": { - "type": "string" - } - }, - "reason": { - "type": "string" - } + "principal": { "$ref": "#/$defs/Principal" }, + "when": { "type": "string" }, + "reason": { "type": "string" } } }, "AccessOperationConfig": { "anyOf": [ { "type": "array", - "items": { - "$ref": "#/$defs/AccessRule" - } - }, - { - "type": "string", - "enum": ["public", "*"] + "items": { "$ref": "#/$defs/AccessRule" } }, - { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*:(?:[A-Za-z_][A-Za-z0-9_]*|\\*)$" - }, - { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - } + { "$ref": "#/$defs/Principal" } ] - }, - "IndexConfig": { - "type": "object", - "required": ["fields"], - "properties": { - "name": { - "type": "string" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/IndexField" - } - }, - "unique": { - "type": "boolean" - }, - "sparse": { - "type": "boolean" - }, - "ttlSeconds": { - "type": "integer" - }, - "text": { - "type": "boolean" - } - } - }, - "IndexField": { - "type": "object", - "required": ["field"], - "properties": { - "field": { - "type": "string" - }, - "order": { - "enum": ["asc", "desc"] - } - } } } } diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index d99120da..71917c0d 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -3,7 +3,11 @@ "$id": "yaml-spec-v0.2.schema.json", "title": "YAML API Generator Structure Spec v0.2", "type": "object", - "required": ["version", "entities"], + "required": [ + "version", + "entities" + ], + "additionalProperties": false, "properties": { "version": { "type": "string", @@ -11,6 +15,7 @@ }, "api": { "type": "object", + "additionalProperties": false, "properties": { "basePath": { "type": "string" @@ -24,35 +29,10 @@ "maximum": 65535 }, "operations": { - "type": "object", - "properties": { - "defaults": { - "$ref": "#/$defs/OperationFlags" - }, - "auth": { - "$ref": "#/$defs/OperationFlags" - } - } + "$ref": "#/$defs/ApiOperations" }, "cors": { - "type": "object", - "properties": { - "origins": { - "type": "array", - "items": { "type": "string" } - }, - "methods": { - "type": "array", - "items": { "type": "string" } - }, - "headers": { - "type": "array", - "items": { "type": "string" } - }, - "credentials": { - "type": "boolean" - } - } + "$ref": "#/$defs/CorsConfig" } } }, @@ -63,15 +43,7 @@ } }, "mongo": { - "type": "object", - "properties": { - "uri": { - "type": "string" - }, - "database": { - "type": "string" - } - } + "$ref": "#/$defs/MongoConfig" }, "datastores": { "type": "object", @@ -122,7 +94,7 @@ } }, "ui": { - "type": "object" + "$ref": "ui.schema.json" }, "deploy": { "$ref": "#/$defs/DeployConfig" @@ -132,35 +104,130 @@ } }, "$defs": { - "SessionConfig": { + "ApiOperations": { "type": "object", + "additionalProperties": false, "properties": { - "algorithm": { "type": "string", "enum": ["HS256", "RS256"], "default": "HS256" }, - "ttl": { "type": "integer", "minimum": 1 }, - "refresh": { "type": "boolean", "default": false }, - "delivery": { "type": "string", "enum": ["token-url", "cookie"], "default": "token-url" } - }, - "additionalProperties": false + "defaults": { + "$ref": "#/$defs/CrudFlags" + }, + "auth": { + "$ref": "#/$defs/AuthFlags" + } + } }, - "OperationFlags": { + "CrudFlags": { "type": "object", + "additionalProperties": false, "properties": { - "list": { "type": "boolean" }, - "get": { "type": "boolean" }, - "create": { "type": "boolean" }, - "update": { "type": "boolean" }, - "replace": { "type": "boolean" }, - "delete": { "type": "boolean" }, - "register": { "type": "boolean" }, - "login": { "type": "boolean" } + "list": { + "type": "boolean" + }, + "get": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "update": { + "type": "boolean" + }, + "replace": { + "type": "boolean" + }, + "delete": { + "type": "boolean" + } + } + }, + "AuthFlags": { + "type": "object", + "additionalProperties": false, + "properties": { + "register": { + "type": "boolean" + }, + "login": { + "type": "boolean" + } + } + }, + "CorsConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "origins": { + "$ref": "#/$defs/StringList" + }, + "methods": { + "$ref": "#/$defs/StringList" + }, + "headers": { + "$ref": "#/$defs/StringList" + }, + "credentials": { + "type": "boolean" + } + } + }, + "StringList": { + "type": "array", + "items": { + "type": "string" + } + }, + "MongoConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "uri": { + "type": "string" + }, + "database": { + "type": "string" + } + } + }, + "SessionConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "algorithm": { + "type": "string", + "enum": [ + "HS256", + "RS256" + ], + "default": "HS256" + }, + "ttl": { + "type": "integer", + "minimum": 1 + }, + "refresh": { + "type": "boolean", + "default": false + }, + "delivery": { + "type": "string", + "enum": [ + "token-url", + "cookie" + ], + "default": "token-url" + } } }, "EntityConfig": { "type": "object", + "additionalProperties": false, "properties": { "type": { "type": "string", - "enum": ["base", "auth"], + "enum": [ + "base", + "auth" + ], "default": "base" }, "extends": { @@ -179,244 +246,517 @@ } }, "auth": { - "type": "object", - "properties": { - "identifier": { "type": "string" }, - "via": { - "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" }, "minItems": 1 }, - { "type": "array", "items": { "$ref": "#/$defs/ProviderBinding" }, "minItems": 1 } - ] - }, - "roles": { "type": "string" } - }, - "additionalProperties": false + "$ref": "#/$defs/EntityAuthConfig" + }, + "roles": { + "$ref": "#/$defs/RolesConfig" } }, "allOf": [ { "if": { - "not": { "required": ["extends"] } + "not": { + "required": [ + "extends" + ] + } }, "then": { - "required": ["fields"] + "required": [ + "fields" + ] } }, { "if": { - "properties": { "type": { "const": "auth" } }, - "required": ["type"] + "properties": { + "type": { + "const": "auth" + } + }, + "required": [ + "type" + ] }, "then": { - "required": ["auth"] + "required": [ + "auth" + ] } } - ] }, + "EntityAuthConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "identifier": { + "type": "string" + }, + "via": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/ProviderBinding" + }, + "minItems": 1 + } + ] + }, + "roles": { + "type": "string" + } + } + }, "RolesConfig": { "type": "object", - "required": ["values", "default"], + "additionalProperties": false, + "required": [ + "values", + "default" + ], "properties": { "values": { "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "minItems": 1 }, - "default": { "type": "string" }, + "default": { + "type": "string" + }, "hierarchy": { "type": "object", "additionalProperties": { - "type": "array", - "items": { "type": "string" } + "$ref": "#/$defs/StringList" } } - }, - "additionalProperties": false + } }, "ProviderBinding": { "type": "object", - "required": ["provider"], + "additionalProperties": false, + "required": [ + "provider" + ], "properties": { - "provider": { "type": "string" }, - "identifier": { "type": "string" }, + "provider": { + "type": "string" + }, + "identifier": { + "type": "string" + }, "claim_mapping": { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } + }, + "auto_provision": { + "type": "boolean" }, - "auto_provision": { "type": "boolean" }, "relations": { "type": "object", "additionalProperties": { "type": "object", - "required": ["collection", "match"], "additionalProperties": false, + "required": [ + "collection", + "match" + ], "properties": { - "collection": { "type": "string" }, - "match": { "type": "object", "additionalProperties": { "type": "string" } } + "collection": { + "type": "string" + }, + "match": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } } } } - }, - "additionalProperties": false + } }, "ProviderConfig": { - "oneOf": [ - { "$ref": "#/$defs/OIDCProviderConfig" }, - { "$ref": "#/$defs/SAMLProviderConfig" } + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "oidc", + "saml" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "oidc" + } + } + }, + "then": { + "$ref": "#/$defs/OIDCProviderConfig" + } + }, + { + "if": { + "properties": { + "type": { + "const": "saml" + } + } + }, + "then": { + "$ref": "#/$defs/SAMLProviderConfig" + } + } ] }, "OIDCProviderConfig": { "type": "object", - "required": ["type", "client_id"], + "additionalProperties": false, + "required": [ + "type", + "client_id", + "discovery_url" + ], "properties": { - "type": { "type": "string", "const": "oidc" }, - "trust": { "type": "string", "enum": ["app", "idp"], "default": "app" }, - "client_id": { "type": "string" }, - "client_secret": { "type": "string" }, - "discovery_url": { "type": "string", "description": "OIDC discovery document URL; issuer is derived from it automatically" }, - "issuer": { "type": "string" }, - "audiences": { "type": "array", "items": { "type": "string" }, "description": "Accepted token audiences; defaults to [client_id] when omitted" } + "type": { + "const": "oidc" + }, + "trust": { + "type": "string", + "enum": [ + "app", + "idp" + ], + "default": "app" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "discovery_url": { + "type": "string", + "description": "OIDC discovery document URL; issuer is derived from it automatically" + }, + "issuer": { + "type": "string" + }, + "audiences": { + "$ref": "#/$defs/StringList", + "description": "Accepted token audiences; defaults to [client_id] when omitted" + } }, "if": { - "properties": { "trust": { "const": "idp" } }, - "required": ["trust"] + "properties": { + "trust": { + "const": "idp" + } + }, + "required": [ + "trust" + ] }, - "then": { "required": ["discovery_url"] }, - "else": { "required": ["discovery_url"] }, - "additionalProperties": false + "else": { + "properties": { + "issuer": false + } + } }, "SAMLProviderConfig": { "type": "object", - "required": ["type"], + "additionalProperties": false, + "required": [ + "type", + "idp_metadata_url", + "sp_entity_id" + ], "properties": { - "type": { "type": "string", "const": "saml" }, - "trust": { "type": "string", "enum": ["app", "idp"], "default": "app" }, - "idp_metadata_url": { "type": "string" }, - "sp_entity_id": { "type": "string" }, - "issuer": { "type": "string" }, - "audience": { "type": "string" } - }, - "if": { - "properties": { "trust": { "const": "idp" } }, - "required": ["trust"] - }, - "then": { "required": ["issuer", "audience"] }, - "else": { "required": ["idp_metadata_url", "sp_entity_id"] }, - "additionalProperties": false + "type": { + "const": "saml" + }, + "idp_metadata_url": { + "type": "string" + }, + "sp_entity_id": { + "type": "string" + } + } }, "DtoConfig": { "type": "object", - "required": ["fields"], + "additionalProperties": false, + "required": [ + "fields" + ], "properties": { "fields": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/FieldConfig" } + "additionalProperties": { + "$ref": "#/$defs/FieldConfig" + } } - }, - "additionalProperties": false + } }, "PolicyConfig": { "type": "object", - "required": ["when"], + "additionalProperties": false, + "required": [ + "when" + ], "properties": { "given": { "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, "params": { - "type": "array", - "items": { "type": "string" } + "$ref": "#/$defs/StringList" }, "when": { - "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" } } - ] + "$ref": "#/$defs/WhenConfig" } - }, - "additionalProperties": false + } + }, + "WhenConfig": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + ] }, "FieldConfig": { "type": "object", - "required": ["type"], + "additionalProperties": false, + "required": [ + "type" + ], "properties": { "type": { - "enum": ["string", "number", "boolean", "datetime", "enum", "array", "object", "ref", "id"] + "enum": [ + "string", + "number", + "boolean", + "datetime", + "enum", + "array", + "object", + "ref", + "id" + ] + }, + "required": { + "type": "boolean" }, - "required": { "type": "boolean" }, "default": {}, - "unique": { "type": "boolean" }, - "min": { "type": "number" }, - "max": { "type": "number" }, - "minLength": { "type": "integer" }, - "maxLength": { "type": "integer" }, + "unique": { + "type": "boolean" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "minLength": { + "type": "integer" + }, + "maxLength": { + "type": "integer" + }, "format": { - "enum": ["email", "e164", "uri", "uuidv7", "uuidv4"] + "enum": [ + "email", + "e164", + "uri", + "uuidv7", + "uuidv4" + ] }, "values": { - "type": "array", - "items": { "type": "string" } + "$ref": "#/$defs/StringList" + }, + "items": { + "$ref": "#/$defs/FieldConfig" }, - "items": { "$ref": "#/$defs/FieldConfig" }, "properties": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/FieldConfig" } + "additionalProperties": { + "$ref": "#/$defs/FieldConfig" + } }, - "additionalProperties": { "type": "boolean" }, - "ref": { "type": "string" }, - "readonly": { "type": "boolean" } + "additionalProperties": { + "type": "boolean" + }, + "ref": { + "type": "string" + }, + "readonly": { + "type": "boolean" + } }, "allOf": [ { - "if": { "properties": { "type": { "const": "enum" } }, "required": ["type"] }, - "then": { "required": ["values"] } + "if": { + "properties": { + "type": { + "const": "enum" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "values" + ] + } }, { - "if": { "properties": { "type": { "const": "array" } }, "required": ["type"] }, - "then": { "required": ["items"] } + "if": { + "properties": { + "type": { + "const": "array" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "items" + ] + } }, { - "if": { "properties": { "type": { "const": "ref" } }, "required": ["type"] }, - "then": { "required": ["ref"] } + "if": { + "properties": { + "type": { + "const": "ref" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "ref" + ] + } } ] }, "OperationConfig": { "type": "object", + "additionalProperties": false, "properties": { "query": { - "type": "object", - "properties": { - "command": { "type": "string" }, - "collection": { "type": "string" }, - "sql": { "type": "string" }, - "params": { "type": "object" } - }, - "additionalProperties": true + "$ref": "#/$defs/QueryConfig" }, - "handler": { "type": "string" } + "handler": { + "type": "string" + } } }, + "QueryConfig": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string" + }, + "collection": { + "type": "string" + }, + "key": { + "type": "string" + }, + "sql": { + "type": "string" + }, + "filter": {}, + "pipeline": {}, + "document": {}, + "documents": {}, + "update": {}, + "replacement": {}, + "requests": {}, + "params": { + "type": "object" + } + }, + "description": "Repository query. Values other than the names may be a {{placeholder}} resolved from the request at call time." + }, "DatastoreConfig": { "type": "object", + "additionalProperties": false, "properties": { "type": { "type": "string", - "enum": ["mongodb", "mongo", "postgres", "postgresql"] + "enum": [ + "mongodb", + "mongo", + "postgres", + "postgresql" + ] + }, + "uri": { + "type": "string" + }, + "database": { + "type": "string" }, - "uri": { "type": "string" }, - "database": { "type": "string" }, - "dsn": { "type": "string" }, - "default": { "type": "boolean" } + "dsn": { + "type": "string" + }, + "default": { + "type": "boolean" + } } }, "RepositoryConfig": { "type": "object", + "additionalProperties": false, "properties": { - "abstract": { "type": "boolean" }, - "extends": { "type": "string" }, + "abstract": { + "type": "boolean" + }, + "extends": { + "type": "string" + }, "entity": { "type": "string" }, @@ -427,7 +767,14 @@ "type": "array", "items": { "type": "string", - "enum": ["list", "get", "create", "update", "replace", "delete"] + "enum": [ + "list", + "get", + "create", + "update", + "replace", + "delete" + ] } }, "operations": { @@ -440,19 +787,40 @@ }, "ServiceConfig": { "type": "object", + "additionalProperties": false, "properties": { - "abstract": { "type": "boolean" }, - "extends": { "type": "string" }, - "repository": { "type": "string" }, + "abstract": { + "type": "boolean" + }, + "extends": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "handler": { + "type": "string" + }, "operations": { "type": "object", "additionalProperties": { "oneOf": [ - { "type": "boolean" }, + { + "type": "boolean" + }, { "type": "object", + "additionalProperties": false, + "required": [ + "handler" + ], "properties": { - "handler": { "type": "string" } + "handler": { + "type": "string" + } } } ] @@ -462,6 +830,7 @@ }, "ControllerConfig": { "type": "object", + "additionalProperties": false, "properties": { "service": { "type": "string" @@ -473,31 +842,105 @@ "$ref": "#/$defs/AccessConfig" }, "operations": { - "type": "object" - }, - "routes": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/RouteConfig" + "oneOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/OperationOverride" + } + ] } + }, + "routes": { + "type": "object" + } + }, + "if": { + "required": [ + "service" + ] + }, + "then": { + "properties": { + "routes": { + "additionalProperties": { + "$ref": "#/$defs/RouteOverride" + } + } + } + }, + "else": { + "properties": { + "access": false, + "operations": false, + "routes": { + "additionalProperties": { + "$ref": "#/$defs/RouteConfig" + } + } + } + } + }, + "OperationOverride": { + "type": "object", + "additionalProperties": false, + "properties": { + "method": { + "$ref": "#/$defs/HttpMethod" + }, + "path": { + "type": "string" + }, + "access": { + "$ref": "#/$defs/AccessConfig" + } + } + }, + "RouteOverride": { + "type": "object", + "additionalProperties": false, + "properties": { + "access": { + "$ref": "#/$defs/AccessConfig" } } }, "RouteConfig": { "type": "object", - "required": ["service", "method", "path"], + "additionalProperties": false, + "required": [ + "service", + "method", + "path" + ], "properties": { - "service": { "type": "string" }, + "service": { + "type": "string" + }, "method": { - "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"] + "$ref": "#/$defs/HttpMethod" + }, + "path": { + "type": "string" }, - "path": { "type": "string" }, "access": { "$ref": "#/$defs/AccessConfig" } } }, + "HttpMethod": { + "type": "string", + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE" + ] + }, "AccessConfig": { "oneOf": [ { @@ -507,113 +950,242 @@ "type": "array", "items": { "oneOf": [ - { "type": "string" }, - { "$ref": "#/$defs/AccessRule" } + { + "type": "string" + }, + { + "$ref": "#/$defs/AccessRule" + } ] } } ] }, + "AccessRule": { + "type": "object", + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "given" + ] + }, + { + "required": [ + "then" + ] + }, + { + "required": [ + "when" + ] + } + ], + "properties": { + "given": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "pattern": "^identity:[A-Za-z_][A-Za-z0-9_]*$" + }, + { + "type": "string", + "pattern": "^request(\\.(body|path|query|header|cookie))?(:[A-Za-z_][A-Za-z0-9_]*)?$" + }, + { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\s+by\\s+[A-Za-z_][A-Za-z0-9_]*)?$" + } + ] + } + }, + "when": { + "$ref": "#/$defs/WhenConfig" + }, + "then": { + "type": "string", + "enum": [ + "allow", + "deny" + ], + "default": "allow" + }, + "reason": { + "type": "string" + } + } + }, "DeployConfig": { "type": "object", - "required": ["runtime"], + "additionalProperties": false, + "required": [ + "runtime" + ], "properties": { - "name": { "type": "string" }, - "port": { "type": "integer", "default": 8000 }, - "runtime": { "type": "string", "enum": ["docker", "systemd"] }, + "name": { + "type": "string" + }, + "port": { + "type": "integer", + "default": 8000 + }, + "runtime": { + "type": "string", + "enum": [ + "docker", + "systemd" + ] + }, "services": { "type": "object", "additionalProperties": { - "type": "object", - "anyOf": [ - { "required": ["image"] }, - { "required": ["build"] } - ], - "properties": { - "image": { "type": "string" }, - "build": { "oneOf": [ - { "type": "string" }, - { "type": "object", "properties": { - "context": { "type": "string" }, - "dockerfile": { "type": "string" }, - "args": { "type": "object", "additionalProperties": { "type": "string" } } - } } - ]}, - "ports": { "type": "array", "items": { "oneOf": [{"type": "integer"}, {"type": "string"}] } }, - "environment": { "oneOf": [ - { "type": "object", "additionalProperties": { "type": "string" } }, - { "type": "array", "items": { "type": "string" } } - ]}, - "volumes": { "type": "array", "items": { "type": "string" } }, - "depends_on": { "oneOf": [ - { "type": "array", "items": { "type": "string" } }, - { "type": "object" } - ]}, - "command": { "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}] } - } + "$ref": "#/$defs/DeployService" } }, "include": { - "type": "array", - "items": { "type": "string" } + "$ref": "#/$defs/StringList" }, "mongo": { "type": "object", + "additionalProperties": false, "properties": { - "local": { "type": "boolean", "default": false } + "local": { + "type": "boolean", + "default": false + } } }, "docker": { "type": "object", + "additionalProperties": false, "properties": { - "image": { "type": "string" } + "image": { + "type": "string" + } } }, "systemd": { "type": "object", + "additionalProperties": false, "properties": { - "user": { "type": "string" }, - "restart": { "type": "string" } + "user": { + "type": "string" + }, + "restart": { + "type": "string" + } } } } }, - "AccessRule": { + "DeployService": { "type": "object", + "additionalProperties": false, "anyOf": [ - { "required": ["given"] }, - { "required": ["then"] }, - { "required": ["when"] } + { + "required": [ + "image" + ] + }, + { + "required": [ + "build" + ] + } ], "properties": { - "given": { - "type": "object", - "minProperties": 1, - "additionalProperties": { - "oneOf": [ - { "type": "string", "pattern": "^identity:[A-Za-z_][A-Za-z0-9_]*$" }, - { "type": "string", "pattern": "^request(\\.(body|path|query|header|cookie))?(:[A-Za-z_][A-Za-z0-9_]*)?$" }, - { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\s+by\\s+[A-Za-z_][A-Za-z0-9_]*)?$" } + "image": { + "type": "string" + }, + "build": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "context": { + "type": "string" + }, + "dockerfile": { + "type": "string" + }, + "args": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + ] + }, + "ports": { + "type": "array", + "items": { + "type": [ + "integer", + "string" ] } }, - "when": { + "environment": { "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" }, "minItems": 1 } + { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + { + "$ref": "#/$defs/StringList" + } ] }, - "then": { - "type": "string", - "enum": ["allow", "deny"], - "default": "allow" + "volumes": { + "$ref": "#/$defs/StringList" }, - "filter": { "type": "string" }, - "writeFields": { - "type": "array", - "items": { "type": "string" } + "depends_on": { + "oneOf": [ + { + "$ref": "#/$defs/StringList" + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": [ + "condition" + ], + "properties": { + "condition": { + "enum": [ + "service_started", + "service_healthy", + "service_completed_successfully" + ] + } + } + } + } + ] }, - "reason": { "type": "string" } + "command": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/StringList" + } + ] + } } } } diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 8cf842cb..79c86eef 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -2,19 +2,21 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "yaml-spec-v0.3.schema.json", "title": "YAML API Generator Structure Spec v0.3", - "$comment": "Superset of v0.2. Adds the where binding form (given: strings), service input:/with:, and controller/access-rule bind:. Everything v0.2 accepted stays valid.", + "$comment": "Extends v0.2 with files, pagination, service input:/with:, the where binding form and access-rule bind:. Definitions v0.3 leaves unchanged are referenced from v0.2.", "type": "object", - "required": ["version", "entities"], + "required": [ + "version", + "entities" + ], + "additionalProperties": false, "properties": { "version": { "type": "string", "const": "0.3" }, - "input": { - "$ref": "#/$defs/InputConfig" - }, "api": { "type": "object", + "additionalProperties": false, "properties": { "basePath": { "type": "string" @@ -28,43 +30,25 @@ "maximum": 65535 }, "operations": { - "type": "object", - "properties": { - "defaults": { - "$ref": "#/$defs/OperationFlags" - }, - "auth": { - "$ref": "#/$defs/OperationFlags" - } - } + "$ref": "yaml-spec-v0.2.schema.json#/$defs/ApiOperations" }, "cors": { - "type": "object", - "properties": { - "origins": { - "type": "array", - "items": { "type": "string" } - }, - "methods": { - "type": "array", - "items": { "type": "string" } - }, - "headers": { - "type": "array", - "items": { "type": "string" } - }, - "credentials": { - "type": "boolean" - } - } + "$ref": "yaml-spec-v0.2.schema.json#/$defs/CorsConfig" }, "files": { "type": "object", "additionalProperties": false, "properties": { - "maxSize": { "$ref": "#/$defs/FileSize" }, - "types": { "$ref": "#/$defs/MimeTypes" }, - "maxFiles": { "type": "integer", "minimum": 1 } + "maxSize": { + "$ref": "#/$defs/FileSize" + }, + "types": { + "$ref": "#/$defs/MimeTypes" + }, + "maxFiles": { + "type": "integer", + "minimum": 1 + } } } } @@ -72,44 +56,29 @@ "imports": { "type": "object", "additionalProperties": { - "oneOf": [ - { - "type": "string" + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "package" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 }, - { - "type": "object", - "required": ["path", "package"], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package": { - "type": "string", - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - }, - "requires": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false + "package": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "requires": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/StringList" } - ] + } } }, "mongo": { - "type": "object", - "properties": { - "uri": { - "type": "string" - }, - "database": { - "type": "string" - } - } + "$ref": "yaml-spec-v0.2.schema.json#/$defs/MongoConfig" }, "datastores": { "type": "object", @@ -156,24 +125,30 @@ "policies": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/PolicyConfig" + "$ref": "yaml-spec-v0.2.schema.json#/$defs/PolicyConfig" } }, "ui": { - "type": "object" + "$ref": "ui.schema.json" }, "deploy": { - "$ref": "#/$defs/DeployConfig" + "$ref": "yaml-spec-v0.2.schema.json#/$defs/DeployConfig" }, "session": { - "$ref": "#/$defs/SessionConfig" + "$ref": "yaml-spec-v0.2.schema.json#/$defs/SessionConfig" } }, "$defs": { "FileSize": { "oneOf": [ - { "type": "integer", "minimum": 1 }, - { "type": "string", "pattern": "^[0-9]+(\\.[0-9]+)?\\s*([kKmMgG]?[bB])?$" } + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+)?\\s*([kKmMgG]?[bB])?$" + } ] }, "MimeTypes": { @@ -188,78 +163,77 @@ "type": "object", "additionalProperties": false, "properties": { - "maxSize": { "$ref": "#/$defs/FileSize" }, - "types": { "$ref": "#/$defs/MimeTypes" } + "maxSize": { + "$ref": "#/$defs/FileSize" + }, + "types": { + "$ref": "#/$defs/MimeTypes" + } } }, - "SessionConfig": { - "type": "object", - "properties": { - "algorithm": { "type": "string", "enum": ["HS256", "RS256"], "default": "HS256" }, - "ttl": { "type": "integer", "minimum": 1 }, - "refresh": { "type": "boolean", "default": false }, - "delivery": { "type": "string", "enum": ["token-url", "cookie"], "default": "token-url" } - }, - "additionalProperties": false - }, - "Symbol": { - "type": "string", - "pattern": "^\\$[A-Za-z_][A-Za-z0-9_]*$", - "description": "v0.3 $-sigil slug: a symbolic reference resolved against the explicit scope of its where, bind, with, or repository-query site (e.g. $userId)." - }, - "WhereOperator": { - "type": "string", - "enum": ["=", "and", "!=", "<", "<=", ">", ">=", "in"], - "description": "v0.3 where-clause operators. '=' and 'and' are active. Reserved tokens '!=', '<', '<=', '>', '>=', and 'in' are rejected by the where parser with WHERE_OPERATOR_UNSUPPORTED." - }, "WhereExpression": { "type": "string", "pattern": "^[\\w$][\\w$.'\"\\s=<>!,()+\\\\-]*$", - "description": "v0.3 where binding form used as a given: value: ' where = [and = ]*'. The where parser is the authority on the grammar; this pattern only bounds the character set. Only '=' and 'and' are active in v0.3; the remaining WhereOperator tokens are reserved." + "description": "Where binding form used as a given: value: ' where = [and = ]*'. The where parser owns the grammar; this pattern only bounds the character set." }, "SymbolBinding": { "type": "object", "minProperties": 1, "additionalProperties": { - "type": ["string", "number", "boolean"] + "type": [ + "string", + "number", + "boolean" + ] }, - "description": "v0.3 map of names to $-sigil symbols or scalar literals. Backs controller/access-rule bind: and service with:." + "description": "Map of names to $-sigil symbols or scalar literals, backing access-rule bind: and service with:." }, "InputConfig": { - "description": "v0.3 declares the named inputs a service requires.", + "description": "Named inputs a service requires.", "oneOf": [ - { "type": "array", "items": { "type": "string" }, "minItems": 1 }, + { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, { "type": "object", "minProperties": 1, "additionalProperties": { "oneOf": [ - { "type": "string", "enum": ["string", "number", "boolean", "datetime", "enum", "array", "object", "ref"] }, - { "$ref": "#/$defs/FieldConfig" } + { + "type": "string", + "enum": [ + "string", + "number", + "boolean", + "datetime", + "enum", + "array", + "object", + "ref" + ] + }, + { + "$ref": "#/$defs/FieldConfig" + } ] } } ] }, - "OperationFlags": { - "type": "object", - "properties": { - "list": { "type": "boolean" }, - "get": { "type": "boolean" }, - "create": { "type": "boolean" }, - "update": { "type": "boolean" }, - "replace": { "type": "boolean" }, - "delete": { "type": "boolean" }, - "register": { "type": "boolean" }, - "login": { "type": "boolean" } - } - }, "EntityConfig": { "type": "object", + "additionalProperties": false, "properties": { "type": { "type": "string", - "enum": ["base", "auth"], + "enum": [ + "base", + "auth" + ], "default": "base" }, "extends": { @@ -278,275 +252,432 @@ } }, "auth": { - "type": "object", - "properties": { - "identifier": { "type": "string" }, - "via": { - "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" }, "minItems": 1 }, - { "type": "array", "items": { "$ref": "#/$defs/ProviderBinding" }, "minItems": 1 } - ] - }, - "roles": { "type": "string" }, - "bindings": { - "type": "array", - "items": { "$ref": "#/$defs/ProviderBinding" } - } - } + "$ref": "yaml-spec-v0.2.schema.json#/$defs/EntityAuthConfig" + }, + "roles": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/RolesConfig" } }, "allOf": [ { "if": { - "not": { "required": ["extends"] } + "not": { + "required": [ + "extends" + ] + } }, "then": { - "required": ["fields"] + "required": [ + "fields" + ] } }, { "if": { - "properties": { "type": { "const": "auth" } }, - "required": ["type"] + "properties": { + "type": { + "const": "auth" + } + }, + "required": [ + "type" + ] }, "then": { - "required": ["auth"] + "required": [ + "auth" + ] } } - ] }, - "RolesConfig": { + "ProviderConfig": { "type": "object", - "required": ["values", "default"], + "required": [ + "type" + ], "properties": { - "values": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1 - }, - "default": { "type": "string" }, - "hierarchy": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { "type": "string" } - } + "type": { + "enum": [ + "oidc", + "saml" + ] } }, - "additionalProperties": false - }, - "ProviderBinding": { - "type": "object", - "required": ["provider"], - "properties": { - "provider": { "type": "string" }, - "identifier": { "type": "string" }, - "claim_mapping": { - "type": "object", - "additionalProperties": { "type": "string" } + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "oidc" + } + } + }, + "then": { + "$ref": "#/$defs/OIDCProviderConfig" + } }, - "auto_provision": { "type": "boolean" }, - "relations": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": ["collection", "match"], - "additionalProperties": false, + { + "if": { "properties": { - "collection": { "type": "string" }, - "match": { "type": "object", "additionalProperties": { "type": "string" } } + "type": { + "const": "saml" + } } + }, + "then": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/SAMLProviderConfig" } } - }, - "additionalProperties": false - }, - "ProviderConfig": { - "oneOf": [ - { "$ref": "#/$defs/OIDCProviderConfig" }, - { "$ref": "#/$defs/SAMLProviderConfig" } ] }, "OIDCProviderConfig": { "type": "object", - "required": ["type", "client_id"], + "additionalProperties": false, + "required": [ + "type", + "client_id", + "discovery_url" + ], "properties": { - "type": { "type": "string", "const": "oidc" }, - "trust": { "type": "string", "enum": ["app", "idp"], "default": "app" }, - "client_id": { "type": "string" }, - "client_secret": { "type": "string" }, - "discovery_url": { "type": "string", "description": "OIDC discovery document URL; issuer is derived from it automatically" }, - "issuer": { "type": "string" }, - "audiences": { "type": "array", "items": { "type": "string" }, "description": "Accepted token audiences; defaults to [client_id] when omitted" }, - "pkce": { "type": "boolean", "default": false, "description": "Emit PKCE (S256 code_challenge on authorization, code_verifier on token exchange) for the OIDC client; enable to match a Keycloak client with Require PKCE" }, - "dpop": { "type": "boolean", "default": false, "description": "Emit DPoP (RFC 9449): a client-held key pair and per-request DPoP proof on the token and userinfo calls, switching those from Bearer to DPoP-bound; enable to match a provider that requires sender-constrained tokens" } + "type": { + "const": "oidc" + }, + "trust": { + "type": "string", + "enum": [ + "app", + "idp" + ], + "default": "app" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "discovery_url": { + "type": "string", + "description": "OIDC discovery document URL; issuer is derived from it automatically" + }, + "issuer": { + "type": "string" + }, + "audiences": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/StringList", + "description": "Accepted token audiences; defaults to [client_id] when omitted" + }, + "pkce": { + "type": "boolean", + "default": false, + "description": "Emit PKCE (S256 code_challenge on authorization, code_verifier on token exchange)" + }, + "dpop": { + "type": "boolean", + "default": false, + "description": "Emit DPoP (RFC 9449) proofs on the token and userinfo calls, switching them from Bearer to DPoP-bound" + } }, "if": { - "properties": { "trust": { "const": "idp" } }, - "required": ["trust"] - }, - "then": { "required": ["discovery_url"] }, - "else": { "required": ["discovery_url"] }, - "additionalProperties": false - }, - "SAMLProviderConfig": { - "type": "object", - "required": ["type"], - "properties": { - "type": { "type": "string", "const": "saml" }, - "trust": { "type": "string", "enum": ["app", "idp"], "default": "app" }, - "idp_metadata_url": { "type": "string" }, - "sp_entity_id": { "type": "string" }, - "issuer": { "type": "string" }, - "audience": { "type": "string" } + "properties": { + "trust": { + "const": "idp" + } + }, + "required": [ + "trust" + ] }, - "if": { - "properties": { "trust": { "const": "idp" } }, - "required": ["trust"] + "then": { + "properties": { + "pkce": false, + "dpop": false + } }, - "then": { "required": ["issuer", "audience"] }, - "else": { "required": ["idp_metadata_url", "sp_entity_id"] }, - "additionalProperties": false + "else": { + "properties": { + "issuer": false + } + } }, "DtoConfig": { "type": "object", - "required": ["fields"], + "additionalProperties": false, + "required": [ + "fields" + ], "properties": { "fields": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/FieldConfig" } - } - }, - "additionalProperties": false - }, - "PolicyConfig": { - "type": "object", - "required": ["when"], - "properties": { - "given": { - "type": "object", - "additionalProperties": { "type": "string" } - }, - "params": { - "type": "array", - "items": { "type": "string" } - }, - "when": { - "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" } } - ] + "additionalProperties": { + "$ref": "#/$defs/FieldConfig" + } } - }, - "additionalProperties": false + } }, "FieldConfig": { "type": "object", - "required": ["type"], + "additionalProperties": false, + "required": [ + "type" + ], "properties": { "type": { - "enum": ["string", "number", "boolean", "datetime", "enum", "array", "object", "ref", "id", "file"] + "enum": [ + "string", + "number", + "boolean", + "datetime", + "enum", + "array", + "object", + "ref", + "id", + "file" + ] + }, + "required": { + "type": "boolean" }, - "required": { "type": "boolean" }, "default": {}, - "unique": { "type": "boolean" }, - "min": { "type": "number" }, - "max": { "type": "number" }, - "minLength": { "type": "integer" }, - "maxLength": { "type": "integer" }, + "unique": { + "type": "boolean" + }, + "min": { + "type": "number" + }, + "max": { + "type": "number" + }, + "minLength": { + "type": "integer" + }, + "maxLength": { + "type": "integer" + }, "format": { - "enum": ["email", "e164", "uri", "uuidv7", "uuidv4"] + "enum": [ + "email", + "e164", + "uri", + "uuidv7", + "uuidv4" + ] }, "values": { - "type": "array", - "items": { "type": "string" } + "$ref": "yaml-spec-v0.2.schema.json#/$defs/StringList" + }, + "items": { + "$ref": "#/$defs/FieldConfig" }, - "items": { "$ref": "#/$defs/FieldConfig" }, "properties": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/FieldConfig" } - }, - "additionalProperties": { "type": "boolean" }, - "ref": { "type": "string" }, - "readonly": { "type": "boolean" }, - "maxSize": { "$ref": "#/$defs/FileSize" }, - "types": { "$ref": "#/$defs/MimeTypes" }, - "disposition": { "enum": ["attachment", "inline"] }, - "maxItems": { "type": "integer", "minimum": 1 } + "additionalProperties": { + "$ref": "#/$defs/FieldConfig" + } + }, + "additionalProperties": { + "type": "boolean" + }, + "ref": { + "type": "string" + }, + "readonly": { + "type": "boolean" + }, + "maxSize": { + "$ref": "#/$defs/FileSize" + }, + "types": { + "$ref": "#/$defs/MimeTypes" + }, + "disposition": { + "enum": [ + "attachment", + "inline" + ] + }, + "maxItems": { + "type": "integer", + "minimum": 1 + } }, "allOf": [ { - "if": { "properties": { "type": { "const": "enum" } }, "required": ["type"] }, - "then": { "required": ["values"] } + "if": { + "properties": { + "type": { + "const": "enum" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "values" + ] + } }, { - "if": { "properties": { "type": { "const": "array" } }, "required": ["type"] }, - "then": { "required": ["items"] } + "if": { + "properties": { + "type": { + "const": "array" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "items" + ] + } }, { - "if": { "properties": { "type": { "const": "ref" } }, "required": ["type"] }, - "then": { "required": ["ref"] } + "if": { + "properties": { + "type": { + "const": "ref" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "ref" + ] + } } ] }, - "OperationConfig": { - "type": "object", - "properties": { - "query": { - "type": "object", - "properties": { - "command": { "type": "string" }, - "collection": { "type": "string" }, - "sql": { "type": "string" }, - "params": { "type": "object" } - }, - "additionalProperties": true - }, - "handler": { "type": "string" } - } - }, "DatastoreConfig": { "type": "object", + "additionalProperties": false, "properties": { "type": { "type": "string", - "enum": ["mongodb", "mongo", "postgres", "postgresql", "filesystem", "s3"] - }, - "uri": { "type": "string" }, - "database": { "type": "string" }, - "dsn": { "type": "string" }, - "default": { "type": "boolean" }, - "path": { "type": "string" }, - "bucket": { "type": "string" }, - "endpoint": { "type": "string" }, - "region": { "type": "string" }, - "accessKey": { "type": "string" }, - "secretKey": { "type": "string" }, - "prefix": { "type": "string" }, - "files": { "$ref": "#/$defs/FileLimits" } + "enum": [ + "mongodb", + "mongo", + "postgres", + "postgresql", + "filesystem", + "s3" + ] + }, + "uri": { + "type": "string" + }, + "database": { + "type": "string" + }, + "dsn": { + "type": "string" + }, + "default": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "bucket": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "region": { + "type": "string" + }, + "accessKey": { + "type": "string" + }, + "secretKey": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "files": { + "$ref": "#/$defs/FileLimits" + } }, "allOf": [ { - "if": { "properties": { "type": { "const": "filesystem" } }, "required": ["type"] }, - "then": { "required": ["path"] } + "if": { + "properties": { + "type": { + "const": "filesystem" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "path" + ] + } }, { - "if": { "properties": { "type": { "const": "s3" } }, "required": ["type"] }, - "then": { "required": ["bucket"] } + "if": { + "properties": { + "type": { + "const": "s3" + } + }, + "required": [ + "type" + ] + }, + "then": { + "required": [ + "bucket" + ] + } }, { - "if": { "properties": { "type": { "enum": ["mongodb", "mongo", "postgres", "postgresql"] } } }, - "then": { "not": { "required": ["files"] } } + "if": { + "properties": { + "type": { + "enum": [ + "mongodb", + "mongo", + "postgres", + "postgresql" + ] + } + } + }, + "then": { + "not": { + "required": [ + "files" + ] + } + } } ] }, "RepositoryConfig": { "type": "object", + "additionalProperties": false, "properties": { - "abstract": { "type": "boolean" }, - "extends": { "type": "string" }, + "abstract": { + "type": "boolean" + }, + "extends": { + "type": "string" + }, "entity": { "type": "string" }, @@ -560,34 +691,66 @@ "type": "array", "items": { "type": "string", - "enum": ["list", "get", "create", "update", "replace", "delete"] + "enum": [ + "list", + "get", + "create", + "update", + "replace", + "delete" + ] } }, "operations": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/OperationConfig" + "$ref": "yaml-spec-v0.2.schema.json#/$defs/OperationConfig" } } } }, "ServiceConfig": { "type": "object", + "additionalProperties": false, "properties": { - "abstract": { "type": "boolean" }, - "extends": { "type": "string" }, - "repository": { "type": "string" }, - "input": { "$ref": "#/$defs/InputConfig" }, - "with": { "$ref": "#/$defs/SymbolBinding" }, + "abstract": { + "type": "boolean" + }, + "extends": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "handler": { + "type": "string" + }, + "input": { + "$ref": "#/$defs/InputConfig" + }, + "with": { + "$ref": "#/$defs/SymbolBinding" + }, "operations": { "type": "object", "additionalProperties": { "oneOf": [ - { "type": "boolean" }, + { + "type": "boolean" + }, { "type": "object", + "additionalProperties": false, + "required": [ + "handler" + ], "properties": { - "handler": { "type": "string" } + "handler": { + "type": "string" + } } } ] @@ -597,6 +760,7 @@ }, "ControllerConfig": { "type": "object", + "additionalProperties": false, "properties": { "service": { "type": "string" @@ -608,26 +772,96 @@ "$ref": "#/$defs/AccessConfig" }, "operations": { - "type": "object" - }, - "routes": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/RouteConfig" + "oneOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/OperationOverride" + } + ] + } + }, + "routes": { + "type": "object" + } + }, + "if": { + "required": [ + "service" + ] + }, + "then": { + "properties": { + "routes": { + "additionalProperties": { + "$ref": "#/$defs/RouteOverride" + } } } + }, + "else": { + "properties": { + "access": false, + "operations": false, + "routes": { + "additionalProperties": { + "$ref": "#/$defs/RouteConfig" + } + } + } + } + }, + "OperationOverride": { + "type": "object", + "additionalProperties": false, + "properties": { + "method": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/HttpMethod" + }, + "path": { + "type": "string" + }, + "access": { + "$ref": "#/$defs/AccessConfig" + }, + "pagination": { + "$ref": "#/$defs/PaginationConfig" + } + } + }, + "RouteOverride": { + "type": "object", + "additionalProperties": false, + "properties": { + "access": { + "$ref": "#/$defs/AccessConfig" + }, + "pagination": { + "$ref": "#/$defs/PaginationConfig" + } } }, "RouteConfig": { "type": "object", - "required": ["service", "method", "path"], + "additionalProperties": false, + "required": [ + "service", + "method", + "path" + ], "properties": { - "service": { "type": "string" }, + "service": { + "type": "string" + }, "method": { - "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"] + "$ref": "yaml-spec-v0.2.schema.json#/$defs/HttpMethod" + }, + "path": { + "type": "string" }, - "path": { "type": "string" }, "access": { "$ref": "#/$defs/AccessConfig" }, @@ -637,42 +871,82 @@ } }, "PaginationConfig": { - "description": "v0.3 controller pagination. Only the offset and keyset strategies and count none/exact are supported; cursor, range and count: estimated are named by the grammar so the parser can reject them by name (PAGINATION_UNSUPPORTED).", + "description": "Controller pagination. The parser accepts the offset and keyset strategies and count none/exact; cursor, range and count: estimated are named so it can reject them by name (PAGINATION_UNSUPPORTED).", "type": "object", "additionalProperties": false, - "required": ["strategy", "size"], + "required": [ + "strategy", + "size" + ], "properties": { "strategy": { "type": "string", - "enum": ["offset", "keyset", "cursor", "range"] + "enum": [ + "offset", + "keyset", + "cursor", + "range" + ] }, "size": { "type": "object", - "required": ["default", "max"], + "additionalProperties": false, + "required": [ + "default", + "max" + ], "properties": { - "default": { "type": "integer", "minimum": 0 }, - "max": { "type": "integer", "minimum": 1 } + "default": { + "type": "integer", + "minimum": 0 + }, + "max": { + "type": "integer", + "minimum": 1 + } } }, "count": { "type": "string", - "enum": ["none", "exact", "estimated"], + "enum": [ + "none", + "exact", + "estimated" + ], "default": "none" }, "onOverflow": { "type": "string", - "enum": ["clamp", "reject"], + "enum": [ + "clamp", + "reject" + ], "default": "clamp" }, - "envelope": { "type": "boolean", "default": false }, + "envelope": { + "type": "boolean", + "default": false + }, "sort": { "type": "array", "items": { "type": "object", - "required": ["field"], + "additionalProperties": false, + "required": [ + "field" + ], "properties": { - "field": { "type": "string" }, - "dir": { "type": "string", "enum": ["asc", "desc"], "default": "asc" } + "field": { + "type": "string" + }, + "dir": { + "type": "string", + "enum": [ + "asc", + "desc" + ], + "default": "asc" + } } } } @@ -687,84 +961,36 @@ "type": "array", "items": { "oneOf": [ - { "type": "string" }, - { "$ref": "#/$defs/AccessRule" } + { + "type": "string" + }, + { + "$ref": "#/$defs/AccessRule" + } ] } } ] }, - "DeployConfig": { + "AccessRule": { "type": "object", - "required": ["runtime"], - "properties": { - "name": { "type": "string" }, - "port": { "type": "integer", "default": 8000 }, - "runtime": { "type": "string", "enum": ["docker", "systemd"] }, - "services": { - "type": "object", - "additionalProperties": { - "type": "object", - "anyOf": [ - { "required": ["image"] }, - { "required": ["build"] } - ], - "properties": { - "image": { "type": "string" }, - "build": { "oneOf": [ - { "type": "string" }, - { "type": "object", "properties": { - "context": { "type": "string" }, - "dockerfile": { "type": "string" }, - "args": { "type": "object", "additionalProperties": { "type": "string" } } - } } - ]}, - "ports": { "type": "array", "items": { "oneOf": [{"type": "integer"}, {"type": "string"}] } }, - "environment": { "oneOf": [ - { "type": "object", "additionalProperties": { "type": "string" } }, - { "type": "array", "items": { "type": "string" } } - ]}, - "volumes": { "type": "array", "items": { "type": "string" } }, - "depends_on": { "oneOf": [ - { "type": "array", "items": { "type": "string" } }, - { "type": "object" } - ]}, - "command": { "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}] } - } - } - }, - "include": { - "type": "array", - "items": { "type": "string" } - }, - "mongo": { - "type": "object", - "properties": { - "local": { "type": "boolean", "default": false } - } + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "given" + ] }, - "docker": { - "type": "object", - "properties": { - "image": { "type": "string" } - } + { + "required": [ + "then" + ] }, - "systemd": { - "type": "object", - "properties": { - "user": { "type": "string" }, - "restart": { "type": "string" } - } + { + "required": [ + "when" + ] } - } - }, - "AccessRule": { - "type": "object", - "anyOf": [ - { "required": ["given"] }, - { "required": ["then"] }, - { "required": ["when"] }, - { "required": ["role"] } ], "properties": { "given": { @@ -772,27 +998,41 @@ "minProperties": 1, "additionalProperties": { "anyOf": [ - { "type": "string", "pattern": "^identity:[A-Za-z_][A-Za-z0-9_]*$" }, - { "type": "string", "pattern": "^request(\\.(body|path|query|header|cookie))?(:[A-Za-z_][A-Za-z0-9_]*)?$" }, - { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\s+by\\s+[A-Za-z_][A-Za-z0-9_]*)?$" }, - { "$ref": "#/$defs/WhereExpression" } + { + "type": "string", + "pattern": "^identity:[A-Za-z_][A-Za-z0-9_]*$" + }, + { + "type": "string", + "pattern": "^request(\\.(body|path|query|header|cookie))?(:[A-Za-z_][A-Za-z0-9_]*)?$" + }, + { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\s+by\\s+[A-Za-z_][A-Za-z0-9_]*)?$" + }, + { + "$ref": "#/$defs/WhereExpression" + } ] } }, "when": { - "oneOf": [ - { "type": "string" }, - { "type": "array", "items": { "type": "string" }, "minItems": 1 } - ] + "$ref": "yaml-spec-v0.2.schema.json#/$defs/WhenConfig" }, "then": { "type": "string", - "enum": ["allow", "deny"], + "enum": [ + "allow", + "deny" + ], "default": "allow" }, - "reason": { "type": "string" }, - "role": { "type": "string" }, - "bind": { "$ref": "#/$defs/SymbolBinding" } + "reason": { + "type": "string" + }, + "bind": { + "$ref": "#/$defs/SymbolBinding" + } } } } diff --git a/src/yaml_api_generator/parsers/transformers/access_validation.py b/src/yaml_api_generator/parsers/transformers/access_validation.py index 606f088a..852370d9 100644 --- a/src/yaml_api_generator/parsers/transformers/access_validation.py +++ b/src/yaml_api_generator/parsers/transformers/access_validation.py @@ -266,15 +266,7 @@ def _check_rule_list( and v.startswith("identity:")), "", ) - role = item.get("role") - if isinstance(role, str) and principal: - _check_role_declared( - role, principal.removeprefix("identity:"), declared_roles, - f"{item_path}.role", errors, - ) - _track_contradiction( - (principal, role, expr), effect, item_path, seen, errors - ) + _track_contradiction((principal, expr), effect, item_path, seen, errors) def _track_contradiction( @@ -316,12 +308,7 @@ def _is_always_allow_rule(rule: Any) -> bool: return False given = rule.get("given") when = rule.get("when") - return ( - not given - and when is None - and rule.get("role") is None - and rule.get("then", "allow") == "allow" - ) + return not given and when is None and rule.get("then", "allow") == "allow" def _warn_access_rules( diff --git a/src/yaml_api_generator/parsers/transformers/shape_validation.py b/src/yaml_api_generator/parsers/transformers/shape_validation.py deleted file mode 100644 index e489c1d0..00000000 --- a/src/yaml_api_generator/parsers/transformers/shape_validation.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Container-shape guards for the pre-desugar v0.2 spec. - -Desugaring dereferences the top-level blocks as mappings (``.items()``, -``.get()``) before JSON-schema validation runs. A block authored as the wrong -container type (a list or scalar where a mapping is expected) would otherwise -crash desugar with a raw ``AttributeError`` instead of a clean -``SCHEMA_VALIDATION_ERROR``. These guards run on the pre-desugar spec, mirroring -``validate_spec_names``. -""" - -from typing import Any - -from yaml_api_generator.parsers.yaml_parser import ConfigError - -_MAPPING_BLOCKS = ("entities", "repositories", "services", "controllers", "api") - - -def _shape_error(path: str, value: Any) -> ConfigError: - return ConfigError( - code="SCHEMA_VALIDATION_ERROR", - message=f"'{path}' must be an object, got {type(value).__name__}", - path=path, - hint="Define it as a mapping of named entries", - ) - - -def validate_spec_shape(spec: dict[str, Any]) -> list[ConfigError]: - """Reject top-level blocks that desugar would dereference as the wrong type.""" - errors = [ - _shape_error(block, spec[block]) - for block in _MAPPING_BLOCKS - if block in spec and not isinstance(spec[block], dict) - ] - api = spec.get("api") - if isinstance(api, dict) and "operations" in api and not isinstance(api["operations"], dict): - errors.append(_shape_error("api.operations", api["operations"])) - return errors diff --git a/src/yaml_api_generator/parsers/transformers/trust_validation.py b/src/yaml_api_generator/parsers/transformers/trust_validation.py index 09d98bb6..9a6fd825 100644 --- a/src/yaml_api_generator/parsers/transformers/trust_validation.py +++ b/src/yaml_api_generator/parsers/transformers/trust_validation.py @@ -1,104 +1,27 @@ -"""Semantic coherence checks for per-provider ``trust`` in a v0.2 spec. +"""Advisory checks for per-provider ``trust`` in a v0.2/v0.3 spec. -A provider's ``trust`` decides who verifies its tokens on each request: - -- ``app`` (the default) -- after login the app mints and verifies its own - token. This is the only meaningful value for password/apiKey-style logins, - since no external IdP sits behind them. -- ``idp`` -- the client keeps carrying the IdP's token and the app verifies it - against the IdP each call, issuing no session of its own. For OIDC providers - this requires ``discovery_url``; the issuer is derived from it automatically. - ``audiences`` is optional and defaults to ``[client_id]`` at runtime. - -These checks run on the pre-desugar spec, mirroring ``validate_spec_auth``, and -reject the spec with coded errors before the schema's structural validation. +Under ``trust: idp`` the app verifies the IdP's token on every request. The +schema rejects incoherent trust settings; this module only warns when an OIDC +provider leaves ``audiences`` to its ``[client_id]`` default. """ from typing import Any from yaml_api_generator.parsers.yaml_parser import ConfigError -_TRUST_VALUES = frozenset({"app", "idp"}) -_IDP_PROVIDER_TYPES = frozenset({"oidc", "saml"}) -_APP_ONLY_KEYS = ( - "algorithm", - "secret", - "ttl", - "refresh", - "delivery", - "pkce", - "dpop", -) - - -def _check_provider( - name: str, config: dict[str, Any], results: list[ConfigError] -) -> None: - trust = config.get("trust", "app") - base = f"providers.{name}" - - if trust not in _TRUST_VALUES: - results.append(ConfigError( - code="TRUST_INVALID_VALUE", - message=f"Provider '{name}' has invalid trust '{trust}'", - path=f"{base}.trust", - hint="Set trust to 'app' (the app mints its own token) or 'idp'", - )) - return - - if trust == "app": - if "issuer" in config: - results.append(ConfigError( - code="TRUST_APP_FIELD_FORBIDDEN", - message=f"Provider '{name}' is app-trust but declares 'issuer'", - path=f"{base}.issuer", - hint="'issuer' only applies to trust: idp; remove it", - )) - return - - if config.get("type") not in _IDP_PROVIDER_TYPES: - results.append(ConfigError( - code="TRUST_IDP_PROVIDER_UNSUPPORTED", - message=f"Provider '{name}' uses trust: idp but is not an OIDC or SAML provider", - path=f"{base}.trust", - hint="trust: idp is only valid for providers fronting an external IdP", - )) - if config.get("type") == "oidc" and not config.get("discovery_url"): - results.append(ConfigError( - code="TRUST_IDP_DISCOVERY_URL_REQUIRED", - message=f"Provider '{name}' uses trust: idp but declares no discovery_url", - path=f"{base}.discovery_url", - hint="Set discovery_url; the issuer is derived from it automatically", - )) - - if config.get("type") == "oidc" and not config.get("audiences"): - results.append(ConfigError( +def validate_spec_trust(spec: dict[str, Any]) -> list[ConfigError]: + return [ + ConfigError( code="TRUST_IDP_AUDIENCE_DEFAULTED", message=( f"Provider '{name}': verifying aud against client_id by default; " "set audiences explicitly to also accept access tokens from sibling services " "(resource-server mode)." ), - path=f"{base}.audiences", + path=f"providers.{name}.audiences", severity="warning", - )) - - for key in _APP_ONLY_KEYS: - if key in config: - results.append(ConfigError( - code="TRUST_IDP_APP_FIELD_FORBIDDEN", - message=f"Provider '{name}' is idp-trust but declares app-signing field '{key}'", - path=f"{base}.{key}", - hint="The app issues no token under trust: idp; remove this field", - )) - - -def validate_spec_trust(spec: dict[str, Any]) -> list[ConfigError]: - """Validate per-provider trust coherence across every provider in the spec.""" - results: list[ConfigError] = [] - providers = spec.get("providers") or {} - for name, config in providers.items(): - if isinstance(config, dict): - _check_provider(name, config, results) - return results + ) + for name, config in spec.get("providers", {}).items() + if config.get("trust") == "idp" and not config.get("audiences") + ] diff --git a/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py index ad200b23..d35956c9 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py @@ -175,7 +175,7 @@ def _parse_access_items( if when_raw is not None: when_expr, when_errors = parse_when_expr(when_raw, f"{item_path}.when") errors.extend(when_errors) - if self._is_bare_authenticated(resolved, when_expr, extras): + if self._is_bare_authenticated(resolved, when_expr): continue rules.append( IRAccessRule( @@ -188,16 +188,10 @@ def _parse_access_items( return is_public, rules, errors @staticmethod - def _is_bare_authenticated(resolved, when_expr, extras: dict[str, Any]) -> bool: + def _is_bare_authenticated(resolved, when_expr) -> bool: """A plain 'authenticated' principal carries no identity, role, or constraint, so it adds nothing beyond requiring authentication; no rule is emitted.""" - return ( - resolved.principal is None - and resolved.required_role is None - and when_expr is None - and not extras.get("filter") - and not extras.get("write_fields") - ) + return resolved.principal is None and resolved.required_role is None and when_expr is None @staticmethod def _unpack_item(item: Any) -> tuple[str | None, str | None, dict[str, Any]]: @@ -205,10 +199,6 @@ def _unpack_item(item: Any) -> tuple[str | None, str | None, dict[str, Any]]: return item, None, {} if isinstance(item, dict): principal = item.get("principal") - extras = { - "filter": item.get("filter"), - "write_fields": item.get("writeFields", []), - "reason": item.get("reason"), - } + extras = {"reason": item.get("reason")} return (principal if isinstance(principal, str) else None), item.get("when"), extras return None, None, {} diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 41e44181..114a3212 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -54,7 +54,6 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: repos_data: dict[str, Any] = resolved.get("repositories", {}) services_data: dict[str, Any] = resolved.get("services", {}) controllers_data: dict[str, Any] = resolved.get("controllers", {}) - roles_data: dict[str, Any] = resolved.get("roles", {}) providers_data: dict[str, Any] = resolved.get("providers", {}) dtos_data: dict[str, Any] = resolved.get("dto", {}) policies_data: dict[str, Any] = resolved.get("policies", {}) @@ -108,14 +107,6 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: if field_errors: raise ConfigErrors(field_errors) - role_hierarchies: dict[str, dict[str, list[str]]] = {} - if roles_data: - role_hierarchies, role_errors = self._identity.parse_role_hierarchies( - roles_data, auth_entities - ) - if role_errors: - raise ConfigErrors(role_errors) - services = self._route.transform_services( services_data, repo_entity_map, import_map, dtos_data ) @@ -155,7 +146,6 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: identityCollections=identity_collections, dtos=dtos, services=services, - role_hierarchies=role_hierarchies, mongo=self._route.build_mongo(mongo_config), datastores=datastores, default_datastore=default_datastore, diff --git a/src/yaml_api_generator/parsers/transformers/v02/access_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/access_transformer.py index e13ce7ca..8e156726 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/access_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/access_transformer.py @@ -108,34 +108,6 @@ def _parse_access_rule_object( entity_to_collection: dict[str, str], ) -> tuple[IRAccessRule | None, list[ConfigError]]: errors: list[ConfigError] = [] - if not self._allow_where and "bind" in rule: - errors.append(ConfigError( - code="BIND_UNSUPPORTED", - message="access-rule 'bind' requires spec version 0.3", - path=f"{path}.bind", - hint='Set version to "0.3"', - )) - if self._allow_where and "filter" in rule: - errors.append(ConfigError( - code="ACCESS_FILTER_REMOVED", - message="access-rule 'filter' key was removed in v0.3", - path=f"{path}.filter", - hint="Bind the record in given: with ' where = '", - )) - if not self._allow_where and "role" in rule: - errors.append(ConfigError( - code="ACCESS_ROLE_UNSUPPORTED", - message="access-rule 'role' requires spec version 0.3", - path=f"{path}.role", - hint='Set version to "0.3", or use the ":" shorthand', - )) - if self._allow_where and "writeFields" in rule: - errors.append(ConfigError( - code="ACCESS_WRITE_FIELDS_REMOVED", - message="access-rule 'writeFields' is not supported in v0.3", - path=f"{path}.writeFields", - hint="Restrict write input with a service input contract or custom handler", - )) given = rule.get("given", {}) effect = rule.get("then", "allow") @@ -167,7 +139,7 @@ def _parse_access_rule_object( bind: dict[str, WhereRhs] = {} bind_raw = rule.get("bind") - if self._allow_where and isinstance(bind_raw, dict): + if isinstance(bind_raw, dict): from yaml_api_generator.parsers.service_input_parser import parse_service_with bind, bind_errors = parse_service_with(bind_raw, f"{path}.bind") @@ -179,25 +151,13 @@ def _parse_access_rule_object( first_identity = next(iter(identity_symbols.values()), None) principal = f"identity:{first_identity.collection_name}" if first_identity else "" - required_role = rule.get("role") if self._allow_where else None - if required_role is not None and first_identity is None: - return None, [ConfigError( - code="ACCESS_ROLE_WITHOUT_IDENTITY", - message="access-rule 'role' requires an identity binding in given:", - path=f"{path}.role", - hint="Add ': identity:' to given:", - )] - return IRAccessRule( principal=principal, - required_role=required_role, identity_symbols=identity_symbols, document_symbols=document_symbols, request_symbols=request_symbols, when=when_expr, effect=effect, - filter=rule.get("filter"), - write_fields=rule.get("writeFields", []), reason=rule.get("reason"), bind=bind, ), [] diff --git a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py index 1c3fc20b..a0b58308 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py @@ -135,40 +135,6 @@ def validate_auth_fields( ) return errors - def parse_role_hierarchies( - self, - roles_data: dict[str, Any], - auth_entities: set[str], - ) -> tuple[dict[str, dict[str, list[str]]], list[ConfigError]]: - """Parse role hierarchy definitions. - - Args: - roles_data: Raw role hierarchy configurations from YAML - auth_entities: Set of entity names marked as auth type - - Returns: - Tuple of (role hierarchies dict, validation errors) - """ - errors: list[ConfigError] = [] - hierarchies: dict[str, dict[str, list[str]]] = {} - for entity_name, role_map in roles_data.items(): - if entity_name not in auth_entities: - errors.append( - ConfigError( - code="ROLES_UNKNOWN_ENTITY", - message=f"Role hierarchy references unknown auth entity '{entity_name}'", - path=f"roles.{entity_name}", - ) - ) - continue - if not isinstance(role_map, dict): - continue - hierarchies[entity_name] = { - role: list(includes) if isinstance(includes, list) else [] - for role, includes in role_map.items() - } - return hierarchies, errors - def _build_roles_config( self, entity_name: str, roles_data: Any ) -> tuple[IRRolesConfig | None, list[ConfigError]]: diff --git a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py index ea806279..4231f36b 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py @@ -26,13 +26,8 @@ class RouteTransformer: """Builds routes from controllers and services.""" - def __init__( - self, - access_transformer: AccessTransformer, - allow_service_inputs: bool = False, - ): + def __init__(self, access_transformer: AccessTransformer): self._access_transformer = access_transformer - self._allow_service_inputs = allow_service_inputs def transform_services( self, @@ -98,16 +93,6 @@ def _transform_input( ) input_ir = None - if not self._allow_service_inputs and ({"input", "with"} & svc_cfg.keys()): - errors.append( - ConfigError( - code="SERVICE_INPUT_UNSUPPORTED", - message="service input and with forwarding require spec version 0.3", - path=f"services.{svc_name}", - hint='Set version to "0.3"', - ) - ) - return None, {} if "input" in svc_cfg: input_ir, input_errors = parse_service_input( svc_cfg["input"], dtos_data, f"services.{svc_name}.input" diff --git a/src/yaml_api_generator/parsers/transformers/v03/__init__.py b/src/yaml_api_generator/parsers/transformers/v03/__init__.py index 74446bcb..8a0c7397 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v03/__init__.py @@ -26,7 +26,7 @@ def __init__(self, spec_dir: Path) -> None: super().__init__() self._spec_dir = spec_dir self._access = AccessTransformer(allow_where=True) - self._route = RouteTransformer(self._access, allow_service_inputs=True) + self._route = RouteTransformer(self._access) def _process_imports( self, resolved: dict[str, Any] diff --git a/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py b/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py index a78c1c31..fa48205d 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py +++ b/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py @@ -54,15 +54,6 @@ def process_vendored_imports( for alias, entry in entries.items(): path = f"imports.{alias}" - if isinstance(entry, str): - errors.append(ConfigError( - code="IMPORT_BARE_STRING", - message=f"Import '{alias}' uses the bare dotted-string form, which v0.3 does not accept", - path=path, - hint="Vendor the package with an object: { path: ./src/handlers, package: name, requires: [...] }", - )) - continue - package = entry["package"] if package in package_owner: errors.append(ConfigError( diff --git a/src/yaml_api_generator/parsers/ui_parser.py b/src/yaml_api_generator/parsers/ui_parser.py index fc7ae75c..5dd3506d 100644 --- a/src/yaml_api_generator/parsers/ui_parser.py +++ b/src/yaml_api_generator/parsers/ui_parser.py @@ -28,15 +28,6 @@ def build_generator_context( ui_block: dict[str, Any], manifest: PackManifest, ) -> GeneratorContext: - frontend_config = ui_block.get("frontend", {}) or {} - pack_config = { - k: v - for k, v in frontend_config.items() - if k not in ("pack", "output", "overwrite", "generate") - } - if "config" in ui_block: - pack_config.update(ui_block["config"]) - entities = _build_entities(ir) identity_entities = [ ir.collections[k].entity_name for k in ir.identityCollections.keys() @@ -61,10 +52,9 @@ def build_generator_context( views=views, pack_components=pack_components, pack_scaffold=manifest.scaffold, - pack_config=pack_config, + pack_config=ui_block.get("config", {}), entities=entities, identity_entities=identity_entities, - role_hierarchies=ir.role_hierarchies, session_delivery=ir.session.delivery, ) diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 78b7486e..8a1258a6 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -16,8 +16,10 @@ - Nested and default-with-reference forms resolve via a fixed-point loop - Literal braces in a default are escaped as \\{ and \\} - Env vars expanded before schema validation +- The resolved spec is schema-validated before any other check runs """ +import json import os import re from dataclasses import dataclass, field @@ -26,7 +28,8 @@ import yaml from jsonschema import Draft202012Validator -from jsonschema.exceptions import ValidationError, best_match +from jsonschema.exceptions import best_match +from referencing import Registry, Resource from ..ir import IRDocument from .errors import ConfigError, ConfigErrors, join_path @@ -36,6 +39,11 @@ INTERPOLATION_TOKEN = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)(?::-(.*))?$", re.DOTALL) INTERPOLATION_MAX_DEPTH = 10 _BRACE_ESCAPES = ((r"\{", "\x00"), (r"\}", "\x01")) +SCHEMA_DIR = Path(__file__).parent / "schemas" +SCHEMA_REGISTRY: Registry = Registry().with_resources( + (path.name, Resource.from_contents(json.loads(path.read_text(encoding="utf-8")))) + for path in SCHEMA_DIR.glob("*.schema.json") +) __all__ = ["ConfigError", "ConfigErrors", "ParseResult", "load_and_generate_ir"] @@ -146,7 +154,7 @@ def _load_schema(schema_file: Path) -> dict[str, Any]: def _validate_schema(config: Any, schema: dict[str, Any]) -> list[ConfigError]: - validator = Draft202012Validator(schema) + validator = Draft202012Validator(schema, registry=SCHEMA_REGISTRY) errors = sorted( validator.iter_errors(config), key=lambda item: (item.json_path, item.message), @@ -154,24 +162,13 @@ def _validate_schema(config: Any, schema: dict[str, Any]) -> list[ConfigError]: return [ ConfigError( code="SCHEMA_VALIDATION_ERROR", - message=_describe_error(error), + message=best_match([error]).message, path=join_path(tuple(str(part) for part in error.path)), ) for error in errors ] -def _describe_error(error: ValidationError) -> str: - """Surface the most specific reason for combinator failures. - - ``oneOf``/``anyOf`` produce an opaque "is not valid under any of the given - schemas"; the actual cause lives in the deepest sub-error. - """ - if error.context: - return best_match(error.context).message - return error.message - - _VERSION_SCHEMA_MAP: dict[str, str] = { "0.1": "yaml-spec-v0.1.schema.json", "0.2": "yaml-spec-v0.2.schema.json", @@ -196,7 +193,7 @@ def _resolve_schema_path( ) ] ) - return Path(__file__).parent / "schemas" / schema_name + return SCHEMA_DIR / schema_name def load_and_generate_ir( @@ -212,15 +209,17 @@ def load_and_generate_ir( raise ConfigErrors(interpolation_errors) version = str(resolved.get("version", "0.1")) + schema_errors = _validate_schema( + resolved, _load_schema(_resolve_schema_path(version, schema_file)) + ) + if schema_errors: + raise ConfigErrors(schema_errors) + + ui_block = resolved.get("ui") trust_warnings: tuple[ConfigError, ...] = () access_warnings: tuple[ConfigError, ...] = () if version in ("0.2", "0.3"): - from .transformers.shape_validation import validate_spec_shape - shape_errors = validate_spec_shape(resolved) - if shape_errors: - raise ConfigErrors(shape_errors) - from .transformers.name_validation import validate_spec_names name_errors = validate_spec_names(resolved) if name_errors: @@ -269,11 +268,7 @@ def load_and_generate_ir( raise ConfigErrors(binding_errors) from .transformers.trust_validation import validate_spec_trust - trust_results = validate_spec_trust(resolved) - trust_errors = [e for e in trust_results if e.severity == "error"] - trust_warnings = tuple(e for e in trust_results if e.severity == "warning") - if trust_errors: - raise ConfigErrors(trust_errors) + trust_warnings = tuple(validate_spec_trust(resolved)) from .desugar import desugar_v02 resolved = desugar_v02(resolved) @@ -288,17 +283,6 @@ def load_and_generate_ir( if request_path_errors: raise ConfigErrors(request_path_errors) - ui_block = resolved.get("ui", None) - - schema_path = _resolve_schema_path(version, schema_file) - - schema = _load_schema(schema_path) - - config_for_validation = {k: v for k, v in resolved.items() if k != "ui"} - schema_errors = _validate_schema(config_for_validation, schema) - if schema_errors: - raise ConfigErrors(schema_errors) - from .transformers.registry import transform_config return ParseResult( diff --git a/tests/fixtures/valid/roles-access.yaml b/tests/fixtures/valid/roles-access.yaml index 46ed862b..970c8606 100644 --- a/tests/fixtures/valid/roles-access.yaml +++ b/tests/fixtures/valid/roles-access.yaml @@ -23,6 +23,12 @@ entities: identifier: email via: password roles: role + roles: + values: [admin, moderator, user] + default: user + hierarchy: + admin: [moderator] + moderator: [user] fields: id: { type: id } email: { type: string, required: true, unique: true } @@ -35,10 +41,6 @@ entities: id: { type: id } title: { type: string, required: true } authorId: { type: ref, ref: user } -roles: - user: - admin: [moderator] - moderator: [user] repositories: users: entity: user diff --git a/tests/fixtures/valid/ui-block.yaml b/tests/fixtures/valid/ui-block.yaml index f6a5b30a..ccc3c367 100644 --- a/tests/fixtures/valid/ui-block.yaml +++ b/tests/fixtures/valid/ui-block.yaml @@ -1,11 +1,14 @@ version: "0.1" ui: - dashboard: - title: My Dashboard - widgets: - - type: chart - - type: table - theme: dark + frontend: + pack: generic + config: + dashboard: + title: My Dashboard + widgets: + - type: chart + - type: table + theme: dark collections: posts: fields: diff --git a/tests/fixtures/valid/v03-service-input.yaml b/tests/fixtures/valid/v03-service-input.yaml index 7595b6d7..31d3c292 100644 --- a/tests/fixtures/valid/v03-service-input.yaml +++ b/tests/fixtures/valid/v03-service-input.yaml @@ -1,6 +1,5 @@ version: "0.3" api: - name: service-input basePath: /api entities: post: diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index cf88226b..d6712202 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -19,7 +19,6 @@ _SPEC = """ version: "0.3" api: - name: generated-where basePath: /api mongo: uri: mongodb://localhost:27017 diff --git a/tests/unit/test_access_reason.py b/tests/unit/test_access_reason.py index 1d8aaf93..e20b5068 100644 --- a/tests/unit/test_access_reason.py +++ b/tests/unit/test_access_reason.py @@ -24,7 +24,6 @@ _SPEC = """ version: "{version}" api: - name: access-reason basePath: /api mongo: uri: mongodb://localhost:27017 diff --git a/tests/unit/test_compiler_context_builder.py b/tests/unit/test_compiler_context_builder.py index a05583d8..238942c4 100644 --- a/tests/unit/test_compiler_context_builder.py +++ b/tests/unit/test_compiler_context_builder.py @@ -50,7 +50,6 @@ def _make_ir(collections=None, identity=None, routes=None, cors=None, repositori collections=cols, routes=routes or [], identityCollections=identity or {}, - role_hierarchies={}, mongo=IRMongo(), repositories=repos, ) diff --git a/tests/unit/test_controller_bind.py b/tests/unit/test_controller_bind.py index da4ed118..bb787685 100644 --- a/tests/unit/test_controller_bind.py +++ b/tests/unit/test_controller_bind.py @@ -47,8 +47,8 @@ def _codes(errors): # --- transformer ---------------------------------------------------------- -def _parse(rule, allow_where=True): - return AccessTransformer(allow_where=allow_where)._parse_access_rule_object( +def _parse(rule): + return AccessTransformer(allow_where=True)._parse_access_rule_object( rule, "p", {"user"}, {"user": "users"} ) @@ -68,12 +68,6 @@ def test_bind_symbol_and_literal_reach_ir(): } -def test_bind_rejected_without_v03_feature_gate(): - rule, errors = _parse({"then": "allow", "bind": {"userId": "$principal.sub"}}, allow_where=False) - assert rule is None - assert [error.code for error in errors] == ["BIND_UNSUPPORTED"] - - # --- build-time symbol resolution ----------------------------------------- @@ -182,7 +176,6 @@ def test_bind_rejects_where_document_rule_on_same_route(): _SPEC = """ version: "0.3" api: - name: test basePath: /api entities: user: @@ -292,7 +285,7 @@ def test_v02_bind_aborts_end_to_end(tmp_path): with pytest.raises(ConfigErrors) as exc_info: load_and_generate_ir(str(spec)) - assert {error.code for error in exc_info.value.errors} == {"BIND_UNSUPPORTED"} + assert {error.code for error in exc_info.value.errors} == {"SCHEMA_VALIDATION_ERROR"} @pytest.mark.parametrize( diff --git a/tests/unit/test_controller_role.py b/tests/unit/test_controller_role.py index 9fca0d2f..47d0b412 100644 --- a/tests/unit/test_controller_role.py +++ b/tests/unit/test_controller_role.py @@ -1,64 +1,18 @@ -"""Tests for v0.3 access-rule ``role:``. +"""Tests for role requirements and where-bound records in access rules. -The ``:`` shorthand carries a role but no ``bind:``, so a route -that both gates on a role and supplies its service inputs from the principal was -previously inexpressible. ``role:`` moves the role requirement onto the long -rule form, where it compiles to the same ``_principal_has_role`` guard and -composes with ``given:``, ``when:`` and ``bind:``. +Roles are granted through the ``:`` shorthand; a long-form rule +carries no ``role:`` key and is rejected by the schema. """ -import compileall -import subprocess -import sys -import textwrap - import pytest import yaml -from yaml_api_generator.generators.backend.context_builder import build_context from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor -from yaml_api_generator.generators.backend.drf.generator import DRFGenerator from yaml_api_generator.generators.backend.fastapi.access_filters import rule_to_python -from yaml_api_generator.generators.backend.fastapi.generator import FastAPIGenerator -from yaml_api_generator.generators.backend.raw.generator import RawASGIGenerator -from yaml_api_generator.parsers.transformers.access_validation import ( - validate_spec_access, - warn_public_routes, -) -from yaml_api_generator.parsers.transformers.v02.access_transformer import AccessTransformer +from yaml_api_generator.parsers.transformers.access_validation import validate_spec_access from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir -def _parse(rule, allow_where=True): - return AccessTransformer(allow_where=allow_where)._parse_access_rule_object( - rule, "p", {"user"}, {"user": "users"} - ) - - -def test_role_reaches_ir_alongside_bind(): - rule, errors = _parse( - { - "given": {"principal": "identity:user"}, - "role": "org-admin", - "bind": {"organizationId": "$principal.record.organizationId"}, - } - ) - assert errors == [] - assert rule.required_role == "org-admin" - assert rule.principal == "identity:users" - - -def test_role_rejected_without_v03_feature_gate(): - _, errors = _parse({"given": {"principal": "identity:user"}, "role": "admin"}, allow_where=False) - assert [error.code for error in errors] == ["ACCESS_ROLE_UNSUPPORTED"] - - -def test_role_requires_an_identity_binding(): - rule, errors = _parse({"then": "allow", "role": "admin"}) - assert rule is None - assert [error.code for error in errors] == ["ACCESS_ROLE_WITHOUT_IDENTITY"] - - def test_role_compiles_to_a_principal_role_guard(): condition = rule_to_python( AccessRuleDescriptor(principal="identity:users", required_role="org-admin") @@ -116,126 +70,34 @@ def test_role_compiles_to_a_principal_role_guard(): access: - given: principal: identity:user - role: ROLE bind: organizationId: $principal.record.organizationId """ -def test_declared_role_passes_validation(): - assert validate_spec_access(yaml.safe_load(_SPEC.replace("ROLE", "org-admin"))) == [] - - -def test_undeclared_role_is_rejected(): - errors = validate_spec_access(yaml.safe_load(_SPEC.replace("ROLE", "ghost-admin"))) - assert [error.code for error in errors] == ["ACCESS_RULE_INVALID"] - assert "ghost-admin" in errors[0].message - - -def test_v03_full_spec_load_preserves_role_and_bind(tmp_path): - path = tmp_path / "api.yaml" - path.write_text(_SPEC.replace("ROLE", "org-admin")) - - rule = load_and_generate_ir(path).ir.routes[0].access_rules[0] - - assert rule.required_role == "org-admin" - assert set(rule.bind) == {"organizationId"} - - -@pytest.mark.parametrize( - ("generator", "route_path"), - [ - (FastAPIGenerator(), "routes/posts.py"), - (DRFGenerator(), "api/views/posts.py"), - (RawASGIGenerator(), "routes/posts.py"), - ], -) -def test_all_generated_backends_enforce_long_form_role( - generator, route_path, tmp_path -): +@pytest.mark.parametrize("version", ["0.2", "0.3"]) +def test_long_form_role_rejected_by_schema(tmp_path, version): + spec = yaml.safe_load(_SPEC) + spec["version"] = version + rule = spec["controllers"]["posts"]["routes"]["listScoped"]["access"][0] + rule["role"] = "org-admin" + if version == "0.2": + spec["services"]["listPosts"].pop("input") + spec["services"]["listPosts"].pop("with") + rule.pop("bind") path = tmp_path / "api.yaml" - path.write_text(_SPEC.replace("ROLE", "org-admin")) - output = tmp_path / generator.__class__.__name__ - - generator.generate(build_context(load_and_generate_ir(path).ir), output) - - assert compileall.compile_dir(str(output), quiet=1) - route_source = (output / route_path).read_text() - assert "_principal_has_role" in route_source - assert '"org-admin"' in route_source - - -def test_generated_fastapi_enforces_role_before_applying_bind(tmp_path): - path = tmp_path / "api.yaml" - path.write_text(_SPEC.replace("ROLE", "org-admin")) - output = tmp_path / "generated" - FastAPIGenerator().generate( - build_context(load_and_generate_ir(path).ir), output - ) - script = textwrap.dedent( - f""" - import os - import sys - - os.environ["JWT_SECRET"] = "test-secret-at-least-thirty-two-bytes" - sys.path.insert(0, {str(output)!r}) - - from fastapi import FastAPI - from fastapi.testclient import TestClient - from dependencies.auth import require_principal - from dependencies.repositories import get_posts_repository - from routes.posts import router - - class Repository: - async def listScoped(self, **kwargs): - return kwargs - - current = {{ - "sub": "u1", - "identity_collection": "users", - "roles": [], - "record": {{"organizationId": "org-1"}}, - }} - - async def principal(): - return current - - app = FastAPI() - app.include_router(router) - app.dependency_overrides[require_principal] = principal - app.dependency_overrides[get_posts_repository] = Repository - client = TestClient(app) - - denied = client.get("/posts") - assert denied.status_code == 403, denied.text - - current["roles"] = ["org-admin"] - allowed = client.get("/posts") - assert allowed.status_code == 200, allowed.text - assert allowed.json() == {{"filter": {{"organizationId": "org-1"}}}} - """ - ) - - subprocess.run([sys.executable, "-c", script], check=True) - - -def test_v02_full_spec_load_rejects_long_form_role(tmp_path): - path = tmp_path / "api.yaml" - spec = yaml.safe_load(_SPEC.replace("ROLE", "org-admin")) - spec["version"] = "0.2" - spec["services"]["listPosts"].pop("input") - spec["services"]["listPosts"].pop("with") - spec["controllers"]["posts"]["routes"]["listScoped"]["access"][0].pop("bind") path.write_text(yaml.safe_dump(spec)) with pytest.raises(ConfigErrors) as exc_info: load_and_generate_ir(path) - assert "ACCESS_ROLE_UNSUPPORTED" in {error.code for error in exc_info.value.errors} + assert [(error.code, "'role' was unexpected" in error.message) for error in exc_info.value.errors] == [ + ("SCHEMA_VALIDATION_ERROR", True) + ] def test_where_bound_document_is_available_to_create_rule(): - spec = yaml.safe_load(_SPEC.replace("ROLE", "org-admin")) + spec = yaml.safe_load(_SPEC) route = spec["controllers"]["posts"]["routes"].pop("listScoped") route["method"] = "POST" route["access"][0]["given"]["scope"] = ( @@ -250,7 +112,7 @@ def test_where_bound_document_is_available_to_create_rule(): def test_implicit_current_document_remains_unavailable_to_create_rule(): - spec = yaml.safe_load(_SPEC.replace("ROLE", "org-admin")) + spec = yaml.safe_load(_SPEC) route = spec["controllers"]["posts"]["routes"].pop("listScoped") route["access"][0]["given"]["scope"] = "posts" route["access"][0]["when"] = ( @@ -261,20 +123,3 @@ def test_implicit_current_document_remains_unavailable_to_create_rule(): assert "ACCESS_RULE_INVALID" in { error.code for error in validate_spec_access(spec) } - - -def test_different_roles_are_not_contradictory_rules(): - spec = yaml.safe_load(_SPEC.replace("ROLE", "org-admin")) - rules = spec["controllers"]["posts"]["routes"]["listScoped"]["access"] - spec["entities"]["user"]["fields"]["roles"]["values"].append("auditor") - rules.append({**rules[0], "role": "auditor", "then": "deny"}) - - assert validate_spec_access(spec) == [] - - -def test_role_only_rule_is_not_reported_as_public(): - spec = yaml.safe_load(_SPEC.replace("ROLE", "org-admin")) - - assert "PUBLIC_ROUTE" not in { - warning.code for warning in warn_public_routes(spec) - } diff --git a/tests/unit/test_dpop.py b/tests/unit/test_dpop.py index 3f3325cb..4956b65c 100644 --- a/tests/unit/test_dpop.py +++ b/tests/unit/test_dpop.py @@ -206,7 +206,7 @@ def test_dpop_is_rejected_for_idp_trust(tmp_path): with pytest.raises(ConfigErrors) as raised: load_and_generate_ir(str(path)) - assert "TRUST_IDP_APP_FIELD_FORBIDDEN" in { + assert "SCHEMA_VALIDATION_ERROR" in { error.code for error in raised.value.errors } diff --git a/tests/unit/test_fastapi_authz_generation.py b/tests/unit/test_fastapi_authz_generation.py index fe891825..f3ad5105 100644 --- a/tests/unit/test_fastapi_authz_generation.py +++ b/tests/unit/test_fastapi_authz_generation.py @@ -60,7 +60,7 @@ def test_principal_includes_roles_extraction(tmp_path): auth_module = (tmp_path / "dependencies" / "auth.py").read_text() assert "_extract_roles" in auth_module assert '"roles": _extract_roles(' in auth_module - assert "_extract_roles(doc, 'role', False)" in auth_module + assert "_extract_roles(doc, 'role', True)" in auth_module def test_roles_field_default_emits_python_none(tmp_path): diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index d4c43cdc..0bfa73be 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -462,7 +462,7 @@ def test_public_only_routes_omit_dead_auth_imports(tmp_path): def test_build_settings_fields_skips_saml_provider_client_secret(): oidc = IROIDCProvider(name="keycloak", client_id="app", client_secret="s3cr3t", trust="app") - saml = IRSAMLProvider(name="corp-saml", trust="app") + saml = IRSAMLProvider(name="corp-saml") auth = AuthConfig( enabled=True, identity_collections=[], diff --git a/tests/unit/test_parser.py b/tests/unit/test_parser.py index acd9bc56..f62eaa8c 100644 --- a/tests/unit/test_parser.py +++ b/tests/unit/test_parser.py @@ -68,10 +68,8 @@ def test_abstract_entity_not_in_collections() -> None: def test_roles_hierarchy_in_ir() -> None: ir = load_and_generate_ir(str(VALID_DIR / "roles-access.yaml")).ir - assert "user" in ir.role_hierarchies - user_h = ir.role_hierarchies["user"] - assert "admin" in user_h - assert "moderator" in user_h["admin"] + hierarchy = ir.identityCollections["users"].roles.hierarchy + assert hierarchy["admin"] == ("moderator",) def test_access_rules_on_routes() -> None: @@ -523,8 +521,8 @@ def test_schema_validation_failure_multiple_errors() -> None: def test_ui_block_extracted_and_preserved() -> None: result = load_and_generate_ir(str(VALID_DIR / "ui-block.yaml")) assert result.ui_block is not None - assert "dashboard" in result.ui_block - assert result.ui_block["theme"] == "dark" + assert "dashboard" in result.ui_block["config"] + assert result.ui_block["config"]["theme"] == "dark" # --- Test 29: No UI block → ui_block=None --- diff --git a/tests/unit/test_pkce.py b/tests/unit/test_pkce.py index 049db6fb..35eefc4e 100644 --- a/tests/unit/test_pkce.py +++ b/tests/unit/test_pkce.py @@ -70,7 +70,7 @@ def test_pkce_is_rejected_for_idp_trust(tmp_path): with pytest.raises(ConfigErrors) as raised: load_and_generate_ir(str(path)) - assert "TRUST_IDP_APP_FIELD_FORBIDDEN" in { + assert "SCHEMA_VALIDATION_ERROR" in { error.code for error in raised.value.errors } diff --git a/tests/unit/test_removed_access_filter.py b/tests/unit/test_removed_access_filter.py deleted file mode 100644 index 45d81109..00000000 --- a/tests/unit/test_removed_access_filter.py +++ /dev/null @@ -1,132 +0,0 @@ -import json -import textwrap -from pathlib import Path - -import pytest - -from yaml_api_generator.parsers.transformers.v02.access_transformer import ( - AccessTransformer, -) -from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir - - -SCHEMAS = Path(__file__).parents[2] / "src/yaml_api_generator/parsers/schemas" - - -def _parse_rule(rule, allow_where): - _, rules, errors = AccessTransformer(allow_where=allow_where).parse_access( - [rule], "controllers.posts.routes.list.access", set(), {} - ) - return rules, errors - - -def test_v03_filter_key_reports_dedicated_error(): - rules, errors = _parse_rule({"then": "allow", "filter": "owner = principal.sub"}, True) - - assert rules is None - assert [error.code for error in errors] == ["ACCESS_FILTER_REMOVED"] - assert errors[0].path == "controllers.posts.routes.list.access[0].filter" - - -def test_v03_write_fields_key_reports_dedicated_error(): - rules, errors = _parse_rule({"then": "allow", "writeFields": ["title"]}, True) - - assert rules is None - assert [error.code for error in errors] == ["ACCESS_WRITE_FIELDS_REMOVED"] - assert errors[0].path == "controllers.posts.routes.list.access[0].writeFields" - - -def test_v02_filter_key_remains_in_ir(): - rules, errors = _parse_rule({"then": "allow", "filter": "owner = principal.sub"}, False) - - assert errors == [] - assert rules[0].filter == "owner = principal.sub" - - -def test_filter_is_removed_only_from_v03_schema(): - v02 = json.loads((SCHEMAS / "yaml-spec-v0.2.schema.json").read_text()) - v03 = json.loads((SCHEMAS / "yaml-spec-v0.3.schema.json").read_text()) - - assert "filter" in v02["$defs"]["AccessRule"]["properties"] - assert "filter" not in v03["$defs"]["AccessRule"]["properties"] - assert "writeFields" in v02["$defs"]["AccessRule"]["properties"] - assert "writeFields" not in v03["$defs"]["AccessRule"]["properties"] - - -def test_v03_filter_aborts_full_spec_load(tmp_path): - spec = tmp_path / "spec.yaml" - spec.write_text( - textwrap.dedent( - """ - version: "0.3" - api: - name: removed-filter - basePath: /api - entities: - post: - fields: - id: {type: id} - repositories: - posts: - entity: post - services: - listPosts: - repository: posts - operation: list - controllers: - posts: - base_path: /posts - routes: - list: - service: listPosts - method: GET - path: "" - access: - - then: allow - filter: owner = principal.sub - """ - ) - ) - - with pytest.raises(ConfigErrors) as raised: - load_and_generate_ir(str(spec)) - - assert "ACCESS_FILTER_REMOVED" in {error.code for error in raised.value.errors} - - -def test_v03_write_fields_aborts_full_spec_load(tmp_path): - spec = tmp_path / "spec.yaml" - spec.write_text( - textwrap.dedent( - """ - version: "0.3" - api: { basePath: /api } - entities: - post: - fields: - id: { type: id } - repositories: - posts: { entity: post } - services: - posts: { repository: posts } - controllers: - posts: - base_path: /posts - routes: - update: - service: posts - method: PATCH - path: /{id} - access: - - then: allow - writeFields: [title] - """ - ) - ) - - with pytest.raises(ConfigErrors) as raised: - load_and_generate_ir(str(spec)) - - assert {error.code for error in raised.value.errors} == { - "ACCESS_WRITE_FIELDS_REMOVED" - } diff --git a/tests/unit/test_repository_operation_validation.py b/tests/unit/test_repository_operation_validation.py index f3524afc..1287e872 100644 --- a/tests/unit/test_repository_operation_validation.py +++ b/tests/unit/test_repository_operation_validation.py @@ -23,7 +23,7 @@ _SPEC = { "version": "0.3", - "api": {"name": "undeclared-operation", "basePath": "/api"}, + "api": {"basePath": "/api"}, "entities": { "post": { "fields": { diff --git a/tests/unit/test_runtime_context_builder.py b/tests/unit/test_runtime_context_builder.py index a0a8081e..ed8feddc 100644 --- a/tests/unit/test_runtime_context_builder.py +++ b/tests/unit/test_runtime_context_builder.py @@ -22,7 +22,6 @@ def _make_ir( collections: dict | None = None, identity: dict | None = None, - role_hierarchies: dict | None = None, routes: list | None = None, ) -> IRDocument: cols = collections or { @@ -41,7 +40,6 @@ def _make_ir( collections=cols, routes=routes or [], identityCollections=identity or {}, - role_hierarchies=role_hierarchies or {}, mongo=IRMongo(), repositories={}, ) diff --git a/tests/unit/test_schema_consumers.py b/tests/unit/test_schema_consumers.py new file mode 100644 index 00000000..6eb53fdd --- /dev/null +++ b/tests/unit/test_schema_consumers.py @@ -0,0 +1,114 @@ +"""Every key a spec schema accepts is read by some consumer: parser, generator or UI pack. + +A key counts as consumed when it occurs as an identifier or a non-docstring string +literal in the package's Python code, or anywhere in its Jinja templates. +""" + +import ast +import json +import re +from collections.abc import Iterator +from functools import cache +from pathlib import Path + +import pytest + +PACKAGE = Path(__file__).parents[2] / "src" / "yaml_api_generator" +SCHEMAS = PACKAGE / "parsers" / "schemas" + +# Accepted by the schema but read by no consumer yet. +UNENFORCED = { + "allowCredentials": "0.1 security.cors", + "allowedMethods": "0.1 security.cors", + "allowedOrigins": "0.1 security.cors", + "defaultPageSize": "0.1 api.pagination", + "maxPageSize": "0.1 api.pagination", + "enableStandard": "0.1 security.headers (D16)", + "softDelete": "0.1 collections (D19)", + "additionalProperties": "object field option", +} + +# Read by an external tool from a block the generator copies verbatim. +VERBATIM = { + "dockerfile": "deploy.services..build, rendered into docker-compose", +} + + +def _schema_keys(node, location: str = "") -> Iterator[tuple[str, str]]: + if isinstance(node, dict): + for key, value in node.items(): + if key == "properties" and isinstance(value, dict): + for name, sub in value.items(): + yield name, f"{location}.{name}" + yield from _schema_keys(sub, f"{location}.{name}") + else: + yield from _schema_keys(value, location) + elif isinstance(node, list): + for item in node: + yield from _schema_keys(item, location) + + +def _docstring_ids(tree: ast.Module) -> set[int]: + scopes = (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + return { + id(node.body[0].value) + for node in ast.walk(tree) + if isinstance(node, scopes) + and node.body + and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + } + + +def _python_words(source: str) -> Iterator[str]: + tree = ast.parse(source) + docstrings = _docstring_ids(tree) + for node in ast.walk(tree): + match node: + case ast.Constant(value=str(text)) if id(node) not in docstrings: + yield text + case ast.Name(id=word) | ast.Attribute(attr=word) | ast.arg(arg=word) | ast.alias(name=word): + yield word + case ast.keyword(arg=str(word)) | ast.FunctionDef(name=word) | ast.ClassDef(name=word): + yield word + + +@cache +def _corpus() -> str: + return "\n".join( + [ + *(word for path in PACKAGE.rglob("*.py") for word in _python_words(path.read_text())), + *(path.read_text() for path in PACKAGE.rglob("*.jinja")), + ] + ) + + +@cache +def _locations() -> dict[str, tuple[str, ...]]: + found: dict[str, list[str]] = {} + for path in sorted(SCHEMAS.glob("*.schema.json")): + for key, location in _schema_keys(json.loads(path.read_text())): + found.setdefault(key, []).append(f"{path.name}{location}") + return {key: tuple(locations) for key, locations in found.items()} + + +def _consumed(key: str) -> bool: + return re.search(rf"(? dict: + *parents, leaf = path + node = spec + for key in parents: + node = node[key] if isinstance(key, int) else node.setdefault(key, {}) + node[leaf] = value + return spec + + +def _load_errors(tmp_path, spec: dict) -> list: + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec)) + with pytest.raises(ConfigErrors) as exc_info: + load_and_generate_ir(str(path)) + return exc_info.value.errors + + +@pytest.mark.parametrize("version", BASE_SPECS) +def test_base_specs_load(tmp_path, version): + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(BASE_SPECS[version])) + assert load_and_generate_ir(str(path)).ir.version == version + + +@pytest.mark.parametrize(("version", "path", "value", "fragment"), CASES) +def test_schema_rejects_unknown_or_removed_key(tmp_path, version, path, value, fragment): + spec = _set_path(copy.deepcopy(BASE_SPECS[version]), path, value) + errors = _load_errors(tmp_path, spec) + assert {error.code for error in errors} == {"SCHEMA_VALIDATION_ERROR"} + assert any(fragment in error.message for error in errors) + + +@pytest.mark.parametrize("spec,path", WRONG_SHAPES.values(), ids=list(WRONG_SHAPES)) +def test_wrong_container_shape_reports_schema_error(tmp_path, spec, path): + errors = _load_errors(tmp_path, {"version": "0.2", **spec}) + assert any(e.code == "SCHEMA_VALIDATION_ERROR" and e.path == path for e in errors) + + +@pytest.mark.parametrize( + "name", ["access-control-patterns.yaml", "advanced-operations.yaml", "multi-tenant.yaml"] +) +def test_invalid_examples_report_schema_errors_instead_of_crashing(name): + with pytest.raises(ConfigErrors) as exc_info: + load_and_generate_ir(str(EXAMPLES_INVALID / name)) + assert {error.code for error in exc_info.value.errors} == {"SCHEMA_VALIDATION_ERROR"} diff --git a/tests/unit/test_service_input.py b/tests/unit/test_service_input.py index eed1a8c5..da881b17 100644 --- a/tests/unit/test_service_input.py +++ b/tests/unit/test_service_input.py @@ -209,7 +209,6 @@ def test_repository_forwarding_rejects_unused_target(): _SPEC = """ version: "0.3" api: - name: test basePath: /api entities: post: @@ -270,7 +269,7 @@ def test_v02_rejects_service_input_syntax(tmp_path): with pytest.raises(ConfigErrors) as exc_info: _load(tmp_path, _SPEC.replace('version: "0.3"', 'version: "0.2"')) - assert any(e.code == "SERVICE_INPUT_UNSUPPORTED" for e in exc_info.value.errors) + assert any(e.code == "SCHEMA_VALIDATION_ERROR" for e in exc_info.value.errors) # --- runtime step --------------------------------------------------------- diff --git a/tests/unit/test_shape_validation.py b/tests/unit/test_shape_validation.py deleted file mode 100644 index 823720b8..00000000 --- a/tests/unit/test_shape_validation.py +++ /dev/null @@ -1,69 +0,0 @@ -import pytest - -from yaml_api_generator.parsers.transformers.shape_validation import validate_spec_shape -from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir - -# Wrong-shape repros: each top-level block desugar dereferences as a mapping, -# authored as a list or scalar, must yield a clean SCHEMA_VALIDATION_ERROR -# instead of a raw AttributeError out of desugar. -WRONG_SHAPES = { - "entities-string": ({"entities": "foo"}, "entities"), - "entities-list": ({"entities": ["a"]}, "entities"), - "repositories-list": ({"repositories": ["a"]}, "repositories"), - "services-list": ({"services": ["a"]}, "services"), - "api-scalar": ({"api": "foo"}, "api"), - "api-operations-list": ({"api": {"operations": ["a"]}}, "api.operations"), -} - - -@pytest.mark.parametrize("spec,path", WRONG_SHAPES.values(), ids=list(WRONG_SHAPES)) -def test_validate_spec_shape_flags_wrong_container(spec, path): - errors = validate_spec_shape(spec) - assert [(e.code, e.path) for e in errors] == [("SCHEMA_VALIDATION_ERROR", path)] - - -def test_validate_spec_shape_accepts_mappings(): - spec = { - "entities": {"items": {}}, - "repositories": {}, - "services": {}, - "controllers": {}, - "api": {"operations": {}}, - } - assert validate_spec_shape(spec) == [] - - -@pytest.mark.parametrize("spec,path", WRONG_SHAPES.values(), ids=list(WRONG_SHAPES)) -def test_load_reports_clean_schema_error(tmp_path, spec, path): - import yaml - - yaml_file = tmp_path / "spec.yaml" - yaml_file.write_text(yaml.safe_dump({"version": "0.2", **spec})) - with pytest.raises(ConfigErrors) as exc_info: - load_and_generate_ir(str(yaml_file)) - errors = exc_info.value.errors - assert any(e.code == "SCHEMA_VALIDATION_ERROR" and e.path == path for e in errors) - - -def test_auth_bindings_key_rejected(tmp_path): - import yaml - - spec = { - "version": "0.2", - "entities": { - "user": { - "type": "auth", - "auth": { - "identifier": "email", - "via": "password", - "bindings": [{"provider": "keycloak"}], - }, - "fields": {"id": {"type": "id"}, "email": {"type": "string"}}, - } - }, - } - yaml_file = tmp_path / "spec.yaml" - yaml_file.write_text(yaml.safe_dump(spec)) - with pytest.raises(ConfigErrors) as exc_info: - load_and_generate_ir(str(yaml_file)) - assert any(e.code == "SCHEMA_VALIDATION_ERROR" for e in exc_info.value.errors) diff --git a/tests/unit/test_trust_validation.py b/tests/unit/test_trust_validation.py index 4e0cf820..3fb73d29 100644 --- a/tests/unit/test_trust_validation.py +++ b/tests/unit/test_trust_validation.py @@ -1,22 +1,20 @@ -from yaml_api_generator.parsers.transformers.trust_validation import ( - validate_spec_trust, -) +import pytest +from jsonschema import Draft202012Validator +from yaml_api_generator.parsers.transformers.trust_validation import validate_spec_trust +from yaml_api_generator.parsers.yaml_parser import SCHEMA_REGISTRY -def _results(providers: dict): - return validate_spec_trust({"providers": providers}) - -def _codes(providers: dict) -> set[str]: - return {e.code for e in _results(providers)} - - -def _error_codes(providers: dict) -> set[str]: - return {e.code for e in _results(providers) if e.severity == "error"} +def _warning_codes(providers: dict) -> set[str]: + return {e.code for e in validate_spec_trust({"providers": providers})} -def _warning_codes(providers: dict) -> set[str]: - return {e.code for e in _results(providers) if e.severity == "warning"} +def _schema_errors(version: str, provider: dict) -> list[str]: + validator = Draft202012Validator( + {"$ref": f"yaml-spec-v{version}.schema.json#/$defs/ProviderConfig"}, + registry=SCHEMA_REGISTRY, + ) + return [error.message for error in validator.iter_errors(provider)] def _oidc(**overrides) -> dict: @@ -39,78 +37,53 @@ def _idp(**overrides) -> dict: } -# ---- defaults ---- - -def test_app_trust_is_the_default_and_accepted(): - assert _error_codes({"keycloak": _oidc()}) == set() - - -def test_explicit_idp_trust_accepted(): - assert _error_codes({"keycloak": _idp()}) == set() - - -def test_idp_trust_without_audiences_produces_warning(): - config = { - "type": "oidc", - "client_id": "id", - "trust": "idp", - "discovery_url": "https://idp/.well-known/openid-configuration", - } - assert "TRUST_IDP_AUDIENCE_DEFAULTED" in _warning_codes({"keycloak": config}) - assert _error_codes({"keycloak": config}) == set() - - -def test_idp_trust_with_explicit_audiences_no_warning(): - assert "TRUST_IDP_AUDIENCE_DEFAULTED" not in _warning_codes( - {"keycloak": _idp(audiences=["api://my-api"])} - ) - assert _error_codes({"keycloak": _idp(audiences=["api://my-api"])}) == set() +VERSIONS = ["0.2", "0.3"] +ACCEPTED = { + "app-default": _oidc(), + "idp": _idp(), + "idp-audiences": _idp(audiences=["api://my-api"]), + "idp-issuer": _idp(issuer="https://idp/"), +} -def test_idp_trust_with_explicit_issuer_accepted(): - assert _error_codes({"keycloak": _idp(issuer="https://idp/")}) == set() +REJECTED = { + "unknown-trust": _oidc(trust="external"), + "idp-without-discovery-url": {"type": "oidc", "client_id": "id", "trust": "idp"}, + "idp-on-password-type": _idp(type="password"), + "idp-with-app-signing-field": _idp(secret="x"), + "app-with-issuer": _oidc(issuer="https://idp/"), +} -def test_idp_trust_audiences_tuple_reaches_ir_correctly(): - """Explicit audiences list suppresses TRUST_IDP_AUDIENCE_DEFAULTED warning.""" - assert "TRUST_IDP_AUDIENCE_DEFAULTED" not in _warning_codes( - {"keycloak": _idp(audiences=["a", "b"])} - ) - +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("provider", ACCEPTED.values(), ids=list(ACCEPTED)) +def test_schema_accepts_coherent_trust(version, provider): + assert _schema_errors(version, provider) == [] -# ---- TRUST_INVALID_VALUE ---- -def test_unknown_trust_value_rejected(): - assert "TRUST_INVALID_VALUE" in _codes({"keycloak": _oidc(trust="external")}) +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("provider", REJECTED.values(), ids=list(REJECTED)) +def test_schema_rejects_incoherent_trust(version, provider): + assert _schema_errors(version, provider) != [] -# ---- TRUST_IDP_DISCOVERY_URL_REQUIRED (OIDC only) ---- +@pytest.mark.parametrize("key", ["pkce", "dpop"]) +def test_v03_schema_rejects_app_only_flags_under_idp_trust(key): + assert _schema_errors("0.3", _idp(**{key: True})) != [] + assert _schema_errors("0.3", _oidc(**{key: True})) == [] -def test_oidc_idp_without_discovery_url_rejected(): - config = {"type": "oidc", "client_id": "id", "trust": "idp"} - assert "TRUST_IDP_DISCOVERY_URL_REQUIRED" in _codes({"keycloak": config}) - -# ---- TRUST_IDP_PROVIDER_UNSUPPORTED ---- - -def test_idp_trust_on_non_idp_type_rejected(): - config = _idp(type="password") - assert "TRUST_IDP_PROVIDER_UNSUPPORTED" in _codes({"local": config}) - - -# ---- TRUST_IDP_APP_FIELD_FORBIDDEN ---- - -def test_idp_with_app_signing_field_rejected(): - assert "TRUST_IDP_APP_FIELD_FORBIDDEN" in _codes({"keycloak": _idp(secret="x")}) +def test_idp_trust_without_audiences_produces_warning(): + assert _warning_codes({"keycloak": _idp()}) == {"TRUST_IDP_AUDIENCE_DEFAULTED"} -# ---- TRUST_APP_FIELD_FORBIDDEN ---- +def test_idp_trust_with_explicit_audiences_no_warning(): + assert _warning_codes({"keycloak": _idp(audiences=["a", "b"])}) == set() -def test_app_trust_with_issuer_rejected(): - assert "TRUST_APP_FIELD_FORBIDDEN" in _codes({"keycloak": _oidc(issuer="https://idp/")}) +def test_app_trust_produces_no_warning(): + assert _warning_codes({"keycloak": _oidc()}) == set() -# ---- scope ---- def test_no_providers_block_is_clean(): assert validate_spec_trust({}) == [] diff --git a/tests/unit/test_v02_transformer.py b/tests/unit/test_v02_transformer.py index 024a7768..2a3f7854 100644 --- a/tests/unit/test_v02_transformer.py +++ b/tests/unit/test_v02_transformer.py @@ -808,25 +808,10 @@ def test_111_document_binding_in_rule_resolved(tmp_path): # ---- Role Hierarchy ---- -def test_113_roles_hierarchy_unknown_entity(): +def test_113_top_level_roles_rejected_by_schema(): with pytest.raises(ConfigErrors) as exc_info: load_and_generate_ir(str(INVALID_DIR / "roles-unknown-entity.yaml")) - assert any(e.code == "ROLES_UNKNOWN_ENTITY" for e in exc_info.value.errors) - - -def test_114_empty_roles_section_produces_empty_hierarchies(): - config = { - "version": "0.2", - "api": {"operations": {"defaults": {}}}, - "entities": {"post": {"fields": {"id": {"type": "id"}, "title": {"type": "string"}}}}, - "repositories": {"posts": {"entity": "post"}}, - "services": {}, - "controllers": {}, - "mongo": {}, - "roles": {}, - } - ir = V02Transformer().transform(config) - assert ir.role_hierarchies == {} + assert {e.code for e in exc_info.value.errors} == {"SCHEMA_VALIDATION_ERROR"} # ---- Route Building ---- diff --git a/tests/unit/test_v03_spec_version.py b/tests/unit/test_v03_spec_version.py index fbd0e297..ade42895 100644 --- a/tests/unit/test_v03_spec_version.py +++ b/tests/unit/test_v03_spec_version.py @@ -1,9 +1,8 @@ """Foundational v0.3 spec-version gate. v0.3 is a superset of v0.2. These tests pin the scaffold behavior: a v0.2-shaped -spec at version 0.3 produces the same IR, dispatch routes to V03Transformer, the -schema accepts the new v0.3 constructs syntactically, and the reserved where -operators are documented. Semantics of the new constructs land in later issues. +spec at version 0.3 produces the same IR, dispatch routes to V03Transformer, and +the schema accepts the new v0.3 constructs syntactically. """ import json import textwrap @@ -13,7 +12,11 @@ import pytest from jsonschema import Draft202012Validator -from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir +from yaml_api_generator.parsers.yaml_parser import ( + SCHEMA_REGISTRY, + ConfigErrors, + load_and_generate_ir, +) from yaml_api_generator.parsers.transformers.registry import _TRANSFORMERS from yaml_api_generator.parsers.transformers.v02 import V02Transformer from yaml_api_generator.parsers.transformers.v03 import V03Transformer @@ -82,7 +85,6 @@ def test_unsupported_version_still_raises(tmp_path): def test_schema_accepts_new_v03_constructs(): spec = { "version": "0.3", - "input": ["orgId"], "api": {"basePath": "/api", "version": "v1"}, "entities": { "post": { @@ -111,13 +113,5 @@ def test_schema_accepts_new_v03_constructs(): } }, } - errors = list(Draft202012Validator(_schema()).iter_errors(spec)) + errors = list(Draft202012Validator(_schema(), registry=SCHEMA_REGISTRY).iter_errors(spec)) assert errors == [] - - -def test_reserved_where_operators_are_documented(): - operator = _schema()["$defs"]["WhereOperator"] - assert set(operator["enum"]) == {"=", "and", "!=", "<", "<=", ">", ">=", "in"} - assert "=" in operator["enum"] - assert "and" in operator["enum"] - assert "Reserved" in operator["description"] diff --git a/tests/unit/test_vendored_imports.py b/tests/unit/test_vendored_imports.py index 578a8efd..a6cfb3bf 100644 --- a/tests/unit/test_vendored_imports.py +++ b/tests/unit/test_vendored_imports.py @@ -286,7 +286,7 @@ def test_handler_referencing_undeclared_package_raises(tmp_path): def test_v03_bare_string_import_rejected(tmp_path): spec = _minimal(tmp_path, "vend: some.author.module", handler="vend.route") - assert _codes(spec) == ["IMPORT_BARE_STRING"] + assert _codes(spec) == ["SCHEMA_VALIDATION_ERROR"] def test_source_escaping_the_spec_directory_is_rejected(tmp_path): From 9998cf252d5dd52a5a82fdf446e049455a2bfc3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 18:36:56 +0100 Subject: [PATCH 02/89] Run repository operations and handlers through one runtime shared by the dev server and every generated backend Declared queries resolve bindings and execute in a vendored runtime (Mongo and SQL), handlers receive a single HandlerRequest, and routes calling an operation no handler or repository provides, or paginating a non-list operation, are rejected at parse time. --- .../generators/backend/drf/generator.py | 16 +- .../backend/drf/templates/handler_call.jinja | 11 ++ .../drf/templates/handler_views.py.jinja | 27 ++-- .../backend/drf/templates/views.py.jinja | 40 ++--- .../generators/backend/fastapi/generator.py | 33 +++-- .../fastapi/templates/handler_call.jinja | 14 ++ .../fastapi/templates/handler_routes.py.jinja | 8 +- .../fastapi/templates/repository.py.jinja | 125 +--------------- .../fastapi/templates/repository_class.jinja | 19 +++ .../fastapi/templates/repository_ops.py.jinja | 6 + .../fastapi/templates/repository_sql.py.jinja | 20 ++- .../templates/repository_sql_ops.py.jinja | 7 + .../backend/fastapi/templates/routes.py.jinja | 26 ++-- .../generators/backend/raw/generator.py | 16 +- .../backend/raw/templates/handler_call.jinja | 11 ++ .../raw/templates/handler_routes.py.jinja | 19 +-- .../backend/raw/templates/routes.py.jinja | 26 +--- .../shared/templates/repository.py.jinja | 110 ++------------ .../templates/repository_operations.jinja | 7 + .../shared/templates/service_inputs.py.jinja | 36 ----- .../generators/backend/utils.py | 14 +- .../generators/backend/validation.py | 49 +------ .../parsers/transformers/v02/__init__.py | 1 + .../transformers/v02/route_transformer.py | 58 +++++++- .../runtime/handler_requests.py | 27 ++++ .../runtime/mongo_operations.py | 138 ++++++++++++++++++ src/yaml_api_generator/runtime/operations.py | 57 ++++++++ .../runtime/sql_operations.py | 12 ++ .../server/common/pipeline_builder.py | 22 +-- .../server/common/repositories/base.py | 9 +- .../server/common/repositories/mongodb.py | 66 +-------- .../server/common/repositories/sql.py | 18 +-- .../server/common/steps/__init__.py | 2 + .../server/common/steps/handlers.py | 29 ++++ .../server/common/steps/repositories.py | 46 ++---- .../server/fastapi/factory.py | 11 +- .../invalid/files-route-conflict.yaml | 2 +- tests/unit/test_conflict_409.py | 4 - tests/unit/test_controller_bind.py | 11 +- tests/unit/test_custom_operations.py | 20 ++- .../unit/test_fastapi_bindings_generation.py | 48 +----- tests/unit/test_generated_mongo_cursors.py | 12 +- tests/unit/test_pipeline.py | 21 +-- .../test_repository_operation_validation.py | 45 ++++-- tests/unit/test_service_input.py | 19 +-- tests/unit/test_service_input_codegen.py | 26 ++-- tests/unit/test_vendored_imports.py | 92 +++++------- 47 files changed, 681 insertions(+), 755 deletions(-) create mode 100644 src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja create mode 100644 src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja create mode 100644 src/yaml_api_generator/generators/backend/fastapi/templates/repository_class.jinja create mode 100644 src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja create mode 100644 src/yaml_api_generator/generators/backend/shared/templates/repository_operations.jinja create mode 100644 src/yaml_api_generator/runtime/handler_requests.py create mode 100644 src/yaml_api_generator/runtime/mongo_operations.py create mode 100644 src/yaml_api_generator/runtime/operations.py create mode 100644 src/yaml_api_generator/runtime/sql_operations.py create mode 100644 src/yaml_api_generator/server/common/steps/handlers.py diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index f94e32f5..58420a42 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -10,7 +10,8 @@ find_identity, group_by_collection, pagination_config, - runtime_module_source, + MONGO_RUNTIME_MODULES, + write_runtime_modules, ) from ..validation import ( reject_cookie_delivery, @@ -19,7 +20,6 @@ reject_idp_trust, reject_pkce, reject_session_refresh, - reject_undeclared_repository_operations, reject_unsigned_session_algorithm, validate_datastore_layout, ) @@ -58,7 +58,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: reject_session_refresh(context, "drf") reject_pkce(context, "drf") reject_unsigned_session_algorithm(context) - reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) env.globals["settings_module"] = f"{self.dir_prefix}.settings" env.filters["drf_field"] = type_mapper.drf_field @@ -91,9 +90,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: self._render_serializers(env, context, collections_by_name, output_dir) self._render_services(env, context, collections_by_name, output_dir) self._render_service_inputs(env, context, output_dir) - (api_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) + write_runtime_modules(api_dir, MONGO_RUNTIME_MODULES) self._render_views(env, context, collections_by_name, output_dir) self._render_repositories(env, context, collections_by_name, output_dir) self._render_authentication(env, context, output_dir) @@ -226,10 +223,6 @@ def _render_views( "collection_name": col_name, "routes": col_data["routes"], "model_index": {m.name: m for m in col_data["models"]}, - "repo_queries": { - op["name"]: op["query"] - for op in ctx.repository_operations.get(col_name, []) - }, "auth": ctx.auth, }, should_render=lambda col_name, col_data, ctx: bool(col_data["routes"]), @@ -275,8 +268,7 @@ def _render_repositories( "id_field": ctx.collections[col_name]["id_field"], "id_field_format": ctx.collections[col_name]["id_format"], "custom_operations": ctx.repository_operations.get(col_name, []), - "conflict_import": "api.conflicts", - "pagination_import": "api.pagination", + "module_prefix": "api.", }, ) ) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja new file mode 100644 index 00000000..de6f609c --- /dev/null +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja @@ -0,0 +1,11 @@ +{% macro invoke_handler(handler) %} + return Response(await invoke({{ handler | py_literal }}, HandlerRequest( + params=path_params, + query=dict(request.query_params), + headers={key.lower(): value for key, value in request.headers.items()}, + cookies=dict(request.COOKIES), + body=body, + principal=_resolved_principal, + inputs=service_inputs, + ))) +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index 108d3dba..cb611883 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -1,10 +1,10 @@ -from importlib import import_module - +{% from "handler_call.jinja" import invoke_handler %} from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView from api.conflicts import ConflictError +from api.handler_requests import HandlerRequest, invoke from api.service_inputs import prepare_service_inputs, resolve_access_bind {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} @@ -23,18 +23,17 @@ from api.authentication.mongodb import get_database class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(APIView): async def {{ route.method | lower }}(self, request, **path_params): {% if route.needs_auth %} - principal = resolve_principal(request, get_database()) - if principal is None: + _resolved_principal = resolve_principal(request, get_database()) + if _resolved_principal is None: return Response( {"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED, ) {% else %} - principal = None + _resolved_principal = None {% endif %} body = request.data if isinstance(request.data, dict) else {} {% if route.access_rules %} - _resolved_principal = principal _request_body = body _request_path = path_params _request_query = dict(request.query_params) @@ -50,7 +49,7 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A {% endfor %} {% endfor %} {% for symbol in ns_symbols.identity %} - {{ symbol }} = principal + {{ symbol }} = _resolved_principal {% endfor %} {% for symbol in ns_symbols.request %} {{ symbol | request_symbol_binding }} @@ -79,24 +78,18 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A bound_inputs = {} {% endif %} try: - request.service_inputs = prepare_service_inputs( + service_inputs = prepare_service_inputs( path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs, ) - except ValueError as exc: - return Response( - {"detail": str(exc)}, - status=status.HTTP_422_UNPROCESSABLE_ENTITY, - ) - module_path, function_name = "{{ route.handler }}".rsplit(".", 1) - try: - result = await getattr(import_module(module_path), function_name)(request) +{{ invoke_handler(route.handler) }} except ConflictError as exc: return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) - return result if isinstance(result, Response) else Response(result) + except ValueError as exc: + return Response({"detail": str(exc)}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 3ac87bab..213866fa 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -1,3 +1,4 @@ +{% from "handler_call.jinja" import invoke_handler %} from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView @@ -27,7 +28,13 @@ from api.repositories.{{ where_collection }} import {{ where_collection | capita {% endfor %} from api.services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from api.services.exceptions import NotFoundError, ValidationError -from api.service_inputs import prepare_service_inputs, resolve_access_bind, resolve_bindings +from api.service_inputs import prepare_service_inputs, resolve_access_bind +{% if routes | selectattr("handler") | list %} +from api.handler_requests import HandlerRequest, invoke +{% endif %} +{% if routes | rejectattr("handler") | rejectattr("operation", "in", crud_op_names + ["login"]) | list %} +from api.operations import operation_bindings +{% endif %} {% if routes | selectattr("pagination") | list %} from api.pagination import build_response, map_request {% endif %} @@ -96,6 +103,8 @@ from api.authentication.auth import resolve_principal, _principal_has_role {{ authorize_rules(route, "None") }} {% endif %} {% endif %} +{% else %} + _resolved_principal = None {% endif %} {% endmacro %} @@ -150,8 +159,10 @@ class {{ class_name }}ListView(APIView): {{ pre_fetch_check(route) }} {{ bind_inputs(route) }} try: - prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.operation == "list" %} + service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) +{% if route.handler %} +{{ invoke_handler(route.handler) }} +{% elif route.operation == "list" %} {% if route.pagination %} pagination_config = {{ route.pagination | pagination_config | py_literal }} pagination_query = dict(request.query_params) @@ -192,23 +203,10 @@ class {{ custom_class }}(APIView): {{ bind_inputs(route) }} try: service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) - params = {**path_params, **service_inputs} {% if route.handler %} - from importlib import import_module - module_path, function_name = "{{ route.handler }}".rsplit(".", 1) - request.service_inputs = service_inputs - result = await getattr(import_module(module_path), function_name)(request) - return result if isinstance(result, Response) else Response(result) +{{ invoke_handler(route.handler) }} {% else %} -{% set query = repo_queries[route.operation] %} - bindings = {**params, "params": params, "body": body} - resolved = resolve_bindings({{ query | py_literal }}, bindings) - kwargs = { - key: value - for key, value in resolved.items() - if key not in ("command", "collection") - } - return Response(await repo.{{ route.operation }}(**kwargs)) + return Response(await repo.{{ route.operation }}(operation_bindings({**path_params, **service_inputs}, _resolved_principal, body))) {% endif %} except NotFoundError: return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) @@ -238,8 +236,10 @@ class {{ class_name }}DetailView(APIView): {{ pre_fetch_check(route) }} {{ bind_inputs(route) }} try: - prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.operation == "get" %} + service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) +{% if route.handler %} +{{ invoke_handler(route.handler) }} +{% elif route.operation == "get" %} {% if route.record_parent_params %} result = await service.get_by({{ record_criteria(route) }}) {% else %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 4c1a7bc8..d4911696 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -13,9 +13,9 @@ pagination_config, runtime_module_source, write_environment_files, + write_runtime_modules, ) from ..validation import ( - reject_undeclared_repository_operations, reject_unsigned_session_algorithm, validate_datastore_layout, ) @@ -44,7 +44,6 @@ class FastAPIGenerator(TemplateBasedGenerator): def generate(self, context: CompileContext, output_dir: Path) -> None: validate_datastore_layout(context, "fastapi") reject_unsigned_session_algorithm(context) - reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) env.filters["python_type"] = type_mapper.python_type env.filters["python_annotation"] = type_mapper.python_annotation @@ -227,11 +226,6 @@ def _build_routes_context( "service_groups": service_groups, "routes": col_data["routes"], "model_index": {m.name: m for m in col_data["models"]}, - "repo_queries": { - op["name"]: op["query"] - for op in ctx.repository_operations.get(col_name, []) - }, - "datastore_type": ctx.collection_datastore_type(col_name), "auth": ctx.auth, "files": ctx.file_fields.get(col_name, {}), "file_routes": [route for route in ctx.file_routes if route.collection == col_name], @@ -354,6 +348,13 @@ def generated_base(name: str) -> dict[str, str] | None: return None return {"module": f".{base}", "class_name": f"{to_pascal_case(base)}Repository"} + def own_queries(name: str) -> list[dict]: + inherited = context.repository_operations.get(context.repository_inherits.get(name), []) + return [ + op for op in context.repository_operations.get(name, []) + if op["query"] is not None and op not in inherited + ] + # Abstract base repositories render as inheritable classes (no collection). for base_name, ds_type in context.repository_abstract.items(): is_sql = ds_type in ("postgres", "postgresql") @@ -362,7 +363,7 @@ def generated_base(name: str) -> dict[str, str] | None: abstract=True, collection_name=base_name, operations=[], - custom_operations=context.repository_operations.get(base_name, []), + custom_operations=own_queries(base_name), generated_base=generated_base(base_name), )) init_imports.append(f"from .{base_name} import {to_pascal_case(base_name)}Repository") @@ -379,7 +380,7 @@ def generated_base(name: str) -> dict[str, str] | None: "id_field_format": meta["id_format"], "collection_type": meta["type"], "fields": meta["fields"], - "custom_operations": ctx_ops if (ctx_ops := context.repository_operations.get(col_name, [])) else [], + "custom_operations": own_queries(col_name), "generated_base": generated_base(col_name), } template = sql_template if is_sql else mongo_template @@ -440,15 +441,15 @@ def _render_dependencies( (deps_dir / "claims.py").write_text( (_STATIC_DIR / "claims.py").read_text() ) - (deps_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) if any(getattr(provider, "dpop", False) for provider in context.auth.providers.values()): (deps_dir / "dpop.py").write_text(runtime_module_source("dpop")) - if any(context.repository_operations.values()): - (deps_dir / "bindings.py").write_text( - (_STATIC_DIR / "bindings.py").read_text() - ) + write_runtime_modules(deps_dir, ( + "pagination", + "operations", + "handler_requests", + *(("mongo_operations",) if uses_mongo else ()), + *(("sql_operations",) if uses_sql else ()), + )) self._render_file_dependencies(env, context, deps_dir) init_imports: list[str] = [] diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja new file mode 100644 index 00000000..d9c45ccf --- /dev/null +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja @@ -0,0 +1,14 @@ +{% macro invoke_handler(handler, principal, body, inputs) %} + try: + return await invoke({{ handler | py_literal }}, HandlerRequest( + params=dict(request.path_params), + query=dict(request.query_params), + headers=dict(request.headers), + cookies=dict(request.cookies), + body={{ body }}, + principal={{ principal }}, + inputs={{ inputs }}, + )) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja index d658ee48..6431747c 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja @@ -1,8 +1,9 @@ -from importlib import import_module +{% from "handler_call.jinja" import invoke_handler %} from typing import Any from fastapi import APIRouter, Depends, HTTPException, Request +from dependencies.handler_requests import HandlerRequest, invoke from dependencies.service_inputs import prepare_service_inputs, resolve_access_bind {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} @@ -85,9 +86,6 @@ async def {{ route.operation | py_identifier }}_{{ loop.index }}( ) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc - request.state.service_inputs = service_inputs - module_path, function_name = "{{ route.handler }}".rsplit(".", 1) - return await getattr(import_module(module_path), function_name)(request) - +{{ invoke_handler(route.handler, "principal" if route.needs_auth else "None", "body", "service_inputs") }} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja index dc41426b..f3dadc38 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja @@ -1,128 +1,15 @@ +{% from "repository_class.jinja" import repository_class with context %} {% set class_name = collection_name | capitalize_collection %} {% set is_abstract = abstract | default(false) %} -{% set has_custom = custom_operations | default([]) | length > 0 %} -{% if has_custom %} +{% if custom_operations %} from typing import Any -from pymongo import ReturnDocument - -{% endif %} -{% if generated_base %} -from {{ generated_base.module }} import {{ generated_base.class_name }} {% endif %} {% if not is_abstract %} -from ._ops import RepositoryBase, _translate_conflict{% for op in operations %}, {{ op | capitalize }}Op{% endfor %} -{% endif %} -{% set ns = namespace(bases=[]) %} -{% if not is_abstract %}{% for op in operations %}{% set _ = ns.bases.append((op | capitalize) ~ "Op") %}{% endfor %}{% endif %} -{% if generated_base %}{% set _ = ns.bases.append(generated_base.class_name) %}{% endif %} -{% if not is_abstract %}{% set _ = ns.bases.append("RepositoryBase") %}{% endif %} - - -class {{ class_name }}Repository{% if ns.bases %}({{ ns.bases | join(", ") }}){% endif %}: -{% if not is_abstract %} - COLLECTION = "{{ collection_name }}" - ID_FIELD = "{{ id_field }}" - ID_FORMAT = "{{ id_field_format }}" +from ._ops import RepositoryBase{% for op in operations %}, {{ op | capitalize }}Op{% endfor %} {% endif %} -{% for op in custom_operations | default([]) %} -{% set q = op.query %} -{% set col_expr = "self._db[\"" + q["collection"] + "\"]" if q and q.get("collection") else "self._col" %} -{% if q %} -{% set cmd = q.command %} -{% if cmd == "find" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> list[dict[str, Any]]: - cursor = {{ col_expr }}.find(filter) - docs = await cursor.to_list(length=None) - return [self._normalize(d) for d in docs] -{% elif cmd == "findOne" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one(filter) - return self._normalize(doc) -{% elif cmd == "aggregate" %} - - async def {{ op.name }}(self, pipeline: list) -> list[dict[str, Any]]: - cursor = await {{ col_expr }}.aggregate(pipeline) - return await cursor.to_list(length=None) -{% elif cmd == "countDocuments" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> int: - return await {{ col_expr }}.count_documents(filter) -{% elif cmd == "estimatedDocumentCount" %} - - async def {{ op.name }}(self) -> int: - return await {{ col_expr }}.estimated_document_count() -{% elif cmd == "distinct" %} - - async def {{ op.name }}(self, key: str, filter: dict[str, Any]) -> list: - return await {{ col_expr }}.distinct(key, filter) -{% elif cmd == "insertOne" %} - - @_translate_conflict - async def {{ op.name }}(self, document: dict[str, Any]) -> dict[str, Any]: - await {{ col_expr }}.insert_one(document) - return self._normalize(document) -{% elif cmd == "insertMany" %} - - @_translate_conflict - async def {{ op.name }}(self, documents: list[dict[str, Any]]) -> list[dict[str, Any]]: - await {{ col_expr }}.insert_many(documents) - return [self._normalize(d) for d in documents] -{% elif cmd == "updateOne" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_update(filter, update, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "updateMany" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> Any: - return await {{ col_expr }}.update_many(filter, update) -{% elif cmd == "replaceOne" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], replacement: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_replace(filter, replacement, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "deleteOne" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_delete(filter) - return self._normalize(doc) -{% elif cmd == "deleteMany" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> Any: - return await {{ col_expr }}.delete_many(filter) -{% elif cmd == "findOneAndUpdate" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_update(filter, update, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "findOneAndReplace" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], replacement: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_replace(filter, replacement, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "findOneAndDelete" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_delete(filter) - return self._normalize(doc) -{% elif cmd == "bulkWrite" %} - - @_translate_conflict - async def {{ op.name }}(self, requests: list) -> Any: - return await {{ col_expr }}.bulk_write(requests) +{% if generated_base %} +from {{ generated_base.module }} import {{ generated_base.class_name }} {% endif %} -{% else %} - async def {{ op.name }}(self, **kwargs: Any) -> Any: - import {{ op.handler.rsplit(".", 1)[0] }} - return await {{ op.handler }}(**kwargs) -{% endif %} -{% endfor %} +{{ repository_class(class_name, "RepositoryBase", operations, generated_base, is_abstract, custom_operations) }} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_class.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_class.jinja new file mode 100644 index 00000000..916e8f4d --- /dev/null +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_class.jinja @@ -0,0 +1,19 @@ +{% from "repository_operations.jinja" import operation_methods %} +{% macro repository_class(class_name, root, crud, generated_base, abstract, operations) %} +{% set ns = namespace(bases=[]) %} +{% if not abstract %}{% for op in crud %}{% set _ = ns.bases.append((op | capitalize) ~ "Op") %}{% endfor %}{% endif %} +{% if generated_base %}{% set _ = ns.bases.append(generated_base.class_name) %}{% endif %} +{% if not abstract %}{% set _ = ns.bases.append(root) %}{% endif %} +class {{ class_name }}Repository{% if ns.bases %}({{ ns.bases | join(", ") }}){% endif %}: +{% if not abstract %} + COLLECTION = "{{ collection_name }}" + ID_FIELD = "{{ id_field }}" + ID_FORMAT = "{{ id_field_format }}" +{% if root == "SQLRepositoryBase" %} + TABLE = {{ collection_name }}_table +{% endif %} +{% elif not operations %} + pass +{% endif %} +{{ operation_methods(operations) }} +{%- endmacro %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja index b8866354..b175f41f 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja @@ -10,6 +10,8 @@ from pymongo.asynchronous.database import AsyncDatabase from pymongo.errors import BulkWriteError, DuplicateKeyError from conflicts import ConflictError +from dependencies.mongo_operations import run_mongo +from dependencies.operations import present, resolve from dependencies.pagination import Page, Primitives, expand_bound @@ -158,6 +160,10 @@ class RepositoryBase: ) return self._normalize(doc) + @_translate_conflict + async def run_operation(self, query: dict[str, Any], bindings: dict[str, Any]) -> Any: + return present(await run_mongo(self._col, resolve(query, bindings)), self.ID_FIELD) + class ListOp: async def list(self) -> list[dict[str, Any]]: diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja index 087ab480..bbfedc0b 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja @@ -1,9 +1,22 @@ +{% from "repository_class.jinja" import repository_class with context %} {% set class_name = collection_name | capitalize_collection %} +{% set is_abstract = abstract | default(false) %} +{% if custom_operations %} +from typing import Any + +{% endif %} +{% if not is_abstract %} import sqlalchemy as sa from dependencies.postgres import metadata from ._sql_ops import SQLRepositoryBase{% for op in operations %}, {{ op | capitalize }}Op{% endfor %} +{% endif %} +{% if generated_base %} +from {{ generated_base.module }} import {{ generated_base.class_name }} +{% endif %} +{% if not is_abstract %} + {{ collection_name }}_table = sa.Table( "{{ collection_name }}", @@ -15,10 +28,7 @@ from ._sql_ops import SQLRepositoryBase{% for op in operations %}, {{ op | capit sa.Column("_auth", sa.JSON(), nullable=True), {% endif %} ) +{% endif %} -class {{ class_name }}Repository({% for op in operations %}{{ op | capitalize }}Op, {% endfor %}SQLRepositoryBase): - COLLECTION = "{{ collection_name }}" - ID_FIELD = "{{ id_field }}" - ID_FORMAT = "{{ id_field_format }}" - TABLE = {{ collection_name }}_table +{{ repository_class(class_name, "SQLRepositoryBase", operations, generated_base, is_abstract, custom_operations) }} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja index a8126b1b..a337d304 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja @@ -9,7 +9,9 @@ import sqlalchemy as sa from sqlalchemy.exc import IntegrityError from conflicts import ConflictError +from dependencies.operations import present, resolve from dependencies.pagination import Page, Primitives, expand_bound +from dependencies.sql_operations import run_sql _PAGINATION_COMPARATORS = { @@ -117,6 +119,11 @@ class SQLRepositoryBase: row = result.first() return dict(row._mapping) if row is not None else None + @_translate_conflict + async def run_operation(self, query: dict[str, Any], bindings: dict[str, Any]) -> Any: + async with self._engine.begin() as conn: + return present(await run_sql(conn, resolve(query, bindings)), self.ID_FIELD) + async def find_by(self, field: str, value: Any) -> dict[str, Any] | None: async with self._engine.connect() as conn: result = await conn.execute( diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 0365f2c8..89e8da18 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -1,3 +1,4 @@ +{% from "handler_call.jinja" import invoke_handler %} {% set crud_op_names = ["list", "get", "create", "update", "replace", "delete", "login"] %} {% set crud_routes = routes | rejectattr("operation", "equalto", "login") | rejectattr("handler") | selectattr("operation", "in", crud_op_names) | list %} {% set custom_routes = routes | rejectattr("operation", "equalto", "login") | selectattr("handler") | list %} @@ -16,9 +17,8 @@ {% set pagination_routes = routes | selectattr("pagination") | list %} from typing import Any -{% set is_mongo = datastore_type not in ["postgres", "postgresql"] %} {% set needs_request = custom_routes or repo_op_routes or (crud_routes | selectattr("needs_request") | list) or (file_routes | selectattr("action", "equalto", "replace") | list) %} -from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} +from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes or custom_routes %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} {% if ns_service_input.enabled or ns_where.collections %} from dependencies.service_inputs import {% if ns_service_input.enabled %}prepare_service_inputs{% endif %}{% if ns_service_input.enabled and (ns_service_input.access_bind or ns_where.collections) %}, {% endif %}{% if ns_service_input.access_bind or ns_where.collections %}resolve_access_bind{% endif +%} @@ -38,9 +38,11 @@ from dependencies.file_requests import RecordUpload, carried_files, discard_on_e from dependencies.file_stores import record_files {% endif %} -{% if repo_op_routes and is_mongo %} -from fastapi.encoders import jsonable_encoder -from dependencies.bindings import resolve_filter +{% if custom_routes %} +from dependencies.handler_requests import HandlerRequest, invoke +{% endif %} +{% if repo_op_routes %} +from dependencies.operations import operation_bindings {% endif %} {% for group in service_groups %} from dependencies.services import get_{{ group }}_service @@ -354,16 +356,10 @@ async def {{ handler_name }}(request: Request{% for name in route.path_params %} {{ pre_fetch_guard(route) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, "dict(request.path_params)", "_body") }} - request.state.service_inputs = _service_inputs - from importlib import import_module - module_path, func_name = "{{ route.handler }}".rsplit(".", 1) - module = import_module(module_path) - return await getattr(module, func_name)(request) +{{ invoke_handler(route.handler, "principal" if auth.enabled and route.needs_auth else "None", "_body", "_service_inputs") }} {% endfor %} -{% if is_mongo %} {% for route in repo_op_routes %} {% set handler_name = route.operation ~ "_" ~ collection_name %} -{% set query = repo_queries[route.operation] %} @router.{{ route.method | lower }}("{{ route.relative_path }}", tags=["{{ collection_name | capitalize_collection }}"]{{ openapi_extra(route) }}) @@ -383,9 +379,5 @@ async def {{ handler_name }}(request: Request{{ auth_param(route) }}, repo=Depen {{ pre_fetch_guard(route) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, "_path", "_body") }} - _params = {**_path, **_service_inputs} - _bindings = {**_params, "principal": _principal, "params": _params, "body": _body} - _resolved = resolve_filter({{ query | py_literal }}, _bindings) - return jsonable_encoder(await repo.{{ route.operation }}(**{k: v for k, v in _resolved.items() if k != "command"})) + return await repo.{{ route.operation }}(operation_bindings({**_path, **_service_inputs}, _principal, _body)) {% endfor %} -{% endif %} diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 9c20543e..4b2d8f09 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -12,8 +12,9 @@ find_identity, group_by_collection, pagination_config, - runtime_module_source, + MONGO_RUNTIME_MODULES, write_environment_files, + write_runtime_modules, ) from ..validation import ( reject_cookie_delivery, @@ -22,7 +23,6 @@ reject_idp_trust, reject_pkce, reject_session_refresh, - reject_undeclared_repository_operations, reject_unsigned_session_algorithm, validate_datastore_layout, ) @@ -49,7 +49,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: reject_session_refresh(context, "raw") reject_pkce(context, "raw") reject_unsigned_session_algorithm(context) - reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) env.globals["settings_module"] = "settings" env.filters["path_to_regex"] = _path_to_regex @@ -81,9 +80,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: output_path="service_inputs.py", ), ) - (output_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) + write_runtime_modules(output_dir, MONGO_RUNTIME_MODULES) collections_by_name = group_by_collection(context) @@ -210,8 +207,7 @@ def _render_repositories( "id_field": ctx.collections[col_name]["id_field"], "id_field_format": ctx.collections[col_name]["id_format"], "custom_operations": ctx.repository_operations.get(col_name, []), - "conflict_import": "conflicts", - "pagination_import": "pagination", + "module_prefix": "", }, ) ) @@ -238,10 +234,6 @@ def _render_routes( collection_name=col_name, routes=col_data["routes"], needs_auth=needs_auth, - repo_queries={ - op["name"]: op["query"] - for op in context.repository_operations.get(col_name, []) - }, auth=context.auth, ) (routes_dir / f"{col_name}.py").write_text(content) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja new file mode 100644 index 00000000..cb4dbac9 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja @@ -0,0 +1,11 @@ +{% macro invoke_handler(handler) %} + return 200, await invoke({{ handler | py_literal }}, HandlerRequest( + params=url_params, + query=query or {}, + headers=headers, + cookies=cookies or {}, + body=body or {}, + principal=principal, + inputs=service_inputs, + )) +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja index cc0ce2d6..f87f98e5 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja @@ -1,8 +1,9 @@ -from importlib import import_module +{% from "handler_call.jinja" import invoke_handler %} from typing import Any from uuid import UUID -from service_inputs import RawRequest, prepare_service_inputs, resolve_access_bind +from handler_requests import HandlerRequest, invoke +from service_inputs import prepare_service_inputs, resolve_access_bind {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} {% if where_collection not in ns_where.collections %}{% set _ = ns_where.collections.append(where_collection) %}{% endif %} @@ -110,18 +111,6 @@ async def handle_custom_{{ loop.index }}( {{ route.service_forwarding | py_literal }}, bound_inputs, ) - request = RawRequest( - url_params, - body, - headers, - principal, - service_inputs, - query, - cookies, - ) - module_path, function_name = "{{ route.handler }}".rsplit(".", 1) - result = await getattr(import_module(module_path), function_name)(request) - return 200, result - +{{ invoke_handler(route.handler) }} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 5e497607..f0e371c8 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -1,3 +1,4 @@ +{% from "handler_call.jinja" import invoke_handler %} from typing import Any from uuid import UUID @@ -11,7 +12,9 @@ from repositories.{{ where_collection }} import {{ where_collection | capitalize {% endfor %} from services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from services.exceptions import NotFoundError, ValidationError -from service_inputs import RawRequest, prepare_service_inputs, resolve_access_bind, resolve_bindings +from handler_requests import HandlerRequest, invoke +from operations import operation_bindings +from service_inputs import prepare_service_inputs, resolve_access_bind {% if routes | selectattr("pagination") | list %} from pagination import build_response, map_request {% endif %} @@ -144,27 +147,10 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {{ pre_fetch_authorize(route) }} {{ bind_inputs(route) }} service_inputs = prepare_service_inputs(url_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) - params = {**url_params, **service_inputs} {% if route.handler %} - from importlib import import_module - module_path, function_name = "{{ route.handler }}".rsplit(".", 1) - request = RawRequest( - url_params, - body, - headers, - principal, - service_inputs, - query, - cookies, - ) - result = await getattr(import_module(module_path), function_name)(request) - return 200, result +{{ invoke_handler(route.handler) }} {% elif route.operation not in crud_op_names %} -{% set query = repo_queries[route.operation] %} - bindings = {**params, "params": params, "body": body or {}} - resolved = resolve_bindings({{ query | py_literal }}, bindings) - result = await repo.{{ route.operation }}(**{key: value for key, value in resolved.items() if key not in ("command", "collection")}) - return 200, result + return 200, await repo.{{ route.operation }}(operation_bindings({**url_params, **service_inputs}, principal, body)) {% else %} try: {% if route.operation == "list" %} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index c784ee7d..4312da74 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -1,5 +1,7 @@ from __future__ import annotations +{% from "repository_operations.jinja" import operation_methods %} + {% if id_field_format == "uuidv4" %} import uuid {% elif id_field_format == "uuidv7" %} @@ -15,8 +17,10 @@ from pymongo import ReturnDocument from pymongo.asynchronous.database import AsyncDatabase from pymongo.errors import BulkWriteError, DuplicateKeyError -from {{ conflict_import }} import ConflictError -from {{ pagination_import }} import Page, Primitives, expand_bound +from {{ module_prefix }}conflicts import ConflictError +from {{ module_prefix }}mongo_operations import run_mongo +from {{ module_prefix }}operations import present, resolve +from {{ module_prefix }}pagination import Page, Primitives, expand_bound _PAGINATION_COMPARATORS = { @@ -186,104 +190,8 @@ class {{ class_name }}Repository: async def delete(self, document_id: str) -> dict[str, Any] | None: doc = await self._col.find_one_and_delete(self._match_filter(document_id)) return self._normalize(doc) -{% for op in custom_operations | default([]) %} -{% set q = op.query %} -{% set col_expr = "self._db[\"" + q["collection"] + "\"]" if q and q.get("collection") else "self._col" %} -{% if q %} -{% set cmd = q.command %} -{% if cmd == "find" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> list[dict[str, Any]]: - cursor = {{ col_expr }}.find(filter) - docs = await cursor.to_list(length=None) - return [self._normalize(d) for d in docs] -{% elif cmd == "findOne" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one(filter) - return self._normalize(doc) -{% elif cmd == "aggregate" %} - - async def {{ op.name }}(self, pipeline: list) -> list[dict[str, Any]]: - cursor = await {{ col_expr }}.aggregate(pipeline) - return await cursor.to_list(length=None) -{% elif cmd == "countDocuments" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> int: - return await {{ col_expr }}.count_documents(filter) -{% elif cmd == "estimatedDocumentCount" %} - - async def {{ op.name }}(self) -> int: - return await {{ col_expr }}.estimated_document_count() -{% elif cmd == "distinct" %} - - async def {{ op.name }}(self, key: str, filter: dict[str, Any]) -> list: - return await {{ col_expr }}.distinct(key, filter) -{% elif cmd == "insertOne" %} - - @_translate_conflict - async def {{ op.name }}(self, document: dict[str, Any]) -> dict[str, Any]: - await {{ col_expr }}.insert_one(document) - return self._normalize(document) -{% elif cmd == "insertMany" %} - - @_translate_conflict - async def {{ op.name }}(self, documents: list[dict[str, Any]]) -> list[dict[str, Any]]: - await {{ col_expr }}.insert_many(documents) - return [self._normalize(d) for d in documents] -{% elif cmd == "updateOne" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_update(filter, update, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "updateMany" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> Any: - return await {{ col_expr }}.update_many(filter, update) -{% elif cmd == "replaceOne" %} @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], replacement: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_replace(filter, replacement, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "deleteOne" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_delete(filter) - return self._normalize(doc) -{% elif cmd == "deleteMany" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> Any: - return await {{ col_expr }}.delete_many(filter) -{% elif cmd == "findOneAndUpdate" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], update: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_update(filter, update, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "findOneAndReplace" %} - - @_translate_conflict - async def {{ op.name }}(self, filter: dict[str, Any], replacement: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_replace(filter, replacement, return_document=ReturnDocument.AFTER) - return self._normalize(doc) -{% elif cmd == "findOneAndDelete" %} - - async def {{ op.name }}(self, filter: dict[str, Any]) -> dict[str, Any] | None: - doc = await {{ col_expr }}.find_one_and_delete(filter) - return self._normalize(doc) -{% elif cmd == "bulkWrite" %} - - @_translate_conflict - async def {{ op.name }}(self, requests: list) -> Any: - return await {{ col_expr }}.bulk_write(requests) -{% endif %} -{% else %} - - async def {{ op.name }}(self, **kwargs: Any) -> Any: - import {{ op.handler.rsplit(".", 1)[0] }} - return await {{ op.handler }}(**kwargs) -{% endif %} -{% endfor %} + async def run_operation(self, query: dict[str, Any], bindings: dict[str, Any]) -> Any: + return present(await run_mongo(self._col, resolve(query, bindings)), self.ID_FIELD) +{{ operation_methods(custom_operations) }} \ No newline at end of file diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository_operations.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository_operations.jinja new file mode 100644 index 00000000..223778a9 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository_operations.jinja @@ -0,0 +1,7 @@ +{% macro operation_methods(operations) %} +{% for op in operations if op.query %} + + async def {{ op.name }}(self, bindings: dict[str, Any]) -> Any: + return await self.run_operation({{ op.query | py_literal }}, bindings) +{% endfor %} +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja index 3043bf58..70f0a77f 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja @@ -1,33 +1,7 @@ from datetime import datetime -from types import SimpleNamespace from typing import Any -class RawRequest: - def __init__( - self, - path_params: dict[str, str], - body: dict[str, Any] | None, - headers: dict[str, str], - principal: dict[str, Any] | None, - service_inputs: dict[str, Any], - query_params: dict[str, Any] | None = None, - cookies: dict[str, str] | None = None, - ) -> None: - self.path_params = path_params - self.body = body - self.headers = headers - self.query_params = query_params or {} - self.cookies = cookies or {} - self.state = SimpleNamespace( - principal=principal, - service_inputs=service_inputs, - ) - - async def json(self) -> dict[str, Any]: - return self.body or {} - - def prepare_service_inputs( path_values: dict[str, Any], body: dict[str, Any] | None, @@ -54,16 +28,6 @@ def resolve_access_bind( return {name: _resolve(binding, namespace) for name, binding in bindings.items()} -def resolve_bindings(value: Any, bindings: dict[str, Any]) -> Any: - if isinstance(value, str) and value.startswith("$"): - return _resolve_path(value[1:].split("."), bindings) - if isinstance(value, dict): - return {key: resolve_bindings(item, bindings) for key, item in value.items()} - if isinstance(value, list): - return [resolve_bindings(item, bindings) for item in value] - return value - - def _resolve(binding: dict[str, Any], values: dict[str, Any]) -> Any: if "symbol" in binding: return _resolve_path(binding["symbol"], values) diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 5a78e60e..25f60aa2 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -1,5 +1,6 @@ """Shared utilities for backend generators.""" +from collections.abc import Iterable from importlib.resources import files from pathlib import Path @@ -12,7 +13,10 @@ _RUNTIME_MODULES = frozenset( - {"dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} + { + "dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store", + "operations", "mongo_operations", "sql_operations", "handler_requests", + } ) @@ -22,6 +26,14 @@ def runtime_module_source(name: str) -> str: return files("yaml_api_generator.runtime").joinpath(f"{name}.py").read_text() +def write_runtime_modules(directory: Path, names: Iterable[str]) -> None: + for name in names: + (directory / f"{name}.py").write_text(runtime_module_source(name)) + + +MONGO_RUNTIME_MODULES = ("pagination", "operations", "mongo_operations", "handler_requests") + + def write_environment_files( env: Environment, context: CompileContext, output_dir: Path ) -> None: diff --git a/src/yaml_api_generator/generators/backend/validation.py b/src/yaml_api_generator/generators/backend/validation.py index dad43e49..b978c987 100644 --- a/src/yaml_api_generator/generators/backend/validation.py +++ b/src/yaml_api_generator/generators/backend/validation.py @@ -5,20 +5,11 @@ that silently skips it. """ -from yaml_api_generator.generators.backend.compile_context import ( - CompileContext, - RouteDescriptor, -) +from yaml_api_generator.generators.backend.compile_context import CompileContext from yaml_api_generator.parsers.transformers import file_validation from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors -# Operations every backend implements itself. Any other operation is dispatched to -# the repository and needs a declared query to filter on. -_BUILT_IN_OPERATIONS = frozenset( - {"list", "get", "create", "update", "replace", "delete", "login"} -) - _SQL_DATASTORES = frozenset({"postgres", "postgresql"}) _MONGO_DATASTORES = frozenset({"mongo", "mongodb"}) @@ -178,41 +169,3 @@ def reject_unsigned_session_algorithm(context: CompileContext) -> None: ), ) ]) - - -def _lacks_repository_operation(context: CompileContext, route: RouteDescriptor) -> bool: - if route.collection is None or route.handler is not None: - return False - if route.operation in _BUILT_IN_OPERATIONS: - return False - declared = context.repository_operations.get(route.collection, []) - return all(operation["name"] != route.operation for operation in declared) - - -def reject_undeclared_repository_operations(context: CompileContext) -> None: - """Reject routes that dispatch to a repository operation nothing declares. - - Such a route is emitted as a call into its repository, filtered by the query - the operation declares. With no declaration there is no query, and the - emitted call would silently drop the filter the route's access rules assume. - """ - errors = [ - ConfigError( - code="REPOSITORY_OPERATION_UNDECLARED", - message=( - f"Route {route.method} {route.path} calls operation " - f"'{route.operation}', which repository '{route.collection}' " - "does not declare" - ), - path=f"repositories.{route.collection}.operations.{route.operation}", - hint=( - f"Declare '{route.operation}' under " - f"repositories.{route.collection}.operations, or point the " - "service at one of the built-in operations" - ), - ) - for route in context.routes - if _lacks_repository_operation(context, route) - ] - if errors: - raise ConfigErrors(errors) diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 114a3212..3bae68c3 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -113,6 +113,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: routes = self._route.transform_routes( controllers_data, services, + repositories, api_data, repo_entity_map, auth_entities, diff --git a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py index 4231f36b..7a15e329 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py @@ -14,6 +14,7 @@ IRDeploySystemd, IRMongo, IRProvider, + IRRepository, IRRoute, IRService, ) @@ -23,6 +24,43 @@ from .file_transformer import LimitLayer +_SESSION_OPERATIONS = frozenset({"login"}) + + +def _provides(repository: IRRepository | None, operation: str, declared) -> bool: + if repository is None: + return False + return ( + operation in repository.crud + or operation in _SESSION_OPERATIONS + or (declared is not None and declared.query is not None) + ) + + +def _undeclared_operation(route_path: str, collection: str | None, operation: str) -> ConfigError: + return ConfigError( + code="REPOSITORY_OPERATION_UNDECLARED", + message=( + f"Route calls operation '{operation}', which no handler implements " + f"and repository '{collection}' does not provide" + ), + path=f"{route_path}.service", + hint=( + "Declare the operation under the repository's operations, list it in " + "the repository's crud, or give the service a handler" + ), + ) + + +def _unpaginated(route_path: str, operation: str) -> ConfigError: + return ConfigError( + code="PAGINATION_UNSUPPORTED", + message=f"Operation '{operation}' does not paginate; only a repository list does", + path=f"{route_path}.pagination", + hint="Remove pagination from this route", + ) + + class RouteTransformer: """Builds routes from controllers and services.""" @@ -111,6 +149,7 @@ def transform_routes( self, controllers_data: dict[str, Any], services: dict[str, IRService], + repositories: dict[str, IRRepository], api_data: dict[str, Any], repo_entity_map: dict[str, str], auth_entities: set[str], @@ -119,9 +158,14 @@ def transform_routes( ) -> list[IRRoute]: """Transform controller routes to IR. + A route whose service names a repository handler operation dispatches to + that handler. Every other route must reach an operation its repository + provides. + Args: controllers_data: Raw controller configurations from YAML services: Already-built service IR objects + repositories: Already-built repository IR objects api_data: API configuration data repo_entity_map: Repository name to entity name mapping auth_entities: Set of entity names marked as auth type @@ -131,7 +175,8 @@ def transform_routes( List of route IR objects Raises: - ConfigErrors: If access rule parsing fails + ConfigErrors: If access rule parsing fails, a route reaches no + operation, or a route paginates anything but a repository list """ base_path = api_data.get("basePath", "/api") version = api_data.get("version", "") @@ -172,9 +217,18 @@ def transform_routes( ) all_errors.extend(access_errors) - handler = service.handler collection = service.repository operation = service.operation or route_name + repository = repositories.get(collection) + declared = repository.operations.get(operation) if repository else None + handler = service.handler or (declared.handler if declared else None) + route_path = f"controllers.{ctrl_name}.routes.{route_name}" + if handler is None and not _provides(repository, operation, declared): + all_errors.append(_undeclared_operation(route_path, collection, operation)) + continue + if route_cfg.get("pagination") is not None and (handler or operation != "list"): + all_errors.append(_unpaginated(route_path, operation)) + continue pagination = None if pagination_parser is not None: diff --git a/src/yaml_api_generator/runtime/handler_requests.py b/src/yaml_api_generator/runtime/handler_requests.py new file mode 100644 index 00000000..5bd7cabe --- /dev/null +++ b/src/yaml_api_generator/runtime/handler_requests.py @@ -0,0 +1,27 @@ +"""The contract every backend uses to call a user handler. + +A handler is a sync or async callable that receives one ``HandlerRequest`` and +returns the JSON response body. Raising ``ValueError`` answers 422. +""" + +from dataclasses import dataclass, field +from importlib import import_module +from inspect import isawaitable +from typing import Any + + +@dataclass(frozen=True) +class HandlerRequest: + params: dict[str, Any] = field(default_factory=dict) + query: dict[str, Any] = field(default_factory=dict) + headers: dict[str, str] = field(default_factory=dict) + cookies: dict[str, str] = field(default_factory=dict) + body: Any = None + principal: dict[str, Any] | None = None + inputs: dict[str, Any] = field(default_factory=dict) + + +async def invoke(handler: str, request: HandlerRequest) -> Any: + module_path, name = handler.rsplit(".", 1) + result = getattr(import_module(module_path), name)(request) + return await result if isawaitable(result) else result diff --git a/src/yaml_api_generator/runtime/mongo_operations.py b/src/yaml_api_generator/runtime/mongo_operations.py new file mode 100644 index 00000000..d6237c22 --- /dev/null +++ b/src/yaml_api_generator/runtime/mongo_operations.py @@ -0,0 +1,138 @@ +"""Run a resolved MongoDB operation query against an async pymongo collection. + +``bulkWrite`` requests use the shell form, e.g. +``{"updateOne": {"filter": ..., "update": ..., "upsert": true}}``. +""" + +from collections.abc import Awaitable, Callable +from typing import Any + +from pymongo import DeleteMany, DeleteOne, InsertOne, ReplaceOne, ReturnDocument, UpdateMany, UpdateOne + +Command = Callable[[Any, dict[str, Any]], Awaitable[Any]] + +_COMMANDS: dict[str, Command] = {} + +_BULK_REQUESTS: dict[str, Callable[[dict[str, Any]], Any]] = { + "insertOne": lambda spec: InsertOne(spec["document"]), + "updateOne": lambda spec: UpdateOne(spec["filter"], spec["update"], upsert=spec.get("upsert", False)), + "updateMany": lambda spec: UpdateMany(spec["filter"], spec["update"], upsert=spec.get("upsert", False)), + "replaceOne": lambda spec: ReplaceOne(spec["filter"], spec["replacement"], upsert=spec.get("upsert", False)), + "deleteOne": lambda spec: DeleteOne(spec["filter"]), + "deleteMany": lambda spec: DeleteMany(spec["filter"]), +} + + +async def run_mongo(collection: Any, query: dict[str, Any]) -> Any: + target = collection.database[query["collection"]] if query.get("collection") else collection + return await _COMMANDS[query["command"]](target, query) + + +def _command(*names: str) -> Callable[[Command], Command]: + def register(run: Command) -> Command: + _COMMANDS.update(dict.fromkeys(names, run)) + return run + + return register + + +def _stringified(document: dict[str, Any] | None) -> dict[str, Any] | None: + if document is None or "_id" not in document: + return document + return {**document, "_id": str(document["_id"])} + + +def _filter(query: dict[str, Any]) -> dict[str, Any]: + return query.get("filter") or {} + + +@_command("find") +async def _find(target: Any, query: dict[str, Any]) -> list[dict[str, Any]]: + return [_stringified(document) async for document in target.find(_filter(query))] + + +@_command("findOne") +async def _find_one(target: Any, query: dict[str, Any]) -> dict[str, Any] | None: + return _stringified(await target.find_one(_filter(query))) + + +@_command("aggregate") +async def _aggregate(target: Any, query: dict[str, Any]) -> list[dict[str, Any]]: + cursor = await target.aggregate(query.get("pipeline") or []) + return [_stringified(document) async for document in cursor] + + +@_command("countDocuments") +async def _count_documents(target: Any, query: dict[str, Any]) -> int: + return await target.count_documents(_filter(query)) + + +@_command("estimatedDocumentCount") +async def _estimated_document_count(target: Any, query: dict[str, Any]) -> int: + return await target.estimated_document_count() + + +@_command("distinct") +async def _distinct(target: Any, query: dict[str, Any]) -> list[Any]: + return await target.distinct(query.get("key"), _filter(query)) + + +@_command("insertOne") +async def _insert_one(target: Any, query: dict[str, Any]) -> dict[str, Any] | None: + document = dict(query.get("document") or {}) + await target.insert_one(document) + return _stringified(document) + + +@_command("insertMany") +async def _insert_many(target: Any, query: dict[str, Any]) -> list[dict[str, Any] | None]: + documents = [dict(document) for document in query.get("documents") or []] + await target.insert_many(documents) + return [_stringified(document) for document in documents] + + +@_command("updateOne", "findOneAndUpdate") +async def _update_one(target: Any, query: dict[str, Any]) -> dict[str, Any] | None: + return _stringified(await target.find_one_and_update( + _filter(query), query.get("update") or {}, return_document=ReturnDocument.AFTER + )) + + +@_command("replaceOne", "findOneAndReplace") +async def _replace_one(target: Any, query: dict[str, Any]) -> dict[str, Any] | None: + return _stringified(await target.find_one_and_replace( + _filter(query), query.get("replacement") or {}, return_document=ReturnDocument.AFTER + )) + + +@_command("deleteOne", "findOneAndDelete") +async def _delete_one(target: Any, query: dict[str, Any]) -> dict[str, Any] | None: + return _stringified(await target.find_one_and_delete(_filter(query))) + + +@_command("updateMany") +async def _update_many(target: Any, query: dict[str, Any]) -> dict[str, int]: + result = await target.update_many(_filter(query), query.get("update") or {}) + return {"matched": result.matched_count, "modified": result.modified_count} + + +@_command("deleteMany") +async def _delete_many(target: Any, query: dict[str, Any]) -> dict[str, int]: + result = await target.delete_many(_filter(query)) + return {"deleted": result.deleted_count} + + +@_command("bulkWrite") +async def _bulk_write(target: Any, query: dict[str, Any]) -> dict[str, int]: + result = await target.bulk_write([ + _BULK_REQUESTS[kind](spec) + for request in query["requests"] + for kind, spec in request.items() + ]) + return { + "inserted": result.inserted_count, + "matched": result.matched_count, + "modified": result.modified_count, + "deleted": result.deleted_count, + "upserted": result.upserted_count, + } diff --git a/src/yaml_api_generator/runtime/operations.py b/src/yaml_api_generator/runtime/operations.py new file mode 100644 index 00000000..1cffa690 --- /dev/null +++ b/src/yaml_api_generator/runtime/operations.py @@ -0,0 +1,57 @@ +"""Bind and present declared repository operations. + +A query value starting with ``$`` is a reference into the request bindings +(``{**params, "principal", "params", "body"}``); every other value passes +through unchanged. Results are presented the way CRUD responses are: the +datastore's own ``_id`` becomes the canonical id field and credential fields +are hidden. +""" + +from typing import Any + +_HIDDEN_FIELDS = ("_auth", "password") + + +def operation_bindings( + params: dict[str, Any], principal: dict[str, Any] | None, body: Any +) -> dict[str, Any]: + return {**params, "principal": principal or {}, "params": params, "body": body or {}} + + +def resolve(value: Any, bindings: dict[str, Any]) -> Any: + if isinstance(value, str) and value.startswith("$"): + return _lookup(value[1:].split("."), bindings) + if isinstance(value, dict): + return {key: resolve(item, bindings) for key, item in value.items()} + if isinstance(value, list): + return [resolve(item, bindings) for item in value] + return value + + +def _lookup(path: list[str], current: Any) -> Any: + for segment in path: + if not isinstance(current, dict) or segment not in current: + return None + current = current[segment] + return current + + +def present_document(document: dict[str, Any] | None, id_field: str) -> dict[str, Any] | None: + if document is None: + return None + presented = dict(document) + if id_field not in presented and presented.get("_id") is not None: + presented[id_field] = str(presented["_id"]) + if id_field != "_id": + presented.pop("_id", None) + for field in _HIDDEN_FIELDS: + presented.pop(field, None) + return presented + + +def present(result: Any, id_field: str) -> Any: + if isinstance(result, list): + return [present(item, id_field) for item in result] + if isinstance(result, dict): + return present_document(result, id_field) + return result diff --git a/src/yaml_api_generator/runtime/sql_operations.py b/src/yaml_api_generator/runtime/sql_operations.py new file mode 100644 index 00000000..5ca1dfef --- /dev/null +++ b/src/yaml_api_generator/runtime/sql_operations.py @@ -0,0 +1,12 @@ +"""Run a resolved SQL operation query on an open SQLAlchemy async connection.""" + +from typing import Any + +import sqlalchemy as sa + + +async def run_sql(connection: Any, query: dict[str, Any]) -> Any: + result = await connection.execute(sa.text(query["sql"]), query.get("params") or {}) + if result.returns_rows: + return [dict(row._mapping) for row in result] + return {"rowcount": result.rowcount} diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 7ac0cf88..15189f7b 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -1,4 +1,3 @@ -import importlib from collections.abc import Iterator, Mapping from yaml_api_generator.ir import IRDocumentBinding @@ -9,6 +8,7 @@ from .repositories import RepositoryRegistry from .steps import ( AuthorizeRouteStep, + HandlerStep, JWKSStep, LoginStep, PostFetchAuthorizeStep, @@ -63,15 +63,6 @@ def _has_access_bind(route: RuntimeRouteDescriptor) -> bool: } -def _import_handler(dotted_path: str) -> Step: - module_path, _, attr_name = dotted_path.rpartition(".") - if not module_path: - raise ValueError(f"Invalid handler path: {dotted_path}") - module = importlib.import_module(module_path) - handler = getattr(module, attr_name) - return handler - - class PipelineBuilder: def __init__( self, @@ -218,7 +209,7 @@ def _build_handler_pipeline(self, route: RuntimeRouteDescriptor) -> Pipeline: service_input_step = self._service_input_step(route) if service_input_step is not None: steps.append(service_input_step) - steps.append(_import_handler(route.handler)) + steps.append(HandlerStep(route.handler)) return self._pipeline(route, steps) def _base_steps(self, route: RuntimeRouteDescriptor) -> list[Step]: @@ -313,10 +304,5 @@ def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: if operation == "delete": return RepositoryDeleteStep(collection_name, id_field, self._locator(route)) - descriptor = self._ctx.operations.get(collection_name, {}).get(operation) - if descriptor is not None: - if descriptor.query is not None: - return RepositoryOperationStep(collection_name, descriptor.query, id_field) - if descriptor.handler is not None: - return _import_handler(descriptor.handler) - raise ValueError(f"Unsupported operation: {operation}") + query = self._ctx.operations[collection_name][operation].query + return RepositoryOperationStep(collection_name, query, id_field) diff --git a/src/yaml_api_generator/server/common/repositories/base.py b/src/yaml_api_generator/server/common/repositories/base.py index f1a91518..42741181 100644 --- a/src/yaml_api_generator/server/common/repositories/base.py +++ b/src/yaml_api_generator/server/common/repositories/base.py @@ -44,13 +44,8 @@ class BaseRepository(ABC): def register_id_field(self, collection_name: str, id_field: str) -> None: pass - async def run_operation( - self, - collection_name: str, - query: dict[str, Any], - bindings: dict[str, Any], - ) -> Any: - """Execute a declarative custom operation. Overridden per backend.""" + async def run_operation(self, collection_name: str, query: dict[str, Any]) -> Any: + """Execute a declared operation query whose bindings are already resolved.""" raise NotImplementedError( f"{type(self).__name__} does not support custom operations" ) diff --git a/src/yaml_api_generator/server/common/repositories/mongodb.py b/src/yaml_api_generator/server/common/repositories/mongodb.py index a32b857c..8a4f90c2 100644 --- a/src/yaml_api_generator/server/common/repositories/mongodb.py +++ b/src/yaml_api_generator/server/common/repositories/mongodb.py @@ -7,8 +7,8 @@ from pymongo import AsyncMongoClient, ReturnDocument from pymongo.errors import BulkWriteError, DuplicateKeyError +from yaml_api_generator.runtime.mongo_operations import run_mongo from .base import BaseRepository, ConflictError -from .bindings import resolve from ..pagination import Page, Primitives, expand_bound @@ -265,69 +265,9 @@ async def delete( ) return self._normalize_document(document) - async def run_operation( - self, - collection_name: str, - query: dict[str, Any], - bindings: dict[str, Any], - ) -> Any: + async def run_operation(self, collection_name: str, query: dict[str, Any]) -> Any: with _as_conflict(): - return await self._run_operation(collection_name, query, bindings) - - async def _run_operation( - self, - collection_name: str, - query: dict[str, Any], - bindings: dict[str, Any], - ) -> Any: - command = query.get("command") - target = self.collection(query.get("collection") or collection_name) - get = lambda field: resolve(query.get(field), bindings) - - if command == "find": - cursor = target.find(get("filter") or {}) - docs = await cursor.to_list(length=None) - return [self._normalize_document_required(d) for d in docs] - if command == "findOne": - return self._normalize_document(await target.find_one(get("filter") or {})) - if command == "aggregate": - cursor = await target.aggregate(get("pipeline") or []) - docs = await cursor.to_list(length=None) - return [self._normalize_document_required(d) for d in docs] - if command == "countDocuments": - return await target.count_documents(get("filter") or {}) - if command == "estimatedDocumentCount": - return await target.estimated_document_count() - if command == "distinct": - return await target.distinct(query.get("key"), get("filter") or {}) - if command == "insertOne": - document = get("document") or {} - await target.insert_one(document) - return self._normalize_document_required(document) - if command == "insertMany": - documents = get("documents") or [] - await target.insert_many(documents) - return [self._normalize_document_required(d) for d in documents] - if command in ("updateOne", "findOneAndUpdate"): - doc = await target.find_one_and_update( - get("filter") or {}, get("update") or {}, return_document=ReturnDocument.AFTER - ) - return self._normalize_document(doc) - if command in ("replaceOne", "findOneAndReplace"): - doc = await target.find_one_and_replace( - get("filter") or {}, get("replacement") or {}, return_document=ReturnDocument.AFTER - ) - return self._normalize_document(doc) - if command in ("deleteOne", "findOneAndDelete"): - return self._normalize_document(await target.find_one_and_delete(get("filter") or {})) - if command == "updateMany": - result = await target.update_many(get("filter") or {}, get("update") or {}) - return {"matched": result.matched_count, "modified": result.modified_count} - if command == "deleteMany": - result = await target.delete_many(get("filter") or {}) - return {"deleted": result.deleted_count} - - raise ValueError(f"Unsupported query command: {command}") + return await run_mongo(self.collection(collection_name), query) @staticmethod def _normalize_document(document: dict[str, Any] | None) -> dict[str, Any] | None: diff --git a/src/yaml_api_generator/server/common/repositories/sql.py b/src/yaml_api_generator/server/common/repositories/sql.py index 89ad9153..f739051a 100644 --- a/src/yaml_api_generator/server/common/repositories/sql.py +++ b/src/yaml_api_generator/server/common/repositories/sql.py @@ -17,8 +17,8 @@ from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from yaml_api_generator.ir import IRCollection +from yaml_api_generator.runtime.sql_operations import run_sql from .base import BaseRepository, ConflictError -from .bindings import resolve from .sql_schema import table_from_collection from ..pagination import Page, Primitives, expand_bound @@ -294,22 +294,10 @@ async def delete(self, collection_name: str, document_id: str) -> dict[str, Any] ) return self._one(result) - async def run_operation( - self, - collection_name: str, - query: dict[str, Any], - bindings: dict[str, Any], - ) -> Any: - sql = query.get("sql") - if not sql: - raise ValueError("SQL custom operations require a 'sql' string") - params = {key: resolve(value, bindings) for key, value in (query.get("params") or {}).items()} + async def run_operation(self, collection_name: str, query: dict[str, Any]) -> Any: with _as_conflict(): async with await self._begin() as conn: - result = await conn.execute(sa.text(sql), params) - if result.returns_rows: - return [dict(row._mapping) for row in result] - return {"rowcount": result.rowcount} + return await run_sql(conn, query) def _table(self, collection_name: str) -> sa.Table: return self._tables[collection_name] diff --git a/src/yaml_api_generator/server/common/steps/__init__.py b/src/yaml_api_generator/server/common/steps/__init__.py index 8d6516cf..0221bdf5 100644 --- a/src/yaml_api_generator/server/common/steps/__init__.py +++ b/src/yaml_api_generator/server/common/steps/__init__.py @@ -32,6 +32,7 @@ class NotFoundError(ValueError): UnauthorizedError, ValidateIdentityCreateStep, ) +from .handlers import HandlerStep from .jwks import JWKSStep from .login import LoginStep from .refresh import RefreshStep @@ -48,6 +49,7 @@ class NotFoundError(ValueError): from .validation import ValidateBodyStep __all__ = [ + "HandlerStep", "ServiceInputStep", "Step", "NotFoundError", diff --git a/src/yaml_api_generator/server/common/steps/handlers.py b/src/yaml_api_generator/server/common/steps/handlers.py new file mode 100644 index 00000000..f29ca71a --- /dev/null +++ b/src/yaml_api_generator/server/common/steps/handlers.py @@ -0,0 +1,29 @@ +"""Call a user handler through the contract shared with every generated backend.""" + +from yaml_api_generator.runtime.handler_requests import HandlerRequest, invoke + +from ..request_context import RequestContext +from . import Step + + +def HandlerStep(handler: str) -> Step: + async def step(ctx: RequestContext) -> RequestContext: + ctx.response = await invoke(handler, handler_request(ctx)) + return ctx + + return step + + +def handler_request(ctx: RequestContext) -> HandlerRequest: + return HandlerRequest( + params=dict(ctx.url_params), + query={ + key: value[-1] if isinstance(value, list) else value + for key, value in (ctx.query or {}).items() + }, + headers=dict(ctx.headers), + cookies=dict(ctx.cookies or {}), + body=ctx.body or {}, + principal=ctx.principal, + inputs=dict(ctx.service_inputs or {}), + ) diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 2fcc7aa9..8a4b886e 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -5,6 +5,12 @@ from ..auth import hash_password from ..request_context import RequestContext from yaml_api_generator.ir.api import IRPagination, IRRolesConfig +from yaml_api_generator.runtime.operations import ( + operation_bindings, + present, + present_document as _inject_canonical_id, + resolve, +) from ..pagination import Page, build_response, map_request from ..repositories.base import BaseRepository from ..utils.ids import generate_id as _generate_id @@ -80,22 +86,6 @@ def _payload(ctx: RequestContext) -> dict[str, Any]: return cast(dict[str, Any], ctx.body) -def _inject_canonical_id( - document: dict[str, Any] | None, - id_field: str, -) -> dict[str, Any] | None: - if document is None: - return None - normalized = dict(document) - if id_field not in normalized and normalized.get("_id") is not None: - normalized[id_field] = str(normalized["_id"]) - if id_field != "_id": - normalized.pop("_id", None) - normalized.pop("_auth", None) - normalized.pop("password", None) - return normalized - - def _inject_canonical_id_list( documents: list[dict[str, Any]], id_field: str, @@ -281,25 +271,11 @@ def RepositoryOperationStep( id_field: str, ) -> Step: async def step(ctx: RequestContext) -> RequestContext: - params = {**ctx.url_params, **(ctx.service_inputs or {})} - bindings = { - **params, - "principal": ctx.principal or {}, - "params": params, - "body": ctx.body or {}, - } - result = await _repository(ctx).run_operation(collection_name, query, bindings) - ctx.response = _normalize_operation_result(result, id_field) + bindings = operation_bindings( + {**ctx.url_params, **(ctx.service_inputs or {})}, ctx.principal, ctx.body + ) + result = await _repository(ctx).run_operation(collection_name, resolve(query, bindings)) + ctx.response = present(result, id_field) return ctx return step - - -def _normalize_operation_result(result: Any, id_field: str) -> Any: - if isinstance(result, list): - if all(isinstance(item, dict) for item in result): - return _inject_canonical_id_list(result, id_field) - return result - if isinstance(result, dict): - return _inject_canonical_id(result, id_field) - return result diff --git a/src/yaml_api_generator/server/fastapi/factory.py b/src/yaml_api_generator/server/fastapi/factory.py index d3e4a1c8..5450d738 100644 --- a/src/yaml_api_generator/server/fastapi/factory.py +++ b/src/yaml_api_generator/server/fastapi/factory.py @@ -618,6 +618,13 @@ def _field_annotation(field_config: dict[str, Any]) -> Any: return Any +async def _json_body(request: Request) -> Any: + try: + return await request.json() + except (ValueError, UnicodeDecodeError): + return None + + def _build_endpoint( pipeline: Pipeline, body_model: type[BaseModel] | None, @@ -634,7 +641,7 @@ async def endpoint_no_body_with_id(request: Request, id: str) -> Any: return await _run_pipeline( pipeline=pipeline, request=request, - body=None, + body=await _json_body(request), url_params={"id": id}, uses_cookie=uses_cookie, operation=route.operation, @@ -646,7 +653,7 @@ async def endpoint_no_body(request: Request) -> Any: return await _run_pipeline( pipeline=pipeline, request=request, - body=None, + body=await _json_body(request), url_params={}, uses_cookie=uses_cookie, operation=route.operation, diff --git a/tests/fixtures/invalid/files-route-conflict.yaml b/tests/fixtures/invalid/files-route-conflict.yaml index 99b3ad99..a97d8c06 100644 --- a/tests/fixtures/invalid/files-route-conflict.yaml +++ b/tests/fixtures/invalid/files-route-conflict.yaml @@ -17,4 +17,4 @@ controllers: routes: get: { service: documents, method: GET, path: "/{id}" } update: { service: documents, method: PATCH, path: "/{id}" } - replaceAvatar: { service: documents, method: PUT, path: "/{id}/files/avatar" } + replace: { service: documents, method: PUT, path: "/{id}/files/avatar" } diff --git a/tests/unit/test_conflict_409.py b/tests/unit/test_conflict_409.py index b9eadcd7..cdd93109 100644 --- a/tests/unit/test_conflict_409.py +++ b/tests/unit/test_conflict_409.py @@ -105,7 +105,6 @@ async def test_sql_custom_write_duplicate_raises_conflict(repo): "sql": "UPDATE members SET email = :email WHERE id = :id", "params": {"email": "a@x.io", "id": "m2"}, }, - {}, ) assert excinfo.value.field == "email" @@ -194,7 +193,6 @@ async def test_mongo_duplicate_key_raises_conflict_naming_field(): lambda repository: repository.run_operation( "members", {"command": "insertOne", "document": {"email": "a@x.io"}}, - {}, ), ], ) @@ -217,7 +215,6 @@ async def test_mongo_bulk_duplicate_raises_conflict_naming_field(): await repository.run_operation( "members", {"command": "insertMany", "documents": [{"email": "a@x.io"}]}, - {}, ) assert excinfo.value.field == "email" @@ -232,7 +229,6 @@ async def test_mongo_non_duplicate_bulk_error_is_not_mapped_to_conflict(): await repository.run_operation( "members", {"command": "insertMany", "documents": [{"email": "a@x.io"}]}, - {}, ) diff --git a/tests/unit/test_controller_bind.py b/tests/unit/test_controller_bind.py index bb787685..4b94428d 100644 --- a/tests/unit/test_controller_bind.py +++ b/tests/unit/test_controller_bind.py @@ -319,11 +319,16 @@ def test_generated_fastapi_applies_authorized_bind_end_to_end(tmp_path): from fastapi.testclient import TestClient from dependencies.auth import require_principal from dependencies.repositories import get_posts_repository + from dependencies.operations import resolve + from repositories.posts import PostsRepository from routes.posts import router - class RecordingRepository: - async def archive(self, **kwargs): - return kwargs + class RecordingRepository(PostsRepository): + def __init__(self): + pass + + async def run_operation(self, query, bindings): + return {{"filter": resolve(query["filter"], bindings)}} async def principal(): return {{ diff --git a/tests/unit/test_custom_operations.py b/tests/unit/test_custom_operations.py index 1c943d7e..e8389080 100644 --- a/tests/unit/test_custom_operations.py +++ b/tests/unit/test_custom_operations.py @@ -11,7 +11,7 @@ import pytest from yaml_api_generator.server import build_raw_app -from yaml_api_generator.server.common.repositories.bindings import resolve +from yaml_api_generator.runtime.operations import resolve from yaml_api_generator.server.common.repositories.mongodb import ( MongoRepository, MongoRepositoryConfig, @@ -91,10 +91,16 @@ async def test_sql_custom_operation_over_http(tmp_path, monkeypatch): class _FakeCursor: def __init__(self, docs): - self._docs = docs + self._docs = iter(docs) + + def __aiter__(self): + return self - async def to_list(self, length=None): - return list(self._docs) + async def __anext__(self): + try: + return next(self._docs) + except StopIteration: + raise StopAsyncIteration from None class _FakeCollection: @@ -110,7 +116,7 @@ def find(self, query_filter): @pytest.mark.asyncio -async def test_mongo_find_operation_resolves_bindings(): +async def test_mongo_find_operation_runs_resolved_query(): repo = MongoRepository(MongoRepositoryConfig(uri="mongodb://x", database="d")) collection = _FakeCollection([ {"_id": "1", "authorId": "u1", "title": "mine"}, @@ -118,10 +124,10 @@ async def test_mongo_find_operation_resolves_bindings(): ]) repo._db = {"posts": collection} - query = {"command": "find", "filter": {"authorId": "$principal.id"}} bindings = {"principal": {"id": "u1"}, "params": {}, "body": {}} + query = resolve({"command": "find", "filter": {"authorId": "$principal.id"}}, bindings) - result = await repo.run_operation("posts", query, bindings) + result = await repo.run_operation("posts", query) assert collection.received_filter == {"authorId": "u1"} assert [d["title"] for d in result] == ["mine"] assert all("_id" in d for d in result) diff --git a/tests/unit/test_fastapi_bindings_generation.py b/tests/unit/test_fastapi_bindings_generation.py index b0e77beb..4d7e1dbc 100644 --- a/tests/unit/test_fastapi_bindings_generation.py +++ b/tests/unit/test_fastapi_bindings_generation.py @@ -37,48 +37,16 @@ """ -def test_bindings_module_emitted_for_custom_operations(tmp_path): +def test_custom_operation_route_calls_the_repository_method(tmp_path): spec_path = tmp_path / "spec.yaml" spec_path.write_text(textwrap.dedent(_SPEC)) - result = load_and_generate_ir(str(spec_path)) - ctx = build_context(result.ir) - gen = FastAPIGenerator() out = tmp_path / "out" - gen.generate(ctx, out) + FastAPIGenerator().generate(build_context(load_and_generate_ir(str(spec_path)).ir), out) - assert (out / "dependencies" / "bindings.py").exists() + assert (out / "dependencies" / "operations.py").exists() + assert (out / "dependencies" / "mongo_operations.py").exists() + repository = (out / "repositories" / "posts.py").read_text() + assert "async def byOrg(self, bindings: dict[str, Any]) -> Any:" in repository + assert "'$principal.record.orgId'" in repository route_content = (out / "routes" / "posts.py").read_text() - assert "resolve_filter" in route_content - assert "dependencies.bindings" in route_content - - -def test_bindings_module_absent_without_custom_operations(tmp_path): - spec_path = tmp_path / "spec.yaml" - spec_path.write_text(textwrap.dedent(""" - version: "0.2" - api: { basePath: "" } - mongo: - uri: mongodb://localhost:27017 - database: testdb - entities: - post: - fields: - id: { type: id } - title: { type: string } - repositories: - posts: { entity: post } - services: - posts: { repository: posts } - controllers: - posts: - base_path: /posts - routes: - list: { service: posts, method: GET, path: "", access: public } - """)) - result = load_and_generate_ir(str(spec_path)) - ctx = build_context(result.ir) - gen = FastAPIGenerator() - out = tmp_path / "out" - gen.generate(ctx, out) - - assert not (out / "dependencies" / "bindings.py").exists() + assert "repo.byOrg(operation_bindings(" in route_content diff --git a/tests/unit/test_generated_mongo_cursors.py b/tests/unit/test_generated_mongo_cursors.py index 6ca5bde5..5f634989 100644 --- a/tests/unit/test_generated_mongo_cursors.py +++ b/tests/unit/test_generated_mongo_cursors.py @@ -50,15 +50,21 @@ repository_module = importlib.import_module({repository_module!r}) class Cursor: - async def to_list(self, length=None): - return [{{"total": 1}}] + def __aiter__(self): + return self + + async def __anext__(self): + if getattr(self, "consumed", False): + raise StopAsyncIteration + self.consumed = True + return {{"total": 1}} class Collection: async def aggregate(self, pipeline): return Cursor() repository = repository_module.MembersRepository({{"members": Collection()}}) - assert asyncio.run(repository.summarize([{{"$count": "total"}}])) == [{{"total": 1}}] + assert asyncio.run(repository.summarize({{}})) == [{{"total": 1}}] """ diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py index 20190387..4d00798e 100644 --- a/tests/unit/test_pipeline.py +++ b/tests/unit/test_pipeline.py @@ -145,9 +145,8 @@ async def test_builder_custom_handler_route_calls_handler(): mod = types.ModuleType("myapp.handlers.items") - def custom_handler(ctx): - ctx.response = {"handled": True} - return ctx + def custom_handler(request): + return {"handled": True, "params": request.params} mod.custom_handler = custom_handler sys.modules["myapp.handlers.items"] = mod @@ -164,7 +163,7 @@ def custom_handler(ctx): repo = InMemoryRepository() pipeline = PipelineBuilder(rc, RepositoryRegistry.single(repo)).build_for_route(route) result = await pipeline.run() - assert result.response == {"handled": True} + assert result.response == {"handled": True, "params": {}} del sys.modules["myapp.handlers.items"] @@ -423,17 +422,3 @@ async def test_builder_identity_create_without_password_raises(): pipeline = PipelineBuilder(rc, RepositoryRegistry.single(repo)).build_for_route(route) with pytest.raises(ValueError): await pipeline.run(body={"email": "a@b.com"}) # missing password - - -def test_builder_unsupported_operation_raises(): - route = RuntimeRouteDescriptor( - method="POST", - path="/items", - operation="custom_op", - collection="items", - is_public=True, - ) - rc = _make_runtime_context(routes=[route]) - repo = InMemoryRepository() - with pytest.raises(ValueError, match="Unsupported operation"): - PipelineBuilder(rc, RepositoryRegistry.single(repo)).build_for_route(route) \ No newline at end of file diff --git a/tests/unit/test_repository_operation_validation.py b/tests/unit/test_repository_operation_validation.py index 1287e872..c857ee1e 100644 --- a/tests/unit/test_repository_operation_validation.py +++ b/tests/unit/test_repository_operation_validation.py @@ -1,12 +1,4 @@ -"""Routes dispatching to an undeclared repository operation are rejected. - -A route whose operation is neither built in nor a custom handler is emitted as a -call into its repository, filtered by the query that operation declares. When no -repository declares it there is no query to emit, and the backends used to -render the Jinja undefined sentinel into the generated source as the bare name -``Undefined`` -- syntactically valid Python that byte-compiles, passes every -syntax-level check, and raises ``NameError`` only once the route is called. -""" +"""Routes must reach an operation their repository provides or a handler.""" import copy import pytest @@ -59,18 +51,41 @@ def _context(tmp_path, *, declare_operation): return build_context(load_and_generate_ir(str(path)).ir) -@pytest.mark.parametrize("generator", _GENERATORS) -def test_undeclared_repository_operation_is_rejected(tmp_path, generator): - context = _context(tmp_path, declare_operation=False) - +def _errors(tmp_path, spec): + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec)) with pytest.raises(ConfigErrors) as excinfo: - generator().generate(context, tmp_path / "out") + load_and_generate_ir(str(path)) + return excinfo.value.errors + - [error] = excinfo.value.errors +def test_undeclared_repository_operation_is_rejected(tmp_path): + [error] = _errors(tmp_path, _SPEC) assert error.code == "REPOSITORY_OPERATION_UNDECLARED" assert "recent" in error.message +def test_operation_excluded_from_crud_is_rejected(tmp_path): + spec = copy.deepcopy(_SPEC) + spec["repositories"]["posts"]["crud"] = ["list", "get"] + spec["controllers"]["posts"]["routes"] = { + "delete": {"service": "posts", "method": "DELETE", "path": "/{id}"} + } + [error] = _errors(tmp_path, spec) + assert error.code == "REPOSITORY_OPERATION_UNDECLARED" + assert "delete" in error.message + + +def test_pagination_outside_a_list_is_rejected(tmp_path): + spec = copy.deepcopy(_SPEC) + spec["repositories"]["posts"]["operations"] = _OPERATION + spec["controllers"]["posts"]["routes"]["recent"]["pagination"] = { + "strategy": "offset", "size": {"default": 10, "max": 50} + } + [error] = _errors(tmp_path, spec) + assert error.code == "PAGINATION_UNSUPPORTED" + + @pytest.mark.parametrize("generator", _GENERATORS) def test_declared_repository_operation_emits_its_query(tmp_path, generator): out = tmp_path / "out" diff --git a/tests/unit/test_service_input.py b/tests/unit/test_service_input.py index da881b17..9aa26cee 100644 --- a/tests/unit/test_service_input.py +++ b/tests/unit/test_service_input.py @@ -339,16 +339,11 @@ def test_runtime_ill_typed_input_is_422(): @pytest.mark.asyncio async def test_runtime_repository_operation_receives_forwarded_values(): class RecordingRepository: - def __init__(self): - self.bindings = None + async def run_operation(self, collection, query): + return query["filter"] - async def run_operation(self, collection, query, bindings): - self.bindings = bindings - return {"id": bindings["lookup"], "state": bindings["state"]} - - repository = RecordingRepository() ctx = _ctx({"postId": "p1"}) - ctx.repository = repository + ctx.repository = RecordingRepository() ctx = ServiceInputStep( {"postId": {"type": "string"}}, { @@ -359,14 +354,8 @@ async def run_operation(self, collection, query, bindings): result = await RepositoryOperationStep( "posts", - {"command": "findOne", "filter": {"id": "$lookup"}}, + {"command": "findOne", "filter": {"id": "$lookup", "state": "$params.state"}}, "id", )(ctx) - assert repository.bindings["lookup"] == "p1" - assert repository.bindings["params"] == { - "postId": "p1", - "lookup": "p1", - "state": "archived", - } assert result.response == {"id": "p1", "state": "archived"} diff --git a/tests/unit/test_service_input_codegen.py b/tests/unit/test_service_input_codegen.py index 1706e48d..a298273f 100644 --- a/tests/unit/test_service_input_codegen.py +++ b/tests/unit/test_service_input_codegen.py @@ -76,8 +76,8 @@ def test_fastapi_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "routes/posts.py").read_text() handler_route = (tmp_path / "routes/_handlers.py").read_text() assert "'state': {'literal': 'archived'}" in repository_route - assert '_bindings = {**_params' in repository_route - assert "request.state.service_inputs = service_inputs" in handler_route + assert "operation_bindings({**_path, **_service_inputs}" in repository_route + assert "inputs=service_inputs" in handler_route def test_fastapi_generated_routes_forward_values_end_to_end(tmp_path): @@ -97,14 +97,20 @@ def test_fastapi_generated_routes_forward_values_end_to_end(tmp_path): handler_module = types.ModuleType("tests.support.service_input_handler") async def notify(request): - return request.state.service_inputs + return request.inputs handler_module.notify = notify sys.modules[handler_module.__name__] = handler_module - class RecordingRepository: - async def archive(self, **kwargs): - return kwargs + from dependencies.operations import resolve + from repositories.posts import PostsRepository + + class RecordingRepository(PostsRepository): + def __init__(self): + pass + + async def run_operation(self, query, bindings): + return {{"filter": resolve(query["filter"], bindings)}} app = FastAPI() app.include_router(posts_router) @@ -130,8 +136,8 @@ def test_raw_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "routes/posts.py").read_text() handler_route = (tmp_path / "routes/handlers.py").read_text() - assert "resolve_bindings" in repository_route - assert "request = RawRequest(" in handler_route + assert "repo.archive(operation_bindings({**url_params, **service_inputs}" in repository_route + assert "inputs=service_inputs" in handler_route def test_drf_emits_repository_and_handler_forwarding(tmp_path): @@ -139,5 +145,5 @@ def test_drf_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "api/views/posts.py").read_text() handler_route = (tmp_path / "api/views/_handlers.py").read_text() - assert "await repo.archive(**kwargs)" in repository_route - assert "request.service_inputs = prepare_service_inputs" in handler_route + assert "repo.archive(operation_bindings({**path_params, **service_inputs}" in repository_route + assert "inputs=service_inputs" in handler_route diff --git a/tests/unit/test_vendored_imports.py b/tests/unit/test_vendored_imports.py index a6cfb3bf..21d2397f 100644 --- a/tests/unit/test_vendored_imports.py +++ b/tests/unit/test_vendored_imports.py @@ -21,17 +21,12 @@ from .helper import build - async def publish(**kwargs): - return build(kwargs) + async def publish(request): + return build(request.body) async def route(request): return build({"via": "route"}) - - - async def serve(ctx): - ctx.response = build({"via": "serve"}) - return ctx """ _HELPER_BODY = """ @@ -62,13 +57,12 @@ def build(data): services: posts: {{ repository: posts }} direct: {{ handler: vend.route }} - served: {{ handler: vend.serve }} controllers: posts: base_path: /posts routes: direct: {{ service: direct, method: POST, path: /direct, access: public }} - served: {{ service: served, method: POST, path: /served, access: public }} + publish: {{ service: posts, method: POST, path: /publish, access: public }} """ _MINIMAL_SPEC = """ @@ -150,27 +144,17 @@ def test_vendored_package_is_a_working_package_inside_every_target(tmp_path, tar sys.path.insert(0, {str(backend)!r}) import vend_pkg assert vend_pkg.__file__.startswith({str(backend)!r}) - assert asyncio.run(vend_pkg.publish(x=1)) == {{"built": {{"x": 1}}}} + assert asyncio.run(vend_pkg.route(None)) == {{"built": {{"via": "route"}}}} """) -@pytest.mark.parametrize( - "target,module", - [("fastapi", "repositories.posts"), ("drf", "api.repositories.posts"), ("raw", "repositories.posts")], -) -def test_generated_repository_operation_calls_the_vendored_handler(tmp_path, target, module): - backend = _compile(tmp_path, target) - - _run(f""" - import asyncio, sys - sys.path.insert(0, {str(backend)!r}) - from {module} import PostsRepository - result = asyncio.run(PostsRepository.publish(None, title="t")) - assert result == {{"built": {{"title": "t"}}}}, result - """) +_EXPECTED = { + "/posts/direct": {"built": {"via": "route"}}, + "/posts/publish": {"built": {"title": "t"}}, +} -def test_generated_fastapi_route_dispatches_to_the_vendored_handler(tmp_path): +def test_generated_fastapi_routes_dispatch_to_the_vendored_handlers(tmp_path): backend = _compile(tmp_path, "fastapi") _run(f""" @@ -178,53 +162,59 @@ def test_generated_fastapi_route_dispatches_to_the_vendored_handler(tmp_path): sys.path.insert(0, {str(backend)!r}) from fastapi import FastAPI from fastapi.testclient import TestClient - from routes._handlers import router + from dependencies.repositories import get_posts_repository + from routes._handlers import router as handler_router + from routes.posts import router as posts_router app = FastAPI() - app.include_router(router) - response = TestClient(app).post("/posts/direct", json={{}}) - assert response.status_code == 200, response.text - assert response.json() == {{"built": {{"via": "route"}}}} + app.include_router(handler_router) + app.include_router(posts_router) + app.dependency_overrides[get_posts_repository] = lambda: None + client = TestClient(app) + for path, expected in {_EXPECTED!r}.items(): + response = client.post(path, json={{"title": "t"}}) + assert response.status_code == 200, response.text + assert response.json() == expected, response.json() """) -def test_dev_server_route_executes_the_vendored_handler(tmp_path, monkeypatch): +def test_dev_server_routes_execute_the_vendored_handlers(tmp_path, monkeypatch): from fastapi.testclient import TestClient from yaml_api_generator.server import build_app monkeypatch.setenv("YAML_API_DB_STORE_DSN", f"sqlite+aiosqlite:///{tmp_path / 'db.sqlite'}") _write_package(tmp_path / "src", "vend_pkg") - spec = _write_spec(tmp_path, _SPEC.format(store=_SQL_STORE)) + client = TestClient(build_app(str(_write_spec(tmp_path, _SPEC.format(store=_SQL_STORE))))) - response = TestClient(build_app(str(spec))).post("/posts/served", json={}) - assert response.status_code == 200, response.text - assert response.json() == {"built": {"via": "serve"}} + for path, expected in _EXPECTED.items(): + response = client.post(path, json={"title": "t"}) + assert response.status_code == 200, response.text + assert response.json() == expected @pytest.mark.asyncio -async def test_raw_dev_server_route_executes_the_vendored_handler(tmp_path, monkeypatch): +async def test_raw_dev_server_routes_execute_the_vendored_handlers(tmp_path, monkeypatch): from yaml_api_generator.server import build_raw_app monkeypatch.setenv("YAML_API_DB_STORE_DSN", f"sqlite+aiosqlite:///{tmp_path / 'db.sqlite'}") _write_package(tmp_path / "src", "vend_pkg") - spec = _write_spec(tmp_path, _SPEC.format(store=_SQL_STORE)) - app = build_raw_app(str(spec)) + app = build_raw_app(str(_write_spec(tmp_path, _SPEC.format(store=_SQL_STORE)))) - scope = {"type": "http", "method": "POST", "path": "/posts/served", "headers": []} - messages = [] + for path, expected in _EXPECTED.items(): + messages = [] - async def receive(): - return {"type": "http.request", "body": b"{}", "more_body": False} + async def receive(): + return {"type": "http.request", "body": b'{"title": "t"}', "more_body": False} - async def send(message): - messages.append(message) + async def send(message): + messages.append(message) - await app(scope, receive, send) - start = next(m for m in messages if m["type"] == "http.response.start") - body = next(m for m in messages if m["type"] == "http.response.body") - assert start["status"] == 200 - assert json.loads(body["body"]) == {"built": {"via": "serve"}} + await app({"type": "http", "method": "POST", "path": path, "headers": []}, receive, send) + start = next(m for m in messages if m["type"] == "http.response.start") + body = next(m for m in messages if m["type"] == "http.response.body") + assert start["status"] == 200 + assert json.loads(body["body"]) == expected @pytest.mark.parametrize("target", ["fastapi", "drf", "raw"]) @@ -434,10 +424,6 @@ def test_v02_output_is_not_vendored_and_stays_stable(tmp_path): compile_project(str(spec), output_dir=str(out_a)) compile_project(str(spec), output_dir=str(out_b)) - repository = (out_a / "backend" / "repositories" / "posts.py").read_text() - assert "import raw_module" in repository - assert "await raw_module.publish(**kwargs)" in repository - assert not (out_a / "backend" / "my_app").exists() requirements_a = (out_a / "backend" / "requirements.txt").read_text() assert requirements_a == (out_b / "backend" / "requirements.txt").read_text() From 60a5c5957ade38867f61912f29b0cb9be920dccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 18:40:20 +0100 Subject: [PATCH 03/89] Share one access runtime across the dev server and generated backends List routes filter rows by the access rules in memory, in Mongo and in SQL; by loads the named record; is null over request input holds only for absent keys; the refusal reason applies in every version. --- .../backend/builders/route_builder.py | 3 +- .../generators/backend/compile_context.py | 1 - .../generators/backend/drf/generator.py | 100 +----- .../drf/templates/handler_views.py.jinja | 25 +- .../backend/drf/templates/views.py.jinja | 43 +-- .../backend/fastapi/access_filters.py | 143 ++++---- .../generators/backend/fastapi/generator.py | 7 +- .../fastapi/templates/authorization.py.jinja | 20 -- .../fastapi/templates/handler_routes.py.jinja | 27 +- .../fastapi/templates/repository_ops.py.jinja | 11 +- .../templates/repository_sql_ops.py.jinja | 20 +- .../backend/fastapi/templates/routes.py.jinja | 45 +-- .../fastapi/templates/service_ops.py.jinja | 7 +- .../generators/backend/raw/generator.py | 9 +- .../raw/templates/handler_routes.py.jinja | 43 +-- .../backend/raw/templates/routes.py.jinja | 64 ++-- .../shared/templates/repository.py.jinja | 11 +- .../backend/shared/templates/service.py.jinja | 13 +- .../generators/backend/utils.py | 2 +- src/yaml_api_generator/ir/when.py | 13 +- .../parsers/transformers/v02/__init__.py | 6 +- .../transformers/v02/document_lookup.py | 56 +++ .../transformers/v02/policy_expansion.py | 13 +- src/yaml_api_generator/runtime/access.py | 327 ++++++++++++++++++ .../server/common/context_builder.py | 2 +- .../server/common/pipeline_builder.py | 12 +- .../server/common/repositories/base.py | 9 +- .../server/common/repositories/mongodb.py | 16 +- .../server/common/repositories/sql.py | 22 +- .../server/common/request_context.py | 3 + .../server/common/steps/authorization.py | 133 +++---- .../server/common/steps/repositories.py | 4 +- .../server/common/utils/value_compare.py | 32 +- .../server/common/when_eval.py | 122 ++++--- .../test_generated_where_authorization.py | 9 +- tests/unit/support/repositories.py | 7 +- tests/unit/test_access_reason.py | 12 +- tests/unit/test_controller_role.py | 14 +- tests/unit/test_fastapi_access_filters.py | 123 ++++--- tests/unit/test_fastapi_authz_emitted.py | 27 -- tests/unit/test_generators.py | 44 --- tests/unit/test_policy_value_coercion.py | 67 +--- 42 files changed, 900 insertions(+), 767 deletions(-) create mode 100644 src/yaml_api_generator/parsers/transformers/v02/document_lookup.py create mode 100644 src/yaml_api_generator/runtime/access.py diff --git a/src/yaml_api_generator/generators/backend/builders/route_builder.py b/src/yaml_api_generator/generators/backend/builders/route_builder.py index 41b1911a..0bc7ba45 100644 --- a/src/yaml_api_generator/generators/backend/builders/route_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/route_builder.py @@ -41,7 +41,6 @@ def _document_descriptors(rule) -> list[DocumentBindingDescriptor]: DocumentBindingDescriptor( name=name, collection=binding.collection, - id_param=binding.id_param, where={conjunct.field: _rhs_config(conjunct.rhs) for conjunct in binding.where}, ) for name, binding in rule.document_symbols.items() @@ -106,7 +105,7 @@ def build_routes( request_symbols=_request_descriptors(rule), when=rule.when, effect=rule.effect, - reason=rule.reason if document.version == "0.3" else None, + reason=rule.reason, bind={name: _rhs_config(rhs) for name, rhs in rule.bind.items()}, ) for rule in ir_route.access_rules diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index f0634b49..463850ff 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -184,7 +184,6 @@ class ModelDescriptor: class DocumentBindingDescriptor: name: str collection: str - id_param: str | None = None where: dict[str, dict[str, Any]] = field(default_factory=dict) @property diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index f94e32f5..42fdf7be 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -26,19 +26,6 @@ from . import type_mapper from ..fastapi import access_filters from .settings_builder import build_drf_settings_fields -from yaml_api_generator.ir.when import ( - And, - Equals, - FieldRef, - In, - Not, - NotEquals, - NotIn, - Or, - StringLiteral, - WhenExpr, - needs_document, -) _BACKEND_GENERATORS = Path(__file__).parent.parent @@ -63,19 +50,18 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: env.globals["settings_module"] = f"{self.dir_prefix}.settings" env.filters["drf_field"] = type_mapper.drf_field env.filters["capitalize_collection"] = to_pascal_case - env.filters["rule_to_python"] = _rule_to_python + env.filters["rule_to_predicate"] = access_filters.rule_to_predicate env.filters["py_literal"] = access_filters.py_literal env.filters["pagination_config"] = pagination_config env.filters["request_symbol_binding"] = access_filters.request_symbol_binding env.filters["role_hierarchies_py"] = access_filters.role_hierarchies_py env.filters["int"] = int - env.tests["needs_document"] = _needs_document_test env.tests["needs_document_rule"] = access_filters.needs_document_rule env.tests["needs_current_document_rule"] = ( access_filters.needs_current_document_rule ) env.tests["needs_post_fetch"] = access_filters.needs_post_fetch - env.tests["unconditional"] = _is_unconditional + env.tests["unconditional"] = access_filters.is_unconditional output_dir.mkdir(parents=True, exist_ok=True) api_dir = output_dir / self.dir_prefix @@ -91,9 +77,8 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: self._render_serializers(env, context, collections_by_name, output_dir) self._render_services(env, context, collections_by_name, output_dir) self._render_service_inputs(env, context, output_dir) - (api_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) + for module in ("access", "pagination"): + (api_dir / f"{module}.py").write_text(runtime_module_source(module)) self._render_views(env, context, collections_by_name, output_dir) self._render_repositories(env, context, collections_by_name, output_dir) self._render_authentication(env, context, output_dir) @@ -206,6 +191,7 @@ def _build_service_context(self, col_name: str, col_data: dict, context: Compile "import_prefix": "api.repositories", "auth_import_prefix": "api.authentication.auth", "service_import_prefix": "api.services", + "access_import": "api.access", } def _render_views( @@ -277,6 +263,7 @@ def _render_repositories( "custom_operations": ctx.repository_operations.get(col_name, []), "conflict_import": "api.conflicts", "pagination_import": "api.pagination", + "access_import": "api.access", }, ) ) @@ -324,78 +311,3 @@ def _render_urls( apps_template = env.get_template("apps.py.jinja") (api_dir / "apps.py").write_text(apps_template.render(ctx=context)) - - -def _build_symbol_map(rule) -> dict[str, str]: - symbol_map: dict[str, str] = {} - if hasattr(rule, 'identity_symbols'): - for symbol_name in rule.identity_symbols: - symbol_map[symbol_name] = symbol_name - if hasattr(rule, 'document_symbols'): - for symbol_name in rule.document_symbols: - symbol_map[symbol_name] = symbol_name - return symbol_map - - -def _operand_to_python(expr: WhenExpr, symbol_map: dict[str, str]) -> str: - if isinstance(expr, StringLiteral): - return f'"{expr.value}"' - if isinstance(expr, FieldRef): - var_name = symbol_map.get(expr.root, expr.root) - if not expr.path: - return f'{var_name}.get("sub")' - if len(expr.path) == 1: - return f'{var_name}.get("{expr.path[0]}")' - if len(expr.path) >= 2 and expr.path[0] == "record": - return f'{var_name}.get("record", {{}}).get("{expr.path[1]}")' - nested = ".".join(f'get("{p}")' for p in expr.path) - return f'{var_name}.{nested}' - return "None" - - -def _when_to_python(expr: WhenExpr, symbol_map: dict[str, str] | None = None) -> str: - if symbol_map is None: - symbol_map = {} - if isinstance(expr, Equals): - return f"{_operand_to_python(expr.left, symbol_map)} == {_operand_to_python(expr.right, symbol_map)}" - if isinstance(expr, NotEquals): - return f"{_operand_to_python(expr.left, symbol_map)} != {_operand_to_python(expr.right, symbol_map)}" - if isinstance(expr, And): - return f"({_when_to_python(expr.left, symbol_map)} and {_when_to_python(expr.right, symbol_map)})" - if isinstance(expr, Or): - return f"({_when_to_python(expr.left, symbol_map)} or {_when_to_python(expr.right, symbol_map)})" - if isinstance(expr, Not): - return f"(not {_when_to_python(expr.operand, symbol_map)})" - if isinstance(expr, In): - return f"{_operand_to_python(expr.left, symbol_map)} in {tuple(expr.values)!r}" - if isinstance(expr, NotIn): - return f"{_operand_to_python(expr.left, symbol_map)} not in {tuple(expr.values)!r}" - return "False" - - -def _rule_to_python(rule) -> str: - conditions: list[str] = [] - if getattr(rule, "principal", "").startswith("identity:"): - collection = rule.principal.removeprefix("identity:") - conditions.append( - f'_resolved_principal.get("identity_collection") == "{collection}"' - ) - if getattr(rule, "required_role", None) is not None: - conditions.append(f'_principal_has_role(_resolved_principal, "{rule.required_role}")') - if getattr(rule, "when", None) is not None: - conditions.append(_when_to_python(rule.when, _build_symbol_map(rule))) - if not conditions: - return "True" - return " and ".join(conditions) - - -def _needs_document_test(expr: WhenExpr) -> bool: - return needs_document(expr) - - -def _is_unconditional(rule) -> bool: - return ( - not getattr(rule, "principal", "") - and getattr(rule, "when", None) is None - and getattr(rule, "required_role", None) is None - ) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index 108d3dba..53a864ab 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -14,6 +14,7 @@ from api.service_inputs import prepare_service_inputs, resolve_access_bind from api.repositories.{{ where_collection }} import {{ where_collection | capitalize_collection }}Repository {% endfor %} {% if routes | selectattr("needs_auth") | list %} +from api.access import AccessDenied, Decision, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, value_at from api.authentication.auth import _principal_has_role, resolve_principal from api.authentication.mongodb import get_database {% endif %} @@ -55,25 +56,19 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A {% for symbol in ns_symbols.request %} {{ symbol | request_symbol_binding }} {% endfor %} - access_granted = False - access_bindings = {} + decisions = [] {% for rule in route.access_rules %} - if not access_granted: {% for binding in rule.document_bindings %} - {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(get_database()).find_one( - resolve_access_bind({{ binding.where | py_literal }}, {**path_params, **locals()}) - ) + {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(get_database()).find_one( + resolve_access_bind({{ binding.where | py_literal }}, {**path_params, **locals()}) + ) {% endfor %} - if {% for binding in rule.document_bindings %}{{ binding.name }} is not None and {% endfor %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - return Response({"detail": {{ (rule.reason or "Forbidden") | py_literal }}}, status=status.HTTP_403_FORBIDDEN) -{% else %} - access_granted = True - access_bindings = {{ rule.bind | py_literal }} -{% endif %} + decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) {% endfor %} - if not access_granted: - return Response({"detail": "Forbidden"}, status=status.HTTP_403_FORBIDDEN) + try: + access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] + except AccessDenied as exc: + return Response({"detail": exc.reason}, status=status.HTTP_403_FORBIDDEN) bound_inputs = resolve_access_bind(access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 3ac87bab..ae9a6acb 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -35,6 +35,7 @@ from api.authentication.mongodb import get_database {% set needs_auth = routes | selectattr("needs_auth") | list %} {% if needs_auth %} from api.authentication.auth import resolve_principal, _principal_has_role +from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at {% endif %} {% set class_name = collection_name | capitalize_collection %} @@ -80,6 +81,7 @@ from api.authentication.auth import resolve_principal, _principal_has_role if _resolved_principal is None: return Response({"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED) {% if route.access_rules %} + principal = _resolved_principal {% set ns = namespace(identity_symbols=[]) %} {% for rule in route.access_rules %} {% for symbol in rule.identity_symbols %} @@ -92,39 +94,37 @@ from api.authentication.auth import resolve_principal, _principal_has_role {{ symbol }} = _resolved_principal {% endfor %} _access_bindings = {} -{% if not (route is needs_post_fetch) %} +{% if route.operation == "list" %} +{{ authorize_rules(route, "Field(())", rows=true) }} +{% elif not (route is needs_post_fetch) %} {{ authorize_rules(route, "None") }} {% endif %} {% endif %} {% endif %} {% endmacro %} -{% macro authorize_rules(route, current_document) %} - _access_granted = False +{% macro authorize_rules(route, current_document, rows=false) %} + _decisions = [] {% for rule in route.access_rules %} - if not _access_granted: {% for binding in rule.document_bindings %} {% if binding.uses_where %} - {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( - resolve_access_bind({{ binding.where | py_literal }}, {**path_params, **locals()}) - ) + {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( + resolve_access_bind({{ binding.where | py_literal }}, {**path_params, **locals()}) + ) {% else %} - {{ binding.name }} = {{ current_document }} + {{ binding.name }} = {{ current_document }} {% endif %} {% endfor %} -{% if rule is needs_current_document_rule %} - document = {{ current_document }} -{% endif %} - if {% for binding in rule.document_bindings if binding.uses_where %}{{ binding.name }} is not None and {% endfor %}{% if rule is needs_current_document_rule %}{{ current_document }} is not None and {% endif %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - return Response({"detail": {{ (rule.reason or "Forbidden") | py_literal }}}, status=status.HTTP_403_FORBIDDEN) + _decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) +{% endfor %} + try: +{% if rows %} + _row_filter = row_filter(_decisions) {% else %} - _access_granted = True - _access_bindings = {{ rule.bind | py_literal }} + _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] {% endif %} -{% endfor %} - if not _access_granted: - return Response({"detail": "Forbidden"}, status=status.HTTP_403_FORBIDDEN) + except AccessDenied as exc: + return Response({"detail": exc.reason}, status=status.HTTP_403_FORBIDDEN) {% endmacro %} {% macro post_fetch_check(route, doc_var) %} @@ -152,13 +152,14 @@ class {{ class_name }}ListView(APIView): try: prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) {% if route.operation == "list" %} +{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} {% if route.pagination %} pagination_config = {{ route.pagination | pagination_config | py_literal }} pagination_query = dict(request.query_params) - page = await service.list(map_request(pagination_config, pagination_query)) + page = await service.list(map_request(pagination_config, pagination_query){{ row_filter_arg }}) return Response(build_response(pagination_config, page, pagination_query)) {% else %} - documents = await service.list() + documents = await service.list({{ "None" ~ row_filter_arg if row_filter_arg }}) return Response(documents) {% endif %} {% elif route.operation == "create" %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/access_filters.py b/src/yaml_api_generator/generators/backend/fastapi/access_filters.py index 5781b22c..a76c49cf 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/access_filters.py +++ b/src/yaml_api_generator/generators/backend/fastapi/access_filters.py @@ -1,8 +1,8 @@ -"""Jinja filters that compile access rules into inline Python boolean expressions. +"""Jinja filters that compile access rules into inline Python predicate expressions. At generation time, ``when`` predicates are fully known, so they are compiled -directly into source strings that reference the runtime ``principal`` dict and -the fetched ``document`` dict. No AST interpreter is needed in the generated app. +directly into calls on the ``access`` runtime module shipped with the app. No AST +interpreter is needed in the generated app. """ from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor, RouteDescriptor @@ -35,79 +35,70 @@ def py_literal(value: object) -> str: return repr(value) -def _safe_nested(root: str, path: tuple[str, ...]) -> str: - chain = "".join(f'.get("{p}", {{}})' for p in path[:-1]) - return f'{root}{chain}.get("{path[-1]}")' - - -def _operand_to_python( - expr: WhenExpr, - record_roots: frozenset[str], - wrapper_roots: frozenset[str], -) -> str: - if isinstance(expr, StringLiteral): - return f'"{expr.value}"' - if isinstance(expr, FieldRef): - if expr.root in record_roots: - if not expr.path: - return expr.root - if len(expr.path) == 1: - return f'{expr.root}.get("{expr.path[0]}")' - return _safe_nested(expr.root, expr.path) - if expr.root in wrapper_roots: - return _safe_nested(expr.root, expr.path) if expr.path else expr.root - if not expr.path: - return f'{expr.root}.get("sub")' - if len(expr.path) == 1: - return f'{expr.root}.get("{expr.path[0]}")' - if len(expr.path) >= 2 and expr.path[0] == "record": - return f'{expr.root}.get("record", {{}}).get("{expr.path[1]}")' - nested = ".".join(f'get("{p}")' for p in expr.path) - return f"{expr.root}.{nested}" - return "None" - - -def _when_to_python( - expr: WhenExpr, - record_roots: frozenset[str], - wrapper_roots: frozenset[str], -) -> str: - operand = lambda e: _operand_to_python(e, record_roots, wrapper_roots) - recurse = lambda e: _when_to_python(e, record_roots, wrapper_roots) - if isinstance(expr, Equals): - return f"_values_equal({operand(expr.left)}, {operand(expr.right)})" - if isinstance(expr, NotEquals): - return f"(not _values_equal({operand(expr.left)}, {operand(expr.right)}))" - if isinstance(expr, And): - return f"({recurse(expr.left)} and {recurse(expr.right)})" - if isinstance(expr, Or): - return f"({recurse(expr.left)} or {recurse(expr.right)})" - if isinstance(expr, Not): - return f"(not {recurse(expr.operand)})" - if isinstance(expr, In): - return f"any(_values_equal({operand(expr.left)}, _v) for _v in {tuple(expr.values)!r})" - if isinstance(expr, NotIn): - return f"(not any(_values_equal({operand(expr.left)}, _v) for _v in {tuple(expr.values)!r}))" - if isinstance(expr, IsNull): - return f"({operand(expr.operand)} is None)" - if isinstance(expr, IsNotNull): - return f"({operand(expr.operand)} is not None)" - return "False" - - -def rule_to_python(rule: AccessRuleDescriptor) -> str: - """Compile a single access rule to an inline Python boolean expression.""" - record_roots = frozenset(sym.name for sym in rule.request_symbols if sym.is_record_shaped) - wrapper_roots = frozenset(sym.name for sym in rule.request_symbols if sym.is_whole) - conditions: list[str] = [] - if rule.principal.startswith("identity:"): - collection = rule.principal.removeprefix("identity:") - conditions.append(f'principal.get("identity_collection") == "{collection}"') - if rule.required_role is not None: - conditions.append(f'_principal_has_role(principal, "{rule.required_role}")') - if rule.when is not None: - conditions.append(_when_to_python(rule.when, record_roots, wrapper_roots)) - return " and ".join(conditions) if conditions else "True" +class _RuleCompiler: + """Compiles a ``when`` expression to a call tree over the shipped ``access`` module. + + The tree evaluates to a constant predicate for a loaded record and to a row + predicate when the record symbol is bound to ``access.Field``. + """ + + def __init__(self, request_roots: frozenset[str], whole_roots: frozenset[str]) -> None: + self._request_roots = request_roots + self._whole_roots = whole_roots + self._rules = { + Equals: lambda expr: f"equals({self._operand(expr.left)}, {self._operand(expr.right)})", + NotEquals: lambda expr: f"negate(equals({self._operand(expr.left)}, {self._operand(expr.right)}))", + And: lambda expr: f"conjoin({self.compile(expr.left)}, {self.compile(expr.right)})", + Or: lambda expr: f"disjoin({self.compile(expr.left)}, {self.compile(expr.right)})", + Not: lambda expr: f"negate({self.compile(expr.operand)})", + In: lambda expr: f"one_of({self._operand(expr.left)}, {tuple(expr.values)!r})", + NotIn: lambda expr: f"negate(one_of({self._operand(expr.left)}, {tuple(expr.values)!r}))", + IsNull: lambda expr: self._is_null(expr.operand), + IsNotNull: lambda expr: f"negate({self._is_null(expr.operand)})", + } + + def compile(self, expr: WhenExpr) -> str: + rule = self._rules.get(type(expr)) + return rule(expr) if rule else "NO_ROW" + + def _is_null(self, operand: WhenExpr) -> str: + if isinstance(operand, FieldRef) and operand.path and operand.root in self._request_roots: + return f"constant(not has_path({operand.root}, {operand.path!r}))" + return f"is_null({self._operand(operand)})" + + def _operand(self, expr: WhenExpr) -> str: + if isinstance(expr, StringLiteral): + return repr(expr.value) + if not isinstance(expr, FieldRef): + return "None" + if not expr.path and expr.root in self._whole_roots: + return expr.root + return f"value_at({expr.root}, {expr.path or ('sub',)!r})" + + +def rule_to_predicate(rule: AccessRuleDescriptor) -> str: + """Compile an access rule to a Python expression of the predicate it admits.""" + compiler = _RuleCompiler( + request_roots=frozenset(sym.name for sym in rule.request_symbols), + whole_roots=frozenset( + sym.name for sym in rule.request_symbols if sym.is_record_shaped or sym.is_whole + ), + ) + conditions = [ + *( + [f'constant(principal.get("identity_collection") == {rule.principal.removeprefix("identity:")!r})'] + if rule.principal.startswith("identity:") + else [] + ), + *( + [f"constant(_principal_has_role(principal, {rule.required_role!r}))"] + if rule.required_role is not None + else [] + ), + *(f"constant({symbol} is not None)" for symbol in rule.document_symbols), + *([compiler.compile(rule.when)] if rule.when is not None else []), + ] + return f"all_of({', '.join(conditions)})" def role_hierarchies_py(hierarchies: dict[str, dict[str, set[str]]]) -> str: diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 4c1a7bc8..b5538a8f 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -56,7 +56,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: env.filters["pagination_config"] = pagination_config env.filters["sql_column"] = sql_filters.sql_column env.filters["py_constructor"] = py_constructor - env.filters["rule_to_python"] = access_filters.rule_to_python + env.filters["rule_to_predicate"] = access_filters.rule_to_predicate env.filters["role_hierarchies_py"] = access_filters.role_hierarchies_py env.filters["request_symbol_binding"] = access_filters.request_symbol_binding env.tests["needs_post_fetch"] = access_filters.needs_post_fetch @@ -440,9 +440,8 @@ def _render_dependencies( (deps_dir / "claims.py").write_text( (_STATIC_DIR / "claims.py").read_text() ) - (deps_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) + for module in ("access", "pagination"): + (deps_dir / f"{module}.py").write_text(runtime_module_source(module)) if any(getattr(provider, "dpop", False) for provider in context.auth.providers.values()): (deps_dir / "dpop.py").write_text(runtime_module_source("dpop")) if any(context.repository_operations.values()): diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/authorization.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/authorization.py.jinja index 3eed7dd8..a35d2a16 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/authorization.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/authorization.py.jinja @@ -1,27 +1,7 @@ from typing import Any -from uuid import UUID from fastapi import HTTPException -try: - from bson import ObjectId - - _IDISH = (UUID, ObjectId) -except ImportError: - _IDISH = (UUID,) - - -def _values_equal(left: Any, right: Any) -> bool: - if left is None or right is None: - return left is None and right is None - if isinstance(left, bool) or isinstance(right, bool): - return isinstance(left, bool) and isinstance(right, bool) and left is right - if left == right: - return True - if isinstance(left, _IDISH) or isinstance(right, _IDISH): - return str(left) == str(right) - return False - ROLE_HIERARCHIES: dict[str, dict[str, set[str]]] = {{ auth.role_hierarchies | role_hierarchies_py }} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja index d658ee48..a846ce4b 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja @@ -15,7 +15,8 @@ from dependencies.repositories import get_{{ where_collection }}_repository from dependencies.auth import require_principal {% endif %} {% if routes | selectattr("access_rules") | list %} -from dependencies.authorization import _principal_has_role, _values_equal +from dependencies.access import AccessDenied, Decision, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, value_at +from dependencies.authorization import _principal_has_role {% endif %} @@ -52,25 +53,19 @@ async def {{ route.operation | py_identifier }}_{{ loop.index }}( {% for symbol in ns_symbols.request %} {{ symbol | request_symbol_binding }} {% endfor %} - _access_granted = False - _access_bindings = {} + _decisions = [] {% for rule in route.access_rules %} - if not _access_granted: {% for binding in rule.document_bindings %} - {{ binding.name }} = await _where_{{ binding.collection }}_repo.find_one( - resolve_access_bind({{ binding.where | py_literal }}, {**dict(request.path_params), **locals()}) - ) + {{ binding.name }} = await _where_{{ binding.collection }}_repo.find_one( + resolve_access_bind({{ binding.where | py_literal }}, {**dict(request.path_params), **locals()}) + ) {% endfor %} - if {% for binding in rule.document_bindings %}{{ binding.name }} is not None and {% endfor %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - raise HTTPException(status_code=403, detail={{ (rule.reason or "Forbidden") | py_literal }}) -{% else %} - _access_granted = True - _access_bindings = {{ rule.bind | py_literal }} -{% endif %} + _decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) {% endfor %} - if not _access_granted: - raise HTTPException(status_code=403, detail="Forbidden") + try: + _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] + except AccessDenied as exc: + raise HTTPException(status_code=403, detail=exc.reason) from exc bound_inputs = resolve_access_bind(_access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja index b8866354..58f06d0d 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja @@ -10,6 +10,7 @@ from pymongo.asynchronous.database import AsyncDatabase from pymongo.errors import BulkWriteError, DuplicateKeyError from conflicts import ConflictError +from dependencies.access import EVERY_ROW, Predicate from dependencies.pagination import Page, Primitives, expand_bound @@ -160,13 +161,13 @@ class RepositoryBase: class ListOp: - async def list(self) -> list[dict[str, Any]]: - cursor = self._col.find({}) + async def list(self, predicate: Predicate = EVERY_ROW) -> list[dict[str, Any]]: + cursor = self._col.find(predicate.mongo()) docs = await cursor.to_list(length=None) return [self._normalize(d) for d in docs] - async def paginate(self, primitives: Primitives) -> Page: - query = _pagination_filter(primitives) + async def paginate(self, primitives: Primitives, predicate: Predicate = EVERY_ROW) -> Page: + query = {"$and": [predicate.mongo(), _pagination_filter(primitives)]} cursor = self._col.find(query) if primitives.sort: cursor = cursor.sort( @@ -178,7 +179,7 @@ class ListOp: cursor = cursor.limit(primitives.limit) documents = await cursor.to_list(length=None) total = ( - await self._col.count_documents({}) + await self._col.count_documents(predicate.mongo()) if primitives.count == "exact" else None ) diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja index a8126b1b..72ffd0b4 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja @@ -9,6 +9,7 @@ import sqlalchemy as sa from sqlalchemy.exc import IntegrityError from conflicts import ConflictError +from dependencies.access import EVERY_ROW, Predicate from dependencies.pagination import Page, Primitives, expand_bound @@ -204,16 +205,15 @@ class SQLRepositoryBase: class ListOp: - async def list(self) -> list[dict[str, Any]]: + async def list(self, predicate: Predicate = EVERY_ROW) -> list[dict[str, Any]]: async with self._engine.connect() as conn: - result = await conn.execute(sa.select(self.TABLE)) + result = await conn.execute(sa.select(self.TABLE).where(predicate.sql(self.TABLE))) return [dict(row._mapping) for row in result] - async def paginate(self, primitives: Primitives) -> Page: - statement = sa.select(self.TABLE) - clauses = _pagination_bounds(self.TABLE, primitives) - if clauses: - statement = statement.where(sa.and_(*clauses)) + async def paginate(self, primitives: Primitives, predicate: Predicate = EVERY_ROW) -> Page: + statement = sa.select(self.TABLE).where( + predicate.sql(self.TABLE), *_pagination_bounds(self.TABLE, primitives) + ) for key in primitives.sort: column = self.TABLE.c[key.field] statement = statement.order_by( @@ -227,7 +227,11 @@ class ListOp: rows = [dict(row._mapping) for row in await conn.execute(statement)] total = None if primitives.count == "exact": - count_statement = sa.select(sa.func.count()).select_from(self.TABLE) + count_statement = ( + sa.select(sa.func.count()) + .select_from(self.TABLE) + .where(predicate.sql(self.TABLE)) + ) total = (await conn.execute(count_statement)).scalar() return Page(rows, total) diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 0365f2c8..d3e32b6a 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -60,7 +60,8 @@ from dependencies.repositories import get_{{ where_collection }}_repository from dependencies.auth import require_principal {% endif %} {% if auth.enabled and ns_auth.needs_role %} -from dependencies.authorization import _principal_has_role, _values_equal +from dependencies.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at +from dependencies.authorization import _principal_has_role {% endif %} {% set ns = namespace(model_imports=[]) %} {% for route in crud_routes %} @@ -196,35 +197,34 @@ _files = record_files[{{ collection_name | tojson }}] {% endfor %} {% endmacro %} -{% macro authorize_rules(route, current_document) %} +{# Collects one decision per rule, then admits a record or filters rows #} +{% macro authorize_rules(route, current_document, rows=false) %} {% if auth.enabled and route.needs_auth and route.access_rules %} {{ identity_locals(route) }} - _access_granted = False - _access_bindings = {} + _decisions = [] {% for rule in route.access_rules %} - if not _access_granted: {% for binding in rule.document_bindings %} {% if binding.uses_where %} - {{ binding.name }} = await _where_{{ binding.collection }}_repo.find_one( - resolve_access_bind({{ binding.where | py_literal }}, locals()) - ) + {{ binding.name }} = await _where_{{ binding.collection }}_repo.find_one( + resolve_access_bind({{ binding.where | py_literal }}, locals()) + ) {% else %} - {{ binding.name }} = {{ current_document }} + {{ binding.name }} = {{ current_document }} {% endif %} {% endfor %} -{% if rule is needs_current_document_rule %} - document = {{ current_document }} + _decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) +{% endfor %} +{% if rows %} + _access_bindings = {} {% endif %} - if {% for binding in rule.document_bindings if binding.uses_where %}{{ binding.name }} is not None and {% endfor %}{% if rule is needs_current_document_rule %}{{ current_document }} is not None and {% endif %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - raise HTTPException(status_code=403, detail={{ (rule.reason or "Forbidden") | py_literal }}) + try: +{% if rows %} + _row_filter = row_filter(_decisions) {% else %} - _access_granted = True - _access_bindings = {{ rule.bind | py_literal }} + _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] {% endif %} -{% endfor %} - if not _access_granted: - raise HTTPException(status_code=403, detail="Forbidden") + except AccessDenied as exc: + raise HTTPException(status_code=403, detail=exc.reason) from exc {% endif %} {% endmacro %} @@ -246,23 +246,24 @@ _files = record_files[{{ collection_name | tojson }}] {% set body_expr = "upload.payload" if files else 'payload.model_dump(mode="json"' ~ (", exclude_unset=True" if partial else "") ~ ")" %} {% set record_expr = "upload.record" if files else body_expr %} {% if route.operation == "list" %} +{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} @router.{{ route.method | lower }}("{{ route.relative_path }}"{{ openapi_extra(route) }}) {% set list_prefix = (extra_path_params(route) ~ request_param(route) ~ auth_param(route)) | trim(chars=", ") %} async def list_{{ collection_name }}({{ list_prefix }}{% if list_prefix %}, {% endif %}{{ svc_dep() }}{{ where_repo_params(route) }}{% if route.pagination %}, limit: int | None = None{% if route.pagination.strategy == "offset" %}, offset: int | None = None{% else %}, after: str | None = None{% endif %}{% endif %}) -> Any: {{ request_locals(route, "None") }} -{{ pre_fetch_guard(route) }} +{{ authorize_rules(route, "Field(())", rows=true) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, path_values(route), "None") }} {% if route.pagination %} _pagination_config = {{ route.pagination | pagination_config | py_literal }} _pagination_query = {"limit": limit{% if route.pagination.strategy == "offset" %}, "offset": offset{% else %}, "after": after{% endif %}} try: - _page = await svc.list(map_request(_pagination_config, _pagination_query)) + _page = await svc.list(map_request(_pagination_config, _pagination_query){{ row_filter_arg }}) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc return build_response(_pagination_config, _page, _pagination_query) {% else %} - return await svc.list() + return await svc.list({{ "None" ~ row_filter_arg if row_filter_arg }}) {% endif %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja index 5af17ece..18686a67 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja @@ -1,5 +1,6 @@ from typing import Any +from dependencies.access import EVERY_ROW, Predicate from services.exceptions import NotFoundError, ValidationError {% if auth.has_password_auth %} from dependencies.auth import hash_password @@ -56,12 +57,12 @@ class ServiceBase: class ListSvc: - async def list(self, primitives: Any = None) -> Any: + async def list(self, primitives: Any = None, predicate: Predicate = EVERY_ROW) -> Any: if primitives is None: - documents = await self._repo.list() + documents = await self._repo.list(predicate) await self._after_list(documents) return documents - page = await self._repo.paginate(primitives) + page = await self._repo.paginate(primitives, predicate) await self._after_list(page.rows) return page diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 9c20543e..4c92f1b2 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -56,7 +56,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: env.filters["capitalize_collection"] = to_pascal_case env.filters["py_literal"] = access_filters.py_literal env.filters["pagination_config"] = pagination_config - env.filters["rule_to_python"] = access_filters.rule_to_python + env.filters["rule_to_predicate"] = access_filters.rule_to_predicate env.filters["request_symbol_binding"] = access_filters.request_symbol_binding env.filters["role_hierarchies_py"] = access_filters.role_hierarchies_py env.tests["unconditional"] = access_filters.is_unconditional @@ -81,9 +81,8 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: output_path="service_inputs.py", ), ) - (output_dir / "pagination.py").write_text( - runtime_module_source("pagination") - ) + for module in ("access", "pagination"): + (output_dir / f"{module}.py").write_text(runtime_module_source(module)) collections_by_name = group_by_collection(context) @@ -190,6 +189,7 @@ def _build_service_context(self, col_name: str, col_data: dict, context: Compile "import_prefix": "repositories", "auth_import_prefix": "dependencies.auth", "service_import_prefix": "services", + "access_import": "access", } def _render_repositories( @@ -212,6 +212,7 @@ def _render_repositories( "custom_operations": ctx.repository_operations.get(col_name, []), "conflict_import": "conflicts", "pagination_import": "pagination", + "access_import": "access", }, ) ) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja index cc0ce2d6..54393e85 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja @@ -1,7 +1,7 @@ from importlib import import_module from typing import Any -from uuid import UUID +from access import AccessDenied, Decision, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, value_at from service_inputs import RawRequest, prepare_service_inputs, resolve_access_bind {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} @@ -15,23 +15,6 @@ from dependencies.auth import resolve_principal {% endif %} -try: - from bson import ObjectId - _IDISH = (UUID, ObjectId) -except ImportError: - _IDISH = (UUID,) - - -def _values_equal(left: Any, right: Any) -> bool: - if left is None or right is None: - return left is None and right is None - if isinstance(left, bool) or isinstance(right, bool): - return isinstance(left, bool) and isinstance(right, bool) and left is right - if left == right: - return True - return str(left) == str(right) if isinstance(left, _IDISH) or isinstance(right, _IDISH) else False - - ROLE_HIERARCHIES = {{ auth.role_hierarchies | role_hierarchies_py }} @@ -80,25 +63,19 @@ async def handle_custom_{{ loop.index }}( {% for symbol in symbols.request %} {{ symbol | request_symbol_binding }} {% endfor %} - access_granted = False - access_bindings = {} + decisions = [] {% for rule in route.access_rules %} - if not access_granted: {% for binding in rule.document_bindings %} - {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( - resolve_access_bind({{ binding.where | py_literal }}, {**url_params, **locals()}) - ) + {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( + resolve_access_bind({{ binding.where | py_literal }}, {**url_params, **locals()}) + ) {% endfor %} - if {% for binding in rule.document_bindings %}{{ binding.name }} is not None and {% endfor %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - return 403, {"detail": {{ (rule.reason or "Forbidden") | py_literal }}} -{% else %} - access_granted = True - access_bindings = {{ rule.bind | py_literal }} -{% endif %} + decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) {% endfor %} - if not access_granted: - return 403, {"detail": "Forbidden"} + try: + access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] + except AccessDenied as exc: + return 403, {"detail": exc.reason} bound_inputs = resolve_access_bind(access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 5e497607..31e76b48 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -1,5 +1,4 @@ from typing import Any -from uuid import UUID from repositories.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository {% set ns_where = namespace(collections=[]) %} @@ -19,23 +18,7 @@ from pagination import build_response, map_request from dependencies.auth import resolve_principal {% endif %} import validation - - -try: - from bson import ObjectId - _IDISH = (UUID, ObjectId) -except ImportError: - _IDISH = (UUID,) - - -def _values_equal(left: Any, right: Any) -> bool: - if left is None or right is None: - return left is None and right is None - if isinstance(left, bool) or isinstance(right, bool): - return isinstance(left, bool) and isinstance(right, bool) and left is right - if left == right: - return True - return str(left) == str(right) if isinstance(left, _IDISH) or isinstance(right, _IDISH) else False +from access import EVERY_ROW, AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at ROLE_HIERARCHIES = {{ auth.role_hierarchies | role_hierarchies_py }} @@ -76,40 +59,43 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: {% endif %} {% endmacro %} -{% macro authorize_rules(route, current_document) %} +{# Collects one decision per rule, then admits a record or filters rows #} +{% macro authorize_rules(route, current_document, rows=false) %} {% if route.needs_auth and route.access_rules %} {{ authorization_locals(route) }} - access_granted = False - access_bindings = {} + decisions = [] {% for rule in route.access_rules %} - if not access_granted: {% for binding in rule.document_bindings %} {% if binding.uses_where %} - {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( - resolve_access_bind({{ binding.where | py_literal }}, {**url_params, **locals()}) - ) + {{ binding.name }} = await {{ binding.collection | capitalize_collection }}Repository(db).find_one( + resolve_access_bind({{ binding.where | py_literal }}, {**url_params, **locals()}) + ) {% else %} - {{ binding.name }} = {{ current_document }} + {{ binding.name }} = {{ current_document }} {% endif %} {% endfor %} -{% if rule is needs_current_document_rule %} - document = {{ current_document }} + decisions.append(Decision({{ rule.effect | py_literal }}, {{ rule | rule_to_predicate }}, {{ rule.reason | py_literal }})) +{% endfor %} +{% if rows %} + access_bindings = {} {% endif %} - if {% for binding in rule.document_bindings if binding.uses_where %}{{ binding.name }} is not None and {% endfor %}{% if rule is needs_current_document_rule %}{{ current_document }} is not None and {% endif %}{{ rule | rule_to_python }}: -{% if rule.effect == "deny" %} - return 403, {"detail": {{ (rule.reason or "Forbidden") | py_literal }}} + try: +{% if rows %} + rows = row_filter(decisions) {% else %} - access_granted = True - access_bindings = {{ rule.bind | py_literal }} + access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] {% endif %} -{% endfor %} - if not access_granted: - return 403, {"detail": "Forbidden"} + except AccessDenied as exc: + return 403, {"detail": exc.reason} +{% elif rows %} + rows = EVERY_ROW {% endif %} {% endmacro %} {% macro pre_fetch_authorize(route) %} -{% if not (route is needs_post_fetch) %} +{% if route.operation == "list" %} +{{ authorize_rules(route, "Field(())", rows=true) }} +{% elif not (route is needs_post_fetch) %} {{ authorize_rules(route, "None") }} {% elif route.access_rules %} access_bindings = {} @@ -171,10 +157,10 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {% if route.pagination %} pagination_config = {{ route.pagination | pagination_config | py_literal }} pagination_query = query or {} - page = await service.list(map_request(pagination_config, pagination_query)) + page = await service.list(map_request(pagination_config, pagination_query), rows) return 200, build_response(pagination_config, page, pagination_query) {% else %} - documents = await service.list() + documents = await service.list(None, rows) return 200, documents {% endif %} {% elif route.operation == "get" %} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index c784ee7d..05d50ad8 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -16,6 +16,7 @@ from pymongo.asynchronous.database import AsyncDatabase from pymongo.errors import BulkWriteError, DuplicateKeyError from {{ conflict_import }} import ConflictError +from {{ access_import }} import EVERY_ROW, Predicate from {{ pagination_import }} import Page, Primitives, expand_bound @@ -93,13 +94,13 @@ class {{ class_name }}Repository: return None return {k: v for k, v in doc.items() if k != "_id"} - async def list(self) -> list[dict[str, Any]]: - cursor = self._col.find({}) + async def list(self, predicate: Predicate = EVERY_ROW) -> list[dict[str, Any]]: + cursor = self._col.find(predicate.mongo()) docs = await cursor.to_list(length=None) return [self._normalize(d) for d in docs] - async def paginate(self, primitives: Primitives) -> Page: - query = _pagination_filter(primitives) + async def paginate(self, primitives: Primitives, predicate: Predicate = EVERY_ROW) -> Page: + query = {"$and": [predicate.mongo(), _pagination_filter(primitives)]} cursor = self._col.find(query) if primitives.sort: cursor = cursor.sort( @@ -111,7 +112,7 @@ class {{ class_name }}Repository: cursor = cursor.limit(primitives.limit) documents = await cursor.to_list(length=None) total = ( - await self._col.count_documents({}) + await self._col.count_documents(predicate.mongo()) if primitives.count == "exact" else None ) diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja index b49d0ea4..7dd6f2ed 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja @@ -1,5 +1,6 @@ from {{ import_prefix }}.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository from {{ service_import_prefix }}.exceptions import NotFoundError, ValidationError +from {{ access_import }} import EVERY_ROW {% if is_identity_collection %} from {{ auth_import_prefix }} import hash_password {% endif %} @@ -10,10 +11,10 @@ class {{ collection_name | capitalize_collection }}Service: self._repo = repo {% if is_async %} - async def list(self, primitives=None): + async def list(self, primitives=None, predicate=EVERY_ROW): if primitives is None: - return await self._repo.list() - return await self._repo.paginate(primitives) + return await self._repo.list(predicate) + return await self._repo.paginate(primitives, predicate) async def get(self, resource_id: str) -> dict: document = await self._repo.get(resource_id) @@ -84,10 +85,10 @@ class {{ collection_name | capitalize_collection }}Service: raise NotFoundError("{{ collection_name }}", criteria) return document {% else %} - def list(self, primitives=None): + def list(self, primitives=None, predicate=EVERY_ROW): if primitives is None: - return self._repo.list() - return self._repo.paginate(primitives) + return self._repo.list(predicate) + return self._repo.paginate(primitives, predicate) def get(self, resource_id: str) -> dict: document = self._repo.get(resource_id) diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 5a78e60e..5f321487 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -12,7 +12,7 @@ _RUNTIME_MODULES = frozenset( - {"dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} + {"access", "dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} ) diff --git a/src/yaml_api_generator/ir/when.py b/src/yaml_api_generator/ir/when.py index 95b26356..e3176187 100644 --- a/src/yaml_api_generator/ir/when.py +++ b/src/yaml_api_generator/ir/when.py @@ -2,7 +2,8 @@ from __future__ import annotations -from dataclasses import dataclass +from collections.abc import Callable +from dataclasses import dataclass, fields, replace from typing import Any @@ -106,3 +107,13 @@ def collect_roots(expr: WhenExpr) -> set[str]: if isinstance(expr, (Not, IsNull, IsNotNull)): return collect_roots(expr.operand) return set() + + +def map_operands(expr: WhenExpr, transform: Callable[[WhenExpr], WhenExpr]) -> WhenExpr: + """Rebuild ``expr`` with ``transform`` applied to each direct sub-expression.""" + children = { + field.name: transform(getattr(expr, field.name)) + for field in fields(expr) + if isinstance(getattr(expr, field.name), WhenExpr) + } + return replace(expr, **children) if children else expr diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 114a3212..2d18223a 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -13,6 +13,7 @@ from .access_transformer import AccessTransformer from .collection_transformer import CollectionTransformer from .datastore_transformer import parse_datastores, parse_file_datastores +from .document_lookup import desugar_document_lookups from .dto_transformer import DtoTransformer from .file_transformer import FileStoreRegistry, FileTransformer from .identity_transformer import IdentityTransformer @@ -126,7 +127,10 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: if policy_errors: raise ConfigErrors(policy_errors) - routes = self._expand_route_policies(routes, policies, entity_to_collection) + routes = desugar_document_lookups( + self._expand_route_policies(routes, policies, entity_to_collection), + collections, + ) file_routes, file_route_errors = self._files.derive(collections, routes) if file_route_errors: raise ConfigErrors(file_route_errors) diff --git a/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py b/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py new file mode 100644 index 00000000..97a97e5b --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py @@ -0,0 +1,56 @@ +"""Desugar `` by `` document bindings into where-bindings. + +``owner: users by ownerId`` loads the ``users`` record whose id field equals the +``ownerId`` route parameter, which is exactly ``users where = $ownerId``. +""" + +from dataclasses import replace + +from yaml_api_generator.ir import ( + IRAccessRule, + IRCollection, + IRDocumentBinding, + IRRoute, + IRSymbolRef, + IRWhereConjunct, +) +from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors + + +def _where_binding( + binding: IRDocumentBinding, collections: dict[str, IRCollection], path: str +) -> IRDocumentBinding: + if binding.id_param is None: + return binding + collection = collections.get(binding.collection) + if collection is None: + raise ConfigErrors([ConfigError( + code="DOCUMENT_BINDING_INVALID", + message=f"document binding references unknown collection '{binding.collection}'", + path=path, + )]) + return IRDocumentBinding( + collection=binding.collection, + where=(IRWhereConjunct(field=collection.idField, rhs=IRSymbolRef(root=binding.id_param)),), + ) + + +def _rule_lookups( + rule: IRAccessRule, collections: dict[str, IRCollection], path: str +) -> IRAccessRule: + return replace(rule, document_symbols={ + name: _where_binding(binding, collections, f"{path}.given.{name}") + for name, binding in rule.document_symbols.items() + }) + + +def desugar_document_lookups( + routes: list[IRRoute], collections: dict[str, IRCollection] +) -> list[IRRoute]: + return [ + replace(route, access_rules=[ + _rule_lookups(rule, collections, f"{route.method} {route.path} access[{index}]") + for index, rule in enumerate(route.access_rules) + ]) if route.access_rules else route + for route in routes + ] diff --git a/src/yaml_api_generator/parsers/transformers/v02/policy_expansion.py b/src/yaml_api_generator/parsers/transformers/v02/policy_expansion.py index 3ac85b23..51cf119f 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/policy_expansion.py +++ b/src/yaml_api_generator/parsers/transformers/v02/policy_expansion.py @@ -36,6 +36,7 @@ StringLiteral, WhenExpr, ) +from yaml_api_generator.ir.when import map_operands from yaml_api_generator.parsers.yaml_parser import ConfigError from yaml_api_generator.parsers.transformers.v02.binding_classifier import classify_binding @@ -66,17 +67,7 @@ def register(self, name: str, binding: IRBinding) -> None: def _rename_roots(expr: WhenExpr, mapping: dict[str, str]) -> WhenExpr: if isinstance(expr, FieldRef): return replace(expr, root=mapping.get(expr.root, expr.root)) - if isinstance(expr, (Equals, NotEquals, And, Or)): - return replace( - expr, - left=_rename_roots(expr.left, mapping), - right=_rename_roots(expr.right, mapping), - ) - if isinstance(expr, (In, NotIn)): - return replace(expr, left=_rename_roots(expr.left, mapping)) - if isinstance(expr, Not): - return replace(expr, operand=_rename_roots(expr.operand, mapping)) - return expr + return map_operands(expr, lambda child: _rename_roots(child, mapping)) def _normalize_param_binding(binding: IRBinding) -> IRBinding: diff --git a/src/yaml_api_generator/runtime/access.py b/src/yaml_api_generator/runtime/access.py new file mode 100644 index 00000000..5caddfcf --- /dev/null +++ b/src/yaml_api_generator/runtime/access.py @@ -0,0 +1,327 @@ +"""Access-rule runtime shared by the built-in server and generated applications. + +``values_equal`` is the equality every ``when`` comparison uses. On list routes +a rule that reads the listed record becomes a row predicate: operands that do not +name a record field fold to constants, and the ordered allow/deny rules fold into +one predicate that a repository applies in memory, as a Mongo query or as a SQL +clause. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial, reduce +from operator import eq +from typing import Any +from uuid import UUID + +try: + from bson import ObjectId + + _IDISH: tuple[type, ...] = (UUID, ObjectId) +except ImportError: + _IDISH = (UUID,) + + +def values_equal(left: Any, right: Any) -> bool: + """Compare by value, matching identifiers across their string and native forms. + + Booleans, numbers and plain scalars never coerce, so ``"1" == 1`` and + ``"True" == True`` stay false. + """ + if left is None or right is None: + return left is None and right is None + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left is right + if left == right: + return True + if isinstance(left, _IDISH) or isinstance(right, _IDISH): + return str(left) == str(right) + return False + + +@dataclass(frozen=True) +class Field: + """A field of the listed record, addressed by its dotted path.""" + + path: tuple[str, ...] + + def value(self, row: dict[str, Any]) -> Any: + return reduce( + lambda current, key: current.get(key) if isinstance(current, dict) else None, + self.path, + row, + ) + + @property + def name(self) -> str: + return ".".join(self.path) + + def column(self, table: Any) -> Any: + if len(self.path) != 1: + raise ValueError(f"Row filters on SQL tables need a top-level field, got '{self.name}'") + return table.c[self.path[0]] + + +class Predicate: + def matches(self, row: dict[str, Any]) -> bool: + raise NotImplementedError + + def mongo(self) -> dict[str, Any]: + raise NotImplementedError + + def sql(self, table: Any) -> Any: + raise NotImplementedError + + +@dataclass(frozen=True) +class Constant(Predicate): + value: bool + + def matches(self, row: dict[str, Any]) -> bool: + return self.value + + def mongo(self) -> dict[str, Any]: + return {} if self.value else {"$expr": False} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + return sa.true() if self.value else sa.false() + + +EVERY_ROW = Constant(True) +NO_ROW = Constant(False) + + +@dataclass(frozen=True) +class FieldEquals(Predicate): + field: Field + value: Any + + def matches(self, row: dict[str, Any]) -> bool: + return values_equal(self.field.value(row), self.value) + + def mongo(self) -> dict[str, Any]: + return {self.field.name: self.value} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + column = self.field.column(table) + if self.value is None: + return column.is_(None) + return sa.and_(column.is_not(None), column == self.value) + + +@dataclass(frozen=True) +class FieldsEqual(Predicate): + left: Field + right: Field + + def matches(self, row: dict[str, Any]) -> bool: + return values_equal(self.left.value(row), self.right.value(row)) + + def mongo(self) -> dict[str, Any]: + operands = [{"$ifNull": [f"${field.name}", None]} for field in (self.left, self.right)] + return {"$expr": {"$eq": operands}} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + left, right = self.left.column(table), self.right.column(table) + return sa.or_( + sa.and_(left.is_(None), right.is_(None)), + sa.and_(left.is_not(None), right.is_not(None), left == right), + ) + + +@dataclass(frozen=True) +class FieldIn(Predicate): + field: Field + values: tuple[Any, ...] + + def matches(self, row: dict[str, Any]) -> bool: + value = self.field.value(row) + return any(values_equal(value, candidate) for candidate in self.values) + + def mongo(self) -> dict[str, Any]: + return {self.field.name: {"$in": list(self.values)}} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + column = self.field.column(table) + return sa.and_(column.is_not(None), column.in_(self.values)) + + +@dataclass(frozen=True) +class Negation(Predicate): + operand: Predicate + + def matches(self, row: dict[str, Any]) -> bool: + return not self.operand.matches(row) + + def mongo(self) -> dict[str, Any]: + return {"$nor": [self.operand.mongo()]} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + return sa.not_(self.operand.sql(table)) + + +@dataclass(frozen=True) +class Conjunction(Predicate): + left: Predicate + right: Predicate + + def matches(self, row: dict[str, Any]) -> bool: + return self.left.matches(row) and self.right.matches(row) + + def mongo(self) -> dict[str, Any]: + return {"$and": [self.left.mongo(), self.right.mongo()]} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + return sa.and_(self.left.sql(table), self.right.sql(table)) + + +@dataclass(frozen=True) +class Disjunction(Predicate): + left: Predicate + right: Predicate + + def matches(self, row: dict[str, Any]) -> bool: + return self.left.matches(row) or self.right.matches(row) + + def mongo(self) -> dict[str, Any]: + return {"$or": [self.left.mongo(), self.right.mongo()]} + + def sql(self, table: Any) -> Any: + import sqlalchemy as sa + + return sa.or_(self.left.sql(table), self.right.sql(table)) + + +def value_at(root: Any, path: tuple[str, ...]) -> Any: + """The value at ``path`` under ``root``, or the record field it names when ``root`` is a ``Field``.""" + if isinstance(root, Field): + return Field(root.path + path) + return Field(path).value(root) + + +def has_path(root: Any, path: tuple[str, ...]) -> bool: + """Whether ``root`` carries the key at ``path``, even with a null value.""" + parent = value_at(root, path[:-1]) + return isinstance(parent, dict) and path[-1] in parent + + +def constant(value: bool) -> Constant: + return EVERY_ROW if value else NO_ROW + + +def equals(left: Any, right: Any) -> Predicate: + if isinstance(left, Field) and isinstance(right, Field): + return FieldsEqual(left, right) + if isinstance(left, Field): + return FieldEquals(left, right) + if isinstance(right, Field): + return FieldEquals(right, left) + return constant(values_equal(left, right)) + + +def one_of(operand: Any, values: tuple[Any, ...]) -> Predicate: + if isinstance(operand, Field): + return FieldIn(operand, tuple(values)) + return constant(any(values_equal(operand, value) for value in values)) + + +def is_null(operand: Any) -> Predicate: + return FieldEquals(operand, None) if isinstance(operand, Field) else constant(operand is None) + + +def all_of(*predicates: Predicate) -> Predicate: + return reduce(conjoin, predicates, EVERY_ROW) + + +def negate(predicate: Predicate) -> Predicate: + if isinstance(predicate, Constant): + return constant(not predicate.value) + if isinstance(predicate, Negation): + return predicate.operand + return Negation(predicate) + + +def conjoin(left: Predicate, right: Predicate) -> Predicate: + if isinstance(left, Constant): + return right if left.value else NO_ROW + if isinstance(right, Constant): + return left if right.value else NO_ROW + return Conjunction(left, right) + + +def disjoin(left: Predicate, right: Predicate) -> Predicate: + if isinstance(left, Constant): + return EVERY_ROW if left.value else right + if isinstance(right, Constant): + return EVERY_ROW if right.value else left + return Disjunction(left, right) + + +class AccessDenied(Exception): + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +@dataclass(frozen=True) +class Decision: + """One access rule: its effect when ``predicate`` holds, and its refusal reason.""" + + effect: str + predicate: Predicate + reason: str | None = None + + def admit(self, rest: Predicate) -> Predicate: + """Fold this rule in front of the rules after it: the first match decides.""" + if self.effect == "deny": + return conjoin(negate(self.predicate), rest) + return disjoin(self.predicate, rest) + + +def refusal_reason(decisions: list[Decision]) -> str: + """The detail of a refusal that no deny rule decided.""" + return next( + (decision.reason for decision in decisions if decision.effect == "allow" and decision.reason), + "Forbidden", + ) + + +def decide(decisions: list[Decision], holds: Callable[[Predicate], bool] = partial(eq, EVERY_ROW)) -> int: + """Index of the first rule whose predicate ``holds``: the rule that decides. + + Raises ``AccessDenied`` when that rule denies or no rule holds. + """ + index, decision = next( + ((index, decision) for index, decision in enumerate(decisions) if holds(decision.predicate)), + (None, None), + ) + if decision is None: + raise AccessDenied(refusal_reason(decisions)) + if decision.effect == "deny": + raise AccessDenied(decision.reason or "Forbidden") + return index + + +def row_filter(decisions: list[Decision]) -> Predicate: + """Combine ordered rules into the predicate of the rows they admit. + + Raises ``AccessDenied`` when the rules admit no row whatever its content. + """ + predicate = reduce(lambda rest, decision: decision.admit(rest), reversed(decisions), NO_ROW) + if predicate == NO_ROW: + decide(decisions) + return predicate diff --git a/src/yaml_api_generator/server/common/context_builder.py b/src/yaml_api_generator/server/common/context_builder.py index def4acca..a739b6ad 100644 --- a/src/yaml_api_generator/server/common/context_builder.py +++ b/src/yaml_api_generator/server/common/context_builder.py @@ -141,7 +141,7 @@ def _runtime_route(route: IRRoute, document: IRDocument) -> RuntimeRouteDescript request_symbols=rule.request_symbols, when=rule.when, effect=rule.effect, - reason=rule.reason if document.version == "0.3" else None, + reason=rule.reason, bind=rule.bind, ) for rule in route.access_rules diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 7ac0cf88..46b441ca 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -37,6 +37,7 @@ remove_file, replace_file, ) +from .steps.authorization import Admission, admit_record, admit_rows, screen_record from .steps.repositories import RecordLocator, record_locator @@ -240,7 +241,9 @@ def _base_steps(self, route: RuntimeRouteDescriptor) -> list[Step]: steps.append(ResolveWhereBindingsStep(route.access_rules)) if not route.is_public: - steps.append(AuthorizeRouteStep(route.access_rules, self._ctx.auth)) + steps.append( + AuthorizeRouteStep(route.access_rules, self._ctx.auth, self._admission(route)) + ) if not route.is_public and _has_access_bind(route): steps.append(ResolveAccessBindStep(route.access_rules)) @@ -263,6 +266,13 @@ def _identity_step(self, route: RuntimeRouteDescriptor) -> Step | None: return None return ValidateIdentityCreateStep(identity_collection) + def _admission(self, route: RuntimeRouteDescriptor) -> Admission: + if route.operation == "list": + return admit_rows + if self._post_fetch_authorize_step(route) is not None: + return screen_record + return admit_record + def _post_fetch_authorize_step(self, route: RuntimeRouteDescriptor) -> Step | None: if route.is_public or not route.access_rules: return None diff --git a/src/yaml_api_generator/server/common/repositories/base.py b/src/yaml_api_generator/server/common/repositories/base.py index f1a91518..f402d084 100644 --- a/src/yaml_api_generator/server/common/repositories/base.py +++ b/src/yaml_api_generator/server/common/repositories/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import Any +from yaml_api_generator.runtime.access import EVERY_ROW, Predicate from ..pagination import Page, Primitives, expand_bound @@ -64,15 +65,19 @@ async def close(self) -> None: raise NotImplementedError @abstractmethod - async def list(self, collection_name: str) -> list[dict[str, Any]]: + async def list( + self, collection_name: str, predicate: Predicate = EVERY_ROW + ) -> list[dict[str, Any]]: + """The records of ``collection_name`` that ``predicate`` admits.""" raise NotImplementedError async def paginate( self, collection_name: str, primitives: Primitives, + predicate: Predicate = EVERY_ROW, ) -> Page: - documents = await self.list(collection_name) + documents = await self.list(collection_name, predicate) total = len(documents) if primitives.count == "exact" else None rows = [ document diff --git a/src/yaml_api_generator/server/common/repositories/mongodb.py b/src/yaml_api_generator/server/common/repositories/mongodb.py index a32b857c..41d3a917 100644 --- a/src/yaml_api_generator/server/common/repositories/mongodb.py +++ b/src/yaml_api_generator/server/common/repositories/mongodb.py @@ -9,6 +9,7 @@ from .base import BaseRepository, ConflictError from .bindings import resolve +from yaml_api_generator.runtime.access import EVERY_ROW, Predicate from ..pagination import Page, Primitives, expand_bound @@ -127,15 +128,18 @@ async def create_unique_index(self, collection_name: str, field_name: str) -> No partialFilterExpression=unique_index_partial_filter(field_name), ) - async def list(self, collection_name: str) -> list[dict[str, Any]]: - cursor = self.collection(collection_name).find({}) + async def list( + self, collection_name: str, predicate: Predicate = EVERY_ROW + ) -> list[dict[str, Any]]: + cursor = self.collection(collection_name).find(predicate.mongo()) documents = await cursor.to_list(length=None) return [self._normalize_document_required(document) for document in documents] - async def paginate(self, collection_name: str, primitives: Primitives) -> Page: + async def paginate( + self, collection_name: str, primitives: Primitives, predicate: Predicate = EVERY_ROW + ) -> Page: target = self.collection(collection_name) - bound_clauses = _mongo_bounds(primitives.bounds) - query = {"$and": bound_clauses} if bound_clauses else {} + query = {"$and": [predicate.mongo(), *_mongo_bounds(primitives.bounds)]} cursor = target.find(query) if primitives.sort: cursor = cursor.sort( @@ -149,7 +153,7 @@ async def paginate(self, collection_name: str, primitives: Primitives) -> Page: rows = [self._normalize_document_required(d) for d in documents] total = None if primitives.count == "exact": - total = await target.count_documents({}) + total = await target.count_documents(predicate.mongo()) return Page(rows=rows, total=total) async def get( diff --git a/src/yaml_api_generator/server/common/repositories/sql.py b/src/yaml_api_generator/server/common/repositories/sql.py index 89ad9153..3329052d 100644 --- a/src/yaml_api_generator/server/common/repositories/sql.py +++ b/src/yaml_api_generator/server/common/repositories/sql.py @@ -20,6 +20,7 @@ from .base import BaseRepository, ConflictError from .bindings import resolve from .sql_schema import table_from_collection +from yaml_api_generator.runtime.access import EVERY_ROW, Predicate from ..pagination import Page, Primitives, expand_bound @@ -104,18 +105,21 @@ async def close(self) -> None: self._engine = None self._schema_ready = False - async def list(self, collection_name: str) -> list[dict[str, Any]]: + async def list( + self, collection_name: str, predicate: Predicate = EVERY_ROW + ) -> list[dict[str, Any]]: table = self._table(collection_name) async with await self._connection() as conn: - result = await conn.execute(sa.select(table)) + result = await conn.execute(sa.select(table).where(predicate.sql(table))) return [dict(row._mapping) for row in result] - async def paginate(self, collection_name: str, primitives: Primitives) -> Page: + async def paginate( + self, collection_name: str, primitives: Primitives, predicate: Predicate = EVERY_ROW + ) -> Page: table = self._table(collection_name) - bound_clauses = self._bounds_clauses(table, primitives.bounds) - stmt = sa.select(table) - if bound_clauses: - stmt = stmt.where(sa.and_(*bound_clauses)) + stmt = sa.select(table).where( + predicate.sql(table), *self._bounds_clauses(table, primitives.bounds) + ) for key in primitives.sort: column = table.c[key.field] stmt = stmt.order_by(column.desc() if key.descending else column.asc()) @@ -127,7 +131,9 @@ async def paginate(self, collection_name: str, primitives: Primitives) -> Page: rows = [dict(row._mapping) for row in await conn.execute(stmt)] total = None if primitives.count == "exact": - count_stmt = sa.select(sa.func.count()).select_from(table) + count_stmt = ( + sa.select(sa.func.count()).select_from(table).where(predicate.sql(table)) + ) total = (await conn.execute(count_stmt)).scalar() return Page(rows=rows, total=total) diff --git a/src/yaml_api_generator/server/common/request_context.py b/src/yaml_api_generator/server/common/request_context.py index 00c4082e..9b87602c 100644 --- a/src/yaml_api_generator/server/common/request_context.py +++ b/src/yaml_api_generator/server/common/request_context.py @@ -13,6 +13,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from yaml_api_generator.runtime.access import EVERY_ROW, Predicate + if TYPE_CHECKING: from .request_body import RequestBody @@ -35,6 +37,7 @@ class RequestContext: cookies: dict[str, str] | None = None where_documents: list[dict[str, Any]] | None = None authorized_rule_index: int | None = None + row_filter: Predicate = EVERY_ROW bound_inputs: dict[str, Any] | None = None service_inputs: dict[str, Any] | None = None upload: "RequestBody | None" = None diff --git a/src/yaml_api_generator/server/common/steps/authorization.py b/src/yaml_api_generator/server/common/steps/authorization.py index a3d20d4f..f5f54b54 100644 --- a/src/yaml_api_generator/server/common/steps/authorization.py +++ b/src/yaml_api_generator/server/common/steps/authorization.py @@ -1,6 +1,6 @@ """Authorization pipeline steps for role-based and condition-based access control.""" -from collections.abc import Iterable +from collections.abc import Callable, Iterable from itertools import repeat from . import Step @@ -12,10 +12,22 @@ RuntimeAuthConfig, RuntimeIdentityDescriptor, ) -from ..when_eval import evaluate_when_with_symbols +from ..when_eval import when_predicate from ..utils.field_access import get_nested_field from yaml_api_generator.ir.api import IRLiteralRef, IRSymbolRef, IRWhereConjunct -from yaml_api_generator.ir.when import needs_document as _when_needs_document +from yaml_api_generator.runtime.access import ( + EVERY_ROW, + NO_ROW, + AccessDenied, + Decision, + Field, + Predicate, + decide, + row_filter, +) + +RECORD = Field(()) +"""The record under authorization, before it is loaded or while rows are filtered.""" WhereDocuments = dict[str, dict | None] @@ -24,8 +36,7 @@ class UnauthorizedError(ValueError): pass -class ForbiddenError(ValueError): - pass +ForbiddenError = AccessDenied def _required_identity_fields( @@ -68,16 +79,6 @@ def step(ctx: RequestContext) -> RequestContext: return step -def _deny(rule: RuntimeAccessRule) -> ForbiddenError: - return ForbiddenError(rule.reason or "Forbidden") - - -def _rule_needs_postfetch(rule: RuntimeAccessRule) -> bool: - return any(not binding.where for binding in rule.document_symbols.values()) or ( - rule.when is not None and _when_needs_document(rule.when) - ) - - def _principal_has_role( principal: dict, required_role: str, @@ -130,11 +131,11 @@ def _resolve_identity_symbols( def _resolve_document_symbols( rule: RuntimeAccessRule, - document: dict | None, + record: dict | Field, where_documents: WhereDocuments, -) -> dict[str, dict | None]: +) -> dict[str, dict | Field | None]: return { - symbol_name: where_documents.get(symbol_name) if binding.where else document + symbol_name: where_documents.get(symbol_name) if binding.where else record for symbol_name, binding in rule.document_symbols.items() } @@ -178,23 +179,37 @@ def _where_documents_found( ) -def _evaluate_rule( +def _rule_predicate( rule: RuntimeAccessRule, - document: dict | None, + record: dict | Field, where_documents: WhereDocuments, ctx: RequestContext, -) -> bool: + auth_config: RuntimeAuthConfig | None, +) -> Predicate: + if not _rule_matches_principal(rule, ctx.principal, auth_config): + return NO_ROW if not _where_documents_found(rule, where_documents): - return False + return NO_ROW if rule.when is None: - return True - - symbols: dict[str, object | None] = { + return EVERY_ROW + symbols: dict[str, object] = { **_resolve_identity_symbols(rule, ctx.principal), - **_resolve_document_symbols(rule, document, where_documents), + **_resolve_document_symbols(rule, record, where_documents), **_resolve_request_symbols(rule, ctx), } - return evaluate_when_with_symbols(rule.when, symbols) + return when_predicate(rule.when, symbols, rule.request_symbols.keys()) + + +def _decisions( + access_rules: list[RuntimeAccessRule], + record: dict | Field, + ctx: RequestContext, + auth_config: RuntimeAuthConfig | None, +) -> list[Decision]: + return [ + Decision(rule.effect, _rule_predicate(rule, record, where_documents, ctx, auth_config), rule.reason) + for rule, where_documents in _rules_with_where_documents(access_rules, ctx) + ] def _rules_with_where_documents( @@ -276,33 +291,35 @@ def step(ctx: RequestContext) -> RequestContext: return step +Admission = Callable[[RequestContext, list[Decision]], None] + + +def admit_record(ctx: RequestContext, decisions: list[Decision]) -> None: + """Routes without a current record: the first rule that holds decides.""" + ctx.authorized_rule_index = decide(decisions) + + +def admit_rows(ctx: RequestContext, decisions: list[Decision]) -> None: + """List routes: the rules become the filter of the rows the caller may see.""" + ctx.row_filter = row_filter(decisions) + + +def screen_record(ctx: RequestContext, decisions: list[Decision]) -> None: + """Routes that load a record: refuse early when no record could be admitted.""" + row_filter(decisions) + + def AuthorizeRouteStep( access_rules: list[RuntimeAccessRule] | None = None, auth_config: RuntimeAuthConfig | None = None, + admission: Admission = admit_record, ) -> Step: def step(ctx: RequestContext) -> RequestContext: if ctx.principal is None: raise UnauthorizedError("Unauthorized") - - if not access_rules: - return ctx - - for index, (rule, where_documents) in enumerate( - _rules_with_where_documents(access_rules, ctx) - ): - if _rule_needs_postfetch(rule): - return ctx - - if not _rule_matches_principal(rule, ctx.principal, auth_config): - continue - - if _evaluate_rule(rule, None, where_documents, ctx): - if rule.effect == "deny": - raise _deny(rule) - ctx.authorized_rule_index = index - return ctx - - raise ForbiddenError("Forbidden") + if access_rules: + admission(ctx, _decisions(access_rules, RECORD, ctx, auth_config)) + return ctx return step @@ -312,25 +329,9 @@ def PostFetchAuthorizeStep( auth_config: RuntimeAuthConfig | None = None, ) -> Step: def step(ctx: RequestContext) -> RequestContext: - document = ctx.response if isinstance(ctx.response, dict) else None - for index, (rule, where_documents) in enumerate( - _rules_with_where_documents(access_rules, ctx) - ): - if not _rule_matches_principal(rule, ctx.principal, auth_config): - continue - - if _rule_needs_postfetch(rule): - if document is None: - continue - - rule_document = document if _rule_needs_postfetch(rule) else None - if _evaluate_rule(rule, rule_document, where_documents, ctx): - if rule.effect == "deny": - raise _deny(rule) - ctx.authorized_rule_index = index - return ctx - - raise ForbiddenError("Forbidden") + record = ctx.response if isinstance(ctx.response, dict) else RECORD + admit_record(ctx, _decisions(access_rules, record, ctx, auth_config)) + return ctx return step diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 2fcc7aa9..b7006a31 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -147,12 +147,12 @@ def RepositoryListStep( ) -> Step: async def step(ctx: RequestContext) -> RequestContext: if pagination is None: - result = await _repository(ctx).list(collection_name) + result = await _repository(ctx).list(collection_name, ctx.row_filter) ctx.response = _inject_canonical_id_list(result, id_field) return ctx query = ctx.query or {} primitives = map_request(pagination, query) - page = await _repository(ctx).paginate(collection_name, primitives) + page = await _repository(ctx).paginate(collection_name, primitives, ctx.row_filter) canonical = Page( rows=_inject_canonical_id_list(page.rows, id_field), total=page.total, diff --git a/src/yaml_api_generator/server/common/utils/value_compare.py b/src/yaml_api_generator/server/common/utils/value_compare.py index e41af78e..b1f544f4 100644 --- a/src/yaml_api_generator/server/common/utils/value_compare.py +++ b/src/yaml_api_generator/server/common/utils/value_compare.py @@ -1,31 +1,3 @@ -from typing import Any -from uuid import UUID +from yaml_api_generator.runtime.access import values_equal -try: - from bson import ObjectId - - _IDISH = (UUID, ObjectId) -except ImportError: - _IDISH = (UUID,) - - -def values_equal(left: Any, right: Any) -> bool: - """Compare two values by value, falling back to identifier string form. - - Request payloads carry identifiers as strings while persisted records hold - them as native types (UUID, ObjectId). Comparing the canonical string forms - lets equal identifiers match across that representational gap. - - The string-form fallback is confined to identifier-like operands. Booleans, - numbers and plain scalars never coerce, so ``"1" == 1`` and ``"True" == True`` - stay false and access rules cannot be bypassed through type confusion. - """ - if left is None or right is None: - return left is None and right is None - if isinstance(left, bool) or isinstance(right, bool): - return isinstance(left, bool) and isinstance(right, bool) and left is right - if left == right: - return True - if isinstance(left, _IDISH) or isinstance(right, _IDISH): - return str(left) == str(right) - return False +__all__ = ["values_equal"] diff --git a/src/yaml_api_generator/server/common/when_eval.py b/src/yaml_api_generator/server/common/when_eval.py index cfd2b47f..9941351c 100644 --- a/src/yaml_api_generator/server/common/when_eval.py +++ b/src/yaml_api_generator/server/common/when_eval.py @@ -1,7 +1,13 @@ -"""Evaluates 'when' condition ASTs against runtime symbol bindings.""" +"""Evaluates 'when' condition ASTs against runtime symbol bindings. + +A symbol bound to a ``Field`` names the record a list route is filtering, so the +expression evaluates to a row predicate instead of a constant. +""" from __future__ import annotations +from collections.abc import Callable, Collection + from yaml_api_generator.ir.when import ( And, Equals, @@ -17,56 +23,74 @@ WhenExpr, needs_document, ) -from .utils.field_access import get_nested_field -from .utils.value_compare import values_equal - -__all__ = ["evaluate_when_with_symbols", "needs_document"] - - -def _resolve_field( - ref: FieldRef, - symbols: dict[str, dict | None], -) -> object: - binding = symbols.get(ref.root) - if binding is None: - return None - if not ref.path: - return binding.get("sub") - return get_nested_field(binding, ".".join(ref.path)) - - -def _evaluate(expr: WhenExpr, symbols: dict[str, dict | None]) -> bool: - if isinstance(expr, Equals): - return values_equal(_resolve(expr.left, symbols), _resolve(expr.right, symbols)) - if isinstance(expr, NotEquals): - return not values_equal(_resolve(expr.left, symbols), _resolve(expr.right, symbols)) - if isinstance(expr, And): - return _evaluate(expr.left, symbols) and _evaluate(expr.right, symbols) - if isinstance(expr, Or): - return _evaluate(expr.left, symbols) or _evaluate(expr.right, symbols) - if isinstance(expr, Not): - return not _evaluate(expr.operand, symbols) - if isinstance(expr, In): - return any(values_equal(_resolve(expr.left, symbols), v) for v in expr.values) - if isinstance(expr, NotIn): - return not any(values_equal(_resolve(expr.left, symbols), v) for v in expr.values) - if isinstance(expr, IsNull): - return _resolve(expr.operand, symbols) is None - if isinstance(expr, IsNotNull): - return _resolve(expr.operand, symbols) is not None - return False - - -def _resolve(expr: WhenExpr, symbols: dict[str, dict | None]) -> object: - if isinstance(expr, FieldRef): - return _resolve_field(expr, symbols) - if isinstance(expr, StringLiteral): - return expr.value - return None +from yaml_api_generator.runtime.access import ( + EVERY_ROW, + NO_ROW, + Predicate, + conjoin, + constant, + disjoin, + equals, + has_path, + is_null, + negate, + one_of, + value_at, +) + +__all__ = ["evaluate_when_with_symbols", "needs_document", "when_predicate"] + +Symbols = dict[str, object] + + +class _Evaluation: + def __init__(self, symbols: Symbols, request_roots: Collection[str]) -> None: + self._symbols = symbols + self._request_roots = request_roots + self._rules: dict[type, Callable[[WhenExpr], Predicate]] = { + Equals: lambda expr: equals(self._operand(expr.left), self._operand(expr.right)), + NotEquals: lambda expr: negate(equals(self._operand(expr.left), self._operand(expr.right))), + And: lambda expr: conjoin(self.predicate(expr.left), self.predicate(expr.right)), + Or: lambda expr: disjoin(self.predicate(expr.left), self.predicate(expr.right)), + Not: lambda expr: negate(self.predicate(expr.operand)), + In: lambda expr: one_of(self._operand(expr.left), expr.values), + NotIn: lambda expr: negate(one_of(self._operand(expr.left), expr.values)), + IsNull: lambda expr: self._is_null(expr.operand), + IsNotNull: lambda expr: negate(self._is_null(expr.operand)), + } + + def predicate(self, expr: WhenExpr) -> Predicate: + rule = self._rules.get(type(expr)) + return rule(expr) if rule else NO_ROW + + def _is_null(self, operand: WhenExpr) -> Predicate: + if isinstance(operand, FieldRef) and operand.path and operand.root in self._request_roots: + return constant(not has_path(self._symbols.get(operand.root), operand.path)) + return is_null(self._operand(operand)) + + def _operand(self, expr: WhenExpr) -> object: + if isinstance(expr, StringLiteral): + return expr.value + if not isinstance(expr, FieldRef): + return None + return value_at(self._symbols.get(expr.root), expr.path or ("sub",)) + + +def when_predicate( + expr: WhenExpr, + symbols: Symbols, + request_roots: Collection[str] = (), +) -> Predicate: + """The rows ``expr`` admits; a constant unless a symbol is bound to a ``Field``. + + ``is null`` over a ``request_roots`` symbol holds only when the key is absent. + """ + return _Evaluation(symbols, request_roots).predicate(expr) def evaluate_when_with_symbols( expr: WhenExpr, - symbols: dict[str, dict | None], + symbols: Symbols, + request_roots: Collection[str] = (), ) -> bool: - return _evaluate(expr, symbols) + return when_predicate(expr, symbols, request_roots) == EVERY_ROW diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index d6712202..50029e37 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -176,8 +176,8 @@ async def find_one(self, criteria): class Reports: organization_id = "org-1" - async def list(self): - return [{{"id": "r1", "title": "visible"}}] + async def list(self, primitives, predicate): + return [row for row in [{{"id": "r1", "title": "visible"}}] if predicate.matches(row)] async def get(self, record_id): return {{"id": record_id, "organizationId": "org-1", "title": "visible"}} @@ -297,8 +297,9 @@ async def find_one(self, criteria): return {{"id": "org-1", "ownerId": "u1", "name": "Example"}} class ReportsService: - async def list(self): - return [{{"id": "report-1", "organizationId": "org-1", "title": "Observed"}}] + async def list(self, primitives, predicate): + rows = [{{"id": "report-1", "organizationId": "org-1", "title": "Observed"}}] + return [row for row in rows if predicate.matches(row)] users = UsersRepository() app = FastAPI() diff --git a/tests/unit/support/repositories.py b/tests/unit/support/repositories.py index 53e0ce11..d382e0f2 100644 --- a/tests/unit/support/repositories.py +++ b/tests/unit/support/repositories.py @@ -1,6 +1,7 @@ from collections.abc import Callable from typing import Any +from yaml_api_generator.runtime.access import EVERY_ROW, Predicate from yaml_api_generator.server.common.repositories.base import BaseRepository Document = dict[str, Any] @@ -23,8 +24,10 @@ async def close(self) -> None: def documents(self, collection_name: str) -> list[Document]: return self._data.setdefault(collection_name, []) - async def list(self, collection_name: str) -> list[Document]: - return [dict(doc) for doc in self.documents(collection_name)] + async def list( + self, collection_name: str, predicate: Predicate = EVERY_ROW + ) -> list[Document]: + return [dict(doc) for doc in self.documents(collection_name) if predicate.matches(doc)] async def get(self, collection_name: str, document_id: str) -> Document | None: index = self._index_of(collection_name, document_id) diff --git a/tests/unit/test_access_reason.py b/tests/unit/test_access_reason.py index e20b5068..fbe16a89 100644 --- a/tests/unit/test_access_reason.py +++ b/tests/unit/test_access_reason.py @@ -100,18 +100,16 @@ def _load(tmp_path, version="0.3"): pytest.fail("\n".join(f"{error.code}: {error.message}" for error in exc.errors)) -def test_reason_is_enabled_only_for_v03_contexts(tmp_path): - v03 = _load(tmp_path, "0.3") - v02 = _load(tmp_path, "0.2") +@pytest.mark.parametrize("version", ["0.2", "0.3"]) +def test_reason_is_carried_in_every_version(tmp_path, version): + ir = _load(tmp_path, version) - assert build_runtime_context(v03).routes[0].access_rules[0].reason == ( + assert build_runtime_context(ir).routes[0].access_rules[0].reason == ( "Blocked accounts cannot create posts." ) - assert build_context(v03).routes[0].access_rules[0].reason == ( + assert build_context(ir).routes[0].access_rules[0].reason == ( "Blocked accounts cannot create posts." ) - assert build_runtime_context(v02).routes[0].access_rules[0].reason is None - assert build_context(v02).routes[0].access_rules[0].reason is None def test_post_fetch_deny_surfaces_only_the_matching_rule_reason(): diff --git a/tests/unit/test_controller_role.py b/tests/unit/test_controller_role.py index 47d0b412..ceda6757 100644 --- a/tests/unit/test_controller_role.py +++ b/tests/unit/test_controller_role.py @@ -8,19 +8,21 @@ import yaml from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor -from yaml_api_generator.generators.backend.fastapi.access_filters import rule_to_python +from yaml_api_generator.generators.backend.fastapi.access_filters import rule_to_predicate +from yaml_api_generator.runtime import access +from yaml_api_generator.runtime.access import EVERY_ROW, NO_ROW from yaml_api_generator.parsers.transformers.access_validation import validate_spec_access from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir def test_role_compiles_to_a_principal_role_guard(): - condition = rule_to_python( + condition = rule_to_predicate( AccessRuleDescriptor(principal="identity:users", required_role="org-admin") ) - assert condition == ( - 'principal.get("identity_collection") == "users" ' - 'and _principal_has_role(principal, "org-admin")' - ) + scope = {**vars(access), "_principal_has_role": lambda principal, role: role in principal["roles"]} + assert eval(condition, {**scope, "principal": {"identity_collection": "users", "roles": ["org-admin"]}}) == EVERY_ROW + assert eval(condition, {**scope, "principal": {"identity_collection": "users", "roles": []}}) == NO_ROW + assert eval(condition, {**scope, "principal": {"identity_collection": "admins", "roles": ["org-admin"]}}) == NO_ROW _SPEC = """ diff --git a/tests/unit/test_fastapi_access_filters.py b/tests/unit/test_fastapi_access_filters.py index 2108dc51..822b0e6a 100644 --- a/tests/unit/test_fastapi_access_filters.py +++ b/tests/unit/test_fastapi_access_filters.py @@ -1,8 +1,9 @@ """Unit tests for the FastAPI access-rule compilation filters. -Verifies that ``rule_to_python`` compiles IR access rules to correct inline -Python boolean expressions, and that ``needs_post_fetch`` correctly identifies -routes requiring a post-fetch authorization check. +Verifies that ``rule_to_predicate`` compiles IR access rules to expressions over +the shipped ``access`` runtime that admit exactly the intended principals and +rows, and that ``needs_post_fetch`` identifies routes requiring a post-fetch +authorization check. """ import pytest @@ -14,9 +15,23 @@ needs_post_fetch, py_bool, role_hierarchies_py, - rule_to_python, + rule_to_predicate, ) from yaml_api_generator.ir.when import And, Equals, FieldRef, In, IsNotNull, IsNull, Not, NotIn, Or, StringLiteral +from yaml_api_generator.runtime import access +from yaml_api_generator.runtime.access import EVERY_ROW, NO_ROW, Field + +USER = {"identity_collection": "users", "sub": "u1", "roles": ["admin"], "record": {"role": "admin"}} + + +def _predicate(rule, **symbols): + namespace = { + **vars(access), + "_principal_has_role": lambda principal, role: role in principal.get("roles", ()), + "principal": USER, + **symbols, + } + return eval(rule_to_predicate(rule), namespace) def test_py_bool(): @@ -26,81 +41,66 @@ def test_py_bool(): assert py_bool(["x"]) == "True" -def test_rule_to_python_unconditional(): +def test_identity_scope_admits_only_its_collection(): rule = AccessRuleDescriptor(principal="identity:admins") - assert rule_to_python(rule) == 'principal.get("identity_collection") == "admins"' + assert _predicate(rule) == NO_ROW + assert _predicate(rule, principal={**USER, "identity_collection": "admins"}) == EVERY_ROW -def test_rule_to_python_role_only(): - rule = AccessRuleDescriptor(principal="identity:admins", required_role="admin") - assert rule_to_python(rule) == ( - 'principal.get("identity_collection") == "admins" and ' - '_principal_has_role(principal, "admin")' - ) +def test_required_role_is_checked(): + assert _predicate(AccessRuleDescriptor(principal="identity:users", required_role="admin")) == EVERY_ROW + assert _predicate(AccessRuleDescriptor(principal="identity:users", required_role="owner")) == NO_ROW -def test_rule_to_python_when_equals(): +def test_when_equals_against_loaded_document(): rule = AccessRuleDescriptor( principal="identity:users", when=Equals(FieldRef("principal", ("sub",)), FieldRef("document", ("created_by",))), ) - result = rule_to_python(rule) - assert '_values_equal(principal.get("sub"), document.get("created_by"))' in result + assert _predicate(rule, document={"created_by": "u1"}) == EVERY_ROW + assert _predicate(rule, document={"created_by": "u2"}) == NO_ROW -def test_rule_to_python_role_and_when(): +def test_when_equals_against_listed_rows_is_a_row_predicate(): rule = AccessRuleDescriptor( principal="identity:users", - required_role="admin", - when=Equals(FieldRef("principal", ("sub",)), FieldRef("document", ("id",))), - ) - result = rule_to_python(rule) - assert result.startswith( - 'principal.get("identity_collection") == "users" and ' - '_principal_has_role(principal, "admin") and ' + when=Equals(FieldRef("principal", ("sub",)), FieldRef("document", ("created_by",))), ) - assert '_values_equal(principal.get("sub"), document.get("id"))' in result + predicate = _predicate(rule, document=Field(())) + assert predicate.mongo() == {"created_by": "u1"} + assert [row["created_by"] for row in ({"created_by": "u1"}, {"created_by": "u2"}) if predicate.matches(row)] == ["u1"] -def test_rule_to_python_in(): - rule = AccessRuleDescriptor( - principal="identity:users", - when=In(FieldRef("principal", ("role",)), ("admin", "moderator")), +def test_in_and_not_in(): + role_in = AccessRuleDescriptor( + principal="identity:users", when=In(FieldRef("principal", ("record", "role")), ("admin", "moderator")) ) - assert "any(_values_equal(principal.get(\"role\"), _v) for _v in ('admin', 'moderator'))" in rule_to_python(rule) - - -def test_rule_to_python_not_in(): - rule = AccessRuleDescriptor( - principal="identity:users", - when=NotIn(FieldRef("principal", ("role",)), ("guest",)), + role_not_in = AccessRuleDescriptor( + principal="identity:users", when=NotIn(FieldRef("principal", ("record", "role")), ("admin",)) ) - assert "not any(_values_equal(principal.get(\"role\"), _v) for _v in ('guest',))" in rule_to_python(rule) + assert _predicate(role_in) == EVERY_ROW + assert _predicate(role_not_in) == NO_ROW -def test_rule_to_python_nested_record_path(): - rule = AccessRuleDescriptor( - principal="identity:users", - when=Equals(FieldRef("principal", ("record", "role")), StringLiteral("admin")), - ) - assert 'principal.get("record", {}).get("role")' in rule_to_python(rule) - - -def test_rule_to_python_and_or(): +def test_and_or_not(): rule = AccessRuleDescriptor( principal="identity:users", when=And( Equals(FieldRef("principal", ("sub",)), FieldRef("document", ("owner",))), Or( - Equals(FieldRef("principal", ("role",)), StringLiteral("admin")), - Not(Equals(FieldRef("principal", ("role",)), StringLiteral("banned"))), + Equals(FieldRef("principal", ("record", "role")), StringLiteral("editor")), + Not(Equals(FieldRef("principal", ("record", "role")), StringLiteral("banned"))), ), ), ) - result = rule_to_python(rule) - assert "and" in result - assert "or" in result - assert "not" in result + assert _predicate(rule, document={"owner": "u1"}) == EVERY_ROW + assert _predicate(rule, document={"owner": "u2"}) == NO_ROW + + +def test_missing_document_admits_nothing(): + rule = AccessRuleDescriptor(principal="identity:users", document_symbols=["document"]) + assert _predicate(rule, document=None) == NO_ROW + assert _predicate(rule, document={}) == EVERY_ROW def test_is_unconditional_true(): @@ -189,18 +189,13 @@ def test_needs_post_fetch_false_for_non_fetching_op(): assert not needs_post_fetch(_route("list", document_symbols=["document"])) -def test_rule_to_python_is_null(): - rule = AccessRuleDescriptor( - principal="identity:users", - when=IsNull(operand=FieldRef("document", ("organizationId",))), +def test_is_null_and_is_not_null(): + is_null = AccessRuleDescriptor( + principal="identity:users", when=IsNull(operand=FieldRef("document", ("organizationId",))) ) - assert 'document.get("organizationId") is None' in rule_to_python(rule) - - -def test_rule_to_python_is_not_null(): - rule = AccessRuleDescriptor( - principal="identity:users", - when=IsNotNull(operand=FieldRef("principal", ("record", "role"))), + is_not_null = AccessRuleDescriptor( + principal="identity:users", when=IsNotNull(operand=FieldRef("principal", ("record", "role"))) ) - result = rule_to_python(rule) - assert "is not None" in result + assert _predicate(is_null, document={}) == EVERY_ROW + assert _predicate(is_null, document={"organizationId": "o1"}) == NO_ROW + assert _predicate(is_not_null) == EVERY_ROW diff --git a/tests/unit/test_fastapi_authz_emitted.py b/tests/unit/test_fastapi_authz_emitted.py index c833dd83..890a59da 100644 --- a/tests/unit/test_fastapi_authz_emitted.py +++ b/tests/unit/test_fastapi_authz_emitted.py @@ -77,33 +77,6 @@ def test_principal_has_role_wrong_collection(authz): assert authz._principal_has_role({"identity_collection": "users", "roles": ["super_admin"]}, "admin") is False -def test_rule_to_python_role_only(): - from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor - rule = AccessRuleDescriptor(principal="identity:admins", required_role="admin") - assert access_filters.rule_to_python(rule) == ( - 'principal.get("identity_collection") == "admins" and ' - '_principal_has_role(principal, "admin")' - ) - - -def test_rule_to_python_unconditional(): - from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor - rule = AccessRuleDescriptor(principal="identity:admins") - assert access_filters.rule_to_python(rule) == ( - 'principal.get("identity_collection") == "admins"' - ) - - -def test_rule_to_python_when_equals(): - from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor - from yaml_api_generator.ir.when import Equals, FieldRef - rule = AccessRuleDescriptor( - principal="identity:users", - when=Equals(FieldRef("principal", ("sub",)), FieldRef("document", ("created_by",))), - ) - assert '_values_equal(principal.get("sub"), document.get("created_by"))' in access_filters.rule_to_python(rule) - - def test_is_unconditional_true(): from yaml_api_generator.generators.backend.compile_context import AccessRuleDescriptor rule = AccessRuleDescriptor(principal="") diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index 0bfa73be..694657f7 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -182,50 +182,6 @@ def test_drf_views_per_collection(tmp_path): assert "ListView" in content or "DetailView" in content -def test_drf_when_expression_to_python(): - from yaml_api_generator.generators.backend.drf.generator import _when_to_python - from yaml_api_generator.ir.when import Equals, FieldRef - - expr = Equals( - left=FieldRef(root="principal", path=("sub",)), - right=FieldRef(root="document", path=("authorId",)), - ) - result = _when_to_python(expr) - assert 'principal.get("sub")' in result - assert 'document.get("authorId")' in result - assert "==" in result - - -def test_drf_in_expression_to_python(): - from yaml_api_generator.generators.backend.drf.generator import _when_to_python - from yaml_api_generator.ir.when import In, FieldRef - - expr = In( - left=FieldRef(root="principal", path=("record", "role")), - values=("admin", "editor"), - ) - result = _when_to_python(expr) - assert 'principal.get("record", {}).get("role")' in result - assert "in ('admin', 'editor')" in result - assert eval(result, {"principal": {"record": {"role": "admin"}}}) is True - assert eval(result, {"principal": {"record": {"role": "viewer"}}}) is False - - -def test_drf_not_in_expression_to_python(): - from yaml_api_generator.generators.backend.drf.generator import _when_to_python - from yaml_api_generator.ir.when import NotIn, FieldRef - - expr = NotIn( - left=FieldRef(root="document", path=("status",)), - values=("draft", "archived"), - ) - result = _when_to_python(expr) - assert 'document.get("status")' in result - assert "not in ('draft', 'archived')" in result - assert eval(result, {"document": {"status": "published"}}) is True - assert eval(result, {"document": {"status": "draft"}}) is False - - def test_drf_access_rule_enforcement_in_views(tmp_path): ctx = _ctx_from_fixture("roles-access.yaml") gen = DRFGenerator() diff --git a/tests/unit/test_policy_value_coercion.py b/tests/unit/test_policy_value_coercion.py index 7ab1ce44..a4337fd5 100644 --- a/tests/unit/test_policy_value_coercion.py +++ b/tests/unit/test_policy_value_coercion.py @@ -1,4 +1,3 @@ -import types import uuid import pytest @@ -8,37 +7,14 @@ from yaml_api_generator.server.common.when_eval import evaluate_when_with_symbols from yaml_api_generator.ir.when import Equals, FieldRef from yaml_api_generator.generators.backend.fastapi import access_filters +from yaml_api_generator.runtime import access +from yaml_api_generator.runtime.access import EVERY_ROW, NO_ROW from yaml_api_generator.generators.backend.compile_context import ( AccessRuleDescriptor, RequestSymbolDescriptor, ) -def _exec_values_equal(): - source = ( - "from typing import Any\n" - "from uuid import UUID\n" - "try:\n" - " from bson import ObjectId\n" - " _IDISH = (UUID, ObjectId)\n" - "except ImportError:\n" - " _IDISH = (UUID,)\n" - "def _values_equal(left: Any, right: Any) -> bool:\n" - " if left is None or right is None:\n" - " return left is None and right is None\n" - " if isinstance(left, bool) or isinstance(right, bool):\n" - " return isinstance(left, bool) and isinstance(right, bool) and left is right\n" - " if left == right:\n" - " return True\n" - " if isinstance(left, _IDISH) or isinstance(right, _IDISH):\n" - " return str(left) == str(right)\n" - " return False\n" - ) - module = types.ModuleType("authorization_fragment") - exec(compile(source, "", "exec"), module.__dict__) - return module._values_equal - - class TestDevValuesEqual: def test_uuid_equals_its_string_form(self): u = uuid.uuid4() @@ -97,29 +73,7 @@ def test_when_eval_equals_unequal_ids_denied(self): assert evaluate_when_with_symbols(expr, symbols) is False -class TestEmittedValuesEqual: - def test_emitted_values_equal_coerces_uuid_string(self): - emitted = _exec_values_equal() - u = uuid.uuid4() - assert emitted(u, str(u)) is True - - def test_emitted_values_equal_unequal(self): - emitted = _exec_values_equal() - assert emitted(uuid.uuid4(), str(uuid.uuid4())) is False - - def test_emitted_values_equal_none(self): - emitted = _exec_values_equal() - assert emitted(None, "x") is False - - def test_emitted_values_equal_no_type_confusion(self): - emitted = _exec_values_equal() - assert emitted("1", 1) is False - assert emitted("True", True) is False - assert emitted(True, 1) is False - assert emitted(True, True) is True - - -class TestRuleToPythonRegression: +class TestRuleToPredicateRegression: def test_target_body_field_equals_user_record_field_grants(self): rule = AccessRuleDescriptor( principal="user", @@ -135,18 +89,11 @@ def test_target_body_field_equals_user_record_field_grants(self): ) binding = access_filters.request_symbol_binding(rule.request_symbols[0]) assert binding == "target = _request_body" - expr = access_filters.rule_to_python(rule) - assert "_values_equal(" in expr - assert 'target.get("ownerId")' in expr - assert 'user.get("record", {}).get("id")' in expr + expr = access_filters.rule_to_predicate(rule) oid = ObjectId() - scope = { - "target": {"ownerId": str(oid)}, - "user": {"record": {"id": oid}}, - "_values_equal": _exec_values_equal(), - } - assert eval(expr, scope) is True + scope = {**vars(access), "target": {"ownerId": str(oid)}, "user": {"record": {"id": oid}}} + assert eval(expr, scope) == EVERY_ROW scope["user"] = {"record": {"id": ObjectId()}} - assert eval(expr, scope) is False + assert eval(expr, scope) == NO_ROW From 72962bd097fccc8192546cabffe3ddb0e41923a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 18:51:35 +0100 Subject: [PATCH 04/89] Validate payloads with one shared field validator in every consumer Nested properties, additionalProperties, format, items, defaults, strict scalars, readonly fields and DTO-typed bodies are enforced by the dev server and the generated FastAPI, DRF and raw backends. Each collection gets one Fields payload that create and replace extend and update makes partial. Field defaults must conform to their own config and ref fields must name a declared entity. An empty PATCH returns 200. --- .../generators/backend/base.py | 14 ++ .../backend/builders/model_builder.py | 197 +++++----------- .../backend/builders/route_builder.py | 20 +- .../generators/backend/compile_context.py | 13 +- .../generators/backend/drf/generator.py | 22 +- .../backend/drf/templates/payloads.py.jinja | 25 ++ .../drf/templates/serializers.py.jinja | 21 +- .../backend/drf/templates/views.py.jinja | 10 +- .../generators/backend/fastapi/generator.py | 26 +- .../backend/fastapi/templates/models.py.jinja | 60 +---- .../fastapi/templates/repository_ops.py.jinja | 8 +- .../templates/repository_sql_ops.py.jinja | 11 +- .../generators/backend/fastapi/type_mapper.py | 141 ++++++++--- .../generators/backend/raw/generator.py | 24 +- .../backend/raw/templates/routes.py.jinja | 6 +- .../backend/raw/templates/validation.py.jinja | 135 ++--------- .../shared/templates/repository.py.jinja | 8 +- .../shared/templates/service_inputs.py.jinja | 82 +------ .../generators/backend/utils.py | 2 +- src/yaml_api_generator/ir/__init__.py | 4 + src/yaml_api_generator/ir/api.py | 33 ++- .../transformers/constraint_validation.py | 52 ++-- .../parsers/transformers/dto_validation.py | 19 +- .../parsers/transformers/ref_validation.py | 42 ++++ src/yaml_api_generator/parsers/yaml_parser.py | 5 + src/yaml_api_generator/runtime/fields.py | 200 ++++++++++++++++ src/yaml_api_generator/runtime/payloads.py | 55 +++++ .../server/common/context_builder.py | 1 + .../server/common/pipeline_builder.py | 2 +- .../server/common/repositories/mongodb.py | 10 +- .../server/common/repositories/sql.py | 14 +- .../server/common/server_context.py | 3 +- .../server/common/steps/services.py | 5 +- .../server/common/steps/validation.py | 223 +++--------------- .../server/fastapi/factory.py | 157 +++++------- tests/fixtures/valid/auth-oidc-relations.yaml | 2 +- .../test_generated_raw_payloads.py | 49 ++++ tests/unit/support/payload_requests.py | 165 +++++++++++++ tests/unit/test_compiler_context_builder.py | 9 +- tests/unit/test_constraint_validation.py | 26 ++ tests/unit/test_fastapi_factory.py | 18 +- tests/unit/test_generators.py | 90 +++---- tests/unit/test_payload_models.py | 138 +++++++++++ tests/unit/test_ref_validation.py | 45 ++++ tests/unit/test_request_access_generation.py | 2 +- tests/unit/test_request_access_validation.py | 5 + tests/unit/test_schema_consumers.py | 1 - tests/unit/test_service_input_codegen.py | 7 +- 48 files changed, 1269 insertions(+), 938 deletions(-) create mode 100644 src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja create mode 100644 src/yaml_api_generator/parsers/transformers/ref_validation.py create mode 100644 src/yaml_api_generator/runtime/fields.py create mode 100644 src/yaml_api_generator/runtime/payloads.py create mode 100644 tests/integration/test_generated_raw_payloads.py create mode 100644 tests/unit/support/payload_requests.py create mode 100644 tests/unit/test_payload_models.py create mode 100644 tests/unit/test_ref_validation.py diff --git a/src/yaml_api_generator/generators/backend/base.py b/src/yaml_api_generator/generators/backend/base.py index 30e84133..0e96785e 100644 --- a/src/yaml_api_generator/generators/backend/base.py +++ b/src/yaml_api_generator/generators/backend/base.py @@ -8,6 +8,8 @@ from yaml_api_generator.generators.backend.compile_context import CompileContext from yaml_api_generator.utils.strings import to_pascal_case +from .utils import runtime_module_source + @dataclass class RenderConfig: @@ -138,6 +140,18 @@ def _generate_import_statement( return f"from .{col_name} import {', '.join(class_names)}" return None + @staticmethod + def _render_service_inputs(env: Environment, output_dir: Path, package: str) -> None: + """Write the field validator and the service-input helper built on it into ``package``.""" + package_dir = output_dir / package + package_dir.mkdir(parents=True, exist_ok=True) + (package_dir / "fields.py").write_text(runtime_module_source("fields")) + (package_dir / "service_inputs.py").write_text( + env.get_template("service_inputs.py.jinja").render( + fields_module=f"{package}.fields" if package else "fields" + ) + ) + def _render_single_file( self, env: Environment, diff --git a/src/yaml_api_generator/generators/backend/builders/model_builder.py b/src/yaml_api_generator/generators/backend/builders/model_builder.py index 003d41c7..e570a1ab 100644 --- a/src/yaml_api_generator/generators/backend/builders/model_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/model_builder.py @@ -1,21 +1,39 @@ -"""Model and field building for backend code generation. +"""Payload model descriptors for backend code generation. -Converts IR collections into Pydantic model descriptors with field validation. +Each collection gets one ``Fields`` model holding its writable +fields; the create and replace payloads extend it and the update payload is +its partial form, so no field is declared twice. """ -from collections.abc import Collection -from typing import Any - -from yaml_api_generator.ir import CRUD_OPERATIONS, IRCollection, IRDto, IRIdentityCollection, IRRepository +from yaml_api_generator.ir import ( + CRUD_OPERATIONS, + IRCollection, + IRDto, + IRIdentityCollection, + IRRepository, + SECRET_FIELD, + writable_fields, +) from yaml_api_generator.utils.strings import to_pascal_case from ..compile_context import FieldDescriptor, ModelDescriptor WRITE_OPERATIONS = ("create", "update", "replace") +_PASSWORD = FieldDescriptor("password", SECRET_FIELD) + + +def payload_name(collection: str, operation: str) -> str: + """``Fields`` for the shared base, else ``Payload``.""" + prefix = "".join(part.capitalize() for part in collection.split("_")) + return f"{prefix}Fields" if operation == "fields" else f"{prefix}{operation.capitalize()}Payload" + + +def _descriptors(fields: dict[str, dict]) -> list[FieldDescriptor]: + return [FieldDescriptor(name, config) for name, config in fields.items()] class ModelBuilder: - """Builds Pydantic model descriptors from IR collections.""" + """Builds payload model descriptors from IR collections and DTOs.""" def build_models( self, @@ -23,36 +41,13 @@ def build_models( identity_map: dict[str, IRIdentityCollection], repositories: dict[str, IRRepository], ) -> list[ModelDescriptor]: - """Build model descriptors for write operations. - - Args: - collections: Collection IR objects - identity_map: Identity collection IR objects - repositories: Repository IR objects - - Returns: - List of model descriptors - """ - models: list[ModelDescriptor] = [] - for name, col in collections.items(): - identity = identity_map.get(name) - repo = repositories.get(name) - enabled_ops = repo.crud if repo is not None else CRUD_OPERATIONS - for op in WRITE_OPERATIONS: - if op not in enabled_ops: - continue - fields = self.build_fields(col, op, identity, excluded=col.files.keys()) - model_name = self._model_name(name, op) - models.append( - ModelDescriptor( - name=model_name, - collection=name, - operation=op, - fields=fields, - ) - ) - - return models + return [ + model + for name, col in collections.items() + for model in self._family( + col, identity_map.get(name), repositories.get(name) + ) + ] def build_dto_models(self, dtos: dict[str, IRDto]) -> list[ModelDescriptor]: return [ @@ -60,110 +55,40 @@ def build_dto_models(self, dtos: dict[str, IRDto]) -> list[ModelDescriptor]: name=to_pascal_case(dto.name), collection=dto.name, operation="dto", - fields=self.build_fields(dto, "dto", None), + fields=_descriptors(writable_fields(dto.fields)), ) for dto in dtos.values() ] - def build_fields( + def _family( self, col: IRCollection, - operation: str, identity: IRIdentityCollection | None, - excluded: Collection[str] = (), - ) -> list[FieldDescriptor]: - """Build field descriptors for a model. - - Args: - col: Collection IR object - operation: Operation name (create, update, replace) - identity: Identity collection IR object if collection is auth - excluded: Field names kept out of the model (file fields travel outside the JSON payload) - - Returns: - List of field descriptors - """ - make_optional = operation == "update" - fields: list[FieldDescriptor] = [] - - for field_name, raw_config in col.fields.items(): - if not isinstance(raw_config, dict): - continue - if raw_config.get("type") == "id": - continue - if field_name == "_auth" or field_name in excluded: - continue - if raw_config.get("readonly", False): - continue - - field_type = raw_config.get("type", "string") - required = bool(raw_config.get("required", False)) - has_default = "default" in raw_config - default_value: Any = None - - if make_optional: - is_optional = True - default_value = raw_config.get("default") - elif has_default: - is_optional = False - default_value = raw_config["default"] - elif required: - is_optional = False - default_value = None - else: - is_optional = True - default_value = None - - constraints: dict[str, Any] = {} - if raw_config.get("min") is not None: - constraints["min"] = raw_config["min"] - if raw_config.get("max") is not None: - constraints["max"] = raw_config["max"] - if raw_config.get("minLength") is not None: - constraints["min_length"] = raw_config["minLength"] - if raw_config.get("maxLength") is not None: - constraints["max_length"] = raw_config["maxLength"] - if raw_config.get("format") is not None: - constraints["format"] = raw_config["format"] - - enum_values = raw_config.get("values") if field_type == "enum" else None - array_item_type = None - if field_type == "array": - items = raw_config.get("items") - if isinstance(items, dict): - array_item_type = items.get("type", "string") - - ref_collection = raw_config.get("ref") if field_type == "ref" else None - - fields.append( - FieldDescriptor( - name=field_name, - type=field_type, - required=required and not make_optional, - is_optional=is_optional, - default=default_value, - constraints=constraints, - enum_values=enum_values, - array_item_type=array_item_type, - ref_collection=ref_collection, - ) - ) - - if identity is not None and operation == "create" and identity.has_password: - has_password_field = any(f.name == "password" for f in fields) - if not has_password_field: - fields.append( - FieldDescriptor( - name="password", - type="string", - required=True, - is_optional=False, - constraints={"min_length": 1}, - ) + repo: IRRepository | None, + ) -> list[ModelDescriptor]: + enabled = repo.crud if repo is not None else CRUD_OPERATIONS + operations = [op for op in WRITE_OPERATIONS if op in enabled] + if not operations: + return [] + base = payload_name(col.name, "fields") + own_fields = { + "create": [_PASSWORD] if identity is not None and identity.has_password else [], + } + return [ + ModelDescriptor( + name=base, + collection=col.name, + operation="fields", + fields=_descriptors(writable_fields(col.fields, col.files.keys())), + ), + *( + ModelDescriptor( + name=payload_name(col.name, op), + collection=col.name, + operation=op, + fields=own_fields.get(op, []), + base=base, ) - - return fields - - def _model_name(self, collection: str, operation: str) -> str: - col_part = "".join(part.capitalize() for part in collection.split("_")) - return f"{col_part}{operation.capitalize()}Payload" + for op in operations + ), + ] diff --git a/src/yaml_api_generator/generators/backend/builders/route_builder.py b/src/yaml_api_generator/generators/backend/builders/route_builder.py index 41b1911a..d506468f 100644 --- a/src/yaml_api_generator/generators/backend/builders/route_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/route_builder.py @@ -7,6 +7,7 @@ from yaml_api_generator.ir import IRDocument, IRDto, IRSymbolRef from yaml_api_generator.utils.strings import to_pascal_case +from .model_builder import payload_name from ..compile_context import ( AccessRuleDescriptor, DocumentBindingDescriptor, @@ -113,7 +114,8 @@ def build_routes( ] needs_request = self._needs_request(access_rules) - typed_body_model = self._typed_body_model(access_rules, dtos) + body_dto = ir_route.body_dto(dtos) + typed_body_model = to_pascal_case(body_dto.name) if body_dto else None if ir_route.handler is not None or ir_route.collection is None: routes.append( @@ -149,7 +151,7 @@ def build_routes( body_model: str | None = None if ir_route.operation in WRITE_OPERATIONS or ir_route.operation == "login": - model_name = self._model_name(ir_route.collection, ir_route.operation) + model_name = payload_name(ir_route.collection, ir_route.operation) if model_name in model_index: body_model = model_name @@ -200,17 +202,3 @@ def _needs_request(rules: list[AccessRuleDescriptor] | None) -> bool: for sym in rule.request_symbols for part in _REQUEST_PARTS ) - - @staticmethod - def _typed_body_model( - rules: list[AccessRuleDescriptor] | None, dtos: dict[str, IRDto] - ) -> str | None: - for rule in rules or (): - for sym in rule.request_symbols: - if sym.part in (None, "body") and sym.type_name in dtos: - return to_pascal_case(sym.type_name) - return None - - def _model_name(self, collection: str, operation: str) -> str: - col_part = "".join(part.capitalize() for part in collection.split("_")) - return f"{col_part}{operation.capitalize()}Payload" diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index f0634b49..cea063cc 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -162,22 +162,19 @@ def _credential(self, key: str, value: str | None) -> SettingField: @dataclass(frozen=True) class FieldDescriptor: name: str - type: str - required: bool - is_optional: bool - default: Any | None = None - constraints: dict[str, Any] = field(default_factory=dict) - enum_values: list[str] | None = None - array_item_type: str | None = None - ref_collection: str | None = None + config: dict[str, Any] @dataclass(frozen=True) class ModelDescriptor: + """A payload model; ``base`` names the model whose fields it extends.""" + name: str collection: str operation: str fields: list[FieldDescriptor] + base: str | None = None + @dataclass(frozen=True) diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index f94e32f5..f82df1c6 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -23,7 +23,6 @@ reject_unsigned_session_algorithm, validate_datastore_layout, ) -from . import type_mapper from ..fastapi import access_filters from .settings_builder import build_drf_settings_fields from yaml_api_generator.ir.when import ( @@ -61,7 +60,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) env.globals["settings_module"] = f"{self.dir_prefix}.settings" - env.filters["drf_field"] = type_mapper.drf_field env.filters["capitalize_collection"] = to_pascal_case env.filters["rule_to_python"] = _rule_to_python env.filters["py_literal"] = access_filters.py_literal @@ -90,7 +88,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: self._render_project_scaffold(env, context, output_dir) self._render_serializers(env, context, collections_by_name, output_dir) self._render_services(env, context, collections_by_name, output_dir) - self._render_service_inputs(env, context, output_dir) + self._render_service_inputs(env, output_dir, self.dir_prefix) (api_dir / "pagination.py").write_text( runtime_module_source("pagination") ) @@ -114,18 +112,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: ) ) - def _render_service_inputs( - self, env: Environment, context: CompileContext, output_dir: Path - ) -> None: - self._render_single_file( - env, - context, - output_dir, - SingleFileRenderConfig( - template_name="service_inputs.py.jinja", - output_path=f"{self.dir_prefix}/service_inputs.py", - ), - ) def _render_settings( self, env: Environment, context: CompileContext, output_dir: Path @@ -174,6 +160,12 @@ def _render_serializers( should_render=lambda col_name, col_data, ctx: bool(col_data["models"]), ) ) + api_dir = output_dir / self.dir_prefix + (api_dir / "payloads.py").write_text(env.get_template("payloads.py.jinja").render()) + if context.dto_models: + (api_dir / "serializers" / "_dto.py").write_text( + env.get_template("serializers.py.jinja").render(models=context.dto_models) + ) def _render_services( self, diff --git a/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja new file mode 100644 index 00000000..674668cc --- /dev/null +++ b/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja @@ -0,0 +1,25 @@ +"""Serializers that check request data against spec field configs.""" + +from typing import Any + +from rest_framework import serializers + +from .fields import conform_object + + +class PayloadSerializer(serializers.Serializer): + """Subclasses declare ``properties``; ``partial=True`` accepts any subset of them.""" + + properties: dict[str, dict[str, Any]] = {} + omittable = False + + def to_internal_value(self, data: Any) -> dict[str, Any]: + try: + return conform_object(self.properties, data, partial=self.partial or self.omittable) + except ValueError as error: + raise serializers.ValidationError({"detail": str(error)}) from error + + +def partial(serializer: type[PayloadSerializer], name: str) -> type[PayloadSerializer]: + """``serializer`` accepting any subset of its properties.""" + return type(name, (serializer,), {"omittable": True}) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/serializers.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/serializers.py.jinja index 5c45609a..77ea2194 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/serializers.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/serializers.py.jinja @@ -1,9 +1,24 @@ -from rest_framework import serializers +from api.payloads import PayloadSerializer{% if models | selectattr("operation", "equalto", "update") | list %}, partial{% endif %} {% for model in models %} -class {{ model.name }}(serializers.Serializer): +{% if model.operation == "update" %} + +{{ model.name }} = partial({{ model.base }}, "{{ model.name }}") +{% else %} + +class {{ model.name }}({{ model.base or "PayloadSerializer" }}): +{% if model.fields %} + properties = { +{% if model.base %} + **{{ model.base }}.properties, +{% endif %} {% for field in model.fields %} - {{ field.name }} = {{ field | drf_field }} + {{ field.name | tojson }}: {{ field.config | py_literal }}, {% endfor %} + } +{% else %} + pass +{% endif %} +{% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 3ac87bab..f37a95b8 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -17,6 +17,10 @@ from api.conflicts import ConflictError {% if ns.needs_models %} from api.serializers.{{ collection_name }} import {{ ns.model_imports | join(", ") }} {% endif %} +{% set dto_imports = crud_routes | selectattr("typed_body_model") | map(attribute="typed_body_model") | unique | list %} +{% if dto_imports %} +from api.serializers._dto import {{ dto_imports | join(", ") }} +{% endif %} from api.repositories.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} @@ -162,7 +166,7 @@ class {{ class_name }}ListView(APIView): return Response(documents) {% endif %} {% elif route.operation == "create" %} - serializer = {{ route.body_model }}(data=request.data) + serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) serializer.is_valid(raise_exception=True) data = serializer.validated_data document = await service.create(data) @@ -248,7 +252,7 @@ class {{ class_name }}DetailView(APIView): {{ post_fetch_check(route, "result") | indent(4, true) }} return Response(result) {% elif route.operation == "update" %} - serializer = {{ route.body_model }}(data=request.data) + serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) serializer.is_valid(raise_exception=True) data = serializer.validated_data {% if route is needs_post_fetch %} @@ -266,7 +270,7 @@ class {{ class_name }}DetailView(APIView): {% endif %} return Response(result) {% elif route.operation == "replace" %} - serializer = {{ route.body_model }}(data=request.data) + serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) serializer.is_valid(raise_exception=True) data = serializer.validated_data {% if route is needs_post_fetch %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 4c1a7bc8..4d702db7 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -46,8 +46,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: reject_unsigned_session_algorithm(context) reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) - env.filters["python_type"] = type_mapper.python_type - env.filters["python_annotation"] = type_mapper.python_annotation env.filters["capitalize_collection"] = to_pascal_case env.filters["py_identifier"] = to_py_identifier env.filters["py_bool"] = access_filters.py_bool @@ -66,9 +64,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: ) env.tests["needs_request_rule"] = access_filters.needs_request_rule env.tests["unconditional"] = access_filters.is_unconditional - env.globals["needs_datetime_import"] = type_mapper.needs_datetime_import - env.globals["needs_literal_import"] = type_mapper.needs_literal_import - env.globals["needs_any_import"] = type_mapper.needs_any_import output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "conflicts.py").write_text( @@ -83,7 +78,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: self._render_models(env, context, collections_by_name, output_dir) self._render_dtos(env, context, output_dir) self._render_services(env, context, collections_by_name, output_dir) - self._render_service_inputs(env, context, output_dir) + self._render_service_inputs(env, output_dir, "dependencies") self._render_routes(env, context, collections_by_name, output_dir) self._render_repositories(env, context, collections_by_name, output_dir) self._render_dependencies(env, context, output_dir) @@ -95,18 +90,6 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: ) ) - def _render_service_inputs( - self, env: Environment, context: CompileContext, output_dir: Path - ) -> None: - self._render_single_file( - env, - context, - output_dir, - SingleFileRenderConfig( - template_name="service_inputs.py.jinja", - output_path="dependencies/service_inputs.py", - ), - ) def _render_settings( self, env: Environment, context: CompileContext, output_dir: Path @@ -140,8 +123,7 @@ def _render_models( template_name="models.py.jinja", output_subdir="models", context_builder=lambda col_name, col_data, ctx: { - "models": col_data["models"], - "all_fields": [f for m in col_data["models"] for f in m.fields], + "module": type_mapper.PayloadModule(col_data["models"]), }, should_render=lambda col_name, col_data, ctx: bool(col_data["models"]), ) @@ -155,8 +137,7 @@ def _render_dtos( models_dir = output_dir / "models" models_dir.mkdir(parents=True, exist_ok=True) content = env.get_template("models.py.jinja").render( - models=context.dto_models, - all_fields=[f for m in context.dto_models for f in m.fields], + module=type_mapper.PayloadModule(context.dto_models) ) (models_dir / "_dto.py").write_text(content) @@ -443,6 +424,7 @@ def _render_dependencies( (deps_dir / "pagination.py").write_text( runtime_module_source("pagination") ) + (deps_dir / "payloads.py").write_text(runtime_module_source("payloads")) if any(getattr(provider, "dpop", False) for provider in context.auth.providers.values()): (deps_dir / "dpop.py").write_text(runtime_module_source("dpop")) if any(context.repository_operations.values()): diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja index c37b3cfe..6c255c9b 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja @@ -1,57 +1,19 @@ -{% set ns = namespace(need_datetime=false, need_literal=false, need_any=false) %} -{% for field in all_fields %} -{% if field.type == "datetime" %}{% set ns.need_datetime = true %}{% endif %} -{% if field.type == "enum" and field.enum_values %}{% set ns.need_literal = true %}{% endif %} -{% if field.type == "object" or (field.type == "array" and "Any" in (field | python_annotation)) %}{% set ns.need_any = true %}{% endif %} +{% for line in module.imports %} +{{ line }} {% endfor %} -{% if ns.need_datetime %} -from datetime import datetime -{% endif %} -{% if ns.need_any or ns.need_literal %} -from typing import {% if ns.need_any %}Any{% endif %}{% if ns.need_any and ns.need_literal %}, {% endif %}{% if ns.need_literal %}Literal{% endif %} - -{% endif %} +{% for declaration in module.declarations %} -from pydantic import BaseModel{% if all_fields | selectattr("constraints") | list %}, Field{% endif %} +{% if declaration.attributes is defined %} -{% for model in models %} +class {{ declaration.name }}({{ declaration.base }}): +{% for attribute in declaration.attributes %} + {{ attribute.name }}: {{ attribute.annotation }}{% if attribute.default is not none %} = {{ attribute.default }}{% endif %} -class {{ model.name }}(BaseModel): -{% if not model.fields %} - pass -{% endif %} -{% for field in model.fields %} -{% set annotation = field | python_annotation %} -{% if field.constraints %} -{% set field_args = [] %} -{% if field.required and not field.is_optional %} -{% set _ = field_args.append("...") %} -{% elif field.is_optional %} -{% set _ = field_args.append("None") %} -{% elif field.default is not none %} -{% set _ = field_args.append(field.default.__repr__()) %} -{% endif %} -{% if field.constraints.get("min") is not none %} -{% set _ = field_args.append("ge=" ~ field.constraints["min"]) %} -{% endif %} -{% if field.constraints.get("max") is not none %} -{% set _ = field_args.append("le=" ~ field.constraints["max"]) %} -{% endif %} -{% if field.constraints.get("min_length") is not none %} -{% set _ = field_args.append("min_length=" ~ field.constraints["min_length"]) %} -{% endif %} -{% if field.constraints.get("max_length") is not none %} -{% set _ = field_args.append("max_length=" ~ field.constraints["max_length"]) %} -{% endif %} - {{ field.name }}: {{ annotation }} = Field({{ field_args | join(", ") }}) {% else %} -{% if field.required and not field.is_optional %} - {{ field.name }}: {{ annotation }} -{% elif field.is_optional %} - {{ field.name }}: {{ annotation }} = {{ field.default.__repr__() if field.default is not none else "None" }} + pass +{% endfor %} {% else %} - {{ field.name }}: {{ annotation }} = {{ field.default.__repr__() }} -{% endif %} + +{{ declaration.name }} = partial({{ declaration.base }}, "{{ declaration.name }}") {% endif %} {% endfor %} -{% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja index b8866354..0525e793 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_ops.py.jinja @@ -197,14 +197,8 @@ class CreateOp: class UpdateOp: - @_translate_conflict async def update(self, document_id: str, data: dict[str, Any]) -> dict[str, Any] | None: - doc = await self._col.find_one_and_update( - self._match_filter(document_id), - {"$set": data}, - return_document=ReturnDocument.AFTER, - ) - return self._normalize(doc) + return await self.update_by(self._match_filter(document_id), data) class ReplaceOp: diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja index a8126b1b..ae04e1ed 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql_ops.py.jinja @@ -247,17 +247,8 @@ class CreateOp: class UpdateOp: - @_translate_conflict async def update(self, document_id: str, data: dict[str, Any]) -> dict[str, Any] | None: - values = self._known(data, drop_id=True) - async with self._engine.begin() as conn: - result = await conn.execute( - sa.update(self.TABLE) - .where(self._id_col() == document_id) - .values(**values) - .returning(self.TABLE) - ) - return self._one(result) + return await self.update_by({self.ID_FIELD: document_id}, data) class ReplaceOp: diff --git a/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py b/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py index 3ecb4ccb..310ad00f 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py +++ b/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py @@ -1,45 +1,126 @@ -from yaml_api_generator.generators.backend.compile_context import FieldDescriptor +"""Field configs rendered as the declarations of a pydantic payload module.""" +from dataclasses import dataclass +from typing import Any -_TYPE_MAP: dict[str, str] = { - "string": "str", - "ref": "str", - "number": "float", - "boolean": "bool", - "datetime": "datetime", - "object": "dict[str, Any]", +from yaml_api_generator.runtime.fields import is_open +from yaml_api_generator.runtime.payloads import CONSTRAINTS, SCALAR_TYPES, STRICT_TYPES +from yaml_api_generator.utils.strings import to_pascal_case +from yaml_api_generator.generators.backend.compile_context import ModelDescriptor + + +_IMPORTS: dict[str, tuple[str, ...]] = { + "datetime": ("datetime",), + "typing": ("Annotated", "Any", "Literal"), + "pydantic": ("Field",), + "dependencies.payloads": ("OpenPayload", "Payload", "formatted", "partial"), } -def python_type(field: FieldDescriptor) -> str: - if field.type == "enum" and field.enum_values: - quoted = ", ".join(repr(v) for v in field.enum_values) - return f"Literal[{quoted}]" +@dataclass(frozen=True) +class Attribute: + name: str + annotation: str + default: str | None + + +@dataclass(frozen=True) +class ClassDeclaration: + name: str + base: str + attributes: tuple[Attribute, ...] + + +@dataclass(frozen=True) +class PartialDeclaration: + """``name`` is ``base`` with every field omittable.""" + + name: str + base: str + + +class PayloadModule: + """The declarations of one generated models module and the imports they use. - if field.type == "array": - inner = _TYPE_MAP.get(field.array_item_type or "string", "Any") - return f"list[{inner}]" + Nested object models are declared before the model that refers to them. + """ - return _TYPE_MAP.get(field.type, "Any") + def __init__(self, models: list[ModelDescriptor]) -> None: + self.declarations: list[ClassDeclaration | PartialDeclaration] = [] + self._used: set[str] = set() + for model in models: + self._declare(model) + @property + def imports(self) -> list[str]: + return [ + f"from {module} import {', '.join(used)}" + for module, names in _IMPORTS.items() + if (used := [name for name in names if name in self._used]) + ] -def python_annotation(field: FieldDescriptor) -> str: - base = python_type(field) - if field.is_optional: - return f"{base} | None" - return base + def _use(self, name: str) -> str: + self._used.add(name) + return name + def _declare(self, model: ModelDescriptor) -> None: + if model.operation == "update": + self._use("partial") + self.declarations.append(PartialDeclaration(model.name, model.base)) + return + self._class( + model.name, + model.base or self._use("Payload"), + {field.name: field.config for field in model.fields}, + owner=to_pascal_case(model.collection), + ) -def needs_datetime_import(fields: list[FieldDescriptor]) -> bool: - return any(f.type == "datetime" for f in fields) + def _class( + self, name: str, base: str, properties: dict[str, dict[str, Any]], owner: str + ) -> None: + if any(declaration.name == name for declaration in self.declarations): + raise ValueError(f"Two payload models are both named '{name}'") + attributes = tuple( + self._attribute(f"{owner}{to_pascal_case(field)}", field, config) + for field, config in properties.items() + ) + self.declarations.append(ClassDeclaration(name, base, attributes)) + def _attribute(self, nested_name: str, name: str, config: dict[str, Any]) -> Attribute: + annotation = self._annotation(config, nested_name) + if config.get("required"): + return Attribute(name, annotation, repr(config["default"]) if "default" in config else None) + return Attribute(name, f"{annotation} | None", repr(config.get("default"))) -def needs_literal_import(fields: list[FieldDescriptor]) -> bool: - return any(f.type == "enum" and f.enum_values for f in fields) + def _annotation(self, config: dict[str, Any], nested_name: str) -> str: + kind = config.get("type", "string") + if kind == "enum": + return f"{self._use('Literal')}[{', '.join(map(repr, config['values']))}]" + if kind == "array": + items = config.get("items") + item = self._annotation(items, f"{nested_name}Item") if items else self._use("Any") + return f"list[{item}]" + if kind == "object": + return self._object(config, nested_name) + return self._scalar(config, SCALAR_TYPES[kind]) + def _object(self, config: dict[str, Any], name: str) -> str: + if "properties" not in config and is_open(config): + return f"dict[str, {self._use('Any')}]" + base = self._use("OpenPayload" if is_open(config) else "Payload") + self._class(name, base, config.get("properties", {}), owner=name) + return name -def needs_any_import(fields: list[FieldDescriptor]) -> bool: - return any( - f.type in ("object", "array") and (f.array_item_type or "string") not in _TYPE_MAP - for f in fields - ) or any(f.type == "object" for f in fields) + def _scalar(self, config: dict[str, Any], scalar: type) -> str: + python_type = self._use(scalar.__name__) + arguments = { + **({"strict": True} if scalar in STRICT_TYPES else {}), + **{CONSTRAINTS[key]: config[key] for key in CONSTRAINTS if key in config}, + } + listed = ", ".join(f"{key}={value!r}" for key, value in arguments.items()) + field = [f"{self._use('Field')}({listed})"] if arguments else [] + formatted = [f"{self._use('formatted')}({config['format']!r})"] if "format" in config else [] + metadata = [*field, *formatted] + if not metadata: + return python_type + return f"{self._use('Annotated')}[{python_type}, {', '.join(metadata)}]" diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 9c20543e..206b949f 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -3,7 +3,10 @@ from jinja2 import Environment -from yaml_api_generator.generators.backend.compile_context import CompileContext, IdentityCollectionDescriptor +from yaml_api_generator.generators.backend.compile_context import ( + CompileContext, + IdentityCollectionDescriptor, +) from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields from yaml_api_generator.utils.strings import to_pascal_case from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment @@ -72,15 +75,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: env.get_template("conflicts.py.jinja").render() ) - self._render_single_file( - env, - context, - output_dir, - SingleFileRenderConfig( - template_name="service_inputs.py.jinja", - output_path="service_inputs.py", - ), - ) + self._render_service_inputs(env, output_dir, "") (output_dir / "pagination.py").write_text( runtime_module_source("pagination") ) @@ -143,9 +138,10 @@ def _render_validation( collections_by_name: dict[str, dict], output_dir: Path, ) -> None: - all_models = [] - for col_data in collections_by_name.values(): - all_models.extend(col_data["models"]) + models = [ + *(model for col_data in collections_by_name.values() for model in col_data["models"]), + *context.dto_models, + ] self._render_single_file( env, context, output_dir, @@ -153,7 +149,7 @@ def _render_validation( template_name="validation.py.jinja", output_path="validation.py", context_builder=lambda ctx: { - "models": all_models, + "models": models, "auth": ctx.auth, }, ) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 5e497607..5fca388c 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -188,13 +188,13 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {% elif route.operation == "create" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate_{{ collection_name }}_create(body) + data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) document = await service.create(data) return 200, document {% elif route.operation == "update" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate_{{ collection_name }}_update(body) + data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) {% if route is needs_post_fetch %} {% if route.record_parent_params %} existing = await service.get_by({{ record_criteria(route) }}) @@ -212,7 +212,7 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {% elif route.operation == "replace" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate_{{ collection_name }}_replace(body) + data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) {% if route is needs_post_fetch %} {% if route.record_parent_params %} existing = await service.get_by({{ record_criteria(route) }}) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja index 79e3f523..06f8a2c4 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja @@ -1,129 +1,36 @@ -from datetime import datetime +from typing import Any, NamedTuple +from fields import conform_object -def _check_string(value, field_name, constraints=None): - if not isinstance(value, str): - return f"{field_name} must be a string" - if constraints: - if "minLength" in constraints and len(value) < constraints["minLength"]: - return f"{field_name} must be at least {constraints['minLength']} characters" - if "maxLength" in constraints and len(value) > constraints["maxLength"]: - return f"{field_name} must be at most {constraints['maxLength']} characters" - return None - -def _check_number(value, field_name, constraints=None): - if not isinstance(value, (int, float)) or isinstance(value, bool): - return f"{field_name} must be a number" - if constraints: - if "min" in constraints and value < constraints["min"]: - return f"{field_name} must be >= {constraints['min']}" - if "max" in constraints and value > constraints["max"]: - return f"{field_name} must be <= {constraints['max']}" - return None - - -def _check_boolean(value, field_name): - if not isinstance(value, bool): - return f"{field_name} must be a boolean" - return None - - -def _check_datetime(value, field_name): - if not isinstance(value, str): - return f"{field_name} must be an ISO 8601 datetime string" - try: - datetime.fromisoformat(value) - except (ValueError, TypeError): - return f"{field_name} must be a valid ISO 8601 datetime" - return None - - -def _check_enum(value, field_name, allowed): - if value not in allowed: - return f"{field_name} must be one of: {', '.join(str(v) for v in allowed)}" - return None - - -def _check_array(value, field_name): - if not isinstance(value, list): - return f"{field_name} must be an array" - return None - - -def _check_object(value, field_name): - if not isinstance(value, dict): - return f"{field_name} must be an object" - return None +class _Payload(NamedTuple): + base: str | None + properties: dict[str, dict[str, Any]] + partial: bool = False +_PAYLOADS = { {% for model in models %} -def validate_{{ model.collection }}_{{ model.operation }}(body: dict) -> dict: - """Validate and return cleaned data for {{ model.collection }} {{ model.operation }}.""" - if not isinstance(body, dict): - raise ValueError("Request body must be a JSON object") - - known_fields = { {{ model.fields | map(attribute='name') | map('tojson') | join(', ') }} } - unknown = set(body.keys()) - known_fields - if unknown: - raise ValueError(f"Unknown fields: {', '.join(sorted(unknown))}") - -{% if model.operation in ("create",) %} -{% for field in model.fields %} -{% if field.required %} - if {{ field.name | tojson }} not in body: - raise ValueError("Missing required field: {{ field.name }}") + {{ model.name | tojson }}: _Payload( + {{ model.base | tojson if model.base else "None" }}, + {{ "{" }}{% for field in model.fields %}{{ field.name | tojson }}: {{ field.config | py_literal }}{{ ", " if not loop.last }}{% endfor %}{{ "}" }}, +{% if model.operation == "update" %} + partial=True, {% endif %} + ), {% endfor %} -{% endif %} +} - errors = [] -{% for field in model.fields %} - if {{ field.name | tojson }} in body: -{% if field.type == "string" or field.type == "ref" %} - err = _check_string(body[{{ field.name | tojson }}], {{ field.name | tojson }}{% if field.constraints %}, {{ field.constraints }}{% endif %}) - if err: - errors.append(err) -{% elif field.type == "number" %} - err = _check_number(body[{{ field.name | tojson }}], {{ field.name | tojson }}{% if field.constraints %}, {{ field.constraints }}{% endif %}) - if err: - errors.append(err) -{% elif field.type == "boolean" %} - err = _check_boolean(body[{{ field.name | tojson }}], {{ field.name | tojson }}) - if err: - errors.append(err) -{% elif field.type == "datetime" %} - err = _check_datetime(body[{{ field.name | tojson }}], {{ field.name | tojson }}) - if err: - errors.append(err) -{% elif field.type == "enum" %} - err = _check_enum(body[{{ field.name | tojson }}], {{ field.name | tojson }}, {{ field.enum_values | tojson }}) - if err: - errors.append(err) -{% elif field.type == "array" %} - err = _check_array(body[{{ field.name | tojson }}], {{ field.name | tojson }}) - if err: - errors.append(err) -{% elif field.type == "object" %} - err = _check_object(body[{{ field.name | tojson }}], {{ field.name | tojson }}) - if err: - errors.append(err) -{% endif %} -{% endfor %} - if errors: - raise ValueError("; ".join(errors)) +def _properties(model: str) -> dict[str, dict[str, Any]]: + payload = _PAYLOADS[model] + return {**(_properties(payload.base) if payload.base else {}), **payload.properties} - data = {k: v for k, v in body.items() if k in known_fields} -{% for field in model.fields %} -{% if field.default is not none and field.name != "password" %} - if {{ field.name | tojson }} not in data: - data[{{ field.name | tojson }}] = {{ field.default | tojson }} -{% endif %} -{% endfor %} - return data -{% endfor %} +def validate(model: str, body: Any) -> dict[str, Any]: + """``body`` checked against the payload model ``model``.""" + return conform_object(_properties(model), body, partial=_PAYLOADS[model].partial) + {% for identity in auth.identity_collections %} {% if identity.has_password and identity.identifier %} def validate_{{ identity.name }}_login(body: dict) -> dict: diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index c784ee7d..aa5ae0c5 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -161,16 +161,10 @@ class {{ class_name }}Repository: await self._col.insert_one(data) return self._normalize(data) - @_translate_conflict async def update( self, document_id: str, data: dict[str, Any] ) -> dict[str, Any] | None: - doc = await self._col.find_one_and_update( - self._match_filter(document_id), - {"$set": data}, - return_document=ReturnDocument.AFTER, - ) - return self._normalize(doc) + return await self.update_by(self._match_filter(document_id), data) @_translate_conflict async def replace( diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja index 3043bf58..d0d34541 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/service_inputs.py.jinja @@ -1,7 +1,8 @@ -from datetime import datetime from types import SimpleNamespace from typing import Any +from {{ fields_module }} import conform_inputs + class RawRequest: def __init__( @@ -35,12 +36,14 @@ def prepare_service_inputs( forwarding: dict[str, dict[str, Any]], bound_inputs: dict[str, Any] | None = None, ) -> dict[str, Any]: - values = { - **(body if isinstance(body, dict) else {}), - **path_values, - **(bound_inputs or {}), - } - _validate(values, field_configs) + values = conform_inputs( + field_configs, + { + **(body if isinstance(body, dict) else {}), + **path_values, + **(bound_inputs or {}), + }, + ) return { name: _resolve(binding, values) for name, binding in forwarding.items() @@ -77,68 +80,3 @@ def _resolve_path(path: list[str], values: dict[str, Any]) -> Any: return None current = current[segment] return current - - -def _validate( - values: dict[str, Any], field_configs: dict[str, dict[str, Any]] -) -> None: - missing = [ - name - for name, config in field_configs.items() - if config.get("required", True) and name not in values - ] - if missing: - raise ValueError(f"Missing required inputs: {', '.join(missing)}") - for name, config in field_configs.items(): - if name not in values or ( - values[name] is None and not config.get("required", True) - ): - continue - _validate_value(name, values[name], config) - - -def _validate_value(name: str, value: Any, config: dict[str, Any]) -> None: - field_type = config.get("type", "string") - if field_type in ("string", "ref"): - if not isinstance(value, str): - raise ValueError(f"Field '{name}' must be a string") - _validate_length(name, value, config) - elif field_type == "number": - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"Field '{name}' must be a number") - _validate_range(name, value, config) - elif field_type == "boolean" and not isinstance(value, bool): - raise ValueError(f"Field '{name}' must be a boolean") - elif field_type == "datetime": - if not isinstance(value, str): - raise ValueError(f"Field '{name}' must be a datetime string") - try: - datetime.fromisoformat(value) - except (TypeError, ValueError) as exc: - raise ValueError(f"Field '{name}' is not a valid ISO datetime") from exc - elif field_type == "enum" and config.get("values") is not None: - if value not in config["values"]: - choices = ", ".join(str(item) for item in config["values"]) - raise ValueError(f"Field '{name}' must be one of: {choices}") - elif field_type == "array" and not isinstance(value, list): - raise ValueError(f"Field '{name}' must be an array") - elif field_type == "object" and not isinstance(value, dict): - raise ValueError(f"Field '{name}' must be an object") - - -def _validate_length(name: str, value: str, config: dict[str, Any]) -> None: - minimum = config.get("minLength") - maximum = config.get("maxLength") - if minimum is not None and len(value) < minimum: - raise ValueError(f"Field '{name}' must have at least {minimum} characters") - if maximum is not None and len(value) > maximum: - raise ValueError(f"Field '{name}' must have at most {maximum} characters") - - -def _validate_range(name: str, value: float, config: dict[str, Any]) -> None: - minimum = config.get("min") - maximum = config.get("max") - if minimum is not None and value < minimum: - raise ValueError(f"Field '{name}' must be >= {minimum}") - if maximum is not None and value > maximum: - raise ValueError(f"Field '{name}' must be <= {maximum}") diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 5a78e60e..10ab1474 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -12,7 +12,7 @@ _RUNTIME_MODULES = frozenset( - {"dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} + {"dpop", "pagination", "fields", "payloads", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} ) diff --git a/src/yaml_api_generator/ir/__init__.py b/src/yaml_api_generator/ir/__init__.py index 410ed256..e84f8959 100644 --- a/src/yaml_api_generator/ir/__init__.py +++ b/src/yaml_api_generator/ir/__init__.py @@ -17,6 +17,8 @@ IRSymbolRef, IRWhereConjunct, WhereRhs, + SECRET_FIELD, + writable_fields, IRMongo, IRMongoDatastore, IRPolicy, @@ -107,6 +109,8 @@ "IRSymbolRef", "IRWhereConjunct", "WhereRhs", + "SECRET_FIELD", + "writable_fields", "IRMongo", "IRMongoDatastore", "IRPolicy", diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 652cd674..b59e7d70 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -19,7 +19,7 @@ eliminating repeated YAML parsing and validation. """ -from collections.abc import Iterable +from collections.abc import Collection, Iterable from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any, ClassVar @@ -248,6 +248,25 @@ class IRCollection: file_store: str | None = None +SECRET_FIELD: dict[str, Any] = {"type": "string", "required": True, "minLength": 1} +"""The config of a credential a request sends, such as a password.""" + + +def writable_fields( + fields: dict[str, Any], excluded: Collection[str] = () +) -> dict[str, dict[str, Any]]: + """The field configs a client may send: no ids, readonly, auth or ``excluded`` fields.""" + return { + name: config + for name, config in fields.items() + if isinstance(config, dict) + and config.get("type") != "id" + and not config.get("readonly", False) + and name != "_auth" + and name not in excluded + } + + @dataclass(frozen=True) class IRSymbolRef: root: str @@ -359,6 +378,18 @@ class IRRoute: handler: str | None = None pagination: IRPagination | None = None + def body_dto(self, dtos: dict[str, IRDto]) -> IRDto | None: + """The DTO that the first DTO-typed request-body binding gives this route's body.""" + return next( + ( + dtos[binding.type_name] + for rule in self.access_rules or () + for binding in rule.request_symbols.values() + if binding.part in (None, "body") and binding.type_name in dtos + ), + None, + ) + @dataclass(frozen=True) class IRFileRoute: diff --git a/src/yaml_api_generator/parsers/transformers/constraint_validation.py b/src/yaml_api_generator/parsers/transformers/constraint_validation.py index 8f9289cf..16caedcc 100644 --- a/src/yaml_api_generator/parsers/transformers/constraint_validation.py +++ b/src/yaml_api_generator/parsers/transformers/constraint_validation.py @@ -13,6 +13,7 @@ from yaml_api_generator.parsers.transformers.file_validation import file_config from yaml_api_generator.parsers.yaml_parser import ConfigError +from yaml_api_generator.runtime.fields import conform # Constraints only meaningful for a given field type. min/max belong to numbers; # minLength/maxLength belong to strings. format is shared by string and id @@ -32,20 +33,6 @@ def _error(message: str, path: str, hint: str) -> ConfigError: ) -def _matches_type(field_type: str, value: Any) -> bool: - if field_type == "string": - return isinstance(value, str) - if field_type == "number": - return isinstance(value, (int, float)) and not isinstance(value, bool) - if field_type == "boolean": - return isinstance(value, bool) - if field_type == "array": - return isinstance(value, list) - if field_type == "object": - return isinstance(value, dict) - return True - - def _check_field(field: dict[str, Any], path: str, errors: list[ConfigError]) -> None: field_type = field.get("type") @@ -118,7 +105,7 @@ def _check_field(field: dict[str, Any], path: str, errors: list[ConfigError]) -> )) if "default" in field: - _check_default(field, field_type, path, errors) + _check_default(field, path, errors) if field_type == "array" and isinstance(field.get("items"), dict): _check_field(field["items"], f"{path}.items", errors) @@ -128,32 +115,23 @@ def _check_field(field: dict[str, Any], path: str, errors: list[ConfigError]) -> _check_field(prop, f"{path}.properties.{name}", errors) -def _check_default( - field: dict[str, Any], field_type: str, path: str, errors: list[ConfigError] -) -> None: +def _check_default(field: dict[str, Any], path: str, errors: list[ConfigError]) -> None: + """A default must satisfy its own field config, nested properties and items included. + + Interpolation has already run, so a numeric or boolean looking string is an + env-driven default (``${PORT:-8080}``) and is accepted for its type. + """ default = field["default"] - if default is None: + if default is None or file_config(field) is not None: return - - if field_type == "enum": - values = field.get("values") - if isinstance(values, list) and values and default not in values: - errors.append(_error( - f"default '{default}' is not one of the enum values {values}", path, - "Set default to one of the listed values", - )) + if isinstance(default, str) and _coerces_to(field.get("type"), default): return - - # Interpolation has already run, so a remaining string is a genuine literal. - # Numbers and booleans coerce cleanly from numeric/boolean-looking strings - # (e.g. an env-driven "${PORT:-8080}" default), so only flag strings that - # cannot be coerced into the declared type. - if not _matches_type(field_type, default): - if isinstance(default, str) and _coerces_to(field_type, default): - return + try: + conform(field, default, path) + except ValueError as error: errors.append(_error( - f"default {default!r} does not match field type '{field_type}'", path, - f"Provide a {field_type} default or remove it", + f"default {default!r} is invalid: {error}", path, + "Make the default satisfy the field's own constraints or remove it", )) diff --git a/src/yaml_api_generator/parsers/transformers/dto_validation.py b/src/yaml_api_generator/parsers/transformers/dto_validation.py index d6416fd3..aade2147 100644 --- a/src/yaml_api_generator/parsers/transformers/dto_validation.py +++ b/src/yaml_api_generator/parsers/transformers/dto_validation.py @@ -7,7 +7,7 @@ - DTO_FORBIDS_PERSISTENCE_ATTR: a DTO field declares ``type: id``, ``unique`` or ``format``. -- DTO_NOT_FOUND: a DTO field references a type that is neither a declared entity +- DTO_NOT_FOUND: a DTO field, at any nesting depth, references a type that is neither a declared entity nor a declared DTO. - FILE_FIELD_CONTEXT_INVALID: a DTO field is, or contains, a ``type: file`` node. """ @@ -15,6 +15,7 @@ from typing import Any from yaml_api_generator.parsers.transformers.file_validation import file_context_errors +from yaml_api_generator.parsers.transformers.ref_validation import field_refs from yaml_api_generator.parsers.yaml_parser import ConfigError _FORBIDDEN_ATTRS = ("unique", "format") @@ -37,13 +38,15 @@ def validate_spec_dtos(spec: dict[str, Any]) -> list[ConfigError]: for attr in _FORBIDDEN_ATTRS: if attr in field: errors.append(_forbidden(attr, f"{base}.{attr}")) - ref = field.get("ref") - if field.get("type") == "ref" and ref is not None and ref not in known_refs: - errors.append(ConfigError( - code="DTO_NOT_FOUND", - message=f"DTO field references undeclared type '{ref}'", - path=f"{base}.ref", - )) + errors.extend( + ConfigError( + code="DTO_NOT_FOUND", + message=f"DTO field references undeclared type '{target}'", + path=path, + ) + for path, target in field_refs(dto.get("fields") or {}, f"dto.{dto_name}.fields") + if target not in known_refs + ) return errors diff --git a/src/yaml_api_generator/parsers/transformers/ref_validation.py b/src/yaml_api_generator/parsers/transformers/ref_validation.py new file mode 100644 index 00000000..514b3c95 --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/ref_validation.py @@ -0,0 +1,42 @@ +"""Reference targets of ``type: ref`` fields in a v0.2 spec. + +REF_TARGET_NOT_FOUND: an entity field, at any nesting depth, references a name +that is not a declared entity. +""" + +from collections.abc import Iterator +from typing import Any + +from yaml_api_generator.parsers.yaml_parser import ConfigError + + +def field_refs(fields: dict[str, Any], base: str) -> Iterator[tuple[str, str]]: + """The ``(path, target)`` of every ref field, inside object properties and array items too.""" + for name, field in fields.items(): + if isinstance(field, dict): + yield from _refs(field, f"{base}.{name}") + + +def _refs(field: dict[str, Any], path: str) -> Iterator[tuple[str, str]]: + if field.get("type") == "ref" and isinstance(field.get("ref"), str): + yield f"{path}.ref", field["ref"] + if isinstance(field.get("items"), dict): + yield from _refs(field["items"], f"{path}.items") + if isinstance(field.get("properties"), dict): + yield from field_refs(field["properties"], f"{path}.properties") + + +def validate_spec_refs(spec: dict[str, Any]) -> list[ConfigError]: + entities = spec.get("entities") or {} + return [ + ConfigError( + code="REF_TARGET_NOT_FOUND", + message=f"Field references undeclared entity '{target}'", + path=path, + hint=f"Declare an entity named '{target}' or reference one of: {', '.join(sorted(entities))}", + ) + for entity_name, entity in entities.items() + if isinstance(entity, dict) + for path, target in field_refs(entity.get("fields") or {}, f"entities.{entity_name}.fields") + if target not in entities + ] diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 8a1258a6..7aed961f 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -225,6 +225,11 @@ def load_and_generate_ir( if name_errors: raise ConfigErrors(name_errors) + from .transformers.ref_validation import validate_spec_refs + ref_errors = validate_spec_refs(resolved) + if ref_errors: + raise ConfigErrors(ref_errors) + from .transformers.constraint_validation import validate_spec_constraints constraint_errors = validate_spec_constraints(resolved) if constraint_errors: diff --git a/src/yaml_api_generator/runtime/fields.py b/src/yaml_api_generator/runtime/fields.py new file mode 100644 index 00000000..228b4b41 --- /dev/null +++ b/src/yaml_api_generator/runtime/fields.py @@ -0,0 +1,200 @@ +"""Validation of values against spec field configs. + +Shared verbatim by the development server and every generated backend, so a +field config means the same thing wherever a request lands. ``conform`` checks +one value, ``conform_object`` a key/value map and ``conform_inputs`` a service's +inputs; each returns the value with the declared defaults filled in. +""" + +import re +from collections.abc import Callable +from copy import deepcopy +from datetime import datetime +from typing import Any +from urllib.parse import urlparse +from uuid import UUID + + +NOT_AN_OBJECT = "Request body must be a JSON object" + + +class FieldError(ValueError): + def __init__(self, path: str, problem: str) -> None: + super().__init__(f"Field '{path}' {problem}") + + +def _is_uuid(version: int) -> Callable[[str], bool]: + def check(value: str) -> bool: + try: + return UUID(value).version == version + except ValueError: + return False + + return check + + +def _is_uri(value: str) -> bool: + parts = urlparse(value) + return bool(parts.scheme and (parts.netloc or parts.path)) + + +FORMATS: dict[str, Callable[[str], object]] = { + "email": re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+").fullmatch, + "e164": re.compile(r"\+[1-9]\d{1,14}").fullmatch, + "uri": _is_uri, + "uuidv4": _is_uuid(4), + "uuidv7": _is_uuid(7), +} + + +def check_format(format_name: str, value: str, path: str = "value") -> str: + if not FORMATS[format_name](value): + raise FieldError(path, f"must be a valid {format_name}") + return value + + +def is_open(config: dict[str, Any]) -> bool: + """An object accepts undeclared keys unless it declares its properties.""" + return config.get("additionalProperties", "properties" not in config) + + +def _string(config: dict[str, Any], value: Any, path: str) -> str: + if not isinstance(value, str): + raise FieldError(path, "must be a string") + minimum, maximum = config.get("minLength"), config.get("maxLength") + if minimum is not None and len(value) < minimum: + raise FieldError(path, f"must have at least {minimum} characters") + if maximum is not None and len(value) > maximum: + raise FieldError(path, f"must have at most {maximum} characters") + if "format" in config: + check_format(config["format"], value, path) + return value + + +def _number(config: dict[str, Any], value: Any, path: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise FieldError(path, "must be a number") + minimum, maximum = config.get("min"), config.get("max") + if minimum is not None and value < minimum: + raise FieldError(path, f"must be >= {minimum}") + if maximum is not None and value > maximum: + raise FieldError(path, f"must be <= {maximum}") + return value + + +def _boolean(config: dict[str, Any], value: Any, path: str) -> bool: + if not isinstance(value, bool): + raise FieldError(path, "must be a boolean") + return value + + +def _datetime(config: dict[str, Any], value: Any, path: str) -> str: + if not isinstance(value, str): + raise FieldError(path, "must be a datetime string") + try: + datetime.fromisoformat(value) + except ValueError as exc: + raise FieldError(path, "is not a valid ISO datetime") from exc + return value + + +def _enum(config: dict[str, Any], value: Any, path: str) -> Any: + choices = config.get("values", []) + if value not in choices: + raise FieldError(path, f"must be one of: {', '.join(map(str, choices))}") + return value + + +def _array(config: dict[str, Any], value: Any, path: str) -> list[Any]: + if not isinstance(value, list): + raise FieldError(path, "must be an array") + if "items" not in config: + return value + return [ + conform(config["items"], item, f"{path}[{index}]") + for index, item in enumerate(value) + ] + + +def _object(config: dict[str, Any], value: Any, path: str) -> dict[str, Any]: + return conform_object( + config.get("properties", {}), value, open=is_open(config), path=path + ) + + +_TYPES: dict[str, Callable[[dict[str, Any], Any, str], Any]] = { + "string": _string, + "id": _string, + "ref": _string, + "number": _number, + "boolean": _boolean, + "datetime": _datetime, + "enum": _enum, + "array": _array, + "object": _object, +} + + +def conform(config: dict[str, Any], value: Any, path: str) -> Any: + return _TYPES[config.get("type", "string")](config, value, path) + + +def _conform_property(config: dict[str, Any], value: Any, path: str) -> Any: + """A property may be null unless it is required.""" + if value is None and not config.get("required"): + return None + if value is None: + raise FieldError(path, "is required and cannot be null") + return conform(config, value, path) + + +def conform_object( + properties: dict[str, dict[str, Any]], + value: dict[str, Any], + *, + open: bool = False, + partial: bool = False, + path: str = "", +) -> dict[str, Any]: + """Check a map against its declared properties. + + ``partial`` accepts any subset of the properties and fills in no defaults. + """ + if not isinstance(value, dict): + raise FieldError(path, "must be an object") if path else ValueError(NOT_AN_OBJECT) + qualified = (lambda name: f"{path}.{name}") if path else str + unknown = [] if open else sorted(value.keys() - properties.keys()) + if unknown: + raise ValueError(f"Unknown fields: {', '.join(map(qualified, unknown))}") + expected = {} if partial else properties + missing = [ + name + for name, config in expected.items() + if config.get("required") and "default" not in config and name not in value + ] + if missing: + raise ValueError(f"Missing required fields: {', '.join(map(qualified, missing))}") + defaults = { + name: deepcopy(config["default"]) + for name, config in expected.items() + if "default" in config and name not in value + } + return { + **value, + **defaults, + **{ + name: _conform_property(properties[name], item, qualified(name)) + for name, item in value.items() + if name in properties + }, + } + + +def conform_inputs( + inputs: dict[str, dict[str, Any]], values: dict[str, Any] +) -> dict[str, Any]: + """Check service inputs, each required unless it says otherwise.""" + required_by_default = { + name: {"required": True, **config} for name, config in inputs.items() + } + return conform_object(required_by_default, values, open=True) diff --git a/src/yaml_api_generator/runtime/payloads.py b/src/yaml_api_generator/runtime/payloads.py new file mode 100644 index 00000000..bedea369 --- /dev/null +++ b/src/yaml_api_generator/runtime/payloads.py @@ -0,0 +1,55 @@ +"""Pydantic building blocks for request payload models.""" + +from datetime import datetime +from typing import Annotated + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, create_model + +from .fields import check_format + + +SCALAR_TYPES: dict[str, type] = { + "string": str, + "id": str, + "ref": str, + "number": float, + "boolean": bool, + "datetime": datetime, +} +STRICT_TYPES = frozenset({float, bool}) +"""Types never coerced from another JSON type.""" +CONSTRAINTS = {"minLength": "min_length", "maxLength": "max_length", "min": "ge", "max": "le"} +"""Field config keys and the ``Field`` arguments that enforce them.""" + + +class Payload(BaseModel): + """A closed object: undeclared keys are rejected.""" + + model_config = ConfigDict(extra="forbid") + + +class OpenPayload(BaseModel): + """An object that keeps the keys it does not declare.""" + + model_config = ConfigDict(extra="allow") + + +def formatted(format_name: str) -> AfterValidator: + return AfterValidator(lambda value: check_format(format_name, value)) + + +def partial(model: type[BaseModel], name: str) -> type[BaseModel]: + """``model`` with every field omittable; types, constraints and nullability are kept.""" + return create_model( + name, + __base__=Payload, + **{ + field_name: ( + Annotated[ + (info.annotation, *info.metadata, Field(**info.asdict()["attributes"])) + ], + None, + ) + for field_name, info in model.model_fields.items() + }, + ) diff --git a/src/yaml_api_generator/server/common/context_builder.py b/src/yaml_api_generator/server/common/context_builder.py index def4acca..78f501e5 100644 --- a/src/yaml_api_generator/server/common/context_builder.py +++ b/src/yaml_api_generator/server/common/context_builder.py @@ -165,6 +165,7 @@ def _runtime_route(route: IRRoute, document: IRDocument) -> RuntimeRouteDescript service_input=service_input, service_forwarding=service_forwarding, pagination=route.pagination, + body_dto=route.body_dto(document.dtos), ) diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 7ac0cf88..1f0e4dad 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -250,7 +250,7 @@ def _base_steps(self, route: RuntimeRouteDescriptor) -> list[Step]: def _validate_body_step(self, route: RuntimeRouteDescriptor) -> Step: collection = self._ctx.collections[route.collection] identity_collection = self._ctx.auth.identity_collections.get(route.collection) - return ValidateBodyStep(collection, route.operation, identity_collection) + return ValidateBodyStep(collection, route.operation, identity_collection, route.body_dto) def _service_input_step(self, route: RuntimeRouteDescriptor) -> Step | None: if route.service_input is None and not route.service_forwarding: diff --git a/src/yaml_api_generator/server/common/repositories/mongodb.py b/src/yaml_api_generator/server/common/repositories/mongodb.py index a32b857c..50d8a75f 100644 --- a/src/yaml_api_generator/server/common/repositories/mongodb.py +++ b/src/yaml_api_generator/server/common/repositories/mongodb.py @@ -234,13 +234,9 @@ async def update( document_id: str, payload: dict[str, Any], ) -> dict[str, Any] | None: - with _as_conflict(): - document = await self.collection(collection_name).find_one_and_update( - self._match_filter(document_id, collection_name), - {"$set": payload}, - return_document=ReturnDocument.AFTER, - ) - return self._normalize_document(document) + return await self.update_by( + collection_name, self._match_filter(document_id, collection_name), payload + ) async def replace( self, diff --git a/src/yaml_api_generator/server/common/repositories/sql.py b/src/yaml_api_generator/server/common/repositories/sql.py index 89ad9153..42b24b4f 100644 --- a/src/yaml_api_generator/server/common/repositories/sql.py +++ b/src/yaml_api_generator/server/common/repositories/sql.py @@ -252,17 +252,9 @@ async def upsert_by( async def update( self, collection_name: str, document_id: str, payload: dict[str, Any] ) -> dict[str, Any] | None: - table = self._table(collection_name) - values = self._known_columns(collection_name, payload, drop_id=True) - with _as_conflict(): - async with await self._begin() as conn: - result = await conn.execute( - sa.update(table) - .where(self._id_column(collection_name) == document_id) - .values(**values) - .returning(table) - ) - return self._one(result) + return await self.update_by( + collection_name, {self._id_fields[collection_name]: document_id}, payload + ) async def replace( self, collection_name: str, document_id: str, payload: dict[str, Any] diff --git a/src/yaml_api_generator/server/common/server_context.py b/src/yaml_api_generator/server/common/server_context.py index 696a613c..45d25866 100644 --- a/src/yaml_api_generator/server/common/server_context.py +++ b/src/yaml_api_generator/server/common/server_context.py @@ -10,7 +10,7 @@ IRRequestBinding, WhereRhs, ) -from yaml_api_generator.ir.api import IRPagination, IRRolesConfig +from yaml_api_generator.ir.api import IRDto, IRPagination, IRRolesConfig from yaml_api_generator.ir.when import WhenExpr @@ -77,6 +77,7 @@ class RuntimeRouteDescriptor: service_input: dict[str, Any] | None = None service_forwarding: dict[str, Any] = field(default_factory=dict) pagination: IRPagination | None = None + body_dto: IRDto | None = None @dataclass(frozen=True) diff --git a/src/yaml_api_generator/server/common/steps/services.py b/src/yaml_api_generator/server/common/steps/services.py index e72a8fd9..42a6a889 100644 --- a/src/yaml_api_generator/server/common/steps/services.py +++ b/src/yaml_api_generator/server/common/steps/services.py @@ -13,10 +13,10 @@ from typing import Any from yaml_api_generator.ir import IRSymbolRef, WhereRhs +from yaml_api_generator.runtime.fields import conform_inputs from ..request_context import RequestContext from . import Step -from .validation import validate_field_values def ServiceInputStep( @@ -24,8 +24,7 @@ def ServiceInputStep( forwarding: dict[str, WhereRhs], ) -> Step: def step(ctx: RequestContext) -> RequestContext: - inputs = _request_namespace(ctx) - validate_field_values(inputs, input_fields) + inputs = conform_inputs(input_fields, _request_namespace(ctx)) forwarded = { name: _resolve(rhs, inputs) for name, rhs in forwarding.items() } diff --git a/src/yaml_api_generator/server/common/steps/validation.py b/src/yaml_api_generator/server/common/steps/validation.py index 79951657..8f23b7ff 100644 --- a/src/yaml_api_generator/server/common/steps/validation.py +++ b/src/yaml_api_generator/server/common/steps/validation.py @@ -1,9 +1,10 @@ """Request body validation pipeline step.""" -from dataclasses import dataclass -from datetime import datetime from typing import Any +from yaml_api_generator.ir import SECRET_FIELD, IRDto, writable_fields +from yaml_api_generator.runtime.fields import NOT_AN_OBJECT, conform_object + from ..request_context import RequestContext from ..server_context import RuntimeCollectionDescriptor, RuntimeIdentityDescriptor from . import Step @@ -12,219 +13,57 @@ _NO_BODY_OPS = frozenset({"list", "get", "delete"}) -@dataclass(frozen=True) -class _FieldSchema: - name: str - field_type: str - required: bool - min: float | None = None - max: float | None = None - min_length: int | None = None - max_length: int | None = None - enum_values: list[str] | None = None - - -def _build_field_schemas( - collection: RuntimeCollectionDescriptor, - operation: str, - identity_collection: RuntimeIdentityDescriptor | None = None, -) -> list[_FieldSchema]: - schemas: list[_FieldSchema] = [] - make_optional = operation == "update" - - for field_name, raw_config in collection.fields.items(): - if not isinstance(raw_config, dict): - continue - if raw_config.get("type") == "id": - continue - if field_name == "_auth" or field_name in collection.files: - continue - - required = not make_optional and bool(raw_config.get("required", False)) - schemas.append( - _FieldSchema( - name=field_name, - field_type=raw_config.get("type", "string"), - required=required, - min=raw_config.get("min"), - max=raw_config.get("max"), - min_length=raw_config.get("minLength"), - max_length=raw_config.get("maxLength"), - enum_values=raw_config.get("values"), - ) - ) - +def credential_properties( + operation: str, identity_collection: RuntimeIdentityDescriptor | None +) -> dict[str, dict[str, Any]]: + """The properties a write operation takes besides the entity's writable fields.""" if ( identity_collection is not None and operation == "create" and identity_collection.has_password - and not any(s.name == "password" for s in schemas) ): - schemas.append( - _FieldSchema( - name="password", - field_type="string", - required=True, - min_length=1, - ) - ) - - return schemas + return {"password": SECRET_FIELD} + return {} -def _build_login_schemas( +def login_properties( identity_collection: RuntimeIdentityDescriptor, -) -> list[_FieldSchema]: - schemas: list[_FieldSchema] = [] - pw_binding = identity_collection.get_binding("password") - if pw_binding and pw_binding.identifier: - schemas.append( - _FieldSchema( - name=pw_binding.identifier, - field_type="string", - required=True, - min_length=1, - ) - ) - schemas.append( - _FieldSchema( - name="password", - field_type="string", - required=True, - min_length=1, - ) - ) - return schemas - - -def _validate_type(name: str, value: Any, schema: _FieldSchema) -> None: - ft = schema.field_type - - if ft in ("string", "ref"): - if not isinstance(value, str): - raise ValueError(f"Field '{name}' must be a string") - if schema.min_length is not None and len(value) < schema.min_length: - raise ValueError( - f"Field '{name}' must have at least {schema.min_length} characters" - ) - if schema.max_length is not None and len(value) > schema.max_length: - raise ValueError( - f"Field '{name}' must have at most {schema.max_length} characters" - ) - - elif ft == "number": - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"Field '{name}' must be a number") - if schema.min is not None and value < schema.min: - raise ValueError(f"Field '{name}' must be >= {schema.min}") - if schema.max is not None and value > schema.max: - raise ValueError(f"Field '{name}' must be <= {schema.max}") - - elif ft == "boolean": - if not isinstance(value, bool): - raise ValueError(f"Field '{name}' must be a boolean") - - elif ft == "datetime": - if not isinstance(value, str): - raise ValueError(f"Field '{name}' must be a datetime string") - try: - datetime.fromisoformat(value) - except (ValueError, TypeError) as exc: - raise ValueError(f"Field '{name}' is not a valid ISO datetime") from exc - - elif ft == "enum": - if schema.enum_values is not None and value not in schema.enum_values: - raise ValueError( - f"Field '{name}' must be one of: {', '.join(schema.enum_values)}" - ) - - elif ft == "array": - if not isinstance(value, list): - raise ValueError(f"Field '{name}' must be an array") - - elif ft == "object": - if not isinstance(value, dict): - raise ValueError(f"Field '{name}' must be an object") - - -def validate_field_values( - values: dict[str, Any], field_configs: dict[str, dict[str, Any]] -) -> None: - """Validate a value map against declared field configs. - - Declared fields are required by default; a missing or ill-typed value raises - ``ValueError`` (mapped to 422). Shares the type checks used for request bodies. - """ - missing = [ - name - for name, cfg in field_configs.items() - if cfg.get("required", True) and name not in values - ] - if missing: - raise ValueError(f"Missing required inputs: {', '.join(missing)}") - - for name, cfg in field_configs.items(): - if name not in values: - continue - if values[name] is None and not cfg.get("required", True): - continue - schema = _FieldSchema( - name=name, - field_type=cfg.get("type", "string"), - required=bool(cfg.get("required", True)), - min=cfg.get("min"), - max=cfg.get("max"), - min_length=cfg.get("minLength"), - max_length=cfg.get("maxLength"), - enum_values=cfg.get("values"), - ) - _validate_type(name, values[name], schema) +) -> dict[str, dict[str, Any]]: + binding = identity_collection.get_binding("password") + identifier = binding.identifier if binding and binding.identifier else "identifier" + return {identifier: SECRET_FIELD, "password": SECRET_FIELD} def ValidateBodyStep( collection: RuntimeCollectionDescriptor, operation: str, identity_collection: RuntimeIdentityDescriptor | None = None, + dto: IRDto | None = None, ) -> Step: + """Check the body against ``dto`` when the route types it, else against the entity.""" + if operation in _NO_BODY_OPS: + return lambda ctx: ctx if operation == "login": if identity_collection is None: raise ValueError("login operation requires an identity collection") - field_schemas = _build_login_schemas(identity_collection) + properties = login_properties(identity_collection) + elif dto is not None: + properties = writable_fields(dto.fields) else: - field_schemas = _build_field_schemas(collection, operation, identity_collection) - - known_fields = {s.name for s in field_schemas} | set(collection.files) - schema_by_name = {s.name: s for s in field_schemas} + properties = { + **credential_properties(operation, identity_collection), + **writable_fields(collection.fields, collection.files), + } + partial = operation == "update" and dto is None + files = set(collection.files) def step(ctx: RequestContext) -> RequestContext: - if operation in _NO_BODY_OPS: - return ctx - body = ctx.body if not isinstance(body, dict): - raise ValueError("Request body must be a JSON object") - - unknown = set(body.keys()) - known_fields - if unknown: - raise ValueError(f"Unknown fields: {', '.join(sorted(unknown))}") - - if operation != "update": - missing = [ - s.name for s in field_schemas if s.required and s.name not in body - ] - if missing: - raise ValueError(f"Missing required fields: {', '.join(missing)}") - - for key, value in body.items(): - schema = schema_by_name.get(key) - if schema is None: - continue - if value is None and not schema.required: - continue - if value is None and schema.required: - raise ValueError(f"Field '{key}' is required and cannot be null") - _validate_type(key, value, schema) - + raise ValueError(NOT_AN_OBJECT) + uploads = {name: value for name, value in body.items() if name in files} + fields = {name: value for name, value in body.items() if name not in files} + ctx.body = {**conform_object(properties, fields, partial=partial), **uploads} return ctx return step diff --git a/src/yaml_api_generator/server/fastapi/factory.py b/src/yaml_api_generator/server/fastapi/factory.py index d3e4a1c8..34a3e1f6 100644 --- a/src/yaml_api_generator/server/fastapi/factory.py +++ b/src/yaml_api_generator/server/fastapi/factory.py @@ -2,8 +2,7 @@ import re import secrets from collections.abc import Callable, Mapping -from datetime import datetime -from typing import Any, Literal +from typing import Annotated, Any, Literal from fastapi import FastAPI, HTTPException, Request, Response from fastapi.encoders import jsonable_encoder @@ -12,6 +11,13 @@ from pydantic import BaseModel, Field, create_model from yaml_api_generator.runtime.dpop import DPoPProofFactory +from yaml_api_generator.runtime.payloads import ( + CONSTRAINTS, + SCALAR_TYPES, + STRICT_TYPES, + Payload, + partial, +) from yaml_api_generator.runtime.files import FileError, RecordFiles from yaml_api_generator.runtime.file_requests import ( json_request_body, @@ -20,7 +26,8 @@ upload_request_body, ) from yaml_api_generator.runtime.multipart import file_download_responses, file_upload_body -from yaml_api_generator.ir.api import IROIDCProvider, IRProviderBinding +from yaml_api_generator.ir import writable_fields +from yaml_api_generator.ir.api import IRDto, IROIDCProvider, IRProviderBinding from yaml_api_generator.utils.strings import to_pascal_case, to_py_identifier from ..common.auth import oidc from ..common.auth.login_codes import login_codes @@ -39,6 +46,7 @@ from ..common.request_body import RecordBody, RequestBody from ..common.request_context import RequestContext from ..common.steps import ConflictError, ForbiddenError, NotFoundError, UnauthorizedError +from ..common.steps.validation import credential_properties, login_properties from .session_cookie import clear_session_cookies, issue_csrf_token, set_session_cookies @@ -55,96 +63,60 @@ } -def create_pydantic_model_fields( - collection: RuntimeCollectionDescriptor, - operation: str, - identity_collection: RuntimeIdentityDescriptor | None = None, -) -> dict[str, tuple[Any, Any]]: - make_optional = operation == "update" - fields: dict[str, tuple[Any, Any]] = {} +def _model_field(config: dict[str, Any]) -> tuple[Any, Any]: + annotation = _field_annotation(config) + constraints = {CONSTRAINTS[key]: config[key] for key in CONSTRAINTS if key in config} + if config.get("required"): + return annotation, Field(config.get("default", ...), **constraints) + return annotation | None, Field(config.get("default"), **constraints) - for field_name, raw_config in collection.fields.items(): - if not isinstance(raw_config, dict): - continue - if raw_config.get("type") == "id": - continue - if field_name == "_auth" or field_name in collection.files: - continue - - annotation = _field_annotation(raw_config) - required = bool(raw_config.get("required", False)) - has_default = "default" in raw_config - default_value: Any - - if make_optional: - annotation = annotation | None - default_value = raw_config.get("default", None) - elif has_default: - default_value = raw_config["default"] - elif required: - default_value = ... - else: - annotation = annotation | None - default_value = None - - field_kwargs: dict[str, Any] = {} - if raw_config.get("min") is not None: - field_kwargs["ge"] = raw_config["min"] - if raw_config.get("max") is not None: - field_kwargs["le"] = raw_config["max"] - if raw_config.get("minLength") is not None: - field_kwargs["min_length"] = raw_config["minLength"] - if raw_config.get("maxLength") is not None: - field_kwargs["max_length"] = raw_config["maxLength"] - - default: Any = ( - Field(default_value, **field_kwargs) if field_kwargs else default_value - ) - fields[field_name] = (annotation, default) - - if ( - identity_collection is not None - and operation == "create" - and identity_collection.has_password - and "password" not in fields - ): - fields["password"] = (str, Field(..., min_length=1)) - return fields +def create_pydantic_model_fields( + properties: dict[str, dict[str, Any]], +) -> dict[str, tuple[Any, Any]]: + return {name: _model_field(config) for name, config in properties.items()} def create_pydantic_model( collection: RuntimeCollectionDescriptor, operation: str, identity_collection: RuntimeIdentityDescriptor | None = None, + dto: IRDto | None = None, ) -> type[BaseModel] | None: + """The body model of ``operation``; nested objects and formats are left to ``ValidateBodyStep``.""" + model_name = to_pascal_case(collection.name) if operation == "login" and identity_collection is not None: - pw_binding = identity_collection.get_binding("password") - identifier_field = pw_binding.identifier if pw_binding else "identifier" - fields = { - identifier_field: (str, Field(..., min_length=1)), - "password": (str, Field(..., min_length=1)), - } - model_name = to_pascal_case(collection.name) - return create_model(f"{model_name}LoginPayload", **fields) - + return create_model( + f"{model_name}LoginPayload", + __base__=Payload, + **create_pydantic_model_fields(login_properties(identity_collection)), + ) if operation == "refresh": return create_model( - f"{to_pascal_case(collection.name)}RefreshPayload", + f"{model_name}RefreshPayload", + __base__=Payload, refresh_token=(str, Field(..., min_length=1)), ) - if operation not in WRITE_OPERATIONS: return None - - model_fields = create_pydantic_model_fields( - collection, - operation, - identity_collection, + if dto is not None: + return create_model( + to_pascal_case(dto.name), + __base__=Payload, + **create_pydantic_model_fields(writable_fields(dto.fields)), + ) + fields = create_model( + f"{model_name}Fields", + __base__=Payload, + **create_pydantic_model_fields(writable_fields(collection.fields, collection.files)), + ) + if operation == "update": + return partial(fields, f"{model_name}UpdatePayload") + return create_model( + f"{model_name}{operation.capitalize()}Payload", + __base__=fields, + **create_pydantic_model_fields(credential_properties(operation, identity_collection)), ) - operation_suffix = operation.capitalize() - model_name = to_pascal_case(collection.name) - return create_model(f"{model_name}{operation_suffix}Payload", **model_fields) def register_fastapi_routes( @@ -169,6 +141,7 @@ def register_fastapi_routes( collection, route.operation, identity_collection, + route.body_dto, ) if ctx.session.uses_cookie and route.operation == "refresh": body_model = None @@ -592,30 +565,16 @@ async def endpoint() -> Response: return endpoint -def _field_annotation(field_config: dict[str, Any]) -> Any: - field_type = field_config.get("type") - - if field_type in {"string", "id", "ref"}: - return str - if field_type == "number": - return float - if field_type == "boolean": - return bool - if field_type == "datetime": - return datetime - if field_type == "enum": - values = field_config.get("values") - if isinstance(values, list) and values: - return Literal[tuple(values)] - return str - if field_type == "array": - item_config = field_config.get("items") - if isinstance(item_config, dict): - return list[_field_annotation(item_config)] - return list[Any] - if field_type == "object": +def _field_annotation(config: dict[str, Any]) -> Any: + kind = config.get("type", "string") + if kind == "enum": + return Literal[tuple(config["values"])] + if kind == "array": + return list[_field_annotation(config["items"])] if "items" in config else list[Any] + if kind == "object": return dict[str, Any] - return Any + scalar = SCALAR_TYPES[kind] + return Annotated[scalar, Field(strict=True)] if scalar in STRICT_TYPES else scalar def _build_endpoint( diff --git a/tests/fixtures/valid/auth-oidc-relations.yaml b/tests/fixtures/valid/auth-oidc-relations.yaml index 96aef87c..fc0ef1e6 100644 --- a/tests/fixtures/valid/auth-oidc-relations.yaml +++ b/tests/fixtures/valid/auth-oidc-relations.yaml @@ -69,7 +69,7 @@ entities: id: { type: id } email: { type: string, required: true, unique: true } name: { type: string, required: true } - organizationId: { type: ref, ref: organizations } + organizationId: { type: ref, ref: organization } orgRole: { type: string } repositories: diff --git a/tests/integration/test_generated_raw_payloads.py b/tests/integration/test_generated_raw_payloads.py new file mode 100644 index 00000000..8afbfaba --- /dev/null +++ b/tests/integration/test_generated_raw_payloads.py @@ -0,0 +1,49 @@ +"""A generated raw-ASGI app validates the shared payload scenario against a real MongoDB.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from uuid import uuid4 + +import pytest + +from tests.unit.support.payload_requests import EXPECTED, JWT_SECRET, MONGO_STORE, spec +from yaml_api_generator.cli.compiler import compile_project + + +ROOT = Path(__file__).parents[2] + +_RUNNER = f""" +import json, sys +sys.path.append({str(ROOT)!r}) +from starlette.testclient import TestClient +from tests.unit.support.payload_requests import scenario +import app + +with TestClient(app.app) as client: + print(json.dumps(scenario(client))) +""" + + +@pytest.mark.integration +def test_generated_raw_validates_payloads(tmp_path, mongo_uri): + spec_file = tmp_path / "spec.yaml" + spec_file.write_text(spec(MONGO_STORE)) + compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "raw"), target="raw") + result = subprocess.run( + [sys.executable, "-c", _RUNNER], + cwd=tmp_path / "raw" / "backend", + env={ + **os.environ, + "MONGO_URI": mongo_uri, + "MONGO_DB": f"payloads_{uuid4().hex}", + "JWT_SECRET": JWT_SECRET, + "PASSWORD_HASH_ROUNDS": "1000", + }, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.splitlines()[-1]) == EXPECTED diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py new file mode 100644 index 00000000..2f6c290d --- /dev/null +++ b/tests/unit/support/payload_requests.py @@ -0,0 +1,165 @@ +"""Valid and invalid request payloads, sent alike to every app under test. + +The spec covers formats, array items, closed nested objects, defaults, readonly +fields, strict scalars, partial updates and a DTO-typed body, which an access +rule binds for any signed-in user. +""" + +import textwrap + +from starlette.testclient import TestClient + + +SQL_STORE = """ +datastores: + store: + type: postgres + dsn: "postgresql+asyncpg://localhost/unused" + default: true +""" + +MONGO_STORE = """ +mongo: + uri: mongodb://localhost:27017 + database: payloads +""" + +JWT_SECRET = "payload-scenario-secret-of-at-least-32-bytes" + +_API = """ +version: "0.2" +api: + basePath: "" + operations: + defaults: { list: true, get: true, create: true, update: true, delete: true } + auth: { register: true, login: true } +""" + +_ENTITIES = """ +dto: + noteInput: + fields: + role: { type: enum, values: [admin, member], required: true } +entities: + user: + type: auth + auth: { identifier: email, via: password } + fields: + id: { type: id, format: uuidv4 } + email: { type: string, required: true, unique: true } + item: + fields: + id: { type: id, format: uuidv4 } + name: { type: string, required: true, minLength: 2 } + email: { type: string, format: email } + status: { type: enum, values: [draft, live], default: draft } + rank: { type: number } + active: { type: boolean } + stamp: { type: string, readonly: true } + tags: { type: array, items: { type: string, maxLength: 3 } } + meta: + type: object + properties: + color: { type: string, required: true } + size: { type: number, min: 1 } + additionalProperties: false + note: + fields: + id: { type: id, format: uuidv4 } + role: { type: string } +repositories: + users: { entity: user } + items: { entity: item } + notes: { entity: note } +services: + users: { repository: users } + items: { repository: items } + notes: { repository: notes } +controllers: + users: + service: users + base_path: /users + access: "*" + operations: + register: { access: public } + login: { access: public } + items: + service: items + base_path: /items + access: public + notes: + service: notes + base_path: /notes + operations: + create: + access: + - given: { caller: "identity:user", note: "request.body:noteInput" } +""" + +CREATES = [ + ({"name": "ab", "rank": 3, "active": True, "tags": ["abc"], "meta": {"color": "r", "size": 2}}, 200), + ({"name": "ab", "email": "not-an-email"}, 422), + ({"name": "ab", "tags": ["abcd"]}, 422), + ({"name": "ab", "meta": {"color": "r", "extra": 1}}, 422), + ({"name": "ab", "meta": {"size": 2}}, 422), + ({"name": "ab", "meta": {"color": "r", "size": 0}}, 422), + ({"name": "ab", "meta": {"color": "r", "size": "2"}}, 422), + ({"name": "ab", "rank": "3"}, 422), + ({"name": "ab", "active": "true"}, 422), + ({"name": "ab", "active": 1}, 422), + ({"name": "ab", "stamp": "x"}, 422), + ({"name": "ab", "unknown": 1}, 422), + ({"name": "a"}, 422), + ({}, 422), +] +UPDATES = [ + ({}, 200), + ({"meta": {"color": "b"}}, 200), + ({"name": "a"}, 422), + ({"name": None}, 422), + ({"email": "x"}, 422), + ({"stamp": "x"}, 422), + ({"meta": {"size": 2}}, 422), +] +NOTES = [ + ({"role": "admin"}, 200), + ({"role": "owner"}, 422), + ({"role": "admin", "unknown": 1}, 422), + ({}, 422), +] +EXPECTED = [ + *(["POST /items", body, status] for body, status in CREATES), + *(["PATCH /items", body, status] for body, status in UPDATES), + *(["POST /notes", body, status] for body, status in NOTES), +] + + +def spec(datastore: str) -> str: + return _API + textwrap.dedent(datastore) + _ENTITIES + + +def _bearer(client: TestClient) -> dict[str, str]: + login = {"email": "caller@example.com", "password": "caller-password"} + assert client.post("/users/register", json=login).status_code == 200 + token = client.post("/users/login", json=login).json()["token"] + return {"Authorization": f"Bearer {token}"} + + +def scenario(client: TestClient) -> list[list]: + """Each payload with the status it got, after one valid item is created for the updates.""" + headers = _bearer(client) + created = client.post("/items", json={"name": "ab"}) + assert created.status_code == 200, created.text + item = created.json() + assert item["status"] == "draft", item + return [ + *(["POST /items", body, client.post("/items", json=body).status_code] for body, _ in CREATES), + *( + ["PATCH /items", body, client.patch(f"/items/{item['id']}", json=body).status_code] + for body, _ in UPDATES + ), + *( + ["POST /notes", body, client.post("/notes", json=body, headers=headers).status_code] + for body, _ in NOTES + ), + ] diff --git a/tests/unit/test_compiler_context_builder.py b/tests/unit/test_compiler_context_builder.py index 238942c4..6cc216c6 100644 --- a/tests/unit/test_compiler_context_builder.py +++ b/tests/unit/test_compiler_context_builder.py @@ -128,18 +128,15 @@ def test_auth_internal_field_excluded_from_generated_model(tmp_path): assert "_auth" not in model_content -def test_update_model_fields_are_optional(tmp_path): - """All fields in an update payload must be Optional (partial update support).""" +def test_update_model_is_the_partial_entity_fields(tmp_path): + """The update payload is the entity's fields model with every field omittable.""" ir = _make_ir(collections={"items": _items_col( name={"type": "string", "required": True}, score={"type": "number"}, )}) _gen(ir, tmp_path) model_content = (tmp_path / "models" / "items.py").read_text() - # Find the UpdatePayload class block and verify Optional usage - if "UpdatePayload" in model_content: - update_section = model_content.split("UpdatePayload")[1].split("class ")[0] - assert "Optional" in update_section or "| None" in update_section + assert 'ItemsUpdatePayload = partial(ItemsFields, "ItemsUpdatePayload")' in model_content # --- Field type mapping --- diff --git a/tests/unit/test_constraint_validation.py b/tests/unit/test_constraint_validation.py index 76be7104..0b2c35b3 100644 --- a/tests/unit/test_constraint_validation.py +++ b/tests/unit/test_constraint_validation.py @@ -131,6 +131,32 @@ def test_valid_string_constraints_accepted(): ) == set() +# ---- Defaults conform to their own field config ---- + +@pytest.mark.parametrize("field", [ + {"type": "string", "format": "email", "default": "not-an-email"}, + {"type": "string", "maxLength": 2, "default": "long"}, + {"type": "number", "min": 5, "default": 1}, + {"type": "array", "items": {"type": "string", "maxLength": 1}, "default": ["ab"]}, + {"type": "array", "items": {"type": "number"}, "default": ["x"]}, + {"type": "object", "properties": {"a": {"type": "number"}}, "default": {"b": 1}}, + {"type": "object", "properties": {"a": {"type": "number", "required": True}}, "default": {}}, + {"type": "object", "properties": {"a": {"type": "number", "max": 1}}, "default": {"a": 2}}, +]) +def test_default_violating_its_own_config_rejected(field): + assert "FIELD_CONSTRAINT_INVALID" in _codes(field) + + +@pytest.mark.parametrize("field", [ + {"type": "string", "format": "email", "default": "a@b.co"}, + {"type": "array", "items": {"type": "string", "maxLength": 1}, "default": ["a"]}, + {"type": "object", "properties": {"a": {"type": "number", "required": True}}, "default": {"a": 1}}, + {"type": "object", "default": {"free": "form"}}, +]) +def test_default_satisfying_its_own_config_accepted(field): + assert _codes(field) == set() + + # ---- End to end ---- def test_invalid_constraint_rejected_end_to_end(tmp_path): diff --git a/tests/unit/test_fastapi_factory.py b/tests/unit/test_fastapi_factory.py index 29c1bccb..d2f8d7e7 100644 --- a/tests/unit/test_fastapi_factory.py +++ b/tests/unit/test_fastapi_factory.py @@ -3,7 +3,7 @@ from typing import Any, Literal import pytest -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from yaml_api_generator.ir import IRProviderBinding from yaml_api_generator.server.fastapi.factory import ( @@ -117,12 +117,20 @@ def test_field_type_mapping_string(): assert _field_annotation({"type": "string"}) is str -def test_field_type_mapping_number(): - assert _field_annotation({"type": "number"}) is float +@pytest.mark.parametrize("value", ["3", True]) +def test_number_field_is_never_coerced(value): + number = TypeAdapter(_field_annotation({"type": "number"})) + assert number.validate_python(3) == 3.0 + with pytest.raises(ValidationError): + number.validate_python(value) -def test_field_type_mapping_boolean(): - assert _field_annotation({"type": "boolean"}) is bool +@pytest.mark.parametrize("value", ["true", 1]) +def test_boolean_field_is_never_coerced(value): + boolean = TypeAdapter(_field_annotation({"type": "boolean"})) + assert boolean.validate_python(True) is True + with pytest.raises(ValidationError): + boolean.validate_python(value) def test_field_type_mapping_datetime(): diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index 0bfa73be..25700014 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -1,9 +1,14 @@ from yaml_api_generator.generators.backend.fastapi.generator import FastAPIGenerator from yaml_api_generator.generators.backend.drf.generator import DRFGenerator from yaml_api_generator.generators.backend.raw.generator import RawASGIGenerator -from yaml_api_generator.generators.backend.fastapi.type_mapper import python_type -from yaml_api_generator.generators.backend.drf.type_mapper import drf_field -from yaml_api_generator.generators.backend.compile_context import AppConfig, AuthConfig, CompileContext, FieldDescriptor +from yaml_api_generator.generators.backend.fastapi.type_mapper import PayloadModule +from yaml_api_generator.generators.backend.compile_context import ( + AppConfig, + AuthConfig, + CompileContext, + FieldDescriptor, + ModelDescriptor, +) from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields from yaml_api_generator.generators.backend.context_builder import build_context from yaml_api_generator.ir.api import IROIDCProvider, IRSAMLProvider @@ -108,43 +113,31 @@ def test_fastapi_cors_middleware_when_configured(tmp_path): assert "CORSMiddleware" in app_content -def test_fastapi_type_mapper_all_field_types(): - fields = [ - FieldDescriptor(name="s", type="string", required=True, is_optional=False), - FieldDescriptor(name="n", type="number", required=True, is_optional=False), - FieldDescriptor(name="b", type="boolean", required=True, is_optional=False), - FieldDescriptor(name="d", type="datetime", required=True, is_optional=False), - FieldDescriptor( - name="e", - type="enum", - required=True, - is_optional=False, - enum_values=["a", "b"], - ), - FieldDescriptor( - name="a", - type="array", - required=True, - is_optional=False, - array_item_type="string", - ), - FieldDescriptor(name="o", type="object", required=True, is_optional=False), - FieldDescriptor( - name="r", - type="ref", - required=True, - is_optional=False, - ref_collection="users", - ), - ] - assert python_type(fields[0]) == "str" - assert python_type(fields[1]) == "float" - assert python_type(fields[2]) == "bool" - assert python_type(fields[3]) == "datetime" - assert "Literal['a', 'b']" in python_type(fields[4]) - assert python_type(fields[5]) == "list[str]" - assert python_type(fields[6]) == "dict[str, Any]" - assert python_type(fields[7]) == "str" +def test_fastapi_payload_module_maps_every_field_type(): + fields = { + "s": {"type": "string", "required": True}, + "n": {"type": "number", "required": True}, + "b": {"type": "boolean", "required": True}, + "d": {"type": "datetime", "required": True}, + "e": {"type": "enum", "values": ["a", "b"], "required": True}, + "a": {"type": "array", "items": {"type": "string"}, "required": True}, + "o": {"type": "object", "required": True}, + "r": {"type": "ref", "ref": "users", "required": True}, + } + model = ModelDescriptor( + "ItemsFields", "items", "fields", [FieldDescriptor(n, c) for n, c in fields.items()] + ) + (declaration,) = PayloadModule([model]).declarations + assert {a.name: a.annotation for a in declaration.attributes} == { + "s": "str", + "n": "Annotated[float, Field(strict=True)]", + "b": "Annotated[bool, Field(strict=True)]", + "d": "datetime", + "e": "Literal['a', 'b']", + "a": "list[str]", + "o": "dict[str, Any]", + "r": "str", + } def test_drf_project_scaffold(tmp_path): @@ -236,18 +229,11 @@ def test_drf_access_rule_enforcement_in_views(tmp_path): assert len(posts_view) > 0 -def test_drf_type_mapper_with_constraints(): - field = FieldDescriptor( - name="score", - type="number", - required=True, - is_optional=False, - constraints={"min": 0, "max": 100}, - ) - result = drf_field(field) - assert "serializers.FloatField" in result - assert "min_value=0" in result - assert "max_value=100" in result +def test_drf_serializers_declare_payload_properties(tmp_path): + DRFGenerator().generate(_ctx_from_fixture("basic-crud.yaml"), tmp_path) + serializers = next((tmp_path / "api" / "serializers").glob("[!_]*.py")).read_text() + assert "(PayloadSerializer):" in serializers + assert "properties = {" in serializers def test_raw_asgi_app_file(tmp_path): diff --git a/tests/unit/test_payload_models.py b/tests/unit/test_payload_models.py new file mode 100644 index 00000000..cc56a70b --- /dev/null +++ b/tests/unit/test_payload_models.py @@ -0,0 +1,138 @@ +"""Request payloads are validated by their field configs in every consumer. + +The shared payload scenario runs against both development servers and a +generated FastAPI app executed in a subprocess. DRF cannot run here, so its +generated serializers are checked to delegate to the shared validator. +""" + +import ast +import json +import os +import subprocess +import sys +from collections import Counter +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from tests.unit.support.payload_requests import ( + EXPECTED, + JWT_SECRET, + MONGO_STORE, + SQL_STORE, + scenario, + spec, +) +from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.server import build_app, build_raw_app + + +ROOT = Path(__file__).parents[2] + + +@pytest.fixture +def spec_file(tmp_path, monkeypatch): + monkeypatch.setenv("YAML_API_DB_STORE_DSN", f"sqlite+aiosqlite:///{tmp_path / 'dev.sqlite'}") + monkeypatch.setenv("JWT_SECRET", JWT_SECRET) + path = tmp_path / "spec.yaml" + path.write_text(spec(SQL_STORE)) + return str(path) + + +def _generate(spec_file: str, target: str) -> Path: + output = Path(spec_file).parent / target + compile_project(yaml_file=spec_file, output_dir=str(output), target=target) + return output / "backend" + + +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["fastapi", "raw"]) +def test_dev_server_validates_payloads(spec_file, build): + with TestClient(build(spec_file)) as client: + assert scenario(client) == EXPECTED + + +_RUNNER = f""" +import json, sys +sys.path.append({str(ROOT)!r}) +from fastapi.testclient import TestClient +from tests.unit.support.payload_requests import scenario +import app + +with TestClient(app.app) as client: + print(json.dumps(scenario(client))) +""" + + +def test_generated_fastapi_validates_payloads(spec_file): + backend = _generate(spec_file, "fastapi") + result = subprocess.run( + [sys.executable, "-c", _RUNNER], + cwd=backend, + env={**os.environ, "DATABASE_URL": f"sqlite+aiosqlite:///{backend / 'generated.sqlite'}"}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.splitlines()[-1]) == EXPECTED + + +@pytest.fixture +def mongo_spec_file(tmp_path): + path = tmp_path / "mongo-spec.yaml" + path.write_text(spec(MONGO_STORE)) + return str(path) + + +def test_generated_drf_serializers_validate_through_the_shared_validator(mongo_spec_file): + api = _generate(mongo_spec_file, "drf") / "api" + assert "conform_object(self.properties, data" in (api / "payloads.py").read_text() + assert (api / "fields.py").read_text() == (ROOT / "src/yaml_api_generator/runtime/fields.py").read_text() + serializers = (api / "serializers" / "items.py").read_text() + assert 'ItemsUpdatePayload = partial(ItemsFields, "ItemsUpdatePayload")' in serializers + assert "class ItemsCreatePayload(ItemsFields):" in serializers + assert "NoteInput(data=request.data)" in (api / "views" / "notes.py").read_text() + + +def _is_field_config(node: ast.expr) -> bool: + return isinstance(node, ast.Dict) and any( + isinstance(key, ast.Constant) and key.value == "type" for key in node.keys + ) + + +def _field_declarations(source: str) -> Counter: + """Each field a module declares, as a class annotation or a field config map entry. + + A declaration is its name with its type, so same-named fields of unrelated + payloads stay distinct while a field copied into a sibling payload repeats. + """ + nodes = list(ast.walk(ast.parse(source))) + annotations = ( + (node.target.id, ast.unparse(node.annotation)) + for node in nodes + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) + and node.target.id != "properties" + ) + configured = ( + (key.value, ast.unparse(value)) + for node in nodes + if isinstance(node, ast.Dict) + for key, value in zip(node.keys, node.values) + if isinstance(key, ast.Constant) and _is_field_config(value) + ) + return Counter(annotations) + Counter(configured) + + +@pytest.mark.parametrize("target,modules", [ + ("fastapi", "models/*.py"), + ("drf", "api/serializers/*.py"), + ("raw", "validation.py"), +]) +def test_no_generated_payload_module_repeats_a_field_declaration(mongo_spec_file, target, modules): + sources = {module.name: module.read_text() for module in _generate(mongo_spec_file, target).glob(modules)} + assert "items.py" in sources or "validation.py" in sources + repeated = { + name: [field for field, count in _field_declarations(source).items() if count > 1] + for name, source in sources.items() + } + assert repeated == {name: [] for name in sources} diff --git a/tests/unit/test_ref_validation.py b/tests/unit/test_ref_validation.py new file mode 100644 index 00000000..2df7e6a6 --- /dev/null +++ b/tests/unit/test_ref_validation.py @@ -0,0 +1,45 @@ +import pytest +import yaml + +from yaml_api_generator.parsers.transformers.ref_validation import validate_spec_refs +from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir + + +def _errors(fields: dict) -> list: + return validate_spec_refs({"entities": {"author": {"fields": {}}, "post": {"fields": fields}}}) + + +def test_ref_to_declared_entity_accepted(): + assert _errors({"authorId": {"type": "ref", "ref": "author"}}) == [] + + +@pytest.mark.parametrize("field,path", [ + ({"type": "ref", "ref": "authors"}, "entities.post.fields.x.ref"), + ({"type": "array", "items": {"type": "ref", "ref": "ghost"}}, "entities.post.fields.x.items.ref"), + ( + {"type": "object", "properties": {"owner": {"type": "ref", "ref": "ghost"}}}, + "entities.post.fields.x.properties.owner.ref", + ), +]) +def test_ref_to_undeclared_entity_rejected(field, path): + errors = _errors({"x": field}) + assert [(e.code, e.path) for e in errors] == [("REF_TARGET_NOT_FOUND", path)] + + +def test_ref_to_undeclared_entity_rejected_end_to_end(tmp_path): + spec = { + "version": "0.2", + "api": {"basePath": "/api", "version": "v1"}, + "mongo": {"uri": "mongodb://localhost:27017", "database": "testdb"}, + "entities": { + "post": {"fields": {"id": {"type": "id"}, "authorId": {"type": "ref", "ref": "posts"}}} + }, + "repositories": {"posts": {"entity": "post"}}, + "services": {"posts": {"repository": "posts"}}, + "controllers": {"posts": {"service": "posts", "base_path": "/posts", "access": "public"}}, + } + yaml_file = tmp_path / "spec.yaml" + yaml_file.write_text(yaml.safe_dump(spec)) + with pytest.raises(ConfigErrors) as exc_info: + load_and_generate_ir(str(yaml_file)) + assert {e.code for e in exc_info.value.errors} == {"REF_TARGET_NOT_FOUND"} diff --git a/tests/unit/test_request_access_generation.py b/tests/unit/test_request_access_generation.py index f213ecee..5ccfec56 100644 --- a/tests/unit/test_request_access_generation.py +++ b/tests/unit/test_request_access_generation.py @@ -24,7 +24,7 @@ def test_dto_model_emitted(tmp_path): ctx = _generate(tmp_path) assert [m.name for m in ctx.dto_models] == ["CreateMembership"] dto_module = (tmp_path / "models" / "_dto.py").read_text() - assert "class CreateMembership(BaseModel):" in dto_module + assert "class CreateMembership(Payload):" in dto_module assert "Literal['admin', 'member', 'viewer']" in dto_module diff --git a/tests/unit/test_request_access_validation.py b/tests/unit/test_request_access_validation.py index eeb3ad73..5c9b1f91 100644 --- a/tests/unit/test_request_access_validation.py +++ b/tests/unit/test_request_access_validation.py @@ -162,3 +162,8 @@ def test_request_path_param_not_declared(): def test_request_path_param_declared_valid(): codes = _request_path_codes("/orgs/{orgId}", "req.path.orgId equals 'a'") assert codes == set() + + +def test_dto_nested_ref_to_undeclared_type(): + dto = {"d": {"fields": {"x": {"type": "array", "items": {"type": "ref", "ref": "ghost"}}}}} + assert "DTO_NOT_FOUND" in _dto_codes(dto) diff --git a/tests/unit/test_schema_consumers.py b/tests/unit/test_schema_consumers.py index 6eb53fdd..ec31516c 100644 --- a/tests/unit/test_schema_consumers.py +++ b/tests/unit/test_schema_consumers.py @@ -25,7 +25,6 @@ "maxPageSize": "0.1 api.pagination", "enableStandard": "0.1 security.headers (D16)", "softDelete": "0.1 collections (D19)", - "additionalProperties": "object field option", } # Read by an external tool from a block the generator copies verbatim. diff --git a/tests/unit/test_service_input_codegen.py b/tests/unit/test_service_input_codegen.py index 1706e48d..006fbacd 100644 --- a/tests/unit/test_service_input_codegen.py +++ b/tests/unit/test_service_input_codegen.py @@ -52,9 +52,12 @@ def test_route_descriptors_carry_service_contract(): ], ) def test_generated_backends_compile_and_validate_service_inputs( - generator, helper, tmp_path + generator, helper, tmp_path, monkeypatch ): generator.generate(_context(), tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + for package in ("dependencies", "api", "fields"): + monkeypatch.delitem(sys.modules, package, raising=False) assert compileall.compile_dir(str(tmp_path), quiet=1) generated = _load_helper(tmp_path / helper) @@ -66,7 +69,7 @@ def test_generated_backends_compile_and_validate_service_inputs( fields, forwarding, ) == {"id": "path-value"} - with pytest.raises(ValueError, match="Missing required inputs"): + with pytest.raises(ValueError, match="Missing required fields: postId"): generated.prepare_service_inputs({}, {}, fields, forwarding) From 69b385bce07835152a915c43a732f68c23cbab44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 18:51:36 +0100 Subject: [PATCH 05/89] Layer controller access, conventions and overrides into every route and reject overrides of unexposed operations --- .../parsers/desugar/controller_desugarer.py | 252 ++++++------------ .../schemas/yaml-spec-v0.2.schema.json | 2 - .../schemas/yaml-spec-v0.3.schema.json | 2 - tests/unit/test_desugar.py | 8 + tests/unit/test_v02_transformer.py | 4 + 5 files changed, 97 insertions(+), 171 deletions(-) diff --git a/src/yaml_api_generator/parsers/desugar/controller_desugarer.py b/src/yaml_api_generator/parsers/desugar/controller_desugarer.py index 81592802..ea0c0781 100644 --- a/src/yaml_api_generator/parsers/desugar/controller_desugarer.py +++ b/src/yaml_api_generator/parsers/desugar/controller_desugarer.py @@ -4,196 +4,114 @@ CRUD_ROUTE_CONVENTIONS = { - "list": ("GET", ""), - "get": ("GET", "/{id}"), - "create": ("POST", ""), - "update": ("PATCH", "/{id}"), - "replace": ("PUT", "/{id}"), - "delete": ("DELETE", "/{id}"), + "list": {"method": "GET", "path": ""}, + "get": {"method": "GET", "path": "/{id}"}, + "create": {"method": "POST", "path": ""}, + "update": {"method": "PATCH", "path": "/{id}"}, + "replace": {"method": "PUT", "path": "/{id}"}, + "delete": {"method": "DELETE", "path": "/{id}"}, } AUTH_ROUTE_CONVENTIONS = { - "register": ("POST", "/register"), - "login": ("POST", "/login"), + "register": {"method": "POST", "path": "/register"}, + "login": {"method": "POST", "path": "/login"}, } ALL_ROUTE_CONVENTIONS = {**CRUD_ROUTE_CONVENTIONS, **AUTH_ROUTE_CONVENTIONS} +_CONTROLLER_DEFAULTS = ("service", "access", "operations") + class ControllerDesugarer: - """ - Handles controller desugaring: expand service references into explicit routes. + """Expand every controller into explicit routes. - Responsibilities: - - Detect minified controller definitions (with "service" key) - - Match service operations to controller - - Apply route conventions (method + path for CRUD/auth ops) - - Resolve access control per operation - - Generate routes dict + Each route is layered from least to most specific: the controller + ``access``, what the route exposes (for a minified controller, the + service operation and its method/path convention), the ``operations`` + override and the ``routes`` entry. ``operations.: false`` drops + the route. """ def desugar( self, controllers: dict[str, Any], expanded_services: dict[str, Any] ) -> None: - """ - Desugar controllers in-place: expand service references into routes. - - Args: - controllers: Controller definitions to desugar - expanded_services: Expanded service definitions (post-service desugaring) - - Raises: - ConfigErrors: If validation fails - """ errors: list[ConfigError] = [] - - for ctrl_name, entry in list(controllers.items()): + for ctrl_name, entry in controllers.items(): if not isinstance(entry, dict): continue - - if not self._is_minified_controller(entry): - continue - try: - self._expand_minified_controller( - ctrl_name, entry, expanded_services - ) - except ConfigError as e: - errors.append(e) - + entry["routes"] = self._routes(ctrl_name, entry, expanded_services) + except ConfigErrors as error: + errors.extend(error.errors) + for key in _CONTROLLER_DEFAULTS: + entry.pop(key, None) if errors: raise ConfigErrors(errors) - def _is_minified_controller(self, entry: dict[str, Any]) -> bool: - """Check if controller entry is minified (needs expansion).""" - return "service" in entry - - def _expand_minified_controller( - self, - ctrl_name: str, - entry: dict[str, Any], - expanded_services: dict[str, Any], - ) -> None: - """Expand a minified controller into explicit route definitions.""" - service_name = entry["service"] - base_access = entry.get("access", "public") - ops_overrides = entry.get("operations", {}) - route_overrides = entry.get("routes", {}) - - matching_services = self._find_matching_services( - service_name, expanded_services - ) - - if not matching_services: - raise ConfigError( - code="DESUGAR_SERVICE_NOT_FOUND", - message=f"No services found for service group '{service_name}'", - path=f"controllers.{ctrl_name}.service", - ) - - routes = self._build_routes( - ctrl_name, matching_services, base_access, ops_overrides, route_overrides - ) - - entry.pop("service", None) - entry.pop("access", None) - entry.pop("operations", None) - entry.pop("routes", None) - entry["routes"] = routes - - def _find_matching_services( - self, service_name: str, expanded_services: dict[str, Any] - ) -> dict[str, str]: - """ - Find expanded service keys matching service_name. - - Returns dict: operation -> service_key (e.g., "list" -> "list-users") - """ - matching_services: dict[str, str] = {} - for svc_key in expanded_services.keys(): - if "-" not in svc_key: - continue - op_part, svc_part = svc_key.split("-", 1) - if svc_part == service_name: - matching_services[op_part] = svc_key - return matching_services - - def _build_routes( - self, - ctrl_name: str, - matching_services: dict[str, str], - base_access: str, - ops_overrides: dict[str, Any], - route_overrides: dict[str, Any] | None = None, + def _routes( + self, ctrl_name: str, entry: dict[str, Any], expanded_services: dict[str, Any] ) -> dict[str, Any]: - """Build routes dict from matching services and overrides.""" - routes: dict[str, Any] = {} - errors: list[ConfigError] = [] - effective_route_overrides = route_overrides or {} - - for op_name, svc_key in matching_services.items(): - op_override = ops_overrides.get(op_name) - if op_override is False: - continue - - route_override = effective_route_overrides.get(op_name) - - try: - route = self._build_single_route( - ctrl_name, op_name, svc_key, base_access, op_override, route_override - ) - routes[op_name] = route - except ConfigError as e: - errors.append(e) - + routes = entry.get("routes") or {} + overrides = entry.get("operations") or {} + if "service" in entry: + base = {"access": entry.get("access", "public")} + exposed = self._exposed(ctrl_name, entry["service"], expanded_services) + else: + base = {"access": entry["access"]} if "access" in entry else {} + exposed = {name: {} for name in routes} + + errors = [ + self._unexposed(ctrl_name, section, name) + for section, names in (("operations", overrides), ("routes", routes)) + for name in names + if name not in exposed + ] + layered = { + name: { + **base, + **route, + **(overrides.get(name) if isinstance(overrides.get(name), dict) else {}), + **routes.get(name, {}), + } + for name, route in exposed.items() + if overrides.get(name) is not False + } + errors += [ + ConfigError( + code="DESUGAR_CUSTOM_OP_MISSING_ROUTE", + message=f"Custom operation '{name}' requires 'method' and 'path'", + path=f"controllers.{ctrl_name}.operations.{name}", + ) + for name, route in layered.items() + if "method" not in route or "path" not in route + ] if errors: raise ConfigErrors(errors) - - return routes - - def _build_single_route( - self, - ctrl_name: str, - op_name: str, - svc_key: str, - base_access: str, - op_override: Any, - route_override: Any = None, - ) -> dict[str, Any]: - """Build a single route definition.""" - if op_name in ALL_ROUTE_CONVENTIONS: - method, path = ALL_ROUTE_CONVENTIONS[op_name] - else: - if ( - not isinstance(op_override, dict) - or "method" not in op_override - or "path" not in op_override - ): - raise ConfigError( - code="DESUGAR_CUSTOM_OP_MISSING_ROUTE", - message=f"Custom operation '{op_name}' requires 'method' and 'path'", - path=f"controllers.{ctrl_name}.operations.{op_name}", - ) - method = op_override["method"] - path = op_override["path"] - - access = base_access - if isinstance(route_override, dict) and "access" in route_override: - access = route_override["access"] - elif isinstance(op_override, dict) and "access" in op_override: - access = op_override["access"] - - route = { - "service": svc_key, - "method": method, - "path": path, - "access": access, + return layered + + def _exposed( + self, ctrl_name: str, service_name: str, expanded_services: dict[str, Any] + ) -> dict[str, dict[str, Any]]: + """Map each operation of the service group to its route convention.""" + exposed = { + operation: {"service": svc_key, **ALL_ROUTE_CONVENTIONS.get(operation, {})} + for svc_key in expanded_services + if "-" in svc_key + for operation, group in (svc_key.split("-", 1),) + if group == service_name } - pagination = None - if isinstance(op_override, dict): - pagination = op_override.get("pagination") - if isinstance(route_override, dict) and "pagination" in route_override: - pagination = route_override["pagination"] - if pagination is not None: - route["pagination"] = pagination - return route + if not exposed: + raise ConfigErrors([ConfigError( + code="DESUGAR_SERVICE_NOT_FOUND", + message=f"No services found for service group '{service_name}'", + path=f"controllers.{ctrl_name}.service", + )]) + return exposed + + @staticmethod + def _unexposed(ctrl_name: str, section: str, name: str) -> ConfigError: + return ConfigError( + code="DESUGAR_OPERATION_NOT_EXPOSED", + message=f"Controller '{ctrl_name}' configures '{name}', which it does not expose", + path=f"controllers.{ctrl_name}.{section}.{name}", + ) diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index 71917c0d..2fc7356c 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -874,8 +874,6 @@ }, "else": { "properties": { - "access": false, - "operations": false, "routes": { "additionalProperties": { "$ref": "#/$defs/RouteConfig" diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 79c86eef..147dff0f 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -804,8 +804,6 @@ }, "else": { "properties": { - "access": false, - "operations": false, "routes": { "additionalProperties": { "$ref": "#/$defs/RouteConfig" diff --git a/tests/unit/test_desugar.py b/tests/unit/test_desugar.py index 02fe9871..78aa4e42 100644 --- a/tests/unit/test_desugar.py +++ b/tests/unit/test_desugar.py @@ -167,6 +167,10 @@ def test_type_inherited_from_parent(tmp_path): api: basePath: /api version: v1 + operations: + auth: + register: true + login: true mongo: uri: mongodb://localhost:27017 database: testdb @@ -521,6 +525,10 @@ def test_access_override_per_operation_applied(tmp_path): api: basePath: /api version: v1 + operations: + auth: + register: true + login: true mongo: uri: mongodb://localhost:27017 database: testdb diff --git a/tests/unit/test_v02_transformer.py b/tests/unit/test_v02_transformer.py index 2a3f7854..a27b03ac 100644 --- a/tests/unit/test_v02_transformer.py +++ b/tests/unit/test_v02_transformer.py @@ -558,6 +558,10 @@ def test_103_unscoped_role_with_multiple_auth_entities_raises_ambiguous_error(tm api: basePath: /api version: v1 + operations: + auth: + register: true + login: true mongo: uri: mongodb://localhost:27017 database: testdb From 480cf09599a94407da9fdc6522193c25edbc1fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Wed, 21 Oct 2026 18:51:36 +0100 Subject: [PATCH 06/89] Enforce platform keys in every consumer Security headers on by default with per-version off switches, CORS and pagination for 0.1, session.ttl as the access-token lifetime, deploy port and mongo.local defaults, api.port as the run default, 0.1 roles and shared validators, mongo beside record datastores rejected, refresh without a MongoDB store rejected, dev API-key lookup on the stored field, and examples that build on every backend. --- .../comparison/auth-api-key-proposed.yaml | 29 ++--- examples/comparison/auth-users-current.yaml | 20 ++- examples/comparison/auth-users-proposed.yaml | 66 +++++----- .../comparison/auth-users/post_actions.py | 14 +++ examples/comparison/auth-users/v0.1.yaml | 38 ++---- examples/comparison/auth-users/v0.2.yaml | 76 ++++++------ examples/comparison/blog-proposed.yaml | 49 ++++---- examples/v02/api.minified.yaml | 7 +- examples/v02/api.yaml | 7 +- examples/v02/myapp/handlers/posts.py | 2 +- src/yaml_api_generator/cli/__init__.py | 53 ++++---- .../generators/backend/builders/__init__.py | 3 +- .../backend/builders/settings_builder.py | 9 +- .../generators/backend/compile_context.py | 1 + .../generators/backend/drf/generator.py | 2 + .../drf/templates/requirements.txt.jinja | 3 + .../backend/drf/templates/settings.py.jinja | 20 +++ .../generators/backend/fastapi/generator.py | 4 + .../backend/fastapi/templates/app.py.jinja | 30 +++-- .../generators/backend/raw/generator.py | 2 + .../backend/raw/templates/app.py.jinja | 18 +++ .../raw/templates/requirements.txt.jinja | 3 + .../shared/templates/jwt_utils.py.jinja | 2 +- .../generators/backend/utils.py | 5 +- .../generators/backend/validation.py | 17 +++ .../generators/deploy/context_builder.py | 7 +- src/yaml_api_generator/ir/api.py | 13 ++ .../schemas/yaml-spec-v0.1.schema.json | 4 +- .../schemas/yaml-spec-v0.2.schema.json | 41 ++++++- .../schemas/yaml-spec-v0.3.schema.json | 35 ++++++ .../parsers/transformers/auth_validation.py | 49 +++++--- .../transformers/constraint_validation.py | 12 +- .../parsers/transformers/name_validation.py | 76 ++++++------ .../parsers/transformers/roles_config.py | 64 ++++++++++ .../parsers/transformers/v01/__init__.py | 51 ++++++-- .../v01/collection_transformer.py | 25 ---- .../transformers/v01/identity_transformer.py | 26 ++-- .../parsers/transformers/v02/__init__.py | 8 +- .../transformers/v02/identity_transformer.py | 65 +--------- .../transformers/v02/route_transformer.py | 12 +- src/yaml_api_generator/parsers/yaml_parser.py | 31 ++--- .../runtime/security_headers.py | 54 ++++++++ src/yaml_api_generator/server/asgi/server.py | 9 +- .../server/common/context_builder.py | 2 +- .../server/common/http_middleware.py | 36 ++++++ .../server/common/steps/authentication.py | 2 +- .../server/fastapi/server.py | 13 +- tests/integration/conftest.py | 12 ++ tests/integration/support/v01_platform.py | 71 +++++++++++ tests/integration/test_session_ttl.py | 16 +++ tests/integration/test_v01_platform.py | 30 +++++ tests/unit/support/generated_apps.py | 48 ++++++++ tests/unit/support/http_probes.py | 38 ++++++ tests/unit/support/session_spec.py | 47 +++++++ tests/unit/test_auth_steps.py | 6 +- tests/unit/test_auth_validation.py | 22 +++- tests/unit/test_cli.py | 26 ++++ tests/unit/test_cors.py | 115 ++++++++++++++++++ tests/unit/test_deploy_defaults.py | 73 +++++++++++ tests/unit/test_example_compatibility.py | 6 +- tests/unit/test_examples_build.py | 38 ++++++ tests/unit/test_mongo_graduation.py | 2 + tests/unit/test_schema_consumers.py | 6 - tests/unit/test_schema_rejection.py | 24 ++++ tests/unit/test_security_headers.py | 115 ++++++++++++++++++ tests/unit/test_session_ttl.py | 56 +++++++++ tests/unit/test_v01_transformer.py | 87 +++++++++++++ 67 files changed, 1525 insertions(+), 428 deletions(-) create mode 100644 examples/comparison/auth-users/post_actions.py create mode 100644 src/yaml_api_generator/parsers/transformers/roles_config.py create mode 100644 src/yaml_api_generator/runtime/security_headers.py create mode 100644 src/yaml_api_generator/server/common/http_middleware.py create mode 100644 tests/integration/support/v01_platform.py create mode 100644 tests/integration/test_session_ttl.py create mode 100644 tests/integration/test_v01_platform.py create mode 100644 tests/unit/support/generated_apps.py create mode 100644 tests/unit/support/http_probes.py create mode 100644 tests/unit/support/session_spec.py create mode 100644 tests/unit/test_cors.py create mode 100644 tests/unit/test_deploy_defaults.py create mode 100644 tests/unit/test_examples_build.py create mode 100644 tests/unit/test_security_headers.py create mode 100644 tests/unit/test_session_ttl.py diff --git a/examples/comparison/auth-api-key-proposed.yaml b/examples/comparison/auth-api-key-proposed.yaml index 90dce635..64a043b3 100644 --- a/examples/comparison/auth-api-key-proposed.yaml +++ b/examples/comparison/auth-api-key-proposed.yaml @@ -2,14 +2,15 @@ version: "0.2" api: operations: - defaults: [list, get, create, update, delete] + defaults: { list: true, get: true, create: true, update: true, delete: true } entities: client: type: auth - via: - - provider: apiKey - identifier: clientId + auth: + via: + - provider: apiKey + identifier: clientId fields: id: type: id @@ -35,18 +36,18 @@ repositories: # Services: only expose the operations we want. # replace is available at the repo level but intentionally not exposed. services: - list-clients: { repository: clients, operation: list } - get-client: { repository: clients, operation: get } - create-client: { repository: clients, operation: create } - update-client: { repository: clients, operation: update } - delete-client: { repository: clients, operation: delete } + list_clients: { repository: clients, operation: list } + get_client: { repository: clients, operation: get } + create_client: { repository: clients, operation: create } + update_client: { repository: clients, operation: update } + delete_client: { repository: clients, operation: delete } controllers: clients: base_path: /clients routes: - list: { service: list-clients, method: GET, path: /, access: authenticated } - get: { service: get-client, method: GET, path: /{id}, access: authenticated } - create: { service: create-client, method: POST, path: /, access: public } - update: { service: update-client, method: PATCH, path: /{id}, access: authenticated } - delete: { service: delete-client, method: DELETE, path: /{id}, access: authenticated } + list: { service: list_clients, method: GET, path: /, access: "*" } + get: { service: get_client, method: GET, path: "/{id}", access: "*" } + create: { service: create_client, method: POST, path: /, access: public } + update: { service: update_client, method: PATCH, path: "/{id}", access: "*" } + delete: { service: delete_client, method: DELETE, path: "/{id}", access: "*" } diff --git a/examples/comparison/auth-users-current.yaml b/examples/comparison/auth-users-current.yaml index 32b6e864..5970a915 100644 --- a/examples/comparison/auth-users-current.yaml +++ b/examples/comparison/auth-users-current.yaml @@ -17,8 +17,8 @@ collections: update: authenticated replace: authenticated delete: - - principal: identity:users - when: "principal.sub == document.id" + - principal: "users:*" + when: "principal.sub equals document.id" fields: id: type: id @@ -42,10 +42,10 @@ collections: get: public update: authenticated delete: - - principal: identity:users - when: "principal.sub == document.authorId" - - principal: identity:users - when: "principal.record.role == 'moderator'" + - principal: "users:*" + when: "principal.sub equals document.authorId" + - principal: "users:*" + when: "principal.record.role equals 'moderator'" fields: id: type: id @@ -57,10 +57,4 @@ collections: authorId: type: ref ref: users - operations: - my-posts: - base: list - path: /posts/mine - access: authenticated - filter: - authorId: $principal + # v0.1 has no custom operations, so a "my posts" listing cannot be declared. diff --git a/examples/comparison/auth-users-proposed.yaml b/examples/comparison/auth-users-proposed.yaml index b5154143..6c26a7c0 100644 --- a/examples/comparison/auth-users-proposed.yaml +++ b/examples/comparison/auth-users-proposed.yaml @@ -4,14 +4,15 @@ api: basePath: /api version: v1 operations: - defaults: [list, get, create, update, replace, delete] + defaults: { list: true, get: true, create: true, update: true, replace: true, delete: true } entities: user: type: auth - via: - - provider: password - identifier: email + auth: + via: + - provider: password + identifier: email fields: id: type: id @@ -42,7 +43,7 @@ entities: ref: user # CRUD is implicit. Custom operations added explicitly. -# posts.by-author is available internally but NOT exposed via any controller. +# posts.by_author is exposed through the my_posts service. repositories: users: entity: user @@ -50,30 +51,29 @@ repositories: posts: entity: post operations: - by-author: - base: list - filter: { authorId: $principal } + by_author: + query: + command: find + filter: { authorId: "{{authorId}}" } # Services: business operations exposed to controllers. # Simple CRUD services delegate to a single repo operation. -# login is a custom service with its own step pipeline. services: register: { repository: users, operation: create } login: repository: users - steps: [validate, authenticate, issue-token] - list-users: { repository: users, operation: list } - get-user: { repository: users, operation: get } - update-user: { repository: users, operation: update } - delete-user: { repository: users, operation: delete } + list_users: { repository: users, operation: list } + get_user: { repository: users, operation: get } + update_user: { repository: users, operation: update } + delete_user: { repository: users, operation: delete } - list-posts: { repository: posts, operation: list } - get-post: { repository: posts, operation: get } - create-post: { repository: posts, operation: create } - update-post: { repository: posts, operation: update } - delete-post: { repository: posts, operation: delete } - my-posts: { repository: posts, operation: by-author } + list_posts: { repository: posts, operation: list } + get_post: { repository: posts, operation: get } + create_post: { repository: posts, operation: create } + update_post: { repository: posts, operation: update } + delete_post: { repository: posts, operation: delete } + my_posts: { repository: posts, operation: by_author } controllers: auth: @@ -85,36 +85,36 @@ controllers: users: base_path: /api/v1/users routes: - list: { service: list-users, method: GET, path: /, access: authenticated } - get: { service: get-user, method: GET, path: /{id}, access: authenticated } - update: { service: update-user, method: PATCH, path: /{id}, access: authenticated } + list: { service: list_users, method: GET, path: /, access: "*" } + get: { service: get_user, method: GET, path: "/{id}", access: "*" } + update: { service: update_user, method: PATCH, path: "/{id}", access: "*" } delete: - service: delete-user + service: delete_user method: DELETE path: /{id} access: - given: - principal: identity:users + principal: identity:user document: users when: principal.sub equals document.id posts: base_path: /api/v1/posts routes: - list: { service: list-posts, method: GET, path: /, access: public } - get: { service: get-post, method: GET, path: /{id}, access: public } - create: { service: create-post, method: POST, path: /, access: public } - update: { service: update-post, method: PATCH, path: /{id}, access: authenticated } + list: { service: list_posts, method: GET, path: /, access: public } + get: { service: get_post, method: GET, path: "/{id}", access: public } + create: { service: create_post, method: POST, path: /, access: public } + update: { service: update_post, method: PATCH, path: "/{id}", access: "*" } delete: - service: delete-post + service: delete_post method: DELETE path: /{id} access: - given: - principal: identity:users + principal: identity:user document: posts when: principal.sub equals document.authorId - given: - principal: identity:users + principal: identity:user when: principal.record.role equals 'moderator' - my-posts: { service: my-posts, method: GET, path: /mine, access: authenticated } + by-author: { service: my_posts, method: GET, path: "/by-author/{authorId}", access: "*" } diff --git a/examples/comparison/auth-users/post_actions.py b/examples/comparison/auth-users/post_actions.py new file mode 100644 index 00000000..ddcbdb50 --- /dev/null +++ b/examples/comparison/auth-users/post_actions.py @@ -0,0 +1,14 @@ +from yaml_api_generator.server.common.request_context import RequestContext + + +async def publish(ctx: RequestContext) -> RequestContext: + """Set a post's published flag to true.""" + post_id = ctx.url_params.get("id") + if post_id is None: + raise ValueError("Missing post id") + + result = await ctx.repository.update( + "posts", post_id, {"published": True} + ) + ctx.response = result + return ctx diff --git a/examples/comparison/auth-users/v0.1.yaml b/examples/comparison/auth-users/v0.1.yaml index 1c828003..7cea9886 100644 --- a/examples/comparison/auth-users/v0.1.yaml +++ b/examples/comparison/auth-users/v0.1.yaml @@ -16,16 +16,6 @@ mongo: uri: mongodb://localhost:27017 database: myapp -include: - - comments.yaml # This yaml is merged into the root level, allowing it to define its own collections and operations while still being part of the same API version. - # CANNOT OVERWRITE EXISTING KEYS (e.g. collections.users.type is already defined as 'auth' here, so it cannot be redefined in comments.yaml) - -import: - repositories: - - ./custom/post_queries.py # @repository_op: DB queries (by-author, search, etc.) - services: - - ./custom/post_actions.py # @step: business logic (publish, archive, etc.) - collections: users: type: auth @@ -39,8 +29,8 @@ collections: update: authenticated replace: authenticated delete: - - principal: identity:users - when: "principal.sub == document.id" + - principal: "users:*" + when: "principal.sub equals document.id" fields: id: type: id @@ -64,10 +54,10 @@ collections: get: public update: authenticated delete: - - principal: identity:users - when: "principal.sub == document.authorId" - - principal: identity:users - when: "principal.record.role == 'moderator'" + - principal: "users:*" + when: "principal.sub equals document.authorId" + - principal: "users:*" + when: "principal.record.role equals 'moderator'" fields: id: type: id @@ -79,16 +69,6 @@ collections: authorId: type: ref ref: users - operations: - my-posts: - path: /posts/mine - access: authenticated - # Is this a DB query (repository) or business logic (service)? - # From the YAML alone, impossible to tell. - # The decorator in custom/posts.py decides, but the YAML is blind to it. - publish: - path: /posts/{id}/publish - access: authenticated - # Same ambiguity. Is publish a simple field update (repo)? - # Or does it send notifications, validate state, etc. (service)? - # The YAML treats both identically. + # v0.1 has no custom operations: "my posts" and "publish" cannot be declared, + # and the YAML could not say whether each is a DB query or business logic. + # v0.2 separates them into repository operations and service handlers. diff --git a/examples/comparison/auth-users/v0.2.yaml b/examples/comparison/auth-users/v0.2.yaml index 8ed3af34..e370b9c1 100644 --- a/examples/comparison/auth-users/v0.2.yaml +++ b/examples/comparison/auth-users/v0.2.yaml @@ -13,7 +13,7 @@ api: delete: true imports: - posts: myapp.handlers.posts + posts: post_actions mongo: uri: mongodb://localhost:27017 @@ -22,9 +22,10 @@ mongo: entities: user: type: auth - via: - - provider: password - identifier: email + auth: + via: + - provider: password + identifier: email fields: id: type: id @@ -53,36 +54,42 @@ entities: authorId: type: ref ref: user - operations: - replace: true # Override default to allow full replacement of post documents. + published: + type: boolean + default: false -# by-author is a DB query — it belongs here. -# Registered by @repository_op("posts", "by-author") in custom/posts.py. +# by_author is a DB query — it belongs here. repositories: users: entity: user posts: entity: post + operations: + by_author: + query: + command: find + filter: { authorId: "{{authorId}}" } -# publish-post is business logic — it belongs here. +# publish_post is business logic — it belongs here. services: register: { repository: users, operation: create } login: repository: users - list-users: { repository: users, operation: list } - get-user: { repository: users, operation: get } - update-user: { repository: users, operation: update } - delete-user: { repository: users, operation: delete } + list_users: { repository: users, operation: list } + get_user: { repository: users, operation: get } + update_user: { repository: users, operation: update } + delete_user: { repository: users, operation: delete } - list-posts: { repository: posts, operation: list } - get-post: { repository: posts, operation: get } - create-post: { repository: posts, operation: create } - update-post: { repository: posts, operation: update } - delete-post: { repository: posts, operation: delete } - my-posts: { repository: posts, operation: by-author } - publish-post: + list_posts: { repository: posts, operation: list } + get_post: { repository: posts, operation: get } + create_post: { repository: posts, operation: create } + update_post: { repository: posts, operation: update } + replace_post: { repository: posts, operation: replace } + delete_post: { repository: posts, operation: delete } + posts_by_author: { repository: posts, operation: by_author } + publish_post: handler: posts.publish controllers: @@ -95,37 +102,38 @@ controllers: users: base_path: /users routes: - list: { service: list-users, method: GET, path: /, access: authenticated } - get: { service: get-user, method: GET, path: "/{id}", access: authenticated } - update: { service: update-user, method: PATCH, path: "/{id}", access: authenticated } + list: { service: list_users, method: GET, path: /, access: "*" } + get: { service: get_user, method: GET, path: "/{id}", access: "*" } + update: { service: update_user, method: PATCH, path: "/{id}", access: "*" } delete: - service: delete-user + service: delete_user method: DELETE path: /{id} access: - given: - principal: identity:users + principal: identity:user document: users when: principal.sub equals document.id posts: base_path: /posts routes: - list: { service: list-posts, method: GET, path: /, access: public } - get: { service: get-post, method: GET, path: "/{id}", access: public } - create: { service: create-post, method: POST, path: /, access: public } - update: { service: update-post, method: PATCH, path: "/{id}", access: authenticated } - my-posts: { service: my-posts, method: GET, path: /mine, access: authenticated } - publish: { service: publish-post, method: POST, path: "/{id}/publish", access: authenticated } + list: { service: list_posts, method: GET, path: /, access: public } + get: { service: get_post, method: GET, path: "/{id}", access: public } + create: { service: create_post, method: POST, path: /, access: public } + update: { service: update_post, method: PATCH, path: "/{id}", access: "*" } + replace: { service: replace_post, method: PUT, path: "/{id}", access: "*" } + by-author: { service: posts_by_author, method: GET, path: "/by-author/{authorId}", access: "*" } + publish: { service: publish_post, method: POST, path: "/{id}/publish", access: "*" } delete: - service: delete-post + service: delete_post method: DELETE path: /{id} access: - given: - principal: identity:users + principal: identity:user document: posts when: principal.sub equals document.authorId - given: - principal: identity:users + principal: identity:user when: principal.record.role equals 'moderator' diff --git a/examples/comparison/blog-proposed.yaml b/examples/comparison/blog-proposed.yaml index 5a6a4726..df829f0e 100644 --- a/examples/comparison/blog-proposed.yaml +++ b/examples/comparison/blog-proposed.yaml @@ -2,7 +2,7 @@ version: "0.2" api: operations: - defaults: [list, get, create, update, delete] + defaults: { list: true, get: true, create: true, update: true, delete: true } # Entities: data shape only. No operations, no HTTP. entities: @@ -47,9 +47,10 @@ repositories: posts: entity: post operations: - by-author: - base: list - filter: { authorId: $param.authorId } + by_author: + query: + command: find + filter: { authorId: "{{authorId}}" } users: entity: user @@ -57,34 +58,34 @@ repositories: # Services: business operations. Each one delegates to a repo op or runs custom logic. # Controllers can only reference services, never repositories directly. services: - list-posts: { repository: posts, operation: list } - get-post: { repository: posts, operation: get } - create-post: { repository: posts, operation: create } - update-post: { repository: posts, operation: update } - delete-post: { repository: posts, operation: delete } + list_posts: { repository: posts, operation: list } + get_post: { repository: posts, operation: get } + create_post: { repository: posts, operation: create } + update_post: { repository: posts, operation: update } + delete_post: { repository: posts, operation: delete } - list-users: { repository: users, operation: list } - get-user: { repository: users, operation: get } - create-user: { repository: users, operation: create } - update-user: { repository: users, operation: update } - delete-user: { repository: users, operation: delete } + list_users: { repository: users, operation: list } + get_user: { repository: users, operation: get } + create_user: { repository: users, operation: create } + update_user: { repository: users, operation: update } + delete_user: { repository: users, operation: delete } # Controllers: HTTP binding. References services only. controllers: posts: base_path: /posts routes: - list: { service: list-posts, method: GET, path: /, access: public } - get: { service: get-post, method: GET, path: /{id}, access: public } - create: { service: create-post, method: POST, path: /, access: public } - update: { service: update-post, method: PATCH, path: /{id}, access: public } - delete: { service: delete-post, method: DELETE, path: /{id}, access: public } + list: { service: list_posts, method: GET, path: /, access: public } + get: { service: get_post, method: GET, path: "/{id}", access: public } + create: { service: create_post, method: POST, path: /, access: public } + update: { service: update_post, method: PATCH, path: "/{id}", access: public } + delete: { service: delete_post, method: DELETE, path: "/{id}", access: public } users: base_path: /users routes: - list: { service: list-users, method: GET, path: /, access: public } - get: { service: get-user, method: GET, path: /{id}, access: public } - create: { service: create-user, method: POST, path: /, access: public } - update: { service: update-user, method: PATCH, path: /{id}, access: public } - delete: { service: delete-user, method: DELETE, path: /{id}, access: public } + list: { service: list_users, method: GET, path: /, access: public } + get: { service: get_user, method: GET, path: "/{id}", access: public } + create: { service: create_user, method: POST, path: /, access: public } + update: { service: update_user, method: PATCH, path: "/{id}", access: public } + delete: { service: delete_user, method: DELETE, path: "/{id}", access: public } diff --git a/examples/v02/api.minified.yaml b/examples/v02/api.minified.yaml index 55ff6e83..e0a38ef0 100644 --- a/examples/v02/api.minified.yaml +++ b/examples/v02/api.minified.yaml @@ -32,9 +32,10 @@ entities: user: extends: base type: auth - via: - - provider: password - identifier: email + auth: + via: + - provider: password + identifier: email roles: values: [admin, moderator, user] default: user diff --git a/examples/v02/api.yaml b/examples/v02/api.yaml index e4358f8a..fb57dacc 100644 --- a/examples/v02/api.yaml +++ b/examples/v02/api.yaml @@ -25,9 +25,10 @@ mongo: entities: user: type: auth - via: - - provider: password - identifier: email + auth: + via: + - provider: password + identifier: email roles: values: [admin, moderator, user] default: user diff --git a/examples/v02/myapp/handlers/posts.py b/examples/v02/myapp/handlers/posts.py index 448dd31a..ddcbdb50 100644 --- a/examples/v02/myapp/handlers/posts.py +++ b/examples/v02/myapp/handlers/posts.py @@ -1,4 +1,4 @@ -from yaml_api_generator.runtime.context import RequestContext +from yaml_api_generator.server.common.request_context import RequestContext async def publish(ctx: RequestContext) -> RequestContext: diff --git a/src/yaml_api_generator/cli/__init__.py b/src/yaml_api_generator/cli/__init__.py index 9bbf208b..e0f1c818 100644 --- a/src/yaml_api_generator/cli/__init__.py +++ b/src/yaml_api_generator/cli/__init__.py @@ -104,50 +104,45 @@ def build(yaml_file: str, what: str, schema_file: str | None, output_dir: str | @click.argument("yaml_file", type=click.Path(exists=True)) @click.option("-s", "--schema", "schema_file", type=click.Path(exists=True), help="JSON schema file") @click.option("--host", default="127.0.0.1", help="Host to bind") -@click.option("-p", "--port", default=8000, type=int, help="Port to bind") +@click.option("-p", "--port", type=int, help="Port to bind (default: api.port)") @click.option("--raw", is_flag=True, help="Use raw ASGI runner") @click.option("-r", "--reload", is_flag=True, help="Reload the app when the spec changes") @click.option("--json", "output_json", is_flag=True, help="Output errors in JSON") -def run(yaml_file: str, schema_file: str | None, host: str, port: int, raw: bool, reload: bool, output_json: bool): +def run(yaml_file: str, schema_file: str | None, host: str, port: int | None, raw: bool, reload: bool, output_json: bool): """Run development server.""" import uvicorn - from ..parsers.yaml_parser import ConfigErrors + from ..parsers.yaml_parser import ConfigErrors, load_and_generate_ir from ..server import build_app, build_raw_app from ..server.factory import RAW_ENV, SCHEMA_ENV, SPEC_ENV - if reload: - import os - from pathlib import Path - - os.environ[SPEC_ENV] = yaml_file - os.environ[RAW_ENV] = "1" if raw else "0" - if schema_file: - os.environ[SCHEMA_ENV] = schema_file - uvicorn.run( - "yaml_api_generator.server.factory:create_app", - factory=True, - reload=True, - reload_dirs=[str(Path(yaml_file).parent)], - reload_includes=["*.yaml", "*.yml"], - host=host, - port=port, - ) - return - try: - app = ( - build_raw_app(yaml_file, schema_file) - if raw - else build_app(yaml_file, schema_file) - ) + port = port or load_and_generate_ir(yaml_file, schema_file).ir.api.port + if reload: + import os + from pathlib import Path + + os.environ[SPEC_ENV] = yaml_file + os.environ[RAW_ENV] = "1" if raw else "0" + if schema_file: + os.environ[SCHEMA_ENV] = schema_file + uvicorn.run( + "yaml_api_generator.server.factory:create_app", + factory=True, + reload=True, + reload_dirs=[str(Path(yaml_file).parent)], + reload_includes=["*.yaml", "*.yml"], + host=host, + port=port, + ) + else: + build = build_raw_app if raw else build_app + uvicorn.run(build(yaml_file, schema_file), host=host, port=port) except ConfigErrors as error: handle_config_errors(error, yaml_file, "json" if output_json else "human") except OSError as error: handle_generic_error(error) - uvicorn.run(app, host=host, port=port) - @cli.command() @click.argument("pack_dir", type=click.Path(exists=True)) diff --git a/src/yaml_api_generator/generators/backend/builders/__init__.py b/src/yaml_api_generator/generators/backend/builders/__init__.py index 1e295228..76ba7da9 100644 --- a/src/yaml_api_generator/generators/backend/builders/__init__.py +++ b/src/yaml_api_generator/generators/backend/builders/__init__.py @@ -52,6 +52,7 @@ def build_context( api_version=document.api.version, cors=cors, port=document.api.port, + security_headers=document.api.security_headers, ) identity_map = document.identityCollections @@ -109,7 +110,7 @@ def _repo_datastore_type(repo) -> str: collections=raw_collections, session=SessionConfig( algorithm=document.session.algorithm, - ttl=document.session.ttl, + ttl=document.session.access_ttl, refresh=document.session.refresh, delivery=document.session.delivery, ), diff --git a/src/yaml_api_generator/generators/backend/builders/settings_builder.py b/src/yaml_api_generator/generators/backend/builders/settings_builder.py index ee596965..62cbc8a3 100644 --- a/src/yaml_api_generator/generators/backend/builders/settings_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/settings_builder.py @@ -77,13 +77,12 @@ def add(f: SettingField) -> None: add(SettingField("JWT_RETIRED_PUBLIC_KEY", "str", "")) else: add(SettingField("JWT_SECRET", "str", None, secret=True, generator=_token)) + if ctx.session.ttl is None: + add(SettingField("JWT_EXPIRATION_HOURS", "int", "24")) + else: + add(SettingField("JWT_ACCESS_TTL_SECONDS", "int", str(ctx.session.ttl))) if ctx.session.refresh: - add(SettingField( - "JWT_ACCESS_TTL_SECONDS", "int", str(ctx.session.ttl or 900) - )) add(SettingField("JWT_REFRESH_TTL_SECONDS", "int", "2592000")) - else: - add(SettingField("JWT_EXPIRATION_HOURS", "int", "24")) if ctx.auth.has_app_oidc_auth: add(SettingField("FRONTEND_URL", "str", "")) diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index f0634b49..902a93df 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -404,6 +404,7 @@ class AppConfig: api_version: str | None cors: CorsConfig | None = None port: int = 8000 + security_headers: bool = True @dataclass(frozen=True) diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index f94e32f5..c2618ef3 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -94,6 +94,8 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: (api_dir / "pagination.py").write_text( runtime_module_source("pagination") ) + if context.app.security_headers: + (api_dir / "security_headers.py").write_text(runtime_module_source("security_headers")) self._render_views(env, context, collections_by_name, output_dir) self._render_repositories(env, context, collections_by_name, output_dir) self._render_authentication(env, context, output_dir) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja index 3fc3b6d5..e906d67d 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja @@ -1,6 +1,9 @@ django djangorestframework drf-spectacular +{% if ctx.app.cors %} +django-cors-headers +{% endif %} gunicorn pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja index 23543202..8c302ab3 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja @@ -15,12 +15,32 @@ INSTALLED_APPS = [ "django.contrib.auth", "rest_framework", "drf_spectacular", +{% if ctx.app.cors %} + "corsheaders", +{% endif %} "api", ] MIDDLEWARE = [ +{% if ctx.app.security_headers %} + "api.security_headers.DjangoSecurityHeadersMiddleware", +{% endif %} +{% if ctx.app.cors %} + "corsheaders.middleware.CorsMiddleware", +{% endif %} "django.middleware.common.CommonMiddleware", ] +{% if ctx.app.cors %} + +{% if "*" in ctx.app.cors.origins %} +CORS_ALLOW_ALL_ORIGINS = True +{% else %} +CORS_ALLOWED_ORIGINS = {{ ctx.app.cors.origins }} +{% endif %} +CORS_ALLOW_METHODS = {{ ctx.app.cors.methods }} +CORS_ALLOW_HEADERS = {{ ctx.app.cors.headers }} +CORS_ALLOW_CREDENTIALS = {{ ctx.app.cors.credentials }} +{% endif %} ROOT_URLCONF = "project.urls" diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 4c1a7bc8..3f8c04c5 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -15,6 +15,7 @@ write_environment_files, ) from ..validation import ( + reject_session_refresh_without_mongo, reject_undeclared_repository_operations, reject_unsigned_session_algorithm, validate_datastore_layout, @@ -43,6 +44,7 @@ def _crud_operations(col_data: dict) -> list[str]: class FastAPIGenerator(TemplateBasedGenerator): def generate(self, context: CompileContext, output_dir: Path) -> None: validate_datastore_layout(context, "fastapi") + reject_session_refresh_without_mongo(context) reject_unsigned_session_algorithm(context) reject_undeclared_repository_operations(context) env = template_environment(_TEMPLATES_DIR, _SHARED_TEMPLATES_DIR) @@ -443,6 +445,8 @@ def _render_dependencies( (deps_dir / "pagination.py").write_text( runtime_module_source("pagination") ) + if context.app.security_headers: + (deps_dir / "security_headers.py").write_text(runtime_module_source("security_headers")) if any(getattr(provider, "dpop", False) for provider in context.auth.providers.values()): (deps_dir / "dpop.py").write_text(runtime_module_source("dpop")) if any(context.repository_operations.values()): diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/app.py.jinja index 902c98e6..f596effb 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/app.py.jinja @@ -90,18 +90,6 @@ async def _file_handler(_: Request, exc: FileError) -> JSONResponse: {% endif %} -{% if ctx.app.cors %} -from fastapi.middleware.cors import CORSMiddleware - -app.add_middleware( - CORSMiddleware, - allow_origins={{ ctx.app.cors.origins }}, - allow_methods={{ ctx.app.cors.methods }}, - allow_headers={{ ctx.app.cors.headers }}, - allow_credentials={{ ctx.app.cors.credentials }}, -) -{% endif %} - {% if ctx.auth.has_idp_auth %} import logging as _logging @@ -146,6 +134,24 @@ class _CsrfMiddleware(_CsrfBase): app.add_middleware(_CsrfMiddleware) {% endif %} +{% if ctx.app.cors %} +from fastapi.middleware.cors import CORSMiddleware + +app.add_middleware( + CORSMiddleware, + allow_origins={{ ctx.app.cors.origins }}, + allow_methods={{ ctx.app.cors.methods }}, + allow_headers={{ ctx.app.cors.headers }}, + allow_credentials={{ ctx.app.cors.credentials }}, +) +{% endif %} + +{% if ctx.app.security_headers %} +from dependencies.security_headers import SecurityHeadersMiddleware + +app.add_middleware(SecurityHeadersMiddleware) +{% endif %} + app.include_router(api_router, prefix="{{ ctx.app.base_path }}{% if ctx.app.api_version %}/{{ ctx.app.api_version }}{% endif %}") {% if ctx.auth.uses_jwt and ctx.session.uses_asymmetric %} app.include_router(jwks_router) diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 9c20543e..46ac0a94 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -84,6 +84,8 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: (output_dir / "pagination.py").write_text( runtime_module_source("pagination") ) + if context.app.security_headers: + (output_dir / "security_headers.py").write_text(runtime_module_source("security_headers")) collections_by_name = group_by_collection(context) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja index 0fdd9a16..53790b78 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja @@ -162,6 +162,24 @@ class ASGIApp: app = ASGIApp() +{% if ctx.app.cors %} + +from starlette.middleware.cors import CORSMiddleware + +app = CORSMiddleware( + app, + allow_origins={{ ctx.app.cors.origins }}, + allow_methods={{ ctx.app.cors.methods }}, + allow_headers={{ ctx.app.cors.headers }}, + allow_credentials={{ ctx.app.cors.credentials }}, +) +{% endif %} +{% if ctx.app.security_headers %} + +from security_headers import SecurityHeadersMiddleware + +app = SecurityHeadersMiddleware(app) +{% endif %} if __name__ == "__main__": import uvicorn diff --git a/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja index b8079569..f9a93d7c 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja @@ -1,6 +1,9 @@ pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} uvicorn +{% if ctx.app.cors %} +starlette +{% endif %} pydantic-settings {% for req in ctx.vendored_requirements %} {{ req }} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/jwt_utils.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/jwt_utils.py.jinja index aab51e4b..a08585e6 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/jwt_utils.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/jwt_utils.py.jinja @@ -91,7 +91,7 @@ def _decode(token: str) -> dict | None: def create_jwt_token(sub: str, identity_collection: str) -> str: return _encode( {"sub": sub, "identity_collection": identity_collection}, - {% if ctx.session.refresh %}settings.JWT_ACCESS_TTL_SECONDS{% else %}settings.JWT_EXPIRATION_HOURS * 3600{% endif %}, + {% if ctx.session.ttl is none %}settings.JWT_EXPIRATION_HOURS * 3600{% else %}settings.JWT_ACCESS_TTL_SECONDS{% endif %}, ) diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 5a78e60e..8c93fbe6 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -12,7 +12,10 @@ _RUNTIME_MODULES = frozenset( - {"dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} + { + "dpop", "pagination", "files", "multipart", "file_requests", "filesystem_store", + "s3_store", "security_headers", + } ) diff --git a/src/yaml_api_generator/generators/backend/validation.py b/src/yaml_api_generator/generators/backend/validation.py index dad43e49..2d07f999 100644 --- a/src/yaml_api_generator/generators/backend/validation.py +++ b/src/yaml_api_generator/generators/backend/validation.py @@ -106,6 +106,23 @@ def reject_session_refresh(context: CompileContext, backend: str) -> None: ) +def reject_session_refresh_without_mongo(context: CompileContext) -> None: + """Reject rotation when every collection is SQL, leaving no MongoDB store for refresh records.""" + collection_types = {context.collection_datastore_type(name) for name in context.collections} + if not (context.session.refresh and collection_types and collection_types <= _SQL_DATASTORES): + return + raise ConfigErrors( + [ + ConfigError( + code="BACKEND_FEATURE_UNSUPPORTED", + message="session.refresh keeps refresh records in MongoDB, but no collection uses it", + path="session.refresh", + hint="Declare a MongoDB datastore, or drop session.refresh", + ) + ] + ) + + def reject_cookie_delivery(context: CompileContext, backend: str) -> None: if not context.session.uses_cookie: return diff --git a/src/yaml_api_generator/generators/deploy/context_builder.py b/src/yaml_api_generator/generators/deploy/context_builder.py index 8b28bfed..bd651b7d 100644 --- a/src/yaml_api_generator/generators/deploy/context_builder.py +++ b/src/yaml_api_generator/generators/deploy/context_builder.py @@ -2,7 +2,7 @@ from sqlalchemy.engine import make_url -from ...ir import IRDocument, IRPostgres +from ...ir import IRDocument, IRMongo, IRPostgres from ..backend.builders import build_context from ..backend.builders.settings_builder import build_settings_fields from .context import DeployContext, DeployDatastore, DockerDeployContext, SystemdDeployContext @@ -89,7 +89,7 @@ def _build_datastores(ir: IRDocument) -> tuple[DeployDatastore, ...]: service = base if type_counts[base] == 1 else f"{base}-{name}" result.append( _postgres_datastore(service, ds) if isinstance(ds, IRPostgres) - else _mongo_datastore(service, ds) + else _mongo_datastore(service, ds, ir.deploy.mongo_local) ) return tuple(result) @@ -112,8 +112,7 @@ def _postgres_datastore(service: str, ds: IRPostgres) -> DeployDatastore: ) -def _mongo_datastore(service: str, ds) -> DeployDatastore: - local = "localhost" in ds.uri or "127.0.0.1" in ds.uri +def _mongo_datastore(service: str, ds: IRMongo, local: bool) -> DeployDatastore: return DeployDatastore( service=service, type="mongodb", diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 652cd674..aabd5d21 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -161,6 +161,7 @@ class IRApi: port: int = 8000 files: IRFileLimits | None = None max_files: int | None = None + security_headers: bool = True @dataclass(frozen=True) @@ -170,10 +171,17 @@ class IRSession: refresh: bool = False delivery: str = "token-url" + REFRESHED_ACCESS_TTL: ClassVar[int] = 900 + @property def uses_asymmetric(self) -> bool: return self.algorithm == "RS256" + @property + def access_ttl(self) -> int | None: + """Access-token lifetime in seconds; None leaves it to JWT_EXPIRATION_HOURS.""" + return self.ttl or (self.REFRESHED_ACCESS_TTL if self.refresh else None) + @dataclass(frozen=True) class IRMongo: @@ -181,6 +189,11 @@ class IRMongo: database: str = "yaml_api" type: str = "mongodb" + @property + def is_local(self) -> bool: + """Whether the URI points at a server on the deploy host itself.""" + return any(host in self.uri for host in ("localhost", "127.0.0.1")) + @dataclass(frozen=True) class IRPostgres: diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json index a86ce97e..9be55db0 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json @@ -44,7 +44,9 @@ "allowedOrigins": { "$ref": "#/$defs/StringList" }, "allowCredentials": { "type": "boolean" }, "allowedMethods": { "$ref": "#/$defs/StringList" } - } + }, + "if": { "properties": { "allowCredentials": { "const": true } }, "required": ["allowCredentials"] }, + "then": { "properties": { "allowedOrigins": { "not": { "contains": { "const": "*" } } } } } }, "headers": { "type": "object", diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index 71917c0d..e3cca1df 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -8,6 +8,38 @@ "entities" ], "additionalProperties": false, + "if": { + "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", + "required": [ + "datastores" + ], + "properties": { + "datastores": { + "not": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } + } + } + } + } + }, + "then": { + "properties": { + "mongo": { + "not": {} + } + } + }, "properties": { "version": { "type": "string", @@ -33,6 +65,9 @@ }, "cors": { "$ref": "#/$defs/CorsConfig" + }, + "securityHeaders": { + "type": "boolean" } } }, @@ -1029,8 +1064,7 @@ "type": "string" }, "port": { - "type": "integer", - "default": 8000 + "type": "integer" }, "runtime": { "type": "string", @@ -1053,8 +1087,7 @@ "additionalProperties": false, "properties": { "local": { - "type": "boolean", - "default": false + "type": "boolean" } } }, diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 79c86eef..22dc2b63 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -9,6 +9,38 @@ "entities" ], "additionalProperties": false, + "if": { + "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", + "required": [ + "datastores" + ], + "properties": { + "datastores": { + "not": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } + } + } + } + } + }, + "then": { + "properties": { + "mongo": { + "not": {} + } + } + }, "properties": { "version": { "type": "string", @@ -50,6 +82,9 @@ "minimum": 1 } } + }, + "securityHeaders": { + "type": "boolean" } } }, diff --git a/src/yaml_api_generator/parsers/transformers/auth_validation.py b/src/yaml_api_generator/parsers/transformers/auth_validation.py index 4d92bbf4..2b661cf8 100644 --- a/src/yaml_api_generator/parsers/transformers/auth_validation.py +++ b/src/yaml_api_generator/parsers/transformers/auth_validation.py @@ -1,4 +1,4 @@ -"""Semantic coherence checks for auth configuration in a v0.2 spec. +"""Semantic coherence checks for auth configuration in a spec. The JSON schema validates each auth key's shape but not its meaning, so an auth entity can build cleanly while its authentication is silently dead. These checks @@ -13,6 +13,8 @@ the existing ``AUTH_ROLES_FIELD_NOT_FOUND`` check. - AUTH_API_KEY_FIELD_MISSING: a ``via: apiKey`` identity whose entity declares no ``apiKey`` field, so no document can ever store a key and auth always fails. +- AUTH_VIA_UNKNOWN_PROVIDER: a ``via`` entry that is neither a built-in provider + nor a key of ``providers``, so no login can ever reach the identity through it. Inheritance is resolved on the fly: ``fields`` and ``auth`` may be supplied by a parent through ``extends``, exactly as the entity desugarer later merges them. @@ -25,12 +27,12 @@ _IDENTIFIER_PROVIDERS = frozenset({"password", "apiKey"}) -def _normalize_via(value: Any) -> list[str]: - if isinstance(value, str): - return [value] - if isinstance(value, list): - return [str(item) for item in value] - return [] +def _via_entries(value: Any) -> list[Any]: + return [value] if isinstance(value, str | dict) else value if isinstance(value, list) else [] + + +def _provider_name(entry: Any) -> str: + return str(entry.get("provider")) if isinstance(entry, dict) else str(entry) def _resolved(entity: str, entities: dict[str, Any], key: str) -> Any: @@ -69,18 +71,34 @@ def _resolved_fields(entity: str, entities: dict[str, Any]) -> dict[str, Any]: def _check_entity( - name: str, entities: dict[str, Any], errors: list[ConfigError] + name: str, + entities: dict[str, Any], + section: str, + providers: frozenset[str], + errors: list[ConfigError], ) -> None: auth = _resolved(name, entities, "auth") if not isinstance(auth, dict): return - via = _normalize_via(auth.get("via")) + base = f"{section}.{name}.auth" + entries = _via_entries(auth.get("via")) + errors.extend( + ConfigError( + code="AUTH_VIA_UNKNOWN_PROVIDER", + message=f"Auth entity '{name}' uses via '{provider}', which is not a declared provider", + path=f"{base}.via", + hint=f"Declare providers.{provider} or use one of: {', '.join(sorted(_IDENTIFIER_PROVIDERS))}", + ) + for provider in map(_provider_name, entries) + if provider not in _IDENTIFIER_PROVIDERS | providers + ) + # A mapping entry carries its own identifier, so only bare names need auth.identifier. + via = [entry for entry in entries if isinstance(entry, str)] if not _IDENTIFIER_PROVIDERS.intersection(via): return identifier = auth.get("identifier") - base = f"entities.{name}.auth" if not identifier: errors.append(ConfigError( code="AUTH_IDENTIFIER_REQUIRED", @@ -103,16 +121,17 @@ def _check_entity( errors.append(ConfigError( code="AUTH_API_KEY_FIELD_MISSING", message=f"Auth entity '{name}' uses via apiKey but declares no apiKey field", - path=f"entities.{name}.fields.apiKey", + path=f"{section}.{name}.fields.apiKey", hint="Declare an 'apiKey' field so a key can be stored and matched", )) -def validate_spec_auth(spec: dict[str, Any]) -> list[ConfigError]: - """Validate auth-configuration coherence across every entity in the spec.""" +def validate_spec_auth(spec: dict[str, Any], section: str = "entities") -> list[ConfigError]: + """Validate auth-configuration coherence across every model of ``section``.""" errors: list[ConfigError] = [] - entities = spec.get("entities") or {} + entities = spec.get(section) or {} + providers = frozenset(spec.get("providers") or {}) for name, entity in entities.items(): if isinstance(entity, dict) and str(_resolved(name, entities, "type") or "base") == "auth": - _check_entity(name, entities, errors) + _check_entity(name, entities, section, providers, errors) return errors diff --git a/src/yaml_api_generator/parsers/transformers/constraint_validation.py b/src/yaml_api_generator/parsers/transformers/constraint_validation.py index 8f9289cf..c715e17d 100644 --- a/src/yaml_api_generator/parsers/transformers/constraint_validation.py +++ b/src/yaml_api_generator/parsers/transformers/constraint_validation.py @@ -169,15 +169,15 @@ def _coerces_to(field_type: str, value: str) -> bool: return False -def validate_spec_constraints(spec: dict[str, Any]) -> list[ConfigError]: - """Validate field-constraint coherence across every entity in the spec.""" +def validate_spec_constraints(spec: dict[str, Any], section: str = "entities") -> list[ConfigError]: + """Validate field-constraint coherence across every model of ``section``.""" errors: list[ConfigError] = [] - for entity_name, entity in (spec.get("entities") or {}).items(): - if not isinstance(entity, dict): + for model_name, model in (spec.get(section) or {}).items(): + if not isinstance(model, dict): continue - for field_name, field in (entity.get("fields") or {}).items(): + for field_name, field in (model.get("fields") or {}).items(): if isinstance(field, dict): _check_field( - field, f"entities.{entity_name}.fields.{field_name}", errors + field, f"{section}.{model_name}.fields.{field_name}", errors ) return errors diff --git a/src/yaml_api_generator/parsers/transformers/name_validation.py b/src/yaml_api_generator/parsers/transformers/name_validation.py index ade2eba6..5864a2bf 100644 --- a/src/yaml_api_generator/parsers/transformers/name_validation.py +++ b/src/yaml_api_generator/parsers/transformers/name_validation.py @@ -67,48 +67,46 @@ def validate_name(name: str, path: str, *, field: bool = False) -> ConfigError | return None -# Top-level spec sections whose keys are emitted verbatim as Python identifiers, -# module names, attributes or classes. Provider names are excluded: they are only -# used as route-path segments and as identifiers after sanitization (py_identifier), -# so hyphenated names such as ``my-oidc`` are valid. -_NAME_BEARING_SECTIONS = ("entities", "repositories", "services", "controllers") +# Top-level spec sections, besides the model section, whose keys are emitted +# verbatim as Python identifiers, module names, attributes or classes. Provider +# names are excluded: they are only used as route-path segments and as identifiers +# after sanitization (py_identifier), so hyphenated names such as ``my-oidc`` are valid. +_NAME_BEARING_SECTIONS = ("repositories", "services", "controllers") -def validate_spec_names(spec: dict[str, Any]) -> list[ConfigError]: - """Validate every name-bearing key in a user-authored v0.2 spec. +def validate_spec_names(spec: dict[str, Any], section: str = "entities") -> list[ConfigError]: + """Validate every name-bearing key in a user-authored spec. - Operates on the pre-desugar spec so that only author-supplied names are - checked; field names are validated per entity in addition to the top-level - section keys. + ``section`` is the model section (``collections`` in v0.1). Operates on the + pre-desugar spec so that only author-supplied names are checked; field names + are validated per model in addition to the top-level section keys. """ - errors: list[ConfigError] = [] - for section in _NAME_BEARING_SECTIONS: - data = spec.get(section) - if not isinstance(data, dict): - continue - for name in data: - error = validate_name(str(name), f"{section}.{name}") - if error is not None: - errors.append(error) - for entity_name, entity in (spec.get("entities") or {}).items(): - if not isinstance(entity, dict): - continue - for field_name in entity.get("fields", {}): - error = validate_name( - str(field_name), - f"entities.{entity_name}.fields.{field_name}", - field=True, - ) - if error is not None: - errors.append(error) - errors.extend(_pascal_case_collisions(spec.get("entities") or {})) - return errors - - -def _pascal_case_collisions(entities: dict[str, Any]) -> list[ConfigError]: - """Reject distinct entity names that map to the same PascalCase class. - - Each entity becomes a generated model class named ``to_pascal_case(name)``. + models = spec.get(section) or {} + return [ + *( + error + for name_section in (section, *_NAME_BEARING_SECTIONS) + if isinstance(spec.get(name_section), dict) + for name in spec[name_section] + if (error := validate_name(str(name), f"{name_section}.{name}")) is not None + ), + *( + error + for model_name, model in models.items() + if isinstance(model, dict) + for field_name in model.get("fields", {}) + if (error := validate_name( + str(field_name), f"{section}.{model_name}.fields.{field_name}", field=True + )) is not None + ), + *_pascal_case_collisions(section, models), + ] + + +def _pascal_case_collisions(section: str, entities: dict[str, Any]) -> list[ConfigError]: + """Reject distinct model names that map to the same PascalCase class. + + Each model becomes a generated model class named ``to_pascal_case(name)``. The conversion is many-to-one (``userProfile`` and ``user_profile`` both yield ``UserProfile``), so colliding names would emit two classes with the same name and clobber each other in the generated package's imports. @@ -125,7 +123,7 @@ def _pascal_case_collisions(entities: dict[str, Any]) -> list[ConfigError]: f"entity '{name}' and '{prior}' both map to the model class " f"'{class_name}'" ), - path=f"entities.{name}", + path=f"{section}.{name}", hint="Rename one entity so their PascalCase model names differ", )) else: diff --git a/src/yaml_api_generator/parsers/transformers/roles_config.py b/src/yaml_api_generator/parsers/transformers/roles_config.py new file mode 100644 index 00000000..90ea7b2b --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/roles_config.py @@ -0,0 +1,64 @@ +"""Role set and hierarchy parsing shared by every spec version.""" + +from typing import Any + +from yaml_api_generator.ir import IRRolesConfig, RoleHierarchyCycleError, flatten_role_hierarchy +from yaml_api_generator.parsers.yaml_parser import ConfigError + + +def parse_roles_config( + section: str, owner: str, roles_data: Any +) -> tuple[IRRolesConfig | None, list[ConfigError]]: + """Build the roles of ``
..roles``, validating its hierarchy.""" + if not isinstance(roles_data, dict): + return None, [] + + values = tuple(str(v) for v in roles_data.get("values", [])) + if not values: + return None, [] + + raw_hierarchy = roles_data.get("hierarchy", {}) + hierarchy = { + str(role): tuple(str(i) for i in (includes or [])) + for role, includes in raw_hierarchy.items() + } if isinstance(raw_hierarchy, dict) else {} + + errors = _hierarchy_errors(section, owner, values, hierarchy) + if errors: + return None, errors + + default = str(roles_data.get("default", values[0])) + return IRRolesConfig(values=values, default=default, hierarchy=hierarchy), [] + + +def _hierarchy_errors( + section: str, + owner: str, + values: tuple[str, ...], + hierarchy: dict[str, tuple[str, ...]], +) -> list[ConfigError]: + path = f"{section}.{owner}.roles.hierarchy" + referenced = {*hierarchy, *(i for includes in hierarchy.values() for i in includes)} + errors = [ + ConfigError( + code="ROLES_HIERARCHY_UNKNOWN_ROLE", + message=f"Role hierarchy references undeclared role '{role}'", + path=path, + hint=f"Add '{role}' to {section}.{owner}.roles.values", + ) + for role in sorted(referenced - set(values)) + ] + if errors: + return errors + try: + flatten_role_hierarchy(hierarchy) + except RoleHierarchyCycleError as exc: + return [ + ConfigError( + code="ROLES_HIERARCHY_CYCLE", + message=f"Role hierarchy for '{owner}' is cyclic through '{exc.role}'", + path=path, + hint="Remove the circular or self-referential role inclusion", + ) + ] + return [] diff --git a/src/yaml_api_generator/parsers/transformers/v01/__init__.py b/src/yaml_api_generator/parsers/transformers/v01/__init__.py index 01ab648a..b2c4ee33 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v01/__init__.py @@ -8,12 +8,17 @@ from yaml_api_generator.ir import ( IRApi, + IRCors, IRDocument, IRMongo, + IRPagination, IRRepository, IRService, ) -from yaml_api_generator.parsers.yaml_parser import ConfigErrors +from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors + +from ..v02.datastore_transformer import DEFAULT_DATASTORE +from ..v03.pagination import parse_pagination from .collection_transformer import CollectionTransformer from .identity_transformer import IdentityTransformer @@ -65,10 +70,10 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: identity_collections, defaults, ) - api_ir = IRApi( - basePath=api.get("basePath", "/api"), - version=api.get("version"), - ) + api_ir = self._build_api(api, resolved.get("security", {})) + pagination, pagination_errors = self._build_pagination(api.get("pagination", {})) + if pagination_errors: + raise ConfigErrors(pagination_errors) route_prefix = api_ir.basePath.rstrip("/") if api_ir.version: route_prefix = f"{route_prefix}/{api_ir.version}" @@ -90,6 +95,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: route, path=f"{route_prefix}{route.path}", service=f"{route.operation}-{route.collection}", + pagination=pagination if route.operation == "list" else None, ) if route.collection is not None else route @@ -97,6 +103,10 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: ] mongo_data = resolved.get("mongo", {}) + mongo = IRMongo( + uri=mongo_data.get("uri", IRMongo.uri), + database=mongo_data.get("database", IRMongo.database), + ) return IRDocument( version=str(resolved["version"]), api=api_ir, @@ -105,8 +115,33 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: services=services, routes=routes, identityCollections=identity_collections, - mongo=IRMongo( - uri=mongo_data.get("uri", "mongodb://localhost:27017"), - database=mongo_data.get("database", "yaml_api"), + mongo=mongo, + datastores={DEFAULT_DATASTORE: mongo}, + default_datastore=DEFAULT_DATASTORE, + ) + + @staticmethod + def _build_api(api: dict[str, Any], security: dict[str, Any]) -> IRApi: + cors = security.get("cors") + return IRApi( + basePath=api.get("basePath", "/api"), + version=api.get("version"), + cors=None if cors is None else IRCors( + origins=cors.get("allowedOrigins", []), + methods=cors.get("allowedMethods", ["*"]), + credentials=cors.get("allowCredentials", False), ), + security_headers=security.get("headers", {}).get("enableStandard", True), + ) + + @staticmethod + def _build_pagination(pagination: dict[str, Any]) -> tuple[IRPagination | None, list[ConfigError]]: + maximum = pagination.get("maxPageSize", 100) + return parse_pagination( + { + "strategy": "offset", + "size": {"default": pagination.get("defaultPageSize", min(20, maximum)), "max": maximum}, + }, + None, + "api.pagination", ) diff --git a/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py b/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py index c28315d6..86436117 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py @@ -3,15 +3,11 @@ Validates collection fields and extracts ID field configurations. """ -import re from typing import Any, cast from yaml_api_generator.ir import IRCollection from yaml_api_generator.parsers.yaml_parser import ConfigError -FIELD_NAME_PATTERN = re.compile(r"^[a-zA-Z][a-zA-Z0-9_]*$") - - class CollectionTransformer: """Transforms v0.1 collection definitions to IR.""" @@ -35,11 +31,6 @@ def transform( collection_type = str(collection_data.get("type", "base")) fields = cast(dict[str, Any], collection_data["fields"]) - field_name_errors = self._validate_field_names(name, fields) - id_field_errors.extend(field_name_errors) - if field_name_errors: - continue - id_field, errors = self._extract_id_field(name, fields) id_field_errors.extend(errors) @@ -120,19 +111,3 @@ def _resolve_id_format( path=f"collections.{collection_name}.fields.{id_field}.format", ) ] - - def _validate_field_names( - self, collection_name: str, fields: dict[str, Any] - ) -> list[ConfigError]: - errors: list[ConfigError] = [] - for field_name in fields: - if not FIELD_NAME_PATTERN.match(field_name): - errors.append( - ConfigError( - code="INVALID_FIELD_NAME", - message=f"Invalid field name '{field_name}'", - path=f"collections.{collection_name}.fields.{field_name}", - hint="Field names must start with a letter and contain only letters, numbers, or underscores", - ) - ) - return errors diff --git a/src/yaml_api_generator/parsers/transformers/v01/identity_transformer.py b/src/yaml_api_generator/parsers/transformers/v01/identity_transformer.py index 1afb2ab7..355712c5 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/identity_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v01/identity_transformer.py @@ -1,12 +1,14 @@ """Identity and auth transformation for v0.1 specifications. -Validates auth collections and implicit auth fields. +Validates auth collections, their roles and implicit auth fields. """ from typing import Any, cast from yaml_api_generator.ir import IRCollection, IRIdentityCollection, IRProviderBinding -from yaml_api_generator.parsers.yaml_parser import ConfigError +from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors + +from ..roles_config import parse_roles_config class IdentityTransformer: @@ -25,22 +27,30 @@ def transform( Returns: Dictionary of identity collections + + Raises: + ConfigErrors: If a role hierarchy is invalid """ identity_collections: dict[str, IRIdentityCollection] = {} + errors: list[ConfigError] = [] for name, collection in collections.items(): if collection.type != "auth": continue collection_data = collections_data.get(name, {}) identifier, via = self._extract_identity_auth(collection_data) - bindings = tuple( - IRProviderBinding(provider=p, identifier=identifier or "") - for p in via - if identifier - ) + roles, roles_errors = parse_roles_config("collections", name, collection_data.get("roles")) + errors.extend(roles_errors) identity_collections[name] = IRIdentityCollection( name=name, - bindings=bindings, + bindings=tuple( + IRProviderBinding(provider=p, identifier=identifier or "") + for p in via + if identifier + ), + roles=roles, ) + if errors: + raise ConfigErrors(errors) return identity_collections def validate_implicit_auth_fields( diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 114a3212..28fed194 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -137,22 +137,22 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: if dto_errors: raise ConfigErrors(dto_errors) - mongo_config = resolved.get("mongo", {}) + api = self._route.build_api(api_data) return IRDocument( version=str(resolved["version"]), - api=self._route.build_api(api_data), + api=api, collections=collections, routes=routes, identityCollections=identity_collections, dtos=dtos, services=services, - mongo=self._route.build_mongo(mongo_config), + mongo=self._route.build_mongo(resolved.get("mongo", {})), datastores=datastores, default_datastore=default_datastore, file_stores=file_stores, default_file_store=default_file_store, file_routes=file_routes, - deploy=self._route.build_deploy(resolved.get("deploy"), mongo_config, providers, datastores), + deploy=self._route.build_deploy(resolved.get("deploy"), api, providers, datastores), repositories=repositories, providers=providers, policies=policies, diff --git a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py index a0b58308..d4aaa73d 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py @@ -10,12 +10,11 @@ IRIdentityCollection, IRProviderBinding, IRProviderRelation, - IRRolesConfig, - RoleHierarchyCycleError, - flatten_role_hierarchy, ) from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors +from ..roles_config import parse_roles_config + class IdentityTransformer: """Transforms auth entities to identity collections and validates roles.""" @@ -62,8 +61,8 @@ def transform( if role_errors: raise ConfigErrors(role_errors) - roles_config, roles_errors = self._build_roles_config( - entity_name, entity_data.get("roles") + roles_config, roles_errors = parse_roles_config( + "entities", entity_name, entity_data.get("roles") ) if roles_errors: raise ConfigErrors(roles_errors) @@ -135,62 +134,6 @@ def validate_auth_fields( ) return errors - def _build_roles_config( - self, entity_name: str, roles_data: Any - ) -> tuple[IRRolesConfig | None, list[ConfigError]]: - if not isinstance(roles_data, dict): - return None, [] - - values = tuple(str(v) for v in roles_data.get("values", [])) - if not values: - return None, [] - - default = str(roles_data.get("default", values[0])) - raw_hierarchy = roles_data.get("hierarchy", {}) - hierarchy = { - str(role): tuple(str(i) for i in (includes or [])) - for role, includes in raw_hierarchy.items() - } if isinstance(raw_hierarchy, dict) else {} - - errors = self._validate_hierarchy(entity_name, values, hierarchy) - if errors: - return None, errors - - return IRRolesConfig(values=values, default=default, hierarchy=hierarchy), [] - - def _validate_hierarchy( - self, - entity_name: str, - values: tuple[str, ...], - hierarchy: dict[str, tuple[str, ...]], - ) -> list[ConfigError]: - declared = set(values) - referenced = {role for role in hierarchy} - referenced.update(i for includes in hierarchy.values() for i in includes) - errors = [ - ConfigError( - code="ROLES_HIERARCHY_UNKNOWN_ROLE", - message=f"Role hierarchy references undeclared role '{role}'", - path=f"entities.{entity_name}.roles.hierarchy", - hint=f"Add '{role}' to entities.{entity_name}.roles.values", - ) - for role in sorted(referenced - declared) - ] - if errors: - return errors - try: - flatten_role_hierarchy(hierarchy) - except RoleHierarchyCycleError as exc: - return [ - ConfigError( - code="ROLES_HIERARCHY_CYCLE", - message=f"Role hierarchy for '{entity_name}' is cyclic through '{exc.role}'", - path=f"entities.{entity_name}.roles.hierarchy", - hint="Remove the circular or self-referential role inclusion", - ) - ] - return [] - def _normalize_via(self, value: Any) -> list[str]: if isinstance(value, str): return [value] diff --git a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py index 4231f36b..0ff44c8c 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py @@ -266,6 +266,7 @@ def build_api(self, api_data: dict[str, Any]) -> IRApi: version=api_data.get("version"), cors=cors, port=int(api_data.get("port", 8000)), + security_headers=api_data.get("securityHeaders", True), files=None if files is None else LimitLayer.parse(files, "api.files")[0].limits(), max_files=(files or {}).get("maxFiles"), ) @@ -287,7 +288,7 @@ def build_mongo(self, mongo_data: dict[str, Any]) -> IRMongo: def build_deploy( self, deploy_data: Any, - mongo_config: dict[str, Any], + api: IRApi, providers: dict[str, IRProvider], datastores: dict[str, IRDatastore], ) -> IRDeploy | None: @@ -295,7 +296,7 @@ def build_deploy( Args: deploy_data: Raw deployment configuration from YAML. - mongo_config: MongoDB configuration to determine if local. + api: Built API IR, whose port is the default deploy port. providers: Already-built provider IR objects (env vars resolved at build time). datastores: Already-built datastore IR objects (env vars resolved at build time). @@ -320,8 +321,9 @@ def build_deploy( else None ) - mongo_uri = mongo_config.get("uri", "mongodb://localhost:27017") - mongo_local = not mongo_config or "localhost" in mongo_uri or "127.0.0.1" in mongo_uri + mongo_local = deploy_data.get("mongo", {}).get( + "local", any(isinstance(ds, IRMongo) and ds.is_local for ds in datastores.values()) + ) env_vars = self._collect_env_vars(deploy_data, providers, datastores) @@ -333,7 +335,7 @@ def build_deploy( return IRDeploy( runtime=str(runtime), name=deploy_data.get("name"), - port=int(deploy_data.get("port", 8000)), + port=int(deploy_data.get("port", api.port)), mongo_local=mongo_local, env_vars=tuple(env_vars), docker=docker, diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 8a1258a6..71b5c06d 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -219,17 +219,23 @@ def load_and_generate_ir( trust_warnings: tuple[ConfigError, ...] = () access_warnings: tuple[ConfigError, ...] = () - if version in ("0.2", "0.3"): - from .transformers.name_validation import validate_spec_names - name_errors = validate_spec_names(resolved) - if name_errors: - raise ConfigErrors(name_errors) - - from .transformers.constraint_validation import validate_spec_constraints - constraint_errors = validate_spec_constraints(resolved) - if constraint_errors: - raise ConfigErrors(constraint_errors) + model_section = "collections" if version == "0.1" else "entities" + from .transformers.name_validation import validate_spec_names + name_errors = validate_spec_names(resolved, model_section) + if name_errors: + raise ConfigErrors(name_errors) + + from .transformers.constraint_validation import validate_spec_constraints + constraint_errors = validate_spec_constraints(resolved, model_section) + if constraint_errors: + raise ConfigErrors(constraint_errors) + + from .transformers.auth_validation import validate_spec_auth + auth_errors = validate_spec_auth(resolved, model_section) + if auth_errors: + raise ConfigErrors(auth_errors) + if version in ("0.2", "0.3"): from .transformers.file_validation import validate_spec_files file_errors = validate_spec_files(resolved) if file_errors: @@ -257,11 +263,6 @@ def load_and_generate_ir( if symbol_errors: raise ConfigErrors(symbol_errors) - from .transformers.auth_validation import validate_spec_auth - auth_errors = validate_spec_auth(resolved) - if auth_errors: - raise ConfigErrors(auth_errors) - from .transformers.binding_validation import validate_spec_bindings binding_errors = validate_spec_bindings(resolved) if binding_errors: diff --git a/src/yaml_api_generator/runtime/security_headers.py b/src/yaml_api_generator/runtime/security_headers.py new file mode 100644 index 00000000..5e39f917 --- /dev/null +++ b/src/yaml_api_generator/runtime/security_headers.py @@ -0,0 +1,54 @@ +"""Standard security response headers shared by runtime and generated backends. + +A header the application already set is left untouched, so a route can still +choose its own framing or referrer policy. +""" + +from typing import Any + +STANDARD_HEADERS: tuple[tuple[str, str], ...] = ( + ("x-content-type-options", "nosniff"), + ("x-frame-options", "DENY"), + ("referrer-policy", "no-referrer"), + ("strict-transport-security", "max-age=31536000; includeSubDomains"), + ("cross-origin-opener-policy", "same-origin"), +) + + +class SecurityHeadersMiddleware: + """ASGI middleware adding STANDARD_HEADERS to every HTTP response.""" + + def __init__(self, app: Any) -> None: + self.app = app + + async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_headers(message: dict[str, Any]) -> None: + if message["type"] == "http.response.start": + headers = list(message.get("headers", [])) + present = {bytes(name).lower() for name, _ in headers} + headers.extend( + (name.encode("latin-1"), value.encode("latin-1")) + for name, value in STANDARD_HEADERS + if name.encode("latin-1") not in present + ) + message = {**message, "headers": headers} + await send(message) + + await self.app(scope, receive, send_with_headers) + + +class DjangoSecurityHeadersMiddleware: + """Django middleware adding STANDARD_HEADERS to every response.""" + + def __init__(self, get_response: Any) -> None: + self.get_response = get_response + + def __call__(self, request: Any) -> Any: + response = self.get_response(request) + for name, value in STANDARD_HEADERS: + response.headers.setdefault(name, value) + return response diff --git a/src/yaml_api_generator/server/asgi/server.py b/src/yaml_api_generator/server/asgi/server.py index 2ae9c261..f96dfd70 100644 --- a/src/yaml_api_generator/server/asgi/server.py +++ b/src/yaml_api_generator/server/asgi/server.py @@ -2,15 +2,18 @@ from pathlib import Path +from starlette import types + from yaml_api_generator.parsers.transformers.file_validation import reject_file_fields from ..common.context_builder import build_runtime_context +from ..common.http_middleware import api_middleware, wrap_asgi from ..common.loader import create_repositories, ensure_spec_dir_importable from ..common.vendored_packages import install_vendored_packages from .runner import ASGIApp -def build_raw_app(yaml_file: str, schema_file: str | Path | None = None) -> ASGIApp: +def build_raw_app(yaml_file: str, schema_file: str | Path | None = None) -> types.ASGIApp: """Build raw ASGI application from YAML specification. Args: @@ -18,7 +21,7 @@ def build_raw_app(yaml_file: str, schema_file: str | Path | None = None) -> ASGI schema_file: Optional path to JSON schema for validation Returns: - Configured ASGI application + ASGI application wrapped in the middleware the spec enables """ ensure_spec_dir_importable(yaml_file) ir, registry = create_repositories(yaml_file, schema_file) @@ -29,4 +32,4 @@ def build_raw_app(yaml_file: str, schema_file: str | Path | None = None) -> ASGI install_vendored_packages(ir.vendored_imports) ctx = build_runtime_context(ir) - return ASGIApp(ctx, registry, ir) + return wrap_asgi(ASGIApp(ctx, registry, ir), api_middleware(ir.api)) diff --git a/src/yaml_api_generator/server/common/context_builder.py b/src/yaml_api_generator/server/common/context_builder.py index def4acca..3496fe9d 100644 --- a/src/yaml_api_generator/server/common/context_builder.py +++ b/src/yaml_api_generator/server/common/context_builder.py @@ -31,7 +31,7 @@ def build_runtime_context(document: IRDocument) -> RuntimeContext: session = RuntimeSessionConfig( algorithm=document.session.algorithm, - ttl=document.session.ttl, + ttl=document.session.access_ttl, refresh=document.session.refresh, delivery=document.session.delivery, ) diff --git a/src/yaml_api_generator/server/common/http_middleware.py b/src/yaml_api_generator/server/common/http_middleware.py new file mode 100644 index 00000000..4cc2cf5a --- /dev/null +++ b/src/yaml_api_generator/server/common/http_middleware.py @@ -0,0 +1,36 @@ +"""ASGI middleware the spec's api block enables, shared by both dev servers.""" + +from functools import reduce + +from starlette import types +from starlette.middleware import Middleware +from starlette.middleware.cors import CORSMiddleware + +from yaml_api_generator.ir import IRApi +from yaml_api_generator.runtime.security_headers import SecurityHeadersMiddleware + + +def api_middleware(api: IRApi) -> tuple[Middleware, ...]: + """Return the enabled middleware, outermost first.""" + return ( + *((Middleware(SecurityHeadersMiddleware),) if api.security_headers else ()), + *( + (Middleware( + CORSMiddleware, + allow_origins=api.cors.origins, + allow_methods=api.cors.methods, + allow_headers=api.cors.headers, + allow_credentials=api.cors.credentials, + ),) + if api.cors else () + ), + ) + + +def wrap_asgi(app: types.ASGIApp, middleware: tuple[Middleware, ...]) -> types.ASGIApp: + """Wrap a bare ASGI app so the first middleware ends up outermost.""" + return reduce( + lambda inner, layer: layer.cls(inner, *layer.args, **layer.kwargs), + reversed(middleware), + app, + ) diff --git a/src/yaml_api_generator/server/common/steps/authentication.py b/src/yaml_api_generator/server/common/steps/authentication.py index 28b38dd4..6b9eb518 100644 --- a/src/yaml_api_generator/server/common/steps/authentication.py +++ b/src/yaml_api_generator/server/common/steps/authentication.py @@ -227,7 +227,7 @@ async def _resolve_identity_for_collection( return await _find_principal( repository, collection_name=identity.name, - field_name="_auth.apiKey", + field_name="apiKey", field_value=token, id_field=id_field, identity=identity, diff --git a/src/yaml_api_generator/server/fastapi/server.py b/src/yaml_api_generator/server/fastapi/server.py index d6508a50..0d485ae7 100644 --- a/src/yaml_api_generator/server/fastapi/server.py +++ b/src/yaml_api_generator/server/fastapi/server.py @@ -11,6 +11,7 @@ from ..common.context_builder import build_runtime_context from ..common.files import build_file_stores, build_record_files +from ..common.http_middleware import api_middleware from ..common.loader import create_repositories, ensure_spec_dir_importable, initialize_indexes from ..common.server_context import RuntimeContext from ..common.repositories import RepositoryRegistry @@ -98,20 +99,12 @@ async def lifespan(_: FastAPI): app = FastAPI(lifespan=lifespan) - if ir.api.cors: - from fastapi.middleware.cors import CORSMiddleware - app.add_middleware( - CORSMiddleware, - allow_origins=ir.api.cors.origins, - allow_methods=ir.api.cors.methods, - allow_headers=ir.api.cors.headers, - allow_credentials=ir.api.cors.credentials, - ) - if _has_idp_providers(ctx): app.add_middleware(_idp_provisioning_middleware(ctx, registry)) if ctx.session.uses_cookie: app.add_middleware(_csrf_middleware()) + for cls, args, kwargs in reversed(api_middleware(ir.api)): + app.add_middleware(cls, *args, **kwargs) register_fastapi_routes(app, ctx, registry, build_record_files(ir, stores)) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 652224f8..d4993d17 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,4 +1,7 @@ +from uuid import uuid4 + import pytest +from pymongo import MongoClient from tests.integration.support.file_apps import BACKENDS, CLIENT_ID, serve from tests.integration.support.minio import MINIO @@ -18,6 +21,15 @@ def mongo_uri(): yield uri +@pytest.fixture +def mongo_database(mongo_uri): + """Name of a fresh database, dropped after the test.""" + name = f"test_{uuid4().hex[:12]}" + yield name + with MongoClient(mongo_uri) as mongo: + mongo.drop_database(name) + + @pytest.fixture(scope="session") def issuer(): with live_issuer(CLIENT_ID) as issuer: diff --git a/tests/integration/support/v01_platform.py b/tests/integration/support/v01_platform.py new file mode 100644 index 00000000..0dff66b4 --- /dev/null +++ b/tests/integration/support/v01_platform.py @@ -0,0 +1,71 @@ +"""One 0.1 scenario exercised identically against the dev server and a generated app.""" + +from pymongo import MongoClient + +PASSWORD = "correct-horse-battery" + +SPEC = """ +version: "0.1" +api: {{ pagination: {{ defaultPageSize: 2, maxPageSize: 3 }} }} +mongo: {{ uri: "{uri}", database: {database} }} +collections: + users: + type: auth + auth: {{ identifier: email, via: password }} + roles: {{ values: [reader, editor, admin], default: reader, hierarchy: {{ admin: [editor], editor: [reader] }} }} + access: {{ create: public, list: "users:editor" }} + fields: + id: {{ type: id }} + email: {{ type: string, required: true, unique: true }} + clients: + type: auth + auth: {{ identifier: clientId, via: [apiKey] }} + access: {{ create: public }} + fields: + id: {{ type: id }} + clientId: {{ type: string, required: true, unique: true }} + apiKey: {{ type: string, required: true }} + posts: + access: {{ list: public, create: "clients:*" }} + fields: + id: {{ type: id }} + title: {{ type: string }} +""" + + +def exercise(client, uri: str, database: str) -> dict: + """Drive pagination, API-key auth and the role hierarchy; return what each request observed.""" + client.post("/api/clients", json={"clientId": "importer", "apiKey": "key-1"}) + keyed = [ + client.post("/api/posts", json={"title": f"post {n}"}, headers={"X-API-Key": "key-1"}).status_code + for n in range(4) + ] + anonymous = client.post("/api/posts", json={"title": "anonymous"}).status_code + + for email in ("admin@example.com", "reader@example.com"): + client.post("/api/users", json={"email": email, "password": PASSWORD}) + with MongoClient(uri) as mongo: + mongo[database]["users"].update_one({"email": "admin@example.com"}, {"$set": {"_auth.roles": ["admin"]}}) + + def users_listing(email: str) -> int: + token = client.post("/api/users/login", json={"email": email, "password": PASSWORD}).json()["token"] + return client.get("/api/users", headers={"Authorization": f"Bearer {token}"}).status_code + + return { + "keyed": keyed, + "anonymous": anonymous, + "default_page": len(client.get("/api/posts").json()), + "clamped_page": len(client.get("/api/posts", params={"limit": 50}).json()), + "admin_lists_users": users_listing("admin@example.com"), + "reader_lists_users": users_listing("reader@example.com"), + } + + +EXPECTED = { + "keyed": [200, 200, 200, 200], + "anonymous": 401, + "default_page": 2, + "clamped_page": 3, + "admin_lists_users": 200, + "reader_lists_users": 403, +} diff --git a/tests/integration/test_session_ttl.py b/tests/integration/test_session_ttl.py new file mode 100644 index 00000000..e3149b58 --- /dev/null +++ b/tests/integration/test_session_ttl.py @@ -0,0 +1,16 @@ +"""Rotating sessions issue access tokens of session.ttl seconds from a generated FastAPI app on MongoDB.""" + +import pytest + +from tests.unit.support.generated_apps import run_generated +from tests.unit.support.http_probes import TOKEN_LIFETIME_PROBE +from tests.unit.support.session_spec import ROTATING, mongo, session_spec + +pytestmark = pytest.mark.integration + + +@pytest.mark.parametrize(("session", "lifetime"), ROTATING.items()) +def test_generated_fastapi_rotating_token_lifetime(tmp_path, mongo_uri, mongo_database, session, lifetime): + spec = session_spec(tmp_path, session, store=mongo(mongo_uri, mongo_database)) + database = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database} + assert run_generated(spec, "fastapi", TOKEN_LIFETIME_PROBE, database) == lifetime diff --git a/tests/integration/test_v01_platform.py b/tests/integration/test_v01_platform.py new file mode 100644 index 00000000..951f4b32 --- /dev/null +++ b/tests/integration/test_v01_platform.py @@ -0,0 +1,30 @@ +"""0.1 pagination, API keys and role hierarchy over HTTP on the dev server and generated FastAPI.""" + +import pytest +from starlette.testclient import TestClient + +from tests.integration.support.v01_platform import EXPECTED, SPEC, exercise +from tests.unit.support.generated_apps import run_generated, write_spec +from yaml_api_generator.server import build_app + +pytestmark = pytest.mark.integration + + +def test_dev_server(tmp_path, monkeypatch, mongo_uri, mongo_database): + monkeypatch.setenv("YAML_API_MONGO_URI", mongo_uri) + monkeypatch.setenv("YAML_API_MONGO_DB", mongo_database) + spec = write_spec(tmp_path, SPEC.format(uri=mongo_uri, database=mongo_database)) + with TestClient(build_app(str(spec))) as client: + assert exercise(client, mongo_uri, mongo_database) == EXPECTED + + +def test_generated_fastapi(tmp_path, mongo_uri, mongo_database): + spec = write_spec(tmp_path, SPEC.format(uri=mongo_uri, database=mongo_database)) + script = f""" + from tests.integration.support.v01_platform import exercise + + with client: + print(json.dumps(exercise(client, {mongo_uri!r}, {mongo_database!r}))) + """ + env = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database} + assert run_generated(spec, "fastapi", script, env) == EXPECTED diff --git a/tests/unit/support/generated_apps.py b/tests/unit/support/generated_apps.py new file mode 100644 index 00000000..75e7c6a0 --- /dev/null +++ b/tests/unit/support/generated_apps.py @@ -0,0 +1,48 @@ +"""Compile a spec for a backend target and drive the generated app over HTTP.""" + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import Any + +from yaml_api_generator.cli.compiler import compile_project + +_REPOSITORY = Path(__file__).parents[3] + +_PRELUDE = """ +import json +from starlette.testclient import TestClient +import app + +client = TestClient(app.app) +""" + + +def write_spec(directory: Path, spec: str) -> Path: + path = directory / "spec.yaml" + path.write_text(textwrap.dedent(spec)) + return path + + +def generate(spec: Path, target: str) -> Path: + compile_project(yaml_file=spec, output_dir=spec.parent / target, target=target) + return spec.parent / target / "backend" + + +def run_generated(spec: Path, target: str, script: str, env: dict[str, str] | None = None) -> Any: + """Run ``script`` against ``client``, a TestClient of the generated app, and return what it prints as JSON. + + The script can import the ``tests`` package, so probes are shared with in-process tests. + """ + result = subprocess.run( + [sys.executable, "-c", _PRELUDE + textwrap.dedent(script)], + cwd=generate(spec, target), + env={**os.environ, "PYTHONPATH": str(_REPOSITORY), **(env or {})}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) diff --git a/tests/unit/support/http_probes.py b/tests/unit/support/http_probes.py new file mode 100644 index 00000000..80eeae26 --- /dev/null +++ b/tests/unit/support/http_probes.py @@ -0,0 +1,38 @@ +"""Requests shared by in-process and generated-app HTTP tests.""" + +import jwt + +_USER = {"email": "ttl@example.com", "password": "correct-horse-battery"} + + +def preflight(client, origin: str, path: str = "/api/posts") -> dict: + response = client.options( + path, + headers={ + "Origin": origin, + "Access-Control-Request-Method": "POST", + "Access-Control-Request-Headers": "X-API-Key", + }, + ) + return { + "status": response.status_code, + "origin": response.headers.get("access-control-allow-origin"), + "credentials": response.headers.get("access-control-allow-credentials"), + "methods": response.headers.get("access-control-allow-methods"), + } + + +def token_lifetime(client, base: str = "/api/users") -> int: + """Register and log in a user, returning the access token's ``exp - iat`` in seconds.""" + client.post(f"{base}/register", json=_USER) + token = client.post(f"{base}/login", json=_USER).json()["token"] + claims = jwt.decode(token, options={"verify_signature": False}) + return claims["exp"] - claims["iat"] + + +TOKEN_LIFETIME_PROBE = """ +from tests.unit.support.http_probes import token_lifetime + +with client: + print(token_lifetime(client)) +""" diff --git a/tests/unit/support/session_spec.py b/tests/unit/support/session_spec.py new file mode 100644 index 00000000..16b145fa --- /dev/null +++ b/tests/unit/support/session_spec.py @@ -0,0 +1,47 @@ +"""A register/login spec whose session block and datastore vary per test.""" + +from pathlib import Path + +from tests.unit.support.generated_apps import write_spec + +_SPEC = """ +version: "{version}" +api: + basePath: /api + operations: {{ auth: {{ register: true, login: true }} }} +{store} +entities: + user: + type: auth + auth: {{ identifier: email, via: password }} + fields: + id: {{ type: id }} + email: {{ type: string, required: true, unique: true }} +repositories: + users: {{ entity: user }} +services: + users: {{ repository: users }} +controllers: + users: + service: users + base_path: /users + access: "*" + operations: + register: {{ access: public }} + login: {{ access: public }} +{session} +""" + +SQL = 'datastores:\n store: { type: postgres, dsn: "postgresql+asyncpg://localhost/ttl", default: true }' + + +def mongo(uri: str = "mongodb://localhost:27017", database: str = "ttl") -> str: + return f'mongo: {{ uri: "{uri}", database: {database} }}' + + +LASTING = {"session: { ttl: 120 }": 120, "": 24 * 3600} +ROTATING = {"session: { ttl: 300, refresh: true }": 300, "session: { refresh: true }": 900} + + +def session_spec(directory: Path, session: str, version: str = "0.2", store: str = SQL) -> Path: + return write_spec(directory, _SPEC.format(version=version, store=store, session=session)) diff --git a/tests/unit/test_auth_steps.py b/tests/unit/test_auth_steps.py index 8fad6810..83fbb1c4 100644 --- a/tests/unit/test_auth_steps.py +++ b/tests/unit/test_auth_steps.py @@ -241,7 +241,7 @@ async def test_204_api_key_auth_resolves_principal(self): { "id": "c1", "clientId": "myclient", - "_auth": {"apiKey": "ak123"}, + "apiKey": "ak123", } ] } @@ -272,7 +272,7 @@ async def test_205_api_key_wrong_key_no_principal(self): { "id": "c1", "clientId": "myclient", - "_auth": {"apiKey": "ak123"}, + "apiKey": "ak123", } ] } @@ -487,7 +487,7 @@ async def test_213_api_key_via_bearer_fallback(self): "clients": [ { "id": "c1", - "_auth": {"apiKey": "ak123"}, + "apiKey": "ak123", } ] } diff --git a/tests/unit/test_auth_validation.py b/tests/unit/test_auth_validation.py index 599c0d0c..35377b81 100644 --- a/tests/unit/test_auth_validation.py +++ b/tests/unit/test_auth_validation.py @@ -3,8 +3,9 @@ ) -def _codes(entities: dict) -> set[str]: - return {error.code for error in validate_spec_auth({"entities": entities})} +def _codes(entities: dict, providers: dict | None = None) -> set[str]: + spec = {"entities": entities, "providers": providers or {}} + return {error.code for error in validate_spec_auth(spec)} def _auth_entity(auth: dict, fields: dict | None = None) -> dict: @@ -94,4 +95,19 @@ def test_non_auth_entity_ignored(): def test_oidc_only_via_needs_no_identifier(): entities = {"user": _auth_entity({"via": ["oidc"]})} - assert _codes(entities) == set() + assert _codes(entities, {"oidc": {"type": "oidc"}}) == set() + + +# ---- AUTH_VIA_UNKNOWN_PROVIDER ---- + +def test_via_undeclared_provider_rejected(): + entities = {"user": _auth_entity({"via": ["keycloak"]})} + assert _codes(entities) == {"AUTH_VIA_UNKNOWN_PROVIDER"} + + +def test_via_undeclared_provider_rejected_in_v01_collections(): + spec = {"collections": {"user": _auth_entity({"via": ["keycloak"], "identifier": "id"})}} + errors = validate_spec_auth(spec, "collections") + assert [(e.code, e.path) for e in errors] == [ + ("AUTH_VIA_UNKNOWN_PROVIDER", "collections.user.auth.via") + ] diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 68f4f366..4179d87c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,10 +1,12 @@ import json from pathlib import Path +import pytest from click.testing import CliRunner from yaml_api_generator.cli import cli from tests.conftest import VALID_DIR, INVALID_DIR +from tests.unit.support.generated_apps import write_spec V03_EXAMPLES = Path(__file__).parents[2] / "examples" / "v03" @@ -125,3 +127,27 @@ def test_frontend_without_ui_block_exits1(): ], ) assert result.exit_code == 1 + + +@pytest.mark.parametrize(("arguments", "port"), [([], 9100), (["-p", "9200"], 9200)], ids=["api-port", "flag"]) +@pytest.mark.parametrize("raw", [False, True], ids=["fastapi", "raw"]) +def test_run_binds_api_port_unless_flag_given(tmp_path, monkeypatch, arguments, port, raw): + import uvicorn + + spec = write_spec(tmp_path, """ + version: "0.3" + api: { port: 9100 } + mongo: { uri: "mongodb://localhost:27017", database: app } + entities: + post: { fields: { id: { type: id }, title: { type: string } } } + repositories: { posts: { entity: post } } + services: { posts: { repository: posts } } + controllers: + posts: + base_path: /posts + routes: { list: { service: posts, method: GET, path: "", access: public } } + """) + bound = [] + monkeypatch.setattr(uvicorn, "run", lambda app, **options: bound.append(options["port"])) + result = CliRunner().invoke(cli, ["run", str(spec), *arguments, *(["--raw"] if raw else [])]) + assert (result.exit_code, bound) == (0, [port]), result.output diff --git a/tests/unit/test_cors.py b/tests/unit/test_cors.py new file mode 100644 index 00000000..9f0ff4a8 --- /dev/null +++ b/tests/unit/test_cors.py @@ -0,0 +1,115 @@ +"""CORS configuration of every spec version answers preflights in every consumer.""" + +import pytest +from starlette.testclient import TestClient + +from tests.unit.support.generated_apps import generate, run_generated, write_spec +from tests.unit.support.http_probes import preflight +from yaml_api_generator.server import build_app, build_raw_app + +ORIGIN = "https://app.example" +FOREIGN = "https://evil.example" +VERSIONS = ["0.1", "0.2", "0.3"] + +_V01 = """ +version: "0.1" +{cors} +collections: + posts: + fields: + id: {{ type: id }} + title: {{ type: string }} +""" + +_V02_V03 = """ +version: "{version}" +api: + basePath: /api + {cors} +mongo: {{ uri: "mongodb://localhost:27017", database: cors }} +entities: + post: + fields: + id: {{ type: id }} + title: {{ type: string }} +repositories: + posts: {{ entity: post }} +services: + posts: {{ repository: posts }} +controllers: + posts: + base_path: /posts + routes: + list: {{ service: posts, method: GET, path: "", access: public }} +""" + +_CORS = { + "0.1": "security: {{ cors: {{ allowedOrigins: [{origins}], allowedMethods: [GET, POST], allowCredentials: {credentials} }} }}", + **dict.fromkeys( + ("0.2", "0.3"), "cors: {{ origins: [{origins}], methods: [GET, POST], credentials: {credentials} }}" + ), +} + + +def _spec(version: str, origin: str | None = ORIGIN, credentials: bool = True) -> str: + cors = "" if origin is None else _CORS[version].format( + origins=f'"{origin}"', credentials=str(credentials).lower() + ) + return (_V01 if version == "0.1" else _V02_V03).format(version=version, cors=cors) + + +ALLOWED = {"status": 200, "origin": ORIGIN, "credentials": "true", "methods": "GET, POST"} +REFUSED = {"status": 400, "origin": None, "credentials": "true", "methods": "GET, POST"} + +_PROBE = f""" +from tests.unit.support.http_probes import preflight + +print(json.dumps([preflight(client, {ORIGIN!r}), preflight(client, {FOREIGN!r})])) +""" + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev-fastapi", "dev-raw"]) +def test_dev_server_answers_preflight(tmp_path, version, build): + client = TestClient(build(str(write_spec(tmp_path, _spec(version))))) + assert [preflight(client, ORIGIN), preflight(client, FOREIGN)] == [ALLOWED, REFUSED] + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("target", ["fastapi", "raw"]) +def test_generated_app_answers_preflight(tmp_path, version, target): + assert run_generated(write_spec(tmp_path, _spec(version)), target, _PROBE) == [ALLOWED, REFUSED] + + +@pytest.mark.parametrize("version", VERSIONS) +def test_generated_drf_configures_cors_headers(tmp_path, version): + backend = generate(write_spec(tmp_path, _spec(version)), "drf") + settings = next(backend.glob("*/settings.py")).read_text() + assert '"corsheaders"' in settings + assert "corsheaders.middleware.CorsMiddleware" in settings + assert f"CORS_ALLOWED_ORIGINS = ['{ORIGIN}']" in settings + assert "CORS_ALLOW_METHODS = ['GET', 'POST']" in settings + assert "CORS_ALLOW_CREDENTIALS = True" in settings + assert "django-cors-headers" in (backend / "requirements.txt").read_text() + + +@pytest.mark.parametrize("version", VERSIONS) +def test_generated_drf_allows_any_origin_for_wildcard(tmp_path, version): + backend = generate(write_spec(tmp_path, _spec(version, origin="*", credentials=False)), "drf") + settings = next(backend.glob("*/settings.py")).read_text() + assert "CORS_ALLOW_ALL_ORIGINS = True" in settings + assert "CORS_ALLOWED_ORIGINS" not in settings + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("target", ["fastapi", "raw", "drf"]) +def test_backend_without_cors_has_no_cors_layer(tmp_path, version, target): + backend = generate(write_spec(tmp_path, _spec(version, origin=None)), target) + assert not any("CorsMiddleware" in path.read_text() for path in backend.rglob("*.py")) + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev-fastapi", "dev-raw"]) +def test_dev_server_without_cors_sends_no_cors_headers(tmp_path, version, build): + client = TestClient(build(str(write_spec(tmp_path, _spec(version, origin=None))))) + assert preflight(client, ORIGIN)["origin"] is None diff --git a/tests/unit/test_deploy_defaults.py b/tests/unit/test_deploy_defaults.py new file mode 100644 index 00000000..4fc227fb --- /dev/null +++ b/tests/unit/test_deploy_defaults.py @@ -0,0 +1,73 @@ +"""deploy.port and deploy.mongo.local default from the rest of the spec and honour explicit values.""" + +import pytest + +from tests.unit.support.generated_apps import write_spec +from yaml_api_generator.generators.deploy.context_builder import build_deploy_context +from yaml_api_generator.parsers.yaml_parser import load_and_generate_ir + +_SPEC = """ +version: "{version}" +api: + basePath: /api + {api} +{store} +entities: + post: + fields: + id: {{ type: id }} + title: {{ type: string }} +repositories: + posts: {{ entity: post }} +services: + posts: {{ repository: posts }} +controllers: + posts: + base_path: /posts + routes: + list: {{ service: posts, method: GET, path: "", access: public }} +deploy: + runtime: docker + {deploy} +""" + +LOCAL_MONGO = 'mongo: { uri: "mongodb://localhost:27017", database: app }' +REMOTE_MONGO = 'mongo: { uri: "mongodb://db.example.com:27017", database: app }' +POSTGRES = 'datastores:\n main: { type: postgres, dsn: "postgresql+asyncpg://db.example.com/app", default: true }' + +_VERSIONS = pytest.mark.parametrize("version", ["0.2", "0.3"]) + + +def _deploy_context(tmp_path, version: str, store: str, api: str = "", deploy: str = ""): + spec = write_spec(tmp_path, _SPEC.format(version=version, api=api, store=store, deploy=deploy)) + return build_deploy_context(load_and_generate_ir(str(spec)).ir, "localhost", "ubuntu", "~/.ssh/id_rsa") + + +@_VERSIONS +@pytest.mark.parametrize( + ("store", "deploy", "local"), + [ + (LOCAL_MONGO, "", True), + (REMOTE_MONGO, "", False), + (POSTGRES, "", False), + (REMOTE_MONGO, "mongo: { local: true }", True), + (LOCAL_MONGO, "mongo: { local: false }", False), + ], + ids=["local-uri", "remote-uri", "postgres-only", "explicit-local", "explicit-remote"], +) +def test_mongo_local(tmp_path, version, store, deploy, local): + context = _deploy_context(tmp_path, version, store, deploy=deploy) + assert context.mongo_local is local + assert [ds.local for ds in context.datastores if ds.type == "mongodb"] == ([local] if "mongo:" in store else []) + + +@_VERSIONS +@pytest.mark.parametrize( + ("api", "deploy", "port"), + [("", "", 8000), ("port: 9100", "", 9100), ("port: 9100", "port: 9200", 9200)], + ids=["default", "api-port", "deploy-port"], +) +def test_port(tmp_path, version, api, deploy, port): + context = _deploy_context(tmp_path, version, LOCAL_MONGO, api=api, deploy=deploy) + assert context.port == port + assert f"PORT=${{PORT:-{port}}}" in context.app_environment diff --git a/tests/unit/test_example_compatibility.py b/tests/unit/test_example_compatibility.py index 354b039c..b1759b95 100644 --- a/tests/unit/test_example_compatibility.py +++ b/tests/unit/test_example_compatibility.py @@ -81,9 +81,9 @@ def test_raw_routes_use_one_prefix_and_include_environment_files(tmp_path): sys.executable, "-c", ( - "from app import app; " - "print(app._match_route('GET', '/api/v1/users')[0] is not None); " - "print(app._match_route('GET', '/api/v1/api/v1/users')[0] is not None)" + "from app import ASGIApp; " + "print(ASGIApp._match_route('GET', '/api/v1/users')[0] is not None); " + "print(ASGIApp._match_route('GET', '/api/v1/api/v1/users')[0] is not None)" ), ], cwd=backend, diff --git a/tests/unit/test_examples_build.py b/tests/unit/test_examples_build.py new file mode 100644 index 00000000..17ef4cae --- /dev/null +++ b/tests/unit/test_examples_build.py @@ -0,0 +1,38 @@ +"""Every shipped example spec builds on the dev server and generates a FastAPI backend.""" + +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.server import build_app + +EXAMPLES = Path(__file__).parents[2] / "examples" +SPECS = sorted( + path for path in EXAMPLES.rglob("*.yaml") if not {"invalid", "packs"} & set(path.relative_to(EXAMPLES).parts) +) + +_EXAMPLES = pytest.mark.parametrize("spec", SPECS, ids=lambda path: str(path.relative_to(EXAMPLES))) + + +@pytest.fixture(autouse=True) +def dpop_key(monkeypatch): + """The oidc-pkce-dpop example's provider needs a persistent DPoP signing key.""" + key = ec.generate_private_key(ec.SECP256R1()).private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + monkeypatch.setenv("IDENTITY_DPOP_PRIVATE_KEY", key.decode()) + + +@_EXAMPLES +def test_example_builds_on_dev_server(spec): + build_app(str(spec)) + + +@_EXAMPLES +def test_example_generates_fastapi(tmp_path, spec): + compile_project(yaml_file=spec, output_dir=tmp_path, target="fastapi") + for source in tmp_path.rglob("*.py"): + compile(source.read_text(), str(source), "exec") diff --git a/tests/unit/test_mongo_graduation.py b/tests/unit/test_mongo_graduation.py index 84e88afb..b2ebde49 100644 --- a/tests/unit/test_mongo_graduation.py +++ b/tests/unit/test_mongo_graduation.py @@ -18,6 +18,7 @@ from fastapi import APIRouter from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.runtime import security_headers from yaml_api_generator.parsers.yaml_parser import ConfigErrors from yaml_api_generator.server.common.loader import SchemaManagement, create_repositories from yaml_api_generator.server.common.repositories import MongoRepository @@ -157,6 +158,7 @@ def _generated_app_dependencies(initialize_indexes) -> dict[str, types.ModuleTyp "dependencies.postgres": _stub( "dependencies.postgres", connect_db=_noop, close_db=_noop ), + "dependencies.security_headers": security_headers, "conflicts": _stub("conflicts", ConflictError=_Conflict), "services": _stub("services"), "services.exceptions": _stub( diff --git a/tests/unit/test_schema_consumers.py b/tests/unit/test_schema_consumers.py index 6eb53fdd..199d7905 100644 --- a/tests/unit/test_schema_consumers.py +++ b/tests/unit/test_schema_consumers.py @@ -18,12 +18,6 @@ # Accepted by the schema but read by no consumer yet. UNENFORCED = { - "allowCredentials": "0.1 security.cors", - "allowedMethods": "0.1 security.cors", - "allowedOrigins": "0.1 security.cors", - "defaultPageSize": "0.1 api.pagination", - "maxPageSize": "0.1 api.pagination", - "enableStandard": "0.1 security.headers (D16)", "softDelete": "0.1 collections (D19)", "additionalProperties": "object field option", } diff --git a/tests/unit/test_schema_rejection.py b/tests/unit/test_schema_rejection.py index 67c50c2e..6f6aaa30 100644 --- a/tests/unit/test_schema_rejection.py +++ b/tests/unit/test_schema_rejection.py @@ -71,6 +71,12 @@ "'bindings' was unexpected", ), "import-string-form": (("0.3",), ("imports", "helpers"), "helpers.py", "is not of type 'object'"), + "cors-credentials-any-origin": ( + ("0.1",), + ("security", "cors"), + {"allowedOrigins": ["*"], "allowCredentials": True}, + "should not be valid under", + ), "typo-root": (("0.1", "0.2", "0.3"), ("colections",), {}, "'colections' was unexpected"), "typo-field": (("0.2", "0.3"), ("entities", "post", *TITLE, "requird"), True, "'requird' was unexpected"), } @@ -124,6 +130,24 @@ def test_schema_rejects_unknown_or_removed_key(tmp_path, version, path, value, f assert any(fragment in error.message for error in errors) +MONGO = {"uri": "mongodb://localhost:27017", "database": "blog"} +POSTGRES = {"type": "postgres", "dsn": "postgresql+asyncpg://localhost/blog", "default": True} +FILESYSTEM = {"type": "filesystem", "path": "./uploads"} + + +@pytest.mark.parametrize("version", ["0.2", "0.3"]) +def test_mongo_block_beside_record_datastores_rejected(tmp_path, version): + spec = {**BASE_SPECS[version], "mongo": MONGO, "datastores": {"main": POSTGRES}} + errors = _load_errors(tmp_path, spec) + assert [(error.code, error.path) for error in errors] == [("SCHEMA_VALIDATION_ERROR", "mongo")] + + +def test_mongo_block_beside_file_datastores_accepted(tmp_path): + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump({**BASE_SPECS["0.3"], "mongo": MONGO, "datastores": {"disk": FILESYSTEM}})) + assert load_and_generate_ir(str(path)).ir.mongo.database == "blog" + + @pytest.mark.parametrize("spec,path", WRONG_SHAPES.values(), ids=list(WRONG_SHAPES)) def test_wrong_container_shape_reports_schema_error(tmp_path, spec, path): errors = _load_errors(tmp_path, {"version": "0.2", **spec}) diff --git a/tests/unit/test_security_headers.py b/tests/unit/test_security_headers.py new file mode 100644 index 00000000..7e0884f8 --- /dev/null +++ b/tests/unit/test_security_headers.py @@ -0,0 +1,115 @@ +"""Standard security headers are on by default and every consumer honours the off switch.""" + +from types import SimpleNamespace + +import pytest +from starlette.applications import Starlette +from starlette.responses import PlainTextResponse +from starlette.routing import Route +from starlette.testclient import TestClient + +from tests.unit.support.generated_apps import generate, run_generated, write_spec +from yaml_api_generator.runtime.security_headers import ( + STANDARD_HEADERS, + DjangoSecurityHeadersMiddleware, + SecurityHeadersMiddleware, +) +from yaml_api_generator.server import build_app, build_raw_app + +_V01 = """ +version: "0.1" +{switch} +collections: + posts: + fields: + id: {{ type: id }} + title: {{ type: string }} +""" + +_V02_V03 = """ +version: "{version}" +api: + basePath: /api + {switch} +mongo: {{ uri: "mongodb://localhost:27017", database: headers }} +entities: + post: + fields: + id: {{ type: id }} + title: {{ type: string }} +repositories: + posts: {{ entity: post }} +services: + posts: {{ repository: posts }} +controllers: + posts: + base_path: /posts + routes: + list: {{ service: posts, method: GET, path: "", access: public }} +""" + +_SPECS = { + ("0.1", True): _V01.format(switch=""), + ("0.1", False): _V01.format(switch="security: { headers: { enableStandard: false } }"), + **{ + (version, enabled): _V02_V03.format( + version=version, switch="" if enabled else "securityHeaders: false" + ) + for version in ("0.2", "0.3") + for enabled in (True, False) + }, +} + +_CASES = pytest.mark.parametrize( + ("version", "enabled"), sorted(_SPECS), ids=lambda value: str(value) +) + +_PROBE = """ +print(json.dumps(dict(client.get("/missing").headers))) +""" + + +def _standard_headers(headers) -> dict[str, str]: + return {name: headers[name] for name, _ in STANDARD_HEADERS if name in headers} + + +def _expected(enabled: bool) -> dict[str, str]: + return dict(STANDARD_HEADERS) if enabled else {} + + +@_CASES +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev-fastapi", "dev-raw"]) +def test_dev_server_response_headers(tmp_path, version, enabled, build): + client = TestClient(build(str(write_spec(tmp_path, _SPECS[version, enabled])))) + assert _standard_headers(client.get("/missing").headers) == _expected(enabled) + + +@_CASES +@pytest.mark.parametrize("target", ["fastapi", "raw"]) +def test_generated_app_response_headers(tmp_path, version, enabled, target): + headers = run_generated(write_spec(tmp_path, _SPECS[version, enabled]), target, _PROBE) + assert _standard_headers(headers) == _expected(enabled) + + +@_CASES +def test_generated_drf_installs_middleware_only_when_enabled(tmp_path, version, enabled): + backend = generate(write_spec(tmp_path, _SPECS[version, enabled]), "drf") + settings = next(backend.glob("*/settings.py")).read_text() + assert ("api.security_headers.DjangoSecurityHeadersMiddleware" in settings) is enabled + assert (backend / "api" / "security_headers.py").exists() is enabled + + +def test_asgi_middleware_keeps_headers_the_route_set(): + def framed(_): + return PlainTextResponse("ok", headers={"x-frame-options": "SAMEORIGIN"}) + + app = SecurityHeadersMiddleware(Starlette(routes=[Route("/", framed)])) + headers = TestClient(app).get("/").headers + assert headers["x-frame-options"] == "SAMEORIGIN" + assert headers["x-content-type-options"] == "nosniff" + + +def test_django_middleware_keeps_headers_the_view_set(): + response = SimpleNamespace(headers={"referrer-policy": "same-origin"}) + middleware = DjangoSecurityHeadersMiddleware(lambda request: response) + assert middleware(object()).headers == {**dict(STANDARD_HEADERS), "referrer-policy": "same-origin"} diff --git a/tests/unit/test_session_ttl.py b/tests/unit/test_session_ttl.py new file mode 100644 index 00000000..07288aaf --- /dev/null +++ b/tests/unit/test_session_ttl.py @@ -0,0 +1,56 @@ +"""session.ttl sets the access-token lifetime in the dev server and every generated backend.""" + +import pytest +from starlette.testclient import TestClient + +from tests.unit.support.generated_apps import generate, run_generated +from tests.unit.support.http_probes import TOKEN_LIFETIME_PROBE, token_lifetime +from tests.unit.support.session_spec import LASTING, ROTATING, mongo, session_spec +from yaml_api_generator.parsers.yaml_parser import ConfigErrors +from yaml_api_generator.server import build_app + + +def _cases(lifetimes: dict[str, int]): + return pytest.mark.parametrize( + ("session", "lifetime"), lifetimes.items(), ids=lambda value: str(value) or "none" + ) + + +_VERSIONS = pytest.mark.parametrize("version", ["0.2", "0.3"]) + + +@_VERSIONS +@_cases(LASTING | ROTATING) +def test_dev_server_login_token_lifetime(tmp_path, monkeypatch, version, session, lifetime): + monkeypatch.setenv("YAML_API_DB_STORE_DSN", f"sqlite+aiosqlite:///{tmp_path / 'dev.db'}") + with TestClient(build_app(str(session_spec(tmp_path, session, version)))) as client: + assert token_lifetime(client) == lifetime + + +@_VERSIONS +@_cases(LASTING) +def test_generated_fastapi_login_token_lifetime(tmp_path, version, session, lifetime): + database = {"DATABASE_URL": f"sqlite+aiosqlite:///{tmp_path / 'generated.db'}"} + spec = session_spec(tmp_path, session, version) + assert run_generated(spec, "fastapi", TOKEN_LIFETIME_PROBE, database) == lifetime + + +@_cases(ROTATING) +def test_generated_fastapi_settings_rotating_lifetime(tmp_path, session, lifetime): + backend = generate(session_spec(tmp_path, session, store=mongo()), "fastapi") + assert f"JWT_ACCESS_TTL_SECONDS: int = {lifetime}" in (backend / "settings.py").read_text() + + +@_cases(ROTATING) +def test_generated_fastapi_rejects_rotation_without_mongo(tmp_path, session, lifetime): + with pytest.raises(ConfigErrors) as rejected: + generate(session_spec(tmp_path, session), "fastapi") + assert [(e.code, e.path) for e in rejected.value.errors] == [("BACKEND_FEATURE_UNSUPPORTED", "session.refresh")] + + +@_cases(LASTING) +@pytest.mark.parametrize(("target", "settings"), [("raw", "settings.py"), ("drf", "api/settings.py")]) +def test_generated_settings_lasting_lifetime(tmp_path, session, lifetime, target, settings): + backend = generate(session_spec(tmp_path, session, store=mongo()), target) + expected = f"JWT_ACCESS_TTL_SECONDS: int = {lifetime}" if session else "JWT_EXPIRATION_HOURS: int = 24" + assert expected in (backend / settings).read_text() diff --git a/tests/unit/test_v01_transformer.py b/tests/unit/test_v01_transformer.py index 055dc316..a5a8646d 100644 --- a/tests/unit/test_v01_transformer.py +++ b/tests/unit/test_v01_transformer.py @@ -1,9 +1,13 @@ import pathlib import pytest +from starlette.testclient import TestClient +from yaml_api_generator.ir import IRCors, IRPageSize, IRRolesConfig from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir +from yaml_api_generator.server import build_app from tests.conftest import INVALID_DIR, VALID_DIR +from tests.unit.support.generated_apps import write_spec EXAMPLES_DIR = pathlib.Path(__file__).parent.parent.parent / "examples" / "valid" @@ -139,3 +143,86 @@ def test_v01_password_hash_field_forbidden() -> None: load_and_generate_ir(str(INVALID_DIR / "v01-password-field-forbidden.yaml")) codes = {e.code for e in exc_info.value.errors} assert "AUTH_PASSWORD_HASH_FIELD_FORBIDDEN" in codes + + +_PLATFORM_SPEC = """ +version: "0.1" +{top} +collections: + users: + type: auth + auth: {{ identifier: email, via: password }} + {roles} + fields: + id: {{ type: id }} + email: {{ type: string, required: true, unique: true }} + posts: + fields: + id: {{ type: id }} + {field}: {{ type: string, minLength: {min_length} }} +""" + + +def _platform_ir(tmp_path, top="", roles="", field="title", min_length=1): + spec = write_spec(tmp_path, _PLATFORM_SPEC.format(top=top, roles=roles, field=field, min_length=min_length)) + return load_and_generate_ir(str(spec)).ir + + +def _platform_errors(tmp_path, **parts) -> list[tuple[str, str]]: + with pytest.raises(ConfigErrors) as exc_info: + _platform_ir(tmp_path, **parts) + return [(e.code, e.path) for e in exc_info.value.errors] + + +def _list_page_size(ir) -> IRPageSize: + return next(r.pagination.size for r in ir.routes if r.collection == "posts" and r.operation == "list") + + +@pytest.mark.parametrize( + ("pagination", "size"), + [ + ("", IRPageSize(default=20, max=100)), + ("api: { pagination: { defaultPageSize: 5, maxPageSize: 50 } }", IRPageSize(default=5, max=50)), + ("api: { pagination: { maxPageSize: 10 } }", IRPageSize(default=10, max=10)), + ], + ids=["defaults", "explicit", "max-below-default"], +) +def test_v01_pagination_sizes_list_routes(tmp_path, pagination, size) -> None: + assert _list_page_size(_platform_ir(tmp_path, top=pagination)) == size + + +def test_v01_security_block_reaches_api_ir(tmp_path) -> None: + security = ( + "security:\n" + " cors: { allowedOrigins: [\"https://app.example\"], allowedMethods: [GET], allowCredentials: true }\n" + " headers: { enableStandard: false }" + ) + api = _platform_ir(tmp_path, top=security).api + assert (api.cors, api.security_headers) == ( + IRCors(origins=["https://app.example"], methods=["GET"], credentials=True), + False, + ) + + +def test_v01_roles_parsed_into_identity_collection(tmp_path) -> None: + roles = "roles: { values: [reader, editor], default: reader, hierarchy: { editor: [reader] } }" + assert _platform_ir(tmp_path, roles=roles).identityCollections["users"].roles == IRRolesConfig( + values=("reader", "editor"), default="reader", hierarchy={"editor": ("reader",)} + ) + + +def test_v01_roles_hierarchy_unknown_role_rejected(tmp_path) -> None: + roles = "roles: { values: [reader], default: reader, hierarchy: { admin: [reader] } }" + assert _platform_errors(tmp_path, roles=roles) == [ + ("ROLES_HIERARCHY_UNKNOWN_ROLE", "collections.users.roles.hierarchy") + ] + + +def test_v01_shared_validators_check_collections(tmp_path) -> None: + assert _platform_errors(tmp_path, field="class") == [("INVALID_FIELD_NAME", "collections.posts.fields.class")] + assert [code for code, _ in _platform_errors(tmp_path, min_length=-1)] == ["FIELD_CONSTRAINT_INVALID"] + + +def test_v01_example_serves_on_dev_server() -> None: + client = TestClient(build_app(str(EXAMPLES_DIR / "blog.yaml"))) + assert "/api/posts" in client.get("/openapi.json").json()["paths"] From f9712207c83052a6554bcadb2875b7e6953e54f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Fri, 23 Oct 2026 17:52:20 +0100 Subject: [PATCH 07/89] Test nested array payloads and the payload scenario on MongoDB development servers --- ...raw_payloads.py => test_mongo_payloads.py} | 25 +++++++++++++++---- tests/unit/support/payload_requests.py | 21 +++++++++++++--- tests/unit/test_payload_models.py | 4 +-- 3 files changed, 40 insertions(+), 10 deletions(-) rename tests/integration/{test_generated_raw_payloads.py => test_mongo_payloads.py} (57%) diff --git a/tests/integration/test_generated_raw_payloads.py b/tests/integration/test_mongo_payloads.py similarity index 57% rename from tests/integration/test_generated_raw_payloads.py rename to tests/integration/test_mongo_payloads.py index 8afbfaba..aca4ec74 100644 --- a/tests/integration/test_generated_raw_payloads.py +++ b/tests/integration/test_mongo_payloads.py @@ -1,4 +1,4 @@ -"""A generated raw-ASGI app validates the shared payload scenario against a real MongoDB.""" +"""Development servers and a generated raw-ASGI app validate the shared payload scenario against a real MongoDB.""" import json import os @@ -8,9 +8,11 @@ from uuid import uuid4 import pytest +from starlette.testclient import TestClient -from tests.unit.support.payload_requests import EXPECTED, JWT_SECRET, MONGO_STORE, spec +from tests.unit.support.payload_requests import EXPECTED, JWT_SECRET, mongo_store, scenario, spec from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.server import build_app, build_raw_app ROOT = Path(__file__).parents[2] @@ -27,10 +29,23 @@ """ +@pytest.fixture +def spec_file(tmp_path, mongo_uri): + path = tmp_path / "spec.yaml" + path.write_text(spec(mongo_store(mongo_uri, f"payloads_{uuid4().hex}"))) + return path + + +@pytest.mark.integration +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["fastapi", "raw"]) +def test_dev_server_validates_payloads(spec_file, build, monkeypatch): + monkeypatch.setenv("JWT_SECRET", JWT_SECRET) + with TestClient(build(str(spec_file))) as client: + assert scenario(client) == EXPECTED + + @pytest.mark.integration -def test_generated_raw_validates_payloads(tmp_path, mongo_uri): - spec_file = tmp_path / "spec.yaml" - spec_file.write_text(spec(MONGO_STORE)) +def test_generated_raw_validates_payloads(spec_file, tmp_path, mongo_uri): compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "raw"), target="raw") result = subprocess.run( [sys.executable, "-c", _RUNNER], diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py index 2f6c290d..fdce2776 100644 --- a/tests/unit/support/payload_requests.py +++ b/tests/unit/support/payload_requests.py @@ -18,12 +18,15 @@ default: true """ -MONGO_STORE = """ + +def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payloads") -> str: + return f""" mongo: - uri: mongodb://localhost:27017 - database: payloads + uri: {uri} + database: {database} """ + JWT_SECRET = "payload-scenario-secret-of-at-least-32-bytes" _API = """ @@ -63,6 +66,13 @@ color: { type: string, required: true } size: { type: number, min: 1 } additionalProperties: false + points: + type: array + items: + type: object + properties: + x: { type: number, required: true } + label: { type: object, properties: { contact: { type: string, format: email } } } note: fields: id: { type: id, format: uuidv4 } @@ -104,6 +114,11 @@ ({"name": "ab", "meta": {"size": 2}}, 422), ({"name": "ab", "meta": {"color": "r", "size": 0}}, 422), ({"name": "ab", "meta": {"color": "r", "size": "2"}}, 422), + ({"name": "ab", "points": [{"x": 1, "label": {"contact": "a@b.co"}}]}, 200), + ({"name": "ab", "points": [{}]}, 422), + ({"name": "ab", "points": [{"x": 1, "y": 2}]}, 422), + ({"name": "ab", "points": [{"x": 1, "label": {"contact": "x"}}]}, 422), + ({"name": "ab", "points": [{"x": 1, "label": {"other": 1}}]}, 422), ({"name": "ab", "rank": "3"}, 422), ({"name": "ab", "active": "true"}, 422), ({"name": "ab", "active": 1}, 422), diff --git a/tests/unit/test_payload_models.py b/tests/unit/test_payload_models.py index cc56a70b..61839638 100644 --- a/tests/unit/test_payload_models.py +++ b/tests/unit/test_payload_models.py @@ -19,8 +19,8 @@ from tests.unit.support.payload_requests import ( EXPECTED, JWT_SECRET, - MONGO_STORE, SQL_STORE, + mongo_store, scenario, spec, ) @@ -80,7 +80,7 @@ def test_generated_fastapi_validates_payloads(spec_file): @pytest.fixture def mongo_spec_file(tmp_path): path = tmp_path / "mongo-spec.yaml" - path.write_text(spec(MONGO_STORE)) + path.write_text(spec(mongo_store())) return str(path) From e804c29f11607a86523ec5ad8f578059d147e448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Fri, 23 Oct 2026 18:11:13 +0100 Subject: [PATCH 08/89] Generate abstract services as base classes and run custom operations through service methods Extending services and repositories merge their ancestors' configuration, minified services share their input and forwarding with every operation, and handlers and repository operations receive the same HandlerRequest in every backend. --- examples/valid/access-control-advanced.yaml | 3 + .../generators/backend/builders/__init__.py | 1 + .../generators/backend/compile_context.py | 22 +++- .../backend/drf/templates/handler_call.jinja | 12 +- .../drf/templates/handler_views.py.jinja | 4 +- .../backend/drf/templates/views.py.jinja | 21 ++-- .../generators/backend/fastapi/generator.py | 28 +++++ .../fastapi/templates/handler_call.jinja | 22 ++-- .../fastapi/templates/handler_routes.py.jinja | 4 +- .../backend/fastapi/templates/routes.py.jinja | 52 ++------- .../fastapi/templates/service.py.jinja | 30 +++-- .../templates/service_operations.jinja | 11 ++ .../backend/raw/templates/handler_call.jinja | 12 +- .../raw/templates/handler_routes.py.jinja | 4 +- .../backend/raw/templates/routes.py.jinja | 10 +- src/yaml_api_generator/ir/__init__.py | 2 + src/yaml_api_generator/ir/api.py | 15 ++- .../parsers/desugar/__init__.py | 69 +----------- .../parsers/desugar/inheritance_desugarer.py | 106 ++++++++---------- .../parsers/desugar/service_desugarer.py | 19 ++-- .../parsers/transformers/v02/__init__.py | 1 + .../transformers/v02/route_transformer.py | 49 ++++---- src/yaml_api_generator/parsers/ui_parser.py | 2 +- src/yaml_api_generator/parsers/yaml_parser.py | 12 +- .../runtime/handler_requests.py | 10 +- src/yaml_api_generator/runtime/operations.py | 8 +- .../server/common/steps/repositories.py | 6 +- tests/unit/test_controller_bind.py | 3 +- .../unit/test_fastapi_bindings_generation.py | 3 +- tests/unit/test_repo_service_inheritance.py | 64 +++++++++++ tests/unit/test_service_input_codegen.py | 10 +- 31 files changed, 337 insertions(+), 278 deletions(-) create mode 100644 src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja diff --git a/examples/valid/access-control-advanced.yaml b/examples/valid/access-control-advanced.yaml index 69a0f313..eacef2f9 100644 --- a/examples/valid/access-control-advanced.yaml +++ b/examples/valid/access-control-advanced.yaml @@ -68,6 +68,9 @@ repositories: services: users: repository: users + operations: + register: true + login: true articles: repository: articles diff --git a/src/yaml_api_generator/generators/backend/builders/__init__.py b/src/yaml_api_generator/generators/backend/builders/__init__.py index 1e295228..ec956ea0 100644 --- a/src/yaml_api_generator/generators/backend/builders/__init__.py +++ b/src/yaml_api_generator/generators/backend/builders/__init__.py @@ -118,6 +118,7 @@ def _repo_datastore_type(repo) -> str: repository_abstract=repository_abstract, repository_inherits=repository_inherits, services=dict(document.services), + service_bases=dict(document.service_bases), dto_models=dto_models, vendored_requirements=document.vendored_requirements, sql_schema_managed=managed.sql, diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index f0634b49..1c41229c 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -18,7 +18,8 @@ from dataclasses import dataclass, field from typing import Any, ClassVar -from ...ir.api import IRFileField, IRPagination, IRProvider, IRRolesConfig, IRService +from ...constants import CRUD_OPERATIONS +from ...ir.api import IRFileField, IRPagination, IRProvider, IRRolesConfig, IRService, IRServiceBase from ...ir.when import WhenExpr from ...utils.strings import to_py_identifier @@ -283,6 +284,13 @@ def record_criteria(self) -> list[tuple[str, str]]: return parents return [*parents, (self.id_field, self.record_id_param)] + @property + def runs_operation(self) -> bool: + """Whether the route runs a handler or a declared repository operation instead of a generated step.""" + return self.operation != "login" and ( + self.handler is not None or self.operation not in CRUD_OPERATIONS + ) + @property def django_path(self) -> str: path = self.relative_path.lstrip("/") @@ -424,6 +432,7 @@ class CompileContext: repository_inherits: dict[str, str] = field(default_factory=dict) # IR services keyed by their expanded flat name (e.g. "list-posts"). services: dict[str, IRService] = field(default_factory=dict) + service_bases: dict[str, IRServiceBase] = field(default_factory=dict) dto_models: list[ModelDescriptor] = field(default_factory=list) vendored_requirements: tuple[str, ...] = () sql_schema_managed: bool = False @@ -470,3 +479,14 @@ def mongo_unique_indexes(self) -> dict[str, list[str]]: def collection_datastore_type(self, collection: str) -> str: return self.collections.get(collection, {}).get("datastore_type", "mongodb") + + def service_base(self, repository: str) -> IRServiceBase | None: + """The abstract service that the services of ``repository`` extend.""" + return next( + ( + self.service_bases[service.inherits] + for service in self.services.values() + if service.repository == repository and service.inherits + ), + None, + ) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja index de6f609c..05f04d9c 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja @@ -1,5 +1,6 @@ -{% macro invoke_handler(handler) %} - return Response(await invoke({{ handler | py_literal }}, HandlerRequest( +{# Runs the route's handler, or the repository operation it declares, over one HandlerRequest #} +{% macro run_operation(route) %} + _request = HandlerRequest( params=path_params, query=dict(request.query_params), headers={key.lower(): value for key, value in request.headers.items()}, @@ -7,5 +8,10 @@ body=body, principal=_resolved_principal, inputs=service_inputs, - ))) + ) +{% if route.handler %} + return Response(await invoke({{ route.handler | py_literal }}, _request)) +{% else %} + return Response(await repo.{{ route.operation }}(_request.bindings)) +{% endif %} {% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index cb611883..853b472f 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -1,4 +1,4 @@ -{% from "handler_call.jinja" import invoke_handler %} +{% from "handler_call.jinja" import run_operation %} from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView @@ -85,7 +85,7 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A {{ route.service_forwarding | py_literal }}, bound_inputs, ) -{{ invoke_handler(route.handler) }} +{{ run_operation(route) }} except ConflictError as exc: return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) except ValueError as exc: diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 213866fa..cb7d5628 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -1,4 +1,4 @@ -{% from "handler_call.jinja" import invoke_handler %} +{% from "handler_call.jinja" import run_operation %} from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView @@ -29,12 +29,9 @@ from api.repositories.{{ where_collection }} import {{ where_collection | capita from api.services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from api.services.exceptions import NotFoundError, ValidationError from api.service_inputs import prepare_service_inputs, resolve_access_bind -{% if routes | selectattr("handler") | list %} +{% if routes | selectattr("runs_operation") | list %} from api.handler_requests import HandlerRequest, invoke {% endif %} -{% if routes | rejectattr("handler") | rejectattr("operation", "in", crud_op_names + ["login"]) | list %} -from api.operations import operation_bindings -{% endif %} {% if routes | selectattr("pagination") | list %} from api.pagination import build_response, map_request {% endif %} @@ -160,8 +157,8 @@ class {{ class_name }}ListView(APIView): {{ bind_inputs(route) }} try: service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.handler %} -{{ invoke_handler(route.handler) }} +{% if route.runs_operation %} +{{ run_operation(route) }} {% elif route.operation == "list" %} {% if route.pagination %} pagination_config = {{ route.pagination | pagination_config | py_literal }} @@ -203,11 +200,7 @@ class {{ custom_class }}(APIView): {{ bind_inputs(route) }} try: service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.handler %} -{{ invoke_handler(route.handler) }} -{% else %} - return Response(await repo.{{ route.operation }}(operation_bindings({**path_params, **service_inputs}, _resolved_principal, body))) -{% endif %} +{{ run_operation(route) }} except NotFoundError: return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) except ConflictError as exc: @@ -237,8 +230,8 @@ class {{ class_name }}DetailView(APIView): {{ bind_inputs(route) }} try: service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.handler %} -{{ invoke_handler(route.handler) }} +{% if route.runs_operation %} +{{ run_operation(route) }} {% elif route.operation == "get" %} {% if route.record_parent_params %} result = await service.get_by({{ record_criteria(route) }}) diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index d4911696..bcb5d6f9 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -3,6 +3,7 @@ from jinja2 import Environment from yaml_api_generator.generators.backend.compile_context import CompileContext, IdentityCollectionDescriptor +from yaml_api_generator.ir import IRServiceBase from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields from yaml_api_generator.utils.strings import to_pascal_case, to_py_identifier from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment @@ -40,6 +41,17 @@ def _crud_operations(col_data: dict) -> list[str]: return [op for op in _CRUD_ORDER if op in present] +def _own_operations(operations: dict[str, str | None], base: IRServiceBase | None) -> dict[str, str | None]: + inherited = base.operations.items() if base else {}.items() + return {name: handler for name, handler in operations.items() if (name, handler) not in inherited} + + +def _generated_service_base(base: IRServiceBase | None) -> dict[str, str] | None: + if base is None: + return None + return {"module": f"services.{base.name}", "class_name": f"{to_pascal_case(base.name)}Service"} + + class FastAPIGenerator(TemplateBasedGenerator): def generate(self, context: CompileContext, output_dir: Path) -> None: validate_datastore_layout(context, "fastapi") @@ -181,13 +193,29 @@ def _render_services( ops_content = ops_template.render(auth=context.auth) (output_dir / "services" / "_ops.py").write_text(ops_content) + template = env.get_template("service.py.jinja") + for base in context.service_bases.values(): + parent = context.service_bases.get(base.inherits) + (output_dir / "services" / f"{base.name}.py").write_text(template.render( + collection_name=base.name, + operations=[], + is_identity_collection=False, + password_via=False, + custom_operations=_own_operations(base.operations, parent), + generated_base=_generated_service_base(parent), + )) + def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: identity = find_identity(context, col_name) is_identity = identity is not None + base = context.service_base(col_name) + operations = {route.operation: route.handler for route in col_data["routes"] if route.runs_operation} return { "collection_name": col_name, "operations": _crud_operations(col_data), + "custom_operations": _own_operations(operations, base), + "generated_base": _generated_service_base(base), "is_identity_collection": is_identity, "password_via": is_identity and identity.has_password, "default_role": identity.roles.default if is_identity and identity.roles else None, diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja index d9c45ccf..f12b7670 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_call.jinja @@ -1,14 +1,16 @@ -{% macro invoke_handler(handler, principal, body, inputs) %} +{# Runs `call`, an awaitable over `_request`; a ValueError it raises answers 422 #} +{% macro call_with_request(call, principal, body, inputs) %} + _request = HandlerRequest( + params=dict(request.path_params), + query=dict(request.query_params), + headers=dict(request.headers), + cookies=dict(request.cookies), + body={{ body }}, + principal={{ principal }}, + inputs={{ inputs }}, + ) try: - return await invoke({{ handler | py_literal }}, HandlerRequest( - params=dict(request.path_params), - query=dict(request.query_params), - headers=dict(request.headers), - cookies=dict(request.cookies), - body={{ body }}, - principal={{ principal }}, - inputs={{ inputs }}, - )) + return await {{ call }} except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc {% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja index 6431747c..4cb7df08 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja @@ -1,4 +1,4 @@ -{% from "handler_call.jinja" import invoke_handler %} +{% from "handler_call.jinja" import call_with_request %} from typing import Any from fastapi import APIRouter, Depends, HTTPException, Request @@ -86,6 +86,6 @@ async def {{ route.operation | py_identifier }}_{{ loop.index }}( ) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc -{{ invoke_handler(route.handler, "principal" if route.needs_auth else "None", "body", "service_inputs") }} +{{ call_with_request("invoke(" ~ (route.handler | py_literal) ~ ", _request)", "principal" if route.needs_auth else "None", "body", "service_inputs") }} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 89e8da18..80dbbd33 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -1,8 +1,6 @@ -{% from "handler_call.jinja" import invoke_handler %} -{% set crud_op_names = ["list", "get", "create", "update", "replace", "delete", "login"] %} -{% set crud_routes = routes | rejectattr("operation", "equalto", "login") | rejectattr("handler") | selectattr("operation", "in", crud_op_names) | list %} -{% set custom_routes = routes | rejectattr("operation", "equalto", "login") | selectattr("handler") | list %} -{% set repo_op_routes = routes | rejectattr("handler") | rejectattr("operation", "in", crud_op_names) | list %} +{% from "handler_call.jinja" import call_with_request %} +{% set crud_routes = routes | rejectattr("operation", "equalto", "login") | rejectattr("runs_operation") | list %} +{% set operation_routes = routes | selectattr("runs_operation") | list %} {% set ns_service_input = namespace(enabled=false, access_bind=false) %} {% set ns_where = namespace(collections=[]) %} {% for route in routes %} @@ -17,8 +15,8 @@ {% set pagination_routes = routes | selectattr("pagination") | list %} from typing import Any -{% set needs_request = custom_routes or repo_op_routes or (crud_routes | selectattr("needs_request") | list) or (file_routes | selectattr("action", "equalto", "replace") | list) %} -from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes or custom_routes %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} +{% set needs_request = operation_routes or (crud_routes | selectattr("needs_request") | list) or (file_routes | selectattr("action", "equalto", "replace") | list) %} +from fastapi import APIRouter, Depends{% if auth.enabled or ns_service_input.enabled or pagination_routes or operation_routes %}, HTTPException{% endif %}{% if needs_request %}, Request{% endif %} {% if ns_service_input.enabled or ns_where.collections %} from dependencies.service_inputs import {% if ns_service_input.enabled %}prepare_service_inputs{% endif %}{% if ns_service_input.enabled and (ns_service_input.access_bind or ns_where.collections) %}, {% endif %}{% if ns_service_input.access_bind or ns_where.collections %}resolve_access_bind{% endif +%} @@ -38,18 +36,12 @@ from dependencies.file_requests import RecordUpload, carried_files, discard_on_e from dependencies.file_stores import record_files {% endif %} -{% if custom_routes %} -from dependencies.handler_requests import HandlerRequest, invoke -{% endif %} -{% if repo_op_routes %} -from dependencies.operations import operation_bindings +{% if operation_routes %} +from dependencies.handler_requests import HandlerRequest {% endif %} {% for group in service_groups %} from dependencies.services import get_{{ group }}_service {% endfor %} -{% if repo_op_routes %} -from dependencies.repositories import get_{{ collection_name }}_repository -{% endif %} {% for where_collection in ns_where.collections %} from dependencies.repositories import get_{{ where_collection }}_repository {% endfor %} @@ -341,12 +333,11 @@ async def {{ file_route.handler_name }}({{ source.record_id_param }}: str{{ extr {% endfor %} -{% for route in custom_routes %} -{% set handler_name = route.operation ~ "_" ~ collection_name %} +{% for route in operation_routes %} @router.{{ route.method | lower }}("{{ route.relative_path }}", tags=["{{ collection_name | capitalize_collection }}"]{{ openapi_extra(route) }}) -async def {{ handler_name }}(request: Request{% for name in route.path_params %}, {{ name }}: str{% endfor %}{{ auth_param(route) }}{{ where_repo_params(route) }}) -> Any: +async def {{ route.operation }}_{{ collection_name }}(request: Request{% for name in route.path_params %}, {{ name }}: str{% endfor %}{{ auth_param(route) }}, {{ svc_dep() }}{{ where_repo_params(route) }}) -> Any: _body: dict[str, Any] = {} try: _body = await request.json() @@ -356,28 +347,5 @@ async def {{ handler_name }}(request: Request{% for name in route.path_params %} {{ pre_fetch_guard(route) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, "dict(request.path_params)", "_body") }} -{{ invoke_handler(route.handler, "principal" if auth.enabled and route.needs_auth else "None", "_body", "_service_inputs") }} -{% endfor %} -{% for route in repo_op_routes %} -{% set handler_name = route.operation ~ "_" ~ collection_name %} - - -@router.{{ route.method | lower }}("{{ route.relative_path }}", tags=["{{ collection_name | capitalize_collection }}"]{{ openapi_extra(route) }}) -async def {{ handler_name }}(request: Request{{ auth_param(route) }}, repo=Depends(get_{{ collection_name }}_repository){{ where_repo_params(route) }}) -> Any: -{% if route.needs_auth %} - _principal: dict[str, Any] = principal -{% else %} - _principal: dict[str, Any] = {} -{% endif %} - _body: dict[str, Any] = {} - try: - _body = await request.json() - except (ValueError, UnicodeDecodeError): - pass - _path = dict(request.path_params) -{{ request_locals(route, "_body") }} -{{ pre_fetch_guard(route) }} -{{ bind_inputs(route) }} -{{ prepare_inputs(route, "_path", "_body") }} - return await repo.{{ route.operation }}(operation_bindings({**_path, **_service_inputs}, _principal, _body)) +{{ call_with_request("svc." ~ route.operation ~ "(_request)", "principal" if auth.enabled and route.needs_auth else "None", "_body", "_service_inputs") }} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja index 51a7827c..fc63283b 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja @@ -1,22 +1,30 @@ -{% set class_name = collection_name | capitalize_collection %} -{% set svc_mixins = operations | map("capitalize") | list %} -{% set identity_mixins = [] %} -{% if is_identity_collection %} -{% set _ = identity_mixins.append("IdentityFieldsMixin") %} +{% from "service_operations.jinja" import operation_methods %} +{% set bases = [] %} +{% for op in operations %}{% set _ = bases.append((op | capitalize) ~ "Svc") %}{% endfor %} +{% if is_identity_collection %}{% set _ = bases.append("IdentityFieldsMixin") %}{% endif %} +{% if password_via %}{% set _ = bases.append("PasswordMixin") %}{% endif %} +{% set ops_bases = bases + ([] if generated_base else ["ServiceBase"]) %} +{% if custom_operations %} +from typing import Any + +from dependencies.handler_requests import HandlerRequest{% if custom_operations.values() | select | list %}, invoke{% endif %} + {% endif %} -{% if password_via %} -{% set _ = identity_mixins.append("PasswordMixin") %} +{% if ops_bases %} +from services._ops import {{ ops_bases | join(", ") }} +{% endif %} +{% if generated_base %} +from {{ generated_base.module }} import {{ generated_base.class_name }} {% endif %} -from services._ops import ServiceBase{% for op in svc_mixins %}, {{ op }}Svc{% endfor %}{% for mixin in identity_mixins %}, {{ mixin }}{% endfor %} - -class {{ class_name }}Service({% for op in svc_mixins %}{{ op }}Svc, {% endfor %}{% for mixin in identity_mixins %}{{ mixin }}, {% endfor %}ServiceBase): +class {{ collection_name | capitalize_collection }}Service({{ (ops_bases + ([generated_base.class_name] if generated_base else [])) | join(", ") }}): {% if is_identity_collection %} IDENTITY_FIELDS = ({% for field_name in identity_fields %}"{{ field_name }}", {% endfor %}) {% if password_via %} DEFAULT_ROLE = {{ default_role | py_literal }} {% endif %} -{% else %} +{% elif not custom_operations %} pass {% endif %} +{{ operation_methods(custom_operations) }} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja new file mode 100644 index 00000000..3adc70a8 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja @@ -0,0 +1,11 @@ +{% macro operation_methods(operations) %} +{% for name, handler in operations.items() %} + + async def {{ name }}(self, request: HandlerRequest) -> Any: +{% if handler %} + return await invoke({{ handler | py_literal }}, request) +{% else %} + return await self._repo.{{ name }}(request.bindings) +{% endif %} +{% endfor %} +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja index cb4dbac9..cecf7164 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja @@ -1,5 +1,6 @@ -{% macro invoke_handler(handler) %} - return 200, await invoke({{ handler | py_literal }}, HandlerRequest( +{# Runs the route's handler, or the repository operation it declares, over one HandlerRequest #} +{% macro run_operation(route) %} + _request = HandlerRequest( params=url_params, query=query or {}, headers=headers, @@ -7,5 +8,10 @@ body=body or {}, principal=principal, inputs=service_inputs, - )) + ) +{% if route.handler %} + return 200, await invoke({{ route.handler | py_literal }}, _request) +{% else %} + return 200, await repo.{{ route.operation }}(_request.bindings) +{% endif %} {% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja index f87f98e5..9ee98ca7 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja @@ -1,4 +1,4 @@ -{% from "handler_call.jinja" import invoke_handler %} +{% from "handler_call.jinja" import run_operation %} from typing import Any from uuid import UUID @@ -111,6 +111,6 @@ async def handle_custom_{{ loop.index }}( {{ route.service_forwarding | py_literal }}, bound_inputs, ) -{{ invoke_handler(route.handler) }} +{{ run_operation(route) }} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index f0e371c8..33e5ed4d 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -1,4 +1,4 @@ -{% from "handler_call.jinja" import invoke_handler %} +{% from "handler_call.jinja" import run_operation %} from typing import Any from uuid import UUID @@ -13,7 +13,6 @@ from repositories.{{ where_collection }} import {{ where_collection | capitalize from services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from services.exceptions import NotFoundError, ValidationError from handler_requests import HandlerRequest, invoke -from operations import operation_bindings from service_inputs import prepare_service_inputs, resolve_access_bind {% if routes | selectattr("pagination") | list %} from pagination import build_response, map_request @@ -52,7 +51,6 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: return any(required_role in hierarchy.get(role, set()) for role in roles) -{% set crud_op_names = ["list", "get", "create", "update", "replace", "delete", "login"] %} {% macro record_criteria(route) %}{{ "{" }}{% for field, param in route.record_criteria %}"{{ field }}": url_params["{{ param }}"]{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} {% macro authorization_locals(route) %} {% if route.access_rules %} @@ -147,10 +145,8 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {{ pre_fetch_authorize(route) }} {{ bind_inputs(route) }} service_inputs = prepare_service_inputs(url_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.handler %} -{{ invoke_handler(route.handler) }} -{% elif route.operation not in crud_op_names %} - return 200, await repo.{{ route.operation }}(operation_bindings({**url_params, **service_inputs}, principal, body)) +{% if route.runs_operation %} +{{ run_operation(route) }} {% else %} try: {% if route.operation == "list" %} diff --git a/src/yaml_api_generator/ir/__init__.py b/src/yaml_api_generator/ir/__init__.py index 410ed256..af0059b2 100644 --- a/src/yaml_api_generator/ir/__init__.py +++ b/src/yaml_api_generator/ir/__init__.py @@ -41,6 +41,7 @@ IRRoute, IRRolesConfig, IRService, + IRServiceBase, IRServiceInput, IRVendoredImport, IROIDCProvider, @@ -131,6 +132,7 @@ "IRRoute", "IRRolesConfig", "IRService", + "IRServiceBase", "IRServiceInput", "IRVendoredImport", "IROIDCProvider", diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 652cd674..e1d5c277 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -341,12 +341,24 @@ class IRService: repository: str | None = None operation: str | None = None handler: str | None = None - abstract: bool = False inherits: str | None = None input: IRServiceInput | None = None forwarding: dict[str, WhereRhs] = field(default_factory=dict) +@dataclass(frozen=True) +class IRServiceBase: + """An abstract service, generated as the base class of the services extending it. + + ``operations`` maps each operation the class implements to its handler, or + to None when the service runs the repository operation of the same name. + """ + + name: str + operations: dict[str, str | None] = field(default_factory=dict) + inherits: str | None = None + + @dataclass(frozen=True) class IRRoute: method: str @@ -463,6 +475,7 @@ class IRDocument: identityCollections: dict[str, IRIdentityCollection] dtos: dict[str, IRDto] = field(default_factory=dict) services: dict[str, IRService] = field(default_factory=dict) + service_bases: dict[str, IRServiceBase] = field(default_factory=dict) providers: dict[str, IRProvider] = field(default_factory=dict) mongo: IRMongo = field(default_factory=IRMongo) datastores: dict[str, IRDatastore] = field(default_factory=dict) diff --git a/src/yaml_api_generator/parsers/desugar/__init__.py b/src/yaml_api_generator/parsers/desugar/__init__.py index b86d45cb..9666a38c 100644 --- a/src/yaml_api_generator/parsers/desugar/__init__.py +++ b/src/yaml_api_generator/parsers/desugar/__init__.py @@ -35,75 +35,12 @@ def desugar_v02(config: dict[str, Any]) -> dict[str, Any]: controllers = config.get("controllers", {}) EntityDesugarer().desugar(entities, repositories) - - _desugar_repositories(repositories, controllers) - - # Resolve service inheritance before expansion so abstract entries and the - # inherited base are computed on the minified service dict. - svc_inherits = _resolve_service_inheritance(services, controllers) - - service_desugarer = ServiceDesugarer() - service_desugarer.desugar(services, repositories, entities, defaults, auth_defaults) - - # Propagate the inherited base to all flat entries from a concrete minified svc. - _attach_service_inherits(services, svc_inherits) - + _REPO_DESUGARER.desugar(repositories, controllers) + _SVC_DESUGARER.desugar(services, controllers) + ServiceDesugarer().desugar(services, repositories, entities, defaults, auth_defaults) ControllerDesugarer().desugar(controllers, services) return config -def _desugar_repositories(repositories: dict[str, Any], controllers: dict[str, Any]) -> None: - """Validate repo inheritance, stamp the inherited base, drop empty abstracts. - - Abstract repositories that declare operations are kept: they are rendered - as generated base classes that concrete repositories inherit. - """ - inherits = _REPO_DESUGARER.resolve(repositories) - _REPO_DESUGARER.validate_not_bound( - repositories, controllers, "repository", - error_code="DESUGAR_ABSTRACT_REPOSITORY_BOUND", - ) - for name, base in inherits.items(): - cfg = repositories.get(name) - if isinstance(cfg, dict): - cfg["_inherits"] = base - _REPO_DESUGARER.remove_abstract(repositories) - - -def _resolve_service_inheritance( - services: dict[str, Any], controllers: dict[str, Any] -) -> dict[str, str | None]: - """Validate service inheritance and return the inherited base keyed by repo name.""" - inherits = _SVC_DESUGARER.resolve(services) - _SVC_DESUGARER.validate_not_bound( - services, controllers, "service", - error_code="DESUGAR_ABSTRACT_SERVICE_BOUND", - ) - # Re-key by repository name so _attach_service_inherits can match flat entries. - inherits_by_repo: dict[str, str | None] = {} - for svc_name, base in inherits.items(): - cfg = services.get(svc_name) - repo = cfg.get("repository", svc_name) if isinstance(cfg, dict) else svc_name - inherits_by_repo[repo] = base - - _SVC_DESUGARER.remove_abstract(services) - return inherits_by_repo - - -def _attach_service_inherits( - services: dict[str, Any], svc_inherits: dict[str, str | None] -) -> None: - """Stamp _inherits onto each expanded flat service that originated from a named svc. - - The flat key format is ``{op}-{svc_name}``, e.g. ``list-posts``. - """ - for cfg in services.values(): - if not isinstance(cfg, dict): - continue - repo = cfg.get("repository") - if repo and repo in svc_inherits: - cfg["_inherits"] = svc_inherits[repo] - - __all__ = ["desugar_v02"] diff --git a/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py b/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py index 6ff206a6..6486f5c4 100644 --- a/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py +++ b/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py @@ -1,17 +1,14 @@ -"""Generic single-inheritance desugarer for repository and service layers. - -Mirrors EntityDesugarer's validate/cycle-detect/resolve pattern but instead of -flattening fields it resolves a chain of ``base`` import paths that the code -generator uses to emit user-authored base-class inheritance. -""" +"""Generic single-inheritance desugarer for repository and service layers.""" from typing import Any from ..yaml_parser import ConfigError, ConfigErrors +_UNINHERITED = frozenset({"abstract", "extends"}) + class InheritanceDesugarer: - """Resolve ``abstract``/``extends``/``base`` for a named config layer. + """Resolve ``abstract``/``extends`` for a named config layer. Args: layer: Human-readable layer name used in error codes/paths @@ -21,77 +18,51 @@ class InheritanceDesugarer: def __init__(self, layer: str) -> None: self._layer = layer - def resolve(self, entries: dict[str, Any]) -> dict[str, str | None]: - """Validate inheritance and resolve each entry's inherited base. + def desugar(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: + """Validate inheritance and fold every ancestor into the entries extending it. - Validates that every ``extends`` target exists and is abstract and - detects cycles, then walks each entry's ``extends`` chain to the - nearest ancestor that declares ``operations``. That ancestor is the - *inherited base*: it is rendered as a generated class which the entry - subclasses. + Every ``extends`` target must exist and be abstract, chains must not + cycle and no binding may reference an abstract entry. Each extending + entry receives its ancestors' configuration: its own keys win and + ``operations`` merge per operation. ``_inherits`` names the nearest + ancestor declaring ``operations``, rendered as the generated class the + entry subclasses. Abstract entries without operations are dropped. Args: - entries: Raw config dict for the layer (repositories or services). - - Returns: - Mapping from entry name to the name of the nearest - operation-bearing ancestor it inherits (or ``None``). + entries: The layer's config dict (repositories or services). + bindings: The controllers, whose ```` key references entries. Raises: - ConfigErrors: On missing/non-abstract extends target or cycle. + ConfigErrors: On a missing or non-abstract target, a cycle or a bound abstract entry. """ self._validate(entries) self._validate_no_cycles(entries) - return { - name: self._resolve_chain(name, entries) + self._validate_not_bound(entries, bindings) + entries.update({ + name: {**self._merged(name, entries), "_inherits": self._resolve_chain(name, entries)} for name, cfg in entries.items() if isinstance(cfg, dict) and cfg.get("extends") - } - - def remove_abstract(self, entries: dict[str, Any]) -> None: - """Remove abstract entries that carry no operations. - - Abstract entries with operations survive: they are rendered as - generated base classes that concrete entries inherit. - """ - removable = [ - k for k, v in entries.items() - if isinstance(v, dict) and v.get("abstract") and not v.get("operations") - ] - for name in removable: + }) + for name in [ + name for name, cfg in entries.items() + if isinstance(cfg, dict) and cfg.get("abstract") and not cfg.get("operations") + ]: del entries[name] - def validate_not_bound( - self, - entries: dict[str, Any], - bindings: dict[str, Any], - binding_field: str, - *, - error_code: str, - ) -> None: - """Ensure abstract entries are not referenced by controllers/routes. - - Args: - entries: The layer dict (repositories or services). - bindings: The dict that references entries (e.g. controllers). - binding_field: Field name in each binding that holds the reference. - error_code: Error code to use if a violation is found. - """ + def _validate_not_bound(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: abstract_names = { name for name, cfg in entries.items() if isinstance(cfg, dict) and cfg.get("abstract") } - errors: list[ConfigError] = [] - for binding_name, binding_cfg in bindings.items(): - if not isinstance(binding_cfg, dict): - continue - ref = binding_cfg.get(binding_field) - if ref in abstract_names: - errors.append(ConfigError( - code=error_code, - message=f"'{binding_name}' references abstract {self._layer} '{ref}'", - path=f"{self._layer}s.{binding_name}.{binding_field}", - )) + errors = [ + ConfigError( + code=f"DESUGAR_ABSTRACT_{self._layer.upper()}_BOUND", + message=f"'{binding_name}' references abstract {self._layer} '{binding_cfg[self._layer]}'", + path=f"controllers.{binding_name}.{self._layer}", + ) + for binding_name, binding_cfg in bindings.items() + if isinstance(binding_cfg, dict) and binding_cfg.get(self._layer) in abstract_names + ] if errors: raise ConfigErrors(errors) @@ -137,6 +108,17 @@ def _validate_no_cycles(self, entries: dict[str, Any]) -> None: if errors: raise ConfigErrors(errors) + def _merged(self, name: str, entries: dict[str, Any]) -> dict[str, Any]: + own = entries[name] + if not own.get("extends"): + return own + parent = { + key: value for key, value in self._merged(own["extends"], entries).items() + if key not in _UNINHERITED + } + operations = {**parent.get("operations", {}), **own.get("operations", {})} + return {**parent, **own, **({"operations": operations} if operations else {})} + def _resolve_chain(self, name: str, entries: dict[str, Any]) -> str | None: """Find the nearest ancestor declaring ``operations`` up the chain.""" current = entries[name].get("extends") if isinstance(entries.get(name), dict) else None diff --git a/src/yaml_api_generator/parsers/desugar/service_desugarer.py b/src/yaml_api_generator/parsers/desugar/service_desugarer.py index 901d1662..2a89315f 100644 --- a/src/yaml_api_generator/parsers/desugar/service_desugarer.py +++ b/src/yaml_api_generator/parsers/desugar/service_desugarer.py @@ -9,6 +9,8 @@ "login": {"operation": "login"}, } +_SHARED_KEYS = ("input", "with", "_inherits") + class ServiceDesugarer: """ @@ -86,7 +88,7 @@ def _is_minified_service( self, key: str, entry: dict[str, Any], repositories: dict[str, Any] ) -> bool: """Check if service entry is minified (needs expansion).""" - if "operation" in entry or "handler" in entry: + if "operation" in entry or "handler" in entry or entry.get("abstract"): return False return "repository" in entry or key in repositories @@ -128,9 +130,10 @@ def _expand_minified_service( entity = entities[entity_name] service_ops = entry.get("operations", {}) + shared = {key: entry[key] for key in _SHARED_KEYS if key in entry} self._expand_crud_operations( - svc_name, repo_name, repo, service_ops, defaults, expanded + svc_name, repo_name, repo, service_ops, shared, defaults, expanded ) is_auth = isinstance(entity, dict) and entity.get("type") == "auth" @@ -145,10 +148,11 @@ def _expand_crud_operations( repo_name: str, repo: dict[str, Any], service_ops: dict[str, Any], + shared: dict[str, Any], defaults: dict[str, Any], expanded: dict[str, Any], ) -> None: - """Expand CRUD operations for a service.""" + """Expand CRUD and custom operations, each sharing the service's input, forwarding and base.""" repo_crud = repo.get("crud") if isinstance(repo, dict) else None repo_ops: dict[str, Any] = {} if repo_crud is not None: @@ -160,11 +164,10 @@ def _expand_crud_operations( for op_name, op_value in resolved.items(): if op_value is False: continue - flat_key = f"{op_name}-{svc_name}" - if op_value is True: - expanded[flat_key] = {"repository": repo_name, "operation": op_name} - elif isinstance(op_value, dict) and "handler" in op_value: - expanded[flat_key] = {"handler": op_value["handler"]} + implementation = op_value if isinstance(op_value, dict) else {} + expanded[f"{op_name}-{svc_name}"] = { + **shared, "repository": repo_name, "operation": op_name, **implementation, + } def _expand_auth_operations( self, diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 3bae68c3..1bacc959 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -147,6 +147,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: identityCollections=identity_collections, dtos=dtos, services=services, + service_bases=self._route.transform_service_bases(services_data, import_map), mongo=self._route.build_mongo(mongo_config), datastores=datastores, default_datastore=default_datastore, diff --git a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py index 7a15e329..92071f71 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py @@ -17,7 +17,9 @@ IRRepository, IRRoute, IRService, + IRServiceBase, ) +from yaml_api_generator.constants import CRUD_OPERATIONS from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors from .access_transformer import AccessTransformer @@ -92,32 +94,41 @@ def transform_services( services: dict[str, IRService] = {} errors: list[ConfigError] = [] for svc_name, svc_cfg in services_data.items(): - if not isinstance(svc_cfg, dict): + if not isinstance(svc_cfg, dict) or svc_cfg.get("abstract"): continue input_ir, forwarding = self._transform_input(svc_name, svc_cfg, dtos_data, errors) handler_ref = svc_cfg.get("handler") - inherits = svc_cfg.get("_inherits") - if handler_ref is not None: - services[svc_name] = IRService( - name=svc_name, - handler=self._resolve_handler_path(handler_ref, import_map), - inherits=inherits, - input=input_ir, - forwarding=forwarding, - ) - else: - services[svc_name] = IRService( - name=svc_name, - repository=svc_cfg.get("repository"), - operation=svc_cfg.get("operation"), - inherits=inherits, - input=input_ir, - forwarding=forwarding, - ) + services[svc_name] = IRService( + name=svc_name, + repository=svc_cfg.get("repository"), + operation=svc_cfg.get("operation"), + handler=None if handler_ref is None else self._resolve_handler_path(handler_ref, import_map), + inherits=svc_cfg.get("_inherits"), + input=input_ir, + forwarding=forwarding, + ) if errors: raise ConfigErrors(errors) return services + def transform_service_bases( + self, services_data: dict[str, Any], import_map: dict[str, str] + ) -> dict[str, IRServiceBase]: + """Build the abstract services that concrete services inherit as generated classes.""" + return { + name: IRServiceBase( + name=name, + operations={ + operation: None if value is True else self._resolve_handler_path(value["handler"], import_map) + for operation, value in cfg["operations"].items() + if isinstance(value, dict) or (value is True and operation not in CRUD_OPERATIONS) + }, + inherits=cfg.get("_inherits"), + ) + for name, cfg in services_data.items() + if isinstance(cfg, dict) and cfg.get("abstract") + } + def _transform_input( self, svc_name: str, diff --git a/src/yaml_api_generator/parsers/ui_parser.py b/src/yaml_api_generator/parsers/ui_parser.py index 5dd3506d..db1ddbcd 100644 --- a/src/yaml_api_generator/parsers/ui_parser.py +++ b/src/yaml_api_generator/parsers/ui_parser.py @@ -76,7 +76,7 @@ def _build_entities(ir: IRDocument) -> dict[str, EntityInfo]: if not route.service or route.operation not in CRUD_OPS: continue service = ir.services.get(route.service) - if not service or not service.repository: + if not service or not service.repository or service.handler: continue repo = service.repository repo_operations.setdefault(repo, set()).add(route.operation) diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 8a1258a6..4bbc61e0 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -251,12 +251,6 @@ def load_and_generate_ir( raise ConfigErrors(access_errors) access_warnings = tuple(warn_public_routes(resolved)) - if version == "0.3": - from .transformers.symbol_validation import validate_spec_symbols - symbol_errors = validate_spec_symbols(resolved) - if symbol_errors: - raise ConfigErrors(symbol_errors) - from .transformers.auth_validation import validate_spec_auth auth_errors = validate_spec_auth(resolved) if auth_errors: @@ -273,6 +267,12 @@ def load_and_generate_ir( from .desugar import desugar_v02 resolved = desugar_v02(resolved) + if version == "0.3": + from .transformers.symbol_validation import validate_spec_symbols + symbol_errors = validate_spec_symbols(resolved) + if symbol_errors: + raise ConfigErrors(symbol_errors) + from .transformers.path_validation import validate_spec_paths path_errors = validate_spec_paths(resolved) if path_errors: diff --git a/src/yaml_api_generator/runtime/handler_requests.py b/src/yaml_api_generator/runtime/handler_requests.py index 5bd7cabe..d4447de9 100644 --- a/src/yaml_api_generator/runtime/handler_requests.py +++ b/src/yaml_api_generator/runtime/handler_requests.py @@ -1,7 +1,8 @@ -"""The contract every backend uses to call a user handler. +"""The request every backend passes to a user handler or a repository operation. A handler is a sync or async callable that receives one ``HandlerRequest`` and -returns the JSON response body. Raising ``ValueError`` answers 422. +returns the JSON response body. Raising ``ValueError`` answers 422. A repository +operation resolves its query's ``$`` references against ``bindings``. """ from dataclasses import dataclass, field @@ -20,6 +21,11 @@ class HandlerRequest: principal: dict[str, Any] | None = None inputs: dict[str, Any] = field(default_factory=dict) + @property + def bindings(self) -> dict[str, Any]: + params = {**self.params, **self.inputs} + return {**params, "principal": self.principal or {}, "params": params, "body": self.body or {}} + async def invoke(handler: str, request: HandlerRequest) -> Any: module_path, name = handler.rsplit(".", 1) diff --git a/src/yaml_api_generator/runtime/operations.py b/src/yaml_api_generator/runtime/operations.py index 1cffa690..1d1382a4 100644 --- a/src/yaml_api_generator/runtime/operations.py +++ b/src/yaml_api_generator/runtime/operations.py @@ -1,6 +1,6 @@ """Bind and present declared repository operations. -A query value starting with ``$`` is a reference into the request bindings +A query value starting with ``$`` is a reference into ``HandlerRequest.bindings`` (``{**params, "principal", "params", "body"}``); every other value passes through unchanged. Results are presented the way CRUD responses are: the datastore's own ``_id`` becomes the canonical id field and credential fields @@ -12,12 +12,6 @@ _HIDDEN_FIELDS = ("_auth", "password") -def operation_bindings( - params: dict[str, Any], principal: dict[str, Any] | None, body: Any -) -> dict[str, Any]: - return {**params, "principal": principal or {}, "params": params, "body": body or {}} - - def resolve(value: Any, bindings: dict[str, Any]) -> Any: if isinstance(value, str) and value.startswith("$"): return _lookup(value[1:].split("."), bindings) diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 8a4b886e..05f52568 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -6,7 +6,6 @@ from ..request_context import RequestContext from yaml_api_generator.ir.api import IRPagination, IRRolesConfig from yaml_api_generator.runtime.operations import ( - operation_bindings, present, present_document as _inject_canonical_id, resolve, @@ -15,6 +14,7 @@ from ..repositories.base import BaseRepository from ..utils.ids import generate_id as _generate_id from . import NotFoundError, Step +from .handlers import handler_request _PATH_TOKEN = re.compile(r"\{(\w+)\}") @@ -271,9 +271,7 @@ def RepositoryOperationStep( id_field: str, ) -> Step: async def step(ctx: RequestContext) -> RequestContext: - bindings = operation_bindings( - {**ctx.url_params, **(ctx.service_inputs or {})}, ctx.principal, ctx.body - ) + bindings = handler_request(ctx).bindings result = await _repository(ctx).run_operation(collection_name, resolve(query, bindings)) ctx.response = present(result, id_field) return ctx diff --git a/tests/unit/test_controller_bind.py b/tests/unit/test_controller_bind.py index 4b94428d..8d697f42 100644 --- a/tests/unit/test_controller_bind.py +++ b/tests/unit/test_controller_bind.py @@ -317,7 +317,7 @@ def test_generated_fastapi_applies_authorized_bind_end_to_end(tmp_path): from fastapi import FastAPI from fastapi.testclient import TestClient - from dependencies.auth import require_principal + from dependencies.auth import get_optional_principal, require_principal from dependencies.repositories import get_posts_repository from dependencies.operations import resolve from repositories.posts import PostsRepository @@ -341,6 +341,7 @@ async def principal(): app = FastAPI() app.include_router(router) app.dependency_overrides[require_principal] = principal + app.dependency_overrides[get_optional_principal] = principal app.dependency_overrides[get_posts_repository] = RecordingRepository response = TestClient(app).post( "/posts/p1/archive", json={{"owner": "spoofed"}} diff --git a/tests/unit/test_fastapi_bindings_generation.py b/tests/unit/test_fastapi_bindings_generation.py index 4d7e1dbc..a3a341a5 100644 --- a/tests/unit/test_fastapi_bindings_generation.py +++ b/tests/unit/test_fastapi_bindings_generation.py @@ -49,4 +49,5 @@ def test_custom_operation_route_calls_the_repository_method(tmp_path): assert "async def byOrg(self, bindings: dict[str, Any]) -> Any:" in repository assert "'$principal.record.orgId'" in repository route_content = (out / "routes" / "posts.py").read_text() - assert "repo.byOrg(operation_bindings(" in route_content + assert "svc.byOrg(_request)" in route_content + assert "self._repo.byOrg(request.bindings)" in (out / "services" / "posts.py").read_text() diff --git a/tests/unit/test_repo_service_inheritance.py b/tests/unit/test_repo_service_inheritance.py index c4847f05..8da3c55b 100644 --- a/tests/unit/test_repo_service_inheritance.py +++ b/tests/unit/test_repo_service_inheritance.py @@ -168,3 +168,67 @@ def test_concrete_repo_inherits_generated_base(tmp_path): assert "from .countable import CountableRepository" in repo_src assert "CountableRepository, RepositoryBase" in repo_src assert "async def countAll" not in repo_src # inherited, not redefined + + +def test_abstract_service_bound_to_controller_raises(): + config = _base_config() + config["services"]["base"] = {"abstract": True, "operations": {"delete": False}} + config["controllers"]["posts"]["service"] = "base" + with pytest.raises(ConfigErrors) as exc_info: + desugar_v02(config) + assert any(e.code == "DESUGAR_ABSTRACT_SERVICE_BOUND" for e in exc_info.value.errors) + + +def test_repository_extends_merges_inherited_and_own_operations(): + config = _base_config() + config["repositories"]["base"] = {"abstract": True, "operations": {"countAll": {"query": {"command": "estimatedDocumentCount"}}}} + config["repositories"]["posts"] |= {"extends": "base", "operations": {"byTitle": {"query": {"command": "find"}}}} + operations = desugar_v02(config)["repositories"]["posts"]["operations"] + assert set(operations) == {"countAll", "byTitle"} + + +def test_service_extends_merges_inherited_and_own_operations(): + config = _base_config() + config["repositories"]["posts"]["operations"] = {"countAll": {"query": {"command": "estimatedDocumentCount"}}} + config["services"]["base"] = {"abstract": True, "operations": {"delete": False, "countAll": True}} + config["services"]["posts"] |= {"extends": "base", "operations": {"update": False}} + config["controllers"]["posts"]["operations"] = {"countAll": {"method": "GET", "path": "/count"}} + services = desugar_v02(config)["services"] + assert {"countAll-posts", "list-posts", "get-posts", "create-posts"} <= set(services) + assert not {"delete-posts", "update-posts"} & set(services) + assert services["countAll-posts"]["_inherits"] == "base" + + +_SERVICE_FIXTURE = _FIXTURE.replace( + """ services: + posts: + repository: posts +""", + """ services: + auditable: + abstract: true + operations: + countAll: true + summary: { handler: hooks.summary } + posts: + repository: posts + extends: auditable +""", +).replace(" access: public\n", " access: public\n operations:\n countAll: { method: GET, path: /count }\n summary: { method: GET, path: /summary }\n") + + +def test_abstract_service_is_a_base_in_ir(tmp_path): + ir = _parse(_SERVICE_FIXTURE, tmp_path) + assert ir.service_bases["auditable"].operations == {"countAll": None, "summary": "hooks.summary"} + assert "auditable" not in ir.services + assert ir.services["countAll-posts"].inherits == "auditable" + + +def test_abstract_service_generates_a_base_class_its_services_extend(tmp_path): + FastAPIGenerator().generate(build_context(_parse(_SERVICE_FIXTURE, tmp_path)), tmp_path / "out") + base_src = (tmp_path / "out" / "services" / "auditable.py").read_text() + service_src = (tmp_path / "out" / "services" / "posts.py").read_text() + assert "class AuditableService(ServiceBase)" in base_src + assert "async def countAll" in base_src and "async def summary" in base_src + assert "from services.auditable import AuditableService" in service_src + assert "async def countAll" not in service_src diff --git a/tests/unit/test_service_input_codegen.py b/tests/unit/test_service_input_codegen.py index a298273f..ed2f76d1 100644 --- a/tests/unit/test_service_input_codegen.py +++ b/tests/unit/test_service_input_codegen.py @@ -76,7 +76,9 @@ def test_fastapi_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "routes/posts.py").read_text() handler_route = (tmp_path / "routes/_handlers.py").read_text() assert "'state': {'literal': 'archived'}" in repository_route - assert "operation_bindings({**_path, **_service_inputs}" in repository_route + assert "inputs=_service_inputs" in repository_route + assert "svc.archive(_request)" in repository_route + assert "self._repo.archive(request.bindings)" in (tmp_path / "services/posts.py").read_text() assert "inputs=service_inputs" in handler_route @@ -136,7 +138,8 @@ def test_raw_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "routes/posts.py").read_text() handler_route = (tmp_path / "routes/handlers.py").read_text() - assert "repo.archive(operation_bindings({**url_params, **service_inputs}" in repository_route + assert "inputs=service_inputs" in repository_route + assert "repo.archive(_request.bindings)" in repository_route assert "inputs=service_inputs" in handler_route @@ -145,5 +148,6 @@ def test_drf_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "api/views/posts.py").read_text() handler_route = (tmp_path / "api/views/_handlers.py").read_text() - assert "repo.archive(operation_bindings({**path_params, **service_inputs}" in repository_route + assert "inputs=service_inputs" in repository_route + assert "repo.archive(_request.bindings)" in repository_route assert "inputs=service_inputs" in handler_route From dbeb3212da6f726558b76d547e5343f3602268a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Fri, 23 Oct 2026 18:11:14 +0100 Subject: [PATCH 09/89] Reject service forwarding to an operation without a query --- .../parsers/transformers/symbol_validation.py | 15 ++-- tests/unit/test_service_input.py | 80 +++++++++++++++++++ 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/yaml_api_generator/parsers/transformers/symbol_validation.py b/src/yaml_api_generator/parsers/transformers/symbol_validation.py index 47603369..6fec1b09 100644 --- a/src/yaml_api_generator/parsers/transformers/symbol_validation.py +++ b/src/yaml_api_generator/parsers/transformers/symbol_validation.py @@ -82,17 +82,12 @@ def _validate_service_with(spec: dict[str, Any]) -> list[ConfigError]: symbols = {rhs.root} if isinstance(rhs, IRSymbolRef) else set() errors.extend(scope.undefined(symbols, path)) repository = repositories.get(service.get("repository")) - operation = service.get("operation") - operation_config = ( - (repository.get("operations") or {}).get(operation) - if isinstance(repository, dict) and operation is not None - else None - ) - if not isinstance(operation_config, dict) or not isinstance( - operation_config.get("query"), dict - ): + if service.get("handler") or not isinstance(repository, dict): continue - referenced = _free_symbols(operation_config["query"]) + operation = service.get("operation") + operation_config = (repository.get("operations") or {}).get(operation) + query = operation_config.get("query") if isinstance(operation_config, dict) else None + referenced = _free_symbols(query) if isinstance(query, dict) else set() errors.extend( ConfigError( code="SERVICE_WITH_TARGET_UNKNOWN", diff --git a/tests/unit/test_service_input.py b/tests/unit/test_service_input.py index 9aa26cee..ec936833 100644 --- a/tests/unit/test_service_input.py +++ b/tests/unit/test_service_input.py @@ -203,6 +203,41 @@ def test_repository_forwarding_rejects_unused_target(): } +def test_forwarding_to_an_operation_without_a_query_is_rejected(): + spec = { + "version": "0.3", + "repositories": {"posts": {"entity": "post"}}, + "services": { + "create-posts": { + "repository": "posts", + "operation": "create", + "input": {"postId": "string"}, + "with": {"id": "$postId"}, + } + }, + } + + assert _codes(validate_spec_symbols(spec)) == {"SERVICE_WITH_TARGET_UNKNOWN"} + + +def test_handler_receives_forwarding_without_a_query(): + spec = { + "version": "0.3", + "repositories": {"posts": {"entity": "post"}}, + "services": { + "notify": { + "repository": "posts", + "operation": "notify", + "handler": "hooks.notify", + "input": {"postId": "string"}, + "with": {"id": "$postId"}, + } + }, + } + + assert _codes(validate_spec_symbols(spec)) == set() + + # --- transformer / IR end to end ------------------------------------------ @@ -359,3 +394,48 @@ async def run_operation(self, collection, query): )(ctx) assert result.response == {"id": "p1", "state": "archived"} + + +_MINIFIED_SPEC = """ +version: "0.3" +api: + basePath: /api +entities: + post: + fields: + id: { type: id } + title: { type: string } +repositories: + posts: + entity: post + crud: [] + operations: + byTitle: + query: { command: find, filter: { title: $title } } +services: + posts: + repository: posts + input: { label: string } + with: { title: $label } + operations: + byTitle: true +controllers: + posts: + service: posts + base_path: /posts + operations: + byTitle: { method: GET, path: "/{label}" } +""" + + +def test_minified_service_input_and_forwarding_reach_each_operation(tmp_path): + service = _load(tmp_path, _MINIFIED_SPEC).ir.services["byTitle-posts"] + assert service.input == IRServiceInput(fields={"label": {"type": "string"}}) + assert service.forwarding == {"title": IRSymbolRef(root="label")} + + +def test_minified_service_forwarding_to_a_crud_operation_is_rejected(tmp_path): + spec = _MINIFIED_SPEC.replace("crud: []", "crud: [list]") + with pytest.raises(ConfigErrors) as raised: + _load(tmp_path, spec) + assert {error.code for error in raised.value.errors} == {"SERVICE_WITH_TARGET_UNKNOWN"} From d68cfec86527853418a501bca3bf3fd5eb7f6a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Fri, 23 Oct 2026 18:11:14 +0100 Subject: [PATCH 10/89] Test services over HTTP against the dev server and generated backends --- .../specs/services_mongo/hooks/__init__.py | 8 ++ .../specs/services_mongo/spec.yaml | 63 +++++++++++ .../specs/services_sql/hooks/__init__.py | 8 ++ .../integration/specs/services_sql/spec.yaml | 104 ++++++++++++++++++ tests/integration/support/spec_apps.py | 68 ++++++++++++ tests/integration/test_services_mongo.py | 69 ++++++++++++ tests/integration/test_services_sql.py | 92 ++++++++++++++++ tests/unit/test_services_spec_builds.py | 19 ++++ 8 files changed, 431 insertions(+) create mode 100644 tests/integration/specs/services_mongo/hooks/__init__.py create mode 100644 tests/integration/specs/services_mongo/spec.yaml create mode 100644 tests/integration/specs/services_sql/hooks/__init__.py create mode 100644 tests/integration/specs/services_sql/spec.yaml create mode 100644 tests/integration/support/spec_apps.py create mode 100644 tests/integration/test_services_mongo.py create mode 100644 tests/integration/test_services_sql.py create mode 100644 tests/unit/test_services_spec_builds.py diff --git a/tests/integration/specs/services_mongo/hooks/__init__.py b/tests/integration/specs/services_mongo/hooks/__init__.py new file mode 100644 index 00000000..5b4a46c2 --- /dev/null +++ b/tests/integration/specs/services_mongo/hooks/__init__.py @@ -0,0 +1,8 @@ +async def echo(request): + return { + "params": request.params, + "query": request.query, + "body": request.body, + "principal": request.principal and {"sub": request.principal["sub"]}, + "inputs": request.inputs, + } diff --git a/tests/integration/specs/services_mongo/spec.yaml b/tests/integration/specs/services_mongo/spec.yaml new file mode 100644 index 00000000..921bd9b8 --- /dev/null +++ b/tests/integration/specs/services_mongo/spec.yaml @@ -0,0 +1,63 @@ +version: "0.3" + +api: + basePath: /api + +imports: + hooks: { path: ./hooks, package: hooks } + +mongo: + uri: ${MONGO_URI} + database: ${MONGO_DB} + +entities: + task: + fields: + id: { type: id } + title: { type: string, required: true } + note: + fields: + id: { type: id } + body: { type: string, required: true } + +repositories: + tasks: + entity: task + operations: + byTitle: + query: { command: find, filter: { title: $title } } + notesByBody: + query: { command: find, collection: notes, filter: { body: $text } } + retitle: + query: + command: bulkWrite + requests: + - updateMany: { filter: { title: $old }, update: { $set: { title: $new } } } + notes: + entity: note + crud: [create] + +services: + tasks: + repository: tasks + operations: + byTitle: true + notesByBody: true + retitle: true + echo: { handler: hooks.echo } + notes: { repository: notes } + +controllers: + tasks: + service: tasks + base_path: /tasks + access: public + operations: + byTitle: { method: GET, path: "/by-title/{title}" } + notesByBody: { method: GET, path: "/notes/{text}" } + retitle: { method: POST, path: "/retitle/{old}/{new}" } + echo: { method: POST, path: "/echo/{id}" } + notes: + service: notes + base_path: /notes + access: public diff --git a/tests/integration/specs/services_sql/hooks/__init__.py b/tests/integration/specs/services_sql/hooks/__init__.py new file mode 100644 index 00000000..5b4a46c2 --- /dev/null +++ b/tests/integration/specs/services_sql/hooks/__init__.py @@ -0,0 +1,8 @@ +async def echo(request): + return { + "params": request.params, + "query": request.query, + "body": request.body, + "principal": request.principal and {"sub": request.principal["sub"]}, + "inputs": request.inputs, + } diff --git a/tests/integration/specs/services_sql/spec.yaml b/tests/integration/specs/services_sql/spec.yaml new file mode 100644 index 00000000..eef80418 --- /dev/null +++ b/tests/integration/specs/services_sql/spec.yaml @@ -0,0 +1,104 @@ +version: "0.3" + +api: + basePath: /api + +imports: + hooks: { path: ./hooks, package: hooks } + +datastores: + store: + type: postgres + dsn: ${SQL_DSN} + default: true + +entities: + user: + type: auth + auth: { identifier: email, via: password } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } + task: + fields: + id: { type: id } + title: { type: string, required: true } + note: + fields: + id: { type: id } + body: { type: string, required: true } + +repositories: + users: { entity: user, crud: [create] } + countable: + abstract: true + operations: + countAll: + query: { sql: "SELECT count(*) AS total FROM tasks" } + tasks: + entity: task + extends: countable + operations: + byTitle: + query: + sql: "SELECT title FROM tasks WHERE title = :title" + params: { title: $title } + notes: + entity: note + crud: [list, create] + titles: + entity: task + crud: [] + operations: + named: + query: + sql: "SELECT title FROM tasks WHERE title = :name" + params: { name: $name } + +services: + users: { repository: users } + auditable: + abstract: true + operations: + delete: false + countAll: true + echo: { handler: hooks.echo } + tasks: + repository: tasks + extends: auditable + operations: + byTitle: true + notes: { repository: notes } + titles: + repository: titles + input: { label: string } + with: { name: $label } + operations: + named: true + +controllers: + users: + service: users + base_path: /users + access: public + tasks: + service: tasks + base_path: /tasks + access: public + operations: + countAll: { method: GET, path: /stats/count } + echo: { method: POST, path: "/echo/{id}", access: "*" } + byTitle: { method: GET, path: "/by-title/{title}" } + notes: + service: notes + base_path: /notes + access: public + operations: + list: { path: /all } + create: { access: "*" } + titles: + service: titles + base_path: /titles + access: public + operations: + named: { method: GET, path: "/{label}" } diff --git a/tests/integration/support/spec_apps.py b/tests/integration/support/spec_apps.py new file mode 100644 index 00000000..f07a939d --- /dev/null +++ b/tests/integration/support/spec_apps.py @@ -0,0 +1,68 @@ +"""A spec served over real HTTP by `stackform run` and by each backend it generates.""" + +import os +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +import httpx + +from .network import free_port, wait_until + +STACKFORM = str(Path(sys.executable).with_name("stackform")) + + +@dataclass(frozen=True) +class Backend: + """`stackform run` when `target` is None, otherwise the app `stackform build` emits for `target`.""" + + name: str + target: str | None = None + + def command(self, spec: Path, env: dict[str, str], port: int) -> tuple[list[str], Path]: + if self.target is None: + return [STACKFORM, "run", str(spec), "--port", str(port)], spec.parent + output = spec.parent / self.target + subprocess.run( + [STACKFORM, "build", str(spec), "backend", "--target", self.target, "--output", str(output)], + env=env, check=True, capture_output=True, + ) + return [sys.executable, "-m", "uvicorn", "app:app", "--port", str(port)], output / "backend" + + +DEV = Backend("run") +FASTAPI = Backend("fastapi", "fastapi") +RAW = Backend("raw", "raw") + +SQL_BACKENDS = (DEV, FASTAPI) +MONGO_BACKENDS = (DEV, FASTAPI, RAW) + + +def _answering(url: str) -> bool: + try: + httpx.get(url, timeout=1) + return True + except httpx.HTTPError: + return False + + +@contextmanager +def serve(backend: Backend, spec: Path, env: dict[str, str] | None = None) -> Iterator[httpx.Client]: + """Serve `spec` with `backend` and yield a client bound to its root URL.""" + environment = {**os.environ, "JWT_SECRET": "spec-apps-secret", **(env or {})} + port = free_port() + command, cwd = backend.command(spec, environment, port) + url, log = f"http://127.0.0.1:{port}", spec.parent / f"{backend.name}.log" + with log.open("wb") as output: + process = subprocess.Popen(command, cwd=cwd, env=environment, stdout=output, stderr=subprocess.STDOUT) + try: + if not wait_until(lambda: process.poll() is not None or _answering(url)) or process.poll() is not None: + raise RuntimeError(f"{backend.name} server did not start:\n{log.read_text()}") + with httpx.Client(base_url=url, timeout=30) as client: + yield client + finally: + process.terminate() + process.wait(timeout=10) diff --git a/tests/integration/test_services_mongo.py b/tests/integration/test_services_mongo.py new file mode 100644 index 00000000..10e20d33 --- /dev/null +++ b/tests/integration/test_services_mongo.py @@ -0,0 +1,69 @@ +"""Services over MongoDB behave the same under `stackform run` and every backend it generates.""" + +import shutil +from pathlib import Path +from uuid import uuid4 + +import pytest + +from tests.integration.support.spec_apps import MONGO_BACKENDS, serve + +pytestmark = pytest.mark.integration + +SPEC = Path(__file__).parent / "specs" / "services_mongo" + + +@pytest.fixture(scope="module", params=MONGO_BACKENDS, ids=lambda backend: backend.name) +def api(request, tmp_path_factory, mongo_uri): + workdir = tmp_path_factory.mktemp(request.param.name) / "spec" + shutil.copytree(SPEC, workdir) + env = {"MONGO_URI": mongo_uri, "MONGO_DB": f"services_{uuid4().hex}"} + with serve(request.param, workdir / "spec.yaml", env) as client: + yield client + + +def _create(api, collection: str, **document) -> dict: + response = api.post(f"/api/{collection}", json=document) + assert response.status_code == 200, response.text + return response.json() + + +def _titles(response) -> list[str]: + assert response.status_code == 200, response.text + return [document["title"] for document in response.json()] + + +def test_find_operation_binds_the_path_parameter(api): + title = uuid4().hex + _create(api, "tasks", title=title) + assert _titles(api.get(f"/api/tasks/by-title/{title}")) == [title] + + +def test_operation_reads_the_collection_its_query_names(api): + body = uuid4().hex + _create(api, "notes", body=body) + response = api.get(f"/api/tasks/notes/{body}") + assert response.status_code == 200, response.text + assert [document["body"] for document in response.json()] == [body] + + +def test_bulk_write_applies_its_requests(api): + old, new = uuid4().hex, uuid4().hex + _create(api, "tasks", title=old) + _create(api, "tasks", title=old) + response = api.post(f"/api/tasks/retitle/{old}/{new}") + assert response.status_code == 200, response.text + assert response.json() == {"inserted": 0, "matched": 2, "modified": 2, "deleted": 0, "upserted": 0} + assert _titles(api.get(f"/api/tasks/by-title/{new}")) == [new, new] + + +def test_handler_receives_the_request_it_documents(api): + response = api.post("/api/tasks/echo/t1?page=2", json={"note": "hi"}) + assert response.status_code == 200, response.text + assert response.json() == { + "params": {"id": "t1"}, + "query": {"page": "2"}, + "body": {"note": "hi"}, + "principal": None, + "inputs": {}, + } diff --git a/tests/integration/test_services_sql.py b/tests/integration/test_services_sql.py new file mode 100644 index 00000000..ccc6ed61 --- /dev/null +++ b/tests/integration/test_services_sql.py @@ -0,0 +1,92 @@ +"""Services over SQL behave the same under `stackform run` and the FastAPI app it generates.""" + +import shutil +from pathlib import Path +from uuid import uuid4 + +import pytest + +from tests.integration.support.spec_apps import SQL_BACKENDS, serve + +SPEC = Path(__file__).parent / "specs" / "services_sql" + + +@pytest.fixture(scope="module", params=SQL_BACKENDS, ids=lambda backend: backend.name) +def api(request, tmp_path_factory): + workdir = tmp_path_factory.mktemp(request.param.name) / "spec" + shutil.copytree(SPEC, workdir) + with serve(request.param, workdir / "spec.yaml", {"SQL_DSN": f"sqlite+aiosqlite:///{workdir}/db.sqlite"}) as client: + yield client + + +@pytest.fixture(scope="module") +def user(api): + credentials = {"email": f"{uuid4().hex}@example.com", "password": "correct-horse-battery"} + registered = api.post("/api/users", json=credentials).json() + token = api.post("/api/users/login", json=credentials).json()["token"] + return {"id": registered["id"], "headers": {"Authorization": f"Bearer {token}"}} + + +def _task(api) -> str: + title = uuid4().hex + assert api.post("/api/tasks", json={"title": title}).status_code == 200 + return title + + +def test_operation_inherited_from_abstract_repository_and_service_runs(api): + before = api.get("/api/tasks/stats/count").json()[0]["total"] + _task(api) + assert api.get("/api/tasks/stats/count").json() == [{"total": before + 1}] + + +def test_declared_sql_operation_binds_the_path_parameter(api): + title = _task(api) + response = api.get(f"/api/tasks/by-title/{title}") + assert response.status_code == 200 + assert response.json() == [{"title": title}] + + +def test_handler_receives_the_request_it_documents(api, user): + response = api.post("/api/tasks/echo/t1?page=2", json={"note": "hi"}, headers=user["headers"]) + assert response.status_code == 200 + assert response.json() == { + "params": {"id": "t1"}, + "query": {"page": "2"}, + "body": {"note": "hi"}, + "principal": {"sub": user["id"]}, + "inputs": {}, + } + + +def test_operation_access_override_guards_an_inherited_handler(api): + assert api.post("/api/tasks/echo/t1", json={}).status_code == 401 + + +def test_operation_disabled_by_the_abstract_service_is_not_routed(api): + task = api.post("/api/tasks", json={"title": uuid4().hex}).json() + assert api.delete(f"/api/tasks/{task['id']}").status_code == 405 + + +def test_repository_crud_limits_the_routes(api): + assert api.get("/api/notes/n1").status_code == 404 + assert api.get("/api/notes").status_code == 405 + + +def test_controller_path_override_moves_the_route(api): + response = api.get("/api/notes/all") + assert response.status_code == 200 + assert isinstance(response.json(), list) + + +def test_controller_access_override_applies_to_one_operation(api, user): + assert api.post("/api/notes", json={"body": "x"}).status_code == 401 + created = api.post("/api/notes", json={"body": "x"}, headers=user["headers"]) + assert created.status_code == 200 + assert created.json()["body"] == "x" + + +def test_minified_service_forwards_its_input_to_every_operation(api): + title = _task(api) + response = api.get(f"/api/titles/{title}") + assert response.status_code == 200 + assert response.json() == [{"title": title}] diff --git a/tests/unit/test_services_spec_builds.py b/tests/unit/test_services_spec_builds.py new file mode 100644 index 00000000..1b4ea0a6 --- /dev/null +++ b/tests/unit/test_services_spec_builds.py @@ -0,0 +1,19 @@ +"""The services behaviour specs build cleanly for targets the integration suite cannot serve.""" + +import compileall +from pathlib import Path + +from yaml_api_generator.cli.compiler import compile_project + +_SPEC = Path(__file__).parents[1] / "integration" / "specs" / "services_mongo" / "spec.yaml" + + +def test_services_spec_builds_for_drf(tmp_path, monkeypatch): + monkeypatch.setenv("MONGO_URI", "mongodb://localhost:27017") + monkeypatch.setenv("MONGO_DB", "services") + compile_project(yaml_file=str(_SPEC), output_dir=str(tmp_path), target="drf") + + views = tmp_path / "backend" / "api" / "views" + assert compileall.compile_dir(str(tmp_path / "backend"), quiet=2) + assert "repo.retitle(_request.bindings)" in (views / "tasks.py").read_text() + assert "invoke('hooks.echo', _request)" in (views / "tasks.py").read_text() From 4e4f0a17d5b0d0aefc092f440aa467910b9cdd5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Fri, 23 Oct 2026 18:24:27 +0100 Subject: [PATCH 11/89] Enforce role grants, readonly fields, login exposure and row filters in every backend Identity creates keep a client-chosen role only on authenticated routes, managed roles start at their default, readonly fields are never written, auth.login: false serves no login route, optional raw fields accept null, and raw handlers and DRF views are named per path so routes sharing a collection keep their own access rules. --- examples/valid/access-control-advanced.yaml | 1 + .../backend/builders/auth_builder.py | 2 + .../generators/backend/compile_context.py | 25 +++ .../generators/backend/drf/generator.py | 3 +- .../backend/drf/templates/api_urls.py.jinja | 26 ++-- .../backend/drf/templates/auth.py.jinja | 16 +- .../drf/templates/login_views.py.jinja | 4 +- .../backend/drf/templates/views.py.jinja | 83 ++++------ .../generators/backend/fastapi/generator.py | 4 +- .../backend/fastapi/templates/auth.py.jinja | 29 +--- .../fastapi/templates/login_routes.py.jinja | 11 +- .../backend/fastapi/templates/routes.py.jinja | 7 + .../fastapi/templates/service.py.jinja | 3 - .../fastapi/templates/service_ops.py.jinja | 4 - .../generators/backend/raw/generator.py | 3 +- .../backend/raw/templates/app.py.jinja | 9 +- .../backend/raw/templates/auth.py.jinja | 7 +- .../raw/templates/login_routes.py.jinja | 4 +- .../backend/raw/templates/routes.py.jinja | 9 +- .../backend/raw/templates/validation.py.jinja | 2 +- src/yaml_api_generator/ir/api.py | 11 ++ .../parsers/desugar/__init__.py | 2 +- .../parsers/transformers/v02/__init__.py | 17 +- .../transformers/v02/identity_transformer.py | 4 + src/yaml_api_generator/runtime/access.py | 33 +++- .../server/common/context_builder.py | 7 +- .../server/common/pipeline_builder.py | 13 +- .../server/common/server_context.py | 19 +++ .../server/common/steps/authentication.py | 26 +--- .../server/common/steps/repositories.py | 33 +--- .../server/common/steps/validation.py | 9 +- .../server/fastapi/factory.py | 9 +- .../test_generated_mongo_access.py | 19 +++ .../test_generated_mongo_identity.py | 19 +++ .../test_generated_where_authorization.py | 20 +-- tests/unit/support/live_apps.py | 116 ++++++++++++++ tests/unit/test_access_http.py | 147 ++++++++++++++++++ tests/unit/test_access_reason.py | 6 +- tests/unit/test_drf_path_views.py | 53 +++++++ tests/unit/test_fastapi_authz_generation.py | 6 +- tests/unit/test_generators.py | 2 +- tests/unit/test_identity_http.py | 140 +++++++++++++++++ tests/unit/test_policy_roles_from_claims.py | 3 +- 43 files changed, 733 insertions(+), 233 deletions(-) create mode 100644 tests/integration/test_generated_mongo_access.py create mode 100644 tests/integration/test_generated_mongo_identity.py create mode 100644 tests/unit/support/live_apps.py create mode 100644 tests/unit/test_access_http.py create mode 100644 tests/unit/test_drf_path_views.py create mode 100644 tests/unit/test_identity_http.py diff --git a/examples/valid/access-control-advanced.yaml b/examples/valid/access-control-advanced.yaml index 69a0f313..ab7ab76f 100644 --- a/examples/valid/access-control-advanced.yaml +++ b/examples/valid/access-control-advanced.yaml @@ -14,6 +14,7 @@ entities: auth: identifier: email via: password + roles: role roles: values: [admin, editor, viewer] default: viewer diff --git a/src/yaml_api_generator/generators/backend/builders/auth_builder.py b/src/yaml_api_generator/generators/backend/builders/auth_builder.py index 9913c39c..a8b98461 100644 --- a/src/yaml_api_generator/generators/backend/builders/auth_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/auth_builder.py @@ -86,6 +86,8 @@ def build_auth_config( roles_field=identity.roles_field, issues_jwt=issues_jwt, datastore_type=datastore_types.get(name, "mongodb"), + login=identity.login, + default_role=identity.default_role(col), ) ) if identity.has_password: diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index 463850ff..f7ad15b0 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -20,6 +20,7 @@ from ...ir.api import IRFileField, IRPagination, IRProvider, IRRolesConfig, IRService from ...ir.when import WhenExpr +from ...runtime.access import Roles from ...utils.strings import to_py_identifier FILE_REQUIREMENTS = ("python-multipart", "puremagic", "anyio") @@ -282,6 +283,16 @@ def record_criteria(self) -> list[tuple[str, str]]: return parents return [*parents, (self.id_field, self.record_id_param)] + @property + def handler_name(self) -> str: + """A module-unique symbol: routes may share an operation and a collection, never a method and a path.""" + return f"handle_{self.method.lower()}{to_py_identifier(self.relative_path)}" + + @property + def view_name(self) -> str: + """The class serving every method of this route's path.""" + return f"View{to_py_identifier(self.relative_path)}" + @property def django_path(self) -> str: path = self.relative_path.lstrip("/") @@ -316,6 +327,8 @@ class IdentityCollectionDescriptor: issues_jwt: bool = False datastore_type: str = "mongodb" id_format: str = "uuidv7" + login: bool = True + default_role: str | None = None @property def is_sql(self) -> bool: @@ -334,6 +347,14 @@ def identifier(self) -> str | None: password_binding = self.get_binding("password") return password_binding.identifier if password_binding else None + @property + def serves_login(self) -> bool: + return self.login and bool(self.identifier) + + @property + def role_store(self) -> Roles: + return Roles(self.roles_field, self.default_role) + def get_binding(self, provider: str) -> "ProviderBindingDescriptor | None": return next((b for b in self.bindings if b.provider == provider), None) @@ -363,6 +384,10 @@ class AuthConfig: providers: dict[str, IRProvider] = field(default_factory=dict) role_hierarchies: dict[str, dict[str, set[str]]] = field(default_factory=dict) + @property + def login_identities(self) -> list[IdentityCollectionDescriptor]: + return [identity for identity in self.identity_collections if identity.serves_login] + @property def uses_jwt(self) -> bool: """JWT helpers and JWT_SECRET are needed whenever the app mints its own tokens. diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index 42fdf7be..ecdfe94a 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -217,12 +217,13 @@ def _render_views( for op in ctx.repository_operations.get(col_name, []) }, "auth": ctx.auth, + "identity": find_identity(ctx, col_name), }, should_render=lambda col_name, col_data, ctx: bool(col_data["routes"]), ) ) - if context.auth.has_password_auth: + if context.auth.login_identities: views_dir = output_dir / self.dir_prefix / "views" login_template = env.get_template("login_views.py.jinja") login_content = login_template.render(auth=context.auth) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja index bdf82d38..de0eba2d 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja @@ -9,17 +9,19 @@ from api.views._handlers import {% for route in handler_routes %}Handler{{ route {% for col_name in collections %} {% set collection_routes = routes | selectattr("collection", "equalto", col_name) | selectattr("operation", "in", crud_op_names) | list %} {% set custom_routes = routes | selectattr("collection", "equalto", col_name) | rejectattr("operation", "in", crud_op_names) | rejectattr("operation", "equalto", "login") | list %} -{% set list_routes = collection_routes | selectattr("has_id_param", "false") | list %} -{% set detail_routes = collection_routes | selectattr("has_id_param") | list %} -from api.views.{{ col_name }} import {% if list_routes %}{{ col_name | capitalize_collection }}ListView{% if detail_routes or custom_routes %}, {% endif %}{% endif %}{% if detail_routes %}{{ col_name | capitalize_collection }}DetailView{% if custom_routes %}, {% endif %}{% endif %}{% for route in custom_routes %}{{ col_name | capitalize_collection }}{{ route.operation | capitalize_collection }}View{% if not loop.last %}, {% endif %}{% endfor %} +{% set crud_views = collection_routes | map(attribute="view_name") | unique | list %} +{% if crud_views %} +from api.views.{{ col_name }} import {{ crud_views | join(", ") }} +{% endif %} +{% for route in custom_routes %} +from api.views.{{ col_name }} import {{ col_name | capitalize_collection }}{{ route.operation | capitalize_collection }}View +{% endfor %} {% endfor %} -{% if auth and auth.has_password_auth %} -{% for identity in auth.identity_collections %} -{% if identity.has_password and identity.identifier %} +{% if auth %} +{% for identity in auth.login_identities %} from api.views.login import {{ identity.name | capitalize_collection }}LoginView -{% endif %} {% endfor %} {% endif %} @@ -27,18 +29,16 @@ urlpatterns = [ {% for route in handler_routes %} path("{{ route.django_path }}", Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View.as_view(), name="handler-{{ route.operation }}-{{ loop.index }}"), {% endfor %} -{% if auth and auth.has_password_auth %} -{% for identity in auth.identity_collections %} -{% if identity.has_password and identity.identifier %} +{% if auth %} +{% for identity in auth.login_identities %} path("{{ identity.name }}/login", {{ identity.name | capitalize_collection }}LoginView.as_view(), name="{{ identity.name }}-login"), -{% endif %} {% endfor %} {% endif %} {% set seen_paths = [] %} -{% for route in routes if route.collection and route.operation in crud_op_names %} +{% for route in routes | sort(attribute="has_id_param") if route.collection and route.operation in crud_op_names %} {% if route.django_path not in seen_paths %} {% set _ = seen_paths.append(route.django_path) %} - path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{% if route.has_id_param %}Detail{% else %}List{% endif %}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), + path("{{ route.django_path }}", {{ route.view_name }}.as_view(), name="{{ route.collection }}-{{ route.operation }}"), {% endif %} {% endfor %} {% for route in routes if route.collection and route.operation not in crud_op_names and route.operation != "login" %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja index 0ab63e2f..b091494c 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja @@ -7,6 +7,7 @@ from typing import Any from rest_framework.authentication import BaseAuthentication from rest_framework.request import Request +from api.access import Roles from api.authentication.mongodb import get_database {% if auth.uses_jwt %} from api.authentication.jwt_utils import decode_jwt_token @@ -82,15 +83,6 @@ def _extract_bearer_token(headers: dict[str, str]) -> str | None: ROLE_HIERARCHIES: dict[str, dict[str, set[str]]] = {{ auth.role_hierarchies | role_hierarchies_py }} -def _extract_roles(doc: dict[str, Any], roles_field: str | None) -> list[str]: - roles = doc.get(roles_field) if roles_field else (doc.get("roles") or doc.get("role")) - if isinstance(roles, list): - return [str(role) for role in roles] - if isinstance(roles, str): - return [roles] - return [] - - def _principal_has_role(principal: dict | None, required_role: str) -> bool: if principal is None: return False @@ -121,7 +113,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: {% for identity in auth.identity_collections %} {% if identity.issues_jwt %} if collection_name == "{{ identity.name }}" and str(doc.get("{{ identity.id_field }}", "")) == sub: - return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} + return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} {% endif %} @@ -137,7 +129,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: if str(doc.get("apiKey", "")) == token: subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% if identity.has_password and identity.identifier %} credentials = _extract_basic_credentials(raw_headers) @@ -154,7 +146,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: continue subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja index a4fa75c2..2d24f165 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja @@ -7,8 +7,7 @@ from api.authentication.auth import verify_password from api.authentication.jwt_utils import create_jwt_token from api.authentication.mongodb import get_database -{% for identity in auth.identity_collections %} -{% if identity.has_password and identity.identifier %} +{% for identity in auth.login_identities %} {% set class_name = identity.name | capitalize_collection %} class {{ class_name }}LoginView(APIView): @@ -53,5 +52,4 @@ class {{ class_name }}LoginView(APIView): status=status.HTTP_401_UNAUTHORIZED, ) -{% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index ae9a6acb..e6001a6f 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -27,6 +27,9 @@ from api.repositories.{{ where_collection }} import {{ where_collection | capita {% endfor %} from api.services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from api.services.exceptions import NotFoundError, ValidationError +{% if identity %} +from api.access import Roles +{% endif %} from api.service_inputs import prepare_service_inputs, resolve_access_bind, resolve_bindings {% if routes | selectattr("pagination") | list %} from api.pagination import build_response, map_request @@ -39,8 +42,6 @@ from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, {% endif %} {% set class_name = collection_name | capitalize_collection %} -{% set list_routes = crud_routes | selectattr("has_id_param", "false") | list %} -{% set detail_routes = crud_routes | selectattr("has_id_param") | list %} {% set custom_routes = routes | rejectattr("operation", "equalto", "login") | rejectattr("operation", "in", crud_op_names) | list %} {% macro record_criteria(route) %}{{ "{" }}{% for field, param in route.record_criteria %}"{{ field }}": {{ param }}{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} @@ -133,52 +134,6 @@ from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, {% endif %} {% endmacro %} -{% if list_routes %} -class {{ class_name }}ListView(APIView): -{% for route in list_routes %} - -{{ security_decorator(route) }} - async def {{ route.method | lower }}(self, request, **path_params): -{% for name in route.path_params %} - {{ name }} = path_params["{{ name }}"] -{% endfor %} - db = get_database() - repo = {{ class_name }}Repository(db) - service = {{ class_name }}Service(repo) - body = request.data if isinstance(request.data, dict) else {} -{{ request_locals(route) }} -{{ pre_fetch_check(route) }} -{{ bind_inputs(route) }} - try: - prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.operation == "list" %} -{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} -{% if route.pagination %} - pagination_config = {{ route.pagination | pagination_config | py_literal }} - pagination_query = dict(request.query_params) - page = await service.list(map_request(pagination_config, pagination_query){{ row_filter_arg }}) - return Response(build_response(pagination_config, page, pagination_query)) -{% else %} - documents = await service.list({{ "None" ~ row_filter_arg if row_filter_arg }}) - return Response(documents) -{% endif %} -{% elif route.operation == "create" %} - serializer = {{ route.body_model }}(data=request.data) - serializer.is_valid(raise_exception=True) - data = serializer.validated_data - document = await service.create(data) - return Response(document, status=status.HTTP_201_CREATED) -{% endif %} - except NotFoundError: - return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) - except ConflictError as exc: - return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) - except (ValidationError, ValueError) as exc: - detail = exc.message if isinstance(exc, ValidationError) else str(exc) - return Response({"detail": detail}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) -{% endfor %} -{% endif %} - {% for route in custom_routes %} {% set custom_class = class_name ~ (route.operation | capitalize_collection) ~ "View" %} class {{ custom_class }}(APIView): @@ -222,9 +177,9 @@ class {{ custom_class }}(APIView): {% endfor %} -{% if detail_routes %} -class {{ class_name }}DetailView(APIView): -{% for route in detail_routes %} +{% for _, path_routes in crud_routes | groupby("relative_path") %} +class {{ path_routes[0].view_name }}(APIView): +{% for route in path_routes %} {{ security_decorator(route) }} async def {{ route.method | lower }}(self, request, **path_params): @@ -240,7 +195,27 @@ class {{ class_name }}DetailView(APIView): {{ bind_inputs(route) }} try: prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.operation == "get" %} +{% if route.operation == "list" %} +{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} +{% if route.pagination %} + pagination_config = {{ route.pagination | pagination_config | py_literal }} + pagination_query = dict(request.query_params) + page = await service.list(map_request(pagination_config, pagination_query){{ row_filter_arg }}) + return Response(build_response(pagination_config, page, pagination_query)) +{% else %} + documents = await service.list({{ "None" ~ row_filter_arg if row_filter_arg }}) + return Response(documents) +{% endif %} +{% elif route.operation == "create" %} + serializer = {{ route.body_model }}(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data +{% if identity %} + data = {{ identity.role_store | py_literal }}.granted(dict(data), assignable={{ route.needs_auth | py_literal }}) +{% endif %} + document = await service.create(data) + return Response(document, status=status.HTTP_201_CREATED) +{% elif route.operation == "get" %} {% if route.record_parent_params %} result = await service.get_by({{ record_criteria(route) }}) {% else %} @@ -308,4 +283,6 @@ class {{ class_name }}DetailView(APIView): detail = exc.message if isinstance(exc, ValidationError) else str(exc) return Response({"detail": detail}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) {% endfor %} -{% endif %} + + +{% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index b5538a8f..5a5c5d0d 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -191,7 +191,6 @@ def _build_service_context(self, col_name: str, col_data: dict, context: Compile "operations": _crud_operations(col_data), "is_identity_collection": is_identity, "password_via": is_identity and identity.has_password, - "default_role": identity.roles.default if is_identity and identity.roles else None, "identity_fields": extract_identity_fields(context, col_name), } @@ -233,6 +232,7 @@ def _build_routes_context( }, "datastore_type": ctx.collection_datastore_type(col_name), "auth": ctx.auth, + "identity": find_identity(ctx, col_name), "files": ctx.file_fields.get(col_name, {}), "file_routes": [route for route in ctx.file_routes if route.collection == col_name], } @@ -284,7 +284,7 @@ def _render_routes( ) auth_imports.append("from ._handlers import router as handlers_router") - if context.auth.has_password_auth: + if context.auth.login_identities: (routes_dir / "login.py").write_text( env.get_template("login_routes.py.jinja").render( auth=context.auth, ctx=context diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja index c36bac15..cc1f61a8 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja @@ -13,6 +13,7 @@ from dependencies.mongodb import get_database {% for identity in auth.identity_collections %} from dependencies.repositories import get_{{ identity.name }}_repository {% endfor %} +from dependencies.access import Roles, role_names from settings import settings {% if auth.uses_jwt %} @@ -92,29 +93,9 @@ def _extract_bearer_token(headers: dict[str, str]) -> str | None: return None -def _extract_roles( - doc: dict[str, Any], roles_field: str | None, managed_roles: bool = False -) -> list[str]: - roles = doc.get(roles_field) if roles_field else (doc.get("roles") or doc.get("role")) - if roles is None and managed_roles: - # The identity declares no role field of its own, so the managed store holds them. - auth_data = doc.get("_auth") - roles = auth_data.get("roles") if isinstance(auth_data, dict) else None - if isinstance(roles, list): - return [str(role) for role in roles] - if isinstance(roles, str): - return [roles] - return [] - - {% if auth.has_idp_auth %} def _roles_from_claims(claims: dict[str, Any], claim_path: str | None) -> list[str]: - roles = resolve_claim_path(claims, claim_path) if claim_path else None - if isinstance(roles, list): - return [str(role) for role in roles] - if isinstance(roles, str): - return [roles] - return [] + return role_names(resolve_claim_path(claims, claim_path) if claim_path else None) {% endif %} @@ -261,7 +242,7 @@ async def resolve_principal( if collection_name == "{{ identity.name }}" and repository is not None: doc = await repository.find_by("{{ identity.id_field }}", sub) if doc is not None: - return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }}, {{ (identity.roles is not none) | py_literal }})} + return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} {% endif %} @@ -274,7 +255,7 @@ async def resolve_principal( if doc is not None: subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }}, {{ (identity.roles is not none) | py_literal }})} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% set pw_binding = identity.bindings | selectattr('provider', 'equalto', 'password') | first | default(none) %} {% if pw_binding %} @@ -290,7 +271,7 @@ async def resolve_principal( if isinstance(stored_hash, str) and verify_password(password, stored_hash): subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }}, {{ (identity.roles is not none) | py_literal }})} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja index a370a2c9..ff33b1ee 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja @@ -15,7 +15,7 @@ from dependencies.session_cookie import issue_csrf_token, refresh_token_from_req from dependencies.session_cookie import issue_csrf_token, set_session_cookies {% endif %} {% endif %} -{% for identity in auth.identity_collections if identity.has_password %} +{% for identity in auth.login_identities %} from dependencies.repositories import get_{{ identity.name }}_repository {% endfor %} {% if ctx.session.refresh %} @@ -32,20 +32,18 @@ class RefreshPayload(BaseModel): refresh_token: str = Field(..., min_length=1) {% endif %} -{% for identity in auth.identity_collections %} -{% set pw_binding = identity.bindings | selectattr('provider', 'equalto', 'password') | first | default(none) %} -{% if pw_binding %} +{% for identity in auth.login_identities %} {% set class_name = identity.name | capitalize_collection %} class {{ class_name }}LoginPayload(BaseModel): - {{ pw_binding.identifier }}: str = Field(..., min_length=1) + {{ identity.identifier }}: str = Field(..., min_length=1) password: str = Field(..., min_length=1) @router.post("/{{ identity.name }}/login", tags=["{{ identity.name | capitalize_collection }}"]) async def login_{{ identity.name }}(payload: {{ class_name }}LoginPayload{% if ctx.session.uses_cookie %}, request: Request{% endif %}, repository=Depends(get_{{ identity.name }}_repository)) -> Any: doc = await repository.find_by( - "{{ pw_binding.identifier }}", payload.{{ pw_binding.identifier }} + "{{ identity.identifier }}", payload.{{ identity.identifier }} ) if doc is not None: auth_data = doc.get("_auth") @@ -93,7 +91,6 @@ async def refresh_{{ identity.name }}(payload: RefreshPayload) -> Any: raise HTTPException(status_code=401, detail="Invalid refresh token") return rotated -{% endif %} {% endif %} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index d3e32b6a..52c84747 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -59,6 +59,9 @@ from dependencies.repositories import get_{{ where_collection }}_repository {% if auth.enabled and ns_auth.needs_require %} from dependencies.auth import require_principal {% endif %} +{% if identity %} +from dependencies.access import Roles +{% endif %} {% if auth.enabled and ns_auth.needs_role %} from dependencies.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at from dependencies.authorization import _principal_has_role @@ -292,7 +295,11 @@ async def create_{{ collection_name }}({{ (payload_param(body_model) ~ extra_pat {{ pre_fetch_guard(route) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, path_values(route), body_expr) }} +{% if identity %} + return await svc.create({{ identity.role_store | py_literal }}.granted({{ record_expr }}, assignable={{ route.needs_auth | py_literal }})) +{% else %} return await svc.create({{ record_expr }}) +{% endif %} {% elif route.operation in ("update", "replace") %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja index 51a7827c..455be3fd 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja @@ -14,9 +14,6 @@ from services._ops import ServiceBase{% for op in svc_mixins %}, {{ op }}Svc{% e class {{ class_name }}Service({% for op in svc_mixins %}{{ op }}Svc, {% endfor %}{% for mixin in identity_mixins %}{{ mixin }}, {% endfor %}ServiceBase): {% if is_identity_collection %} IDENTITY_FIELDS = ({% for field_name in identity_fields %}"{{ field_name }}", {% endfor %}) -{% if password_via %} - DEFAULT_ROLE = {{ default_role | py_literal }} -{% endif %} {% else %} pass {% endif %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja index 18686a67..d984c32e 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja @@ -162,8 +162,6 @@ class IdentityFieldsMixin: class PasswordMixin: - DEFAULT_ROLE: str | None = None - async def _before_write(self, data: dict) -> None: raw_password = data.pop("password", None) if raw_password is None: @@ -174,7 +172,5 @@ class PasswordMixin: if not isinstance(auth_data, dict): auth_data = {} auth_data["passwordHash"] = hash_password(raw_password) - if self.DEFAULT_ROLE is not None: - auth_data.setdefault("roles", self.DEFAULT_ROLE) data["_auth"] = auth_data {% endif %} diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 4c92f1b2..a5696eb7 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -89,7 +89,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: self._render_settings(env, context, output_dir) write_environment_files(env, context, output_dir) self._render_routes(env, context, collections_by_name, output_dir) - if context.auth.has_password_auth: + if context.auth.login_identities: self._render_login_routes(env, context, output_dir) if context.session.uses_asymmetric: self._render_single_file( @@ -244,6 +244,7 @@ def _render_routes( for op in context.repository_operations.get(col_name, []) }, auth=context.auth, + identity=find_identity(context, col_name), ) (routes_dir / f"{col_name}.py").write_text(content) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja index 0fdd9a16..a7b1c4a2 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja @@ -18,7 +18,7 @@ from routes.{{ col_name }} import * {% if handler_routes %} from routes.handlers import * {% endif %} -{% if ctx.auth.has_password_auth %} +{% if ctx.auth.login_identities %} from routes.login import * {% endif %} {% if ctx.session.uses_asymmetric %} @@ -41,17 +41,14 @@ def _path_to_regex(path: str) -> str: ROUTES: dict[str, list[tuple[re.Pattern[str], Any]]] = {} {% for route in crud_routes %} {% if route.collection %} -{% set handler_name = "handle_" ~ route.operation ~ "_" ~ route.collection %} -ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ handler_name }})) +ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ route.handler_name }})) {% endif %} {% endfor %} {% for route in handler_routes %} ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), handle_custom_{{ loop.index }})) {% endfor %} -{% for identity in ctx.auth.identity_collections %} -{% if identity.has_password and identity.identifier %} +{% for identity in ctx.auth.login_identities %} ROUTES.setdefault("POST", []).append((re.compile(r"^" + _PREFIX + r"/{{ identity.name }}/login$"), handle_login_{{ identity.name }})) -{% endif %} {% endfor %} {% if ctx.session.uses_asymmetric %} ROUTES.setdefault("GET", []).append((re.compile(r"^/\.well-known/jwks\.json$"), handle_jwks)) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja index bfd986da..b6285ca1 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja @@ -4,6 +4,7 @@ import hmac import secrets from typing import Any +from access import Roles from settings import settings {% if auth.uses_jwt %} from dependencies.jwt_utils import decode_jwt_token @@ -91,7 +92,7 @@ async def resolve_principal(headers: dict[str, str], db) -> dict[str, Any] | Non {% for identity in auth.identity_collections %} {% if identity.issues_jwt %} if collection_name == "{{ identity.name }}" and str(doc.get("{{ identity.id_field }}", "")) == sub: - return {"sub": sub, "identity_collection": collection_name, "record": doc} + return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} {% endif %} @@ -109,7 +110,7 @@ async def resolve_principal(headers: dict[str, str], db) -> dict[str, Any] | Non if str(doc.get("apiKey", "")) == token: subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% if identity.has_password and identity.identifier %} credentials = _extract_basic_credentials(headers) @@ -128,7 +129,7 @@ async def resolve_principal(headers: dict[str, str], db) -> dict[str, Any] | Non continue subject = doc.get("{{ identity.id_field }}") if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc} + return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": {{ identity.role_store | py_literal }}.of(doc)} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja index f535760b..92217173 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja @@ -5,8 +5,7 @@ from dependencies.jwt_utils import create_jwt_token import validation -{% for identity in ctx.auth.identity_collections %} -{% if identity.has_password and identity.identifier %} +{% for identity in ctx.auth.login_identities %} async def handle_login_{{ identity.name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: if body is None: return 422, {"detail": "Request body is required"} @@ -41,5 +40,4 @@ async def handle_login_{{ identity.name }}(url_params: dict, body: dict | None, return 401, {"detail": "Invalid credentials"} -{% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 31e76b48..39ac6f15 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -10,6 +10,9 @@ from repositories.{{ where_collection }} import {{ where_collection | capitalize {% endfor %} from services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from services.exceptions import NotFoundError, ValidationError +{% if identity %} +from access import Roles +{% endif %} from service_inputs import RawRequest, prepare_service_inputs, resolve_access_bind, resolve_bindings {% if routes | selectattr("pagination") | list %} from pagination import build_response, map_request @@ -117,8 +120,7 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: {% endmacro %} {% for route in routes %} {% if route.operation != "login" %} -{% set handler_name = "handle_" ~ route.operation ~ "_" ~ collection_name %} -async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: +async def {{ route.handler_name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: repo = {{ collection_name | capitalize_collection }}Repository(db) service = {{ collection_name | capitalize_collection }}Service(repo) principal = None @@ -175,6 +177,9 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ if body is None: return 422, {"detail": "Request body is required"} data = validation.validate_{{ collection_name }}_create(body) +{% if identity %} + data = {{ identity.role_store | py_literal }}.granted(data, assignable={{ route.needs_auth | py_literal }}) +{% endif %} document = await service.create(data) return 200, document {% elif route.operation == "update" %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja index 79e3f523..81feb263 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja @@ -79,7 +79,7 @@ def validate_{{ model.collection }}_{{ model.operation }}(body: dict) -> dict: errors = [] {% for field in model.fields %} - if {{ field.name | tojson }} in body: + if {% if field.required %}{{ field.name | tojson }} in body{% else %}body.get({{ field.name | tojson }}) is not None{% endif %}: {% if field.type == "string" or field.type == "ref" %} err = _check_string(body[{{ field.name | tojson }}], {{ field.name | tojson }}{% if field.constraints %}, {{ field.constraints }}{% endif %}) if err: diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 652cd674..c49c6917 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -441,11 +441,22 @@ class IRIdentityCollection: bindings: tuple[IRProviderBinding, ...] roles: IRRolesConfig | None = None roles_field: str | None = None + login: bool = True @property def has_password(self) -> bool: return any(b.provider == "password" for b in self.bindings) + @property + def serves_login(self) -> bool: + return self.login and self.has_password + + def default_role(self, collection: IRCollection) -> str | None: + """The role of a record whose creator may not choose one.""" + if self.roles is not None: + return self.roles.default + return collection.fields[self.roles_field].get("default") if self.roles_field else None + @property def has_api_key(self) -> bool: return any(b.provider == "apiKey" for b in self.bindings) diff --git a/src/yaml_api_generator/parsers/desugar/__init__.py b/src/yaml_api_generator/parsers/desugar/__init__.py index b86d45cb..c7411148 100644 --- a/src/yaml_api_generator/parsers/desugar/__init__.py +++ b/src/yaml_api_generator/parsers/desugar/__init__.py @@ -28,7 +28,7 @@ def desugar_v02(config: dict[str, Any]) -> dict[str, Any]: api_ops = config.get("api", {}).get("operations", {}) defaults = api_ops.get("defaults", {}) - auth_defaults = api_ops.pop("auth", {}) + auth_defaults = api_ops.get("auth", {}) repositories = config.get("repositories", {}) entities = config.get("entities", {}) services = config.get("services", {}) diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 2d18223a..ba282c1f 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -101,7 +101,8 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: raise ConfigErrors(binding_errors) identity_collections = self._identity.transform( - entities_data, collections, repo_entity_map + entities_data, collections, repo_entity_map, + _login_collections(api_data, services_data, repo_entity_map), ) field_errors = self._identity.validate_auth_fields(collections, identity_collections) @@ -228,3 +229,17 @@ def _parse_providers(self, providers_data: dict[str, Any]) -> dict[str, IRProvid kwargs["audiences"] = tuple(kwargs["audiences"]) providers[name] = ctor(name=name, **kwargs) return providers + + +def _login_collections( + api_data: dict[str, Any], services_data: dict[str, Any], repo_entity_map: dict[str, str] +) -> set[str]: + """Every collection serves login unless `api.operations.auth.login` is false; then only + collections with an explicit login service do.""" + if api_data.get("operations", {}).get("auth", {}).get("login") is not False: + return set(repo_entity_map) + return { + service["repository"] + for service in services_data.values() + if isinstance(service, dict) and service.get("operation") == "login" + } diff --git a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py index a0b58308..cc2367c2 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/identity_transformer.py @@ -3,6 +3,7 @@ Validates auth entities, role hierarchies, and implicit auth fields. """ +from collections.abc import Container from typing import Any, cast from yaml_api_generator.ir import ( @@ -25,6 +26,7 @@ def transform( entities_data: dict[str, Any], collections: dict[str, IRCollection], repo_entity_map: dict[str, str], + login_collections: Container[str], ) -> dict[str, IRIdentityCollection]: """Transform auth entities to identity collections. @@ -32,6 +34,7 @@ def transform( entities_data: Raw entity configurations from YAML collections: Already-built collection IR objects repo_entity_map: Repository name to entity name mapping + login_collections: Collections whose password identities serve a login route Returns: Dictionary of identity collections @@ -93,6 +96,7 @@ def transform( bindings=bindings, roles=roles_config, roles_field=roles_field, + login=repo_name in login_collections, ) return identity_collections diff --git a/src/yaml_api_generator/runtime/access.py b/src/yaml_api_generator/runtime/access.py index 5caddfcf..1e61e3ac 100644 --- a/src/yaml_api_generator/runtime/access.py +++ b/src/yaml_api_generator/runtime/access.py @@ -9,7 +9,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from functools import partial, reduce from operator import eq @@ -325,3 +325,34 @@ def row_filter(decisions: list[Decision]) -> Predicate: if predicate == NO_ROW: decide(decisions) return predicate + + +def role_names(held: Any) -> list[str]: + """Roles held as one name or a list of names, as a list of names.""" + return [held] if isinstance(held, str) else [str(role) for role in held or ()] + + +@dataclass(frozen=True) +class Roles: + """Where an identity record keeps its roles: its ``field``, or the managed store under + ``_auth`` when it declares none, and the role a new record gets when its creator may + not choose one.""" + + field: str | None = None + default: str | None = None + + def of(self, document: Mapping[str, Any]) -> list[str]: + return role_names( + document.get(self.field) if self.field else (document.get("_auth") or {}).get("roles") + ) + + def granted(self, payload: dict[str, Any], assignable: bool) -> dict[str, Any]: + """``payload`` of a new record, keeping the roles it names only when ``assignable``.""" + if assignable and self.field in payload: + return payload + record = {key: value for key, value in payload.items() if key != self.field} + if self.default is None: + return record + if self.field: + return {**record, self.field: self.default} + return {**record, "_auth": {**record.get("_auth", {}), "roles": self.default}} diff --git a/src/yaml_api_generator/server/common/context_builder.py b/src/yaml_api_generator/server/common/context_builder.py index a739b6ad..13b79b4e 100644 --- a/src/yaml_api_generator/server/common/context_builder.py +++ b/src/yaml_api_generator/server/common/context_builder.py @@ -111,8 +111,8 @@ def _build_routes(document: IRDocument) -> list[RuntimeRouteDescriptor]: def _password_login_routes( document: IRDocument, routes: list[RuntimeRouteDescriptor] ) -> list[RuntimeRouteDescriptor]: - """A generated project serves one login route per password identity, whether or not a - controller declares the operation; the development server serves the same surface.""" + """A generated project serves one login route per identity that serves login, whether or + not a controller declares the operation; the development server serves the same surface.""" declared = {route.collection for route in routes if route.operation == "login"} base_path = document.api.basePath if document.api.version: @@ -126,7 +126,7 @@ def _password_login_routes( is_public=True, ) for name, identity in document.identityCollections.items() - if identity.has_password and name not in declared + if identity.serves_login and name not in declared ] @@ -213,6 +213,7 @@ def _build_auth_config( id_field=id_field, roles=identity.roles, roles_field=identity.roles_field, + default_role=identity.default_role(collection), ) identity_id_fields[name] = id_field diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 46b441ca..2daf63c3 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -1,4 +1,5 @@ import importlib +from functools import partial from collections.abc import Iterator, Mapping from yaml_api_generator.ir import IRDocumentBinding @@ -304,13 +305,13 @@ def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: if operation == "get": return RepositoryGetStep(collection_name, id_field, self._locator(route)) if operation == "create": - roles_config = identity_collection.roles if identity_collection is not None else None + grant = ( + partial(identity_collection.role_store.granted, assignable=not route.is_public) + if identity_collection is not None + else dict + ) return RepositoryCreateStep( - collection_name, - id_field, - collection.id_format, - uses_password, - roles_config=roles_config, + collection_name, id_field, collection.id_format, uses_password, grant ) if operation == "update": return RepositoryUpdateStep( diff --git a/src/yaml_api_generator/server/common/server_context.py b/src/yaml_api_generator/server/common/server_context.py index 696a613c..809222e2 100644 --- a/src/yaml_api_generator/server/common/server_context.py +++ b/src/yaml_api_generator/server/common/server_context.py @@ -12,6 +12,7 @@ ) from yaml_api_generator.ir.api import IRPagination, IRRolesConfig from yaml_api_generator.ir.when import WhenExpr +from yaml_api_generator.runtime.access import Roles @dataclass(frozen=True) @@ -24,6 +25,19 @@ class RuntimeCollectionDescriptor: datastore: str | None = None files: Mapping[str, IRFileField] = field(default_factory=dict) + @property + def writable_fields(self) -> dict[str, dict[str, Any]]: + """The fields a client may send: not the id, a readonly or file field, nor ``_auth``.""" + return { + name: config + for name, config in self.fields.items() + if isinstance(config, dict) + and config.get("type") != "id" + and not config.get("readonly", False) + and name != "_auth" + and name not in self.files + } + @dataclass(frozen=True) class RuntimeOperationDescriptor: @@ -39,6 +53,11 @@ class RuntimeIdentityDescriptor: id_field: str roles: IRRolesConfig | None = None roles_field: str | None = None + default_role: str | None = None + + @property + def role_store(self) -> Roles: + return Roles(self.roles_field, self.default_role) @property def has_password(self) -> bool: diff --git a/src/yaml_api_generator/server/common/steps/authentication.py b/src/yaml_api_generator/server/common/steps/authentication.py index 28b38dd4..154b32b3 100644 --- a/src/yaml_api_generator/server/common/steps/authentication.py +++ b/src/yaml_api_generator/server/common/steps/authentication.py @@ -2,6 +2,7 @@ from typing import Any from yaml_api_generator.ir.api import IROIDCProvider, IRProviderBinding +from yaml_api_generator.runtime.access import Roles, role_names from ..auth import idp, oidc, verify_password from ..auth.jwt import SessionTokens from ..auth.utils import extract_bearer_token, extract_api_key, extract_basic_credentials @@ -114,17 +115,11 @@ def _build_service_principal( binding: IRProviderBinding | None = None, ) -> dict[str, Any]: sub = str(claims["sub"]) - roles = _roles_from_claims(claims, identity, binding) - roles_list: list[str] = [] - if isinstance(roles, list): - roles_list = [str(r) for r in roles] - elif isinstance(roles, str): - roles_list = [roles] return { "sub": sub, "client_id": claims.get("azp") or claims.get("client_id") or sub, "is_service": True, - "roles": roles_list, + "roles": role_names(_roles_from_claims(claims, identity, binding)), "record": None, "identity_collection": None, } @@ -342,19 +337,6 @@ def _build_principal( "record": record, } - if roles is None: - roles_config = identity.roles if identity is not None else None - roles_field = identity.roles_field if identity is not None else None - auth_data = document.get("_auth") - if roles_config is not None and isinstance(auth_data, dict): - roles = auth_data.get("roles") - elif roles_field is not None: - roles = document.get(roles_field) - else: - roles = document.get("roles") or document.get("role") - - if isinstance(roles, list): - principal["roles"] = [str(role) for role in roles] - elif isinstance(roles, str): - principal["roles"] = [roles] + store = identity.role_store if identity is not None else Roles() + principal["roles"] = store.of(document) if roles is None else role_names(roles) return principal diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index b7006a31..e97efac7 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -1,10 +1,11 @@ import re +from collections.abc import Callable from dataclasses import dataclass from typing import Any, cast from ..auth import hash_password from ..request_context import RequestContext -from yaml_api_generator.ir.api import IRPagination, IRRolesConfig +from yaml_api_generator.ir.api import IRPagination from ..pagination import Page, build_response, map_request from ..repositories.base import BaseRepository from ..utils.ids import generate_id as _generate_id @@ -106,19 +107,9 @@ def _inject_canonical_id_list( ] -def _prepare_payload( - payload: dict[str, Any], - *, - password_via: bool, - roles_config: IRRolesConfig | None = None, - is_create: bool = False, -) -> dict[str, Any]: +def _prepare_payload(payload: dict[str, Any], *, password_via: bool) -> dict[str, Any]: normalized = dict(payload) - # Roles are managed exclusively through _auth namespace — never writable via CRUD - normalized.pop("roles", None) - normalized.pop("role", None) - if password_via: raw_password = normalized.pop("password", None) if raw_password is not None: @@ -130,13 +121,6 @@ def _prepare_payload( auth_data["passwordHash"] = hash_password(raw_password) normalized["_auth"] = auth_data - if is_create and roles_config is not None: - auth_data = normalized.get("_auth", {}) - if not isinstance(auth_data, dict): - auth_data = {} - auth_data["roles"] = roles_config.default - normalized["_auth"] = auth_data - return normalized @@ -183,15 +167,12 @@ def RepositoryCreateStep( id_field: str, id_format: str, password_via: bool, - roles_config: IRRolesConfig | None = None, + grant: Callable[[dict[str, Any]], dict[str, Any]] = dict, ) -> Step: + """``grant`` sets the roles of a new identity record.""" + async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload( - _payload(ctx), - password_via=password_via, - roles_config=roles_config, - is_create=True, - ) + payload = _prepare_payload(grant(_payload(ctx)), password_via=password_via) if id_format != "natural": payload[id_field] = _generate_id(id_format) result = await _repository(ctx).create(collection_name, payload) diff --git a/src/yaml_api_generator/server/common/steps/validation.py b/src/yaml_api_generator/server/common/steps/validation.py index 79951657..24794549 100644 --- a/src/yaml_api_generator/server/common/steps/validation.py +++ b/src/yaml_api_generator/server/common/steps/validation.py @@ -32,14 +32,7 @@ def _build_field_schemas( schemas: list[_FieldSchema] = [] make_optional = operation == "update" - for field_name, raw_config in collection.fields.items(): - if not isinstance(raw_config, dict): - continue - if raw_config.get("type") == "id": - continue - if field_name == "_auth" or field_name in collection.files: - continue - + for field_name, raw_config in collection.writable_fields.items(): required = not make_optional and bool(raw_config.get("required", False)) schemas.append( _FieldSchema( diff --git a/src/yaml_api_generator/server/fastapi/factory.py b/src/yaml_api_generator/server/fastapi/factory.py index d3e4a1c8..ecfa3497 100644 --- a/src/yaml_api_generator/server/fastapi/factory.py +++ b/src/yaml_api_generator/server/fastapi/factory.py @@ -63,14 +63,7 @@ def create_pydantic_model_fields( make_optional = operation == "update" fields: dict[str, tuple[Any, Any]] = {} - for field_name, raw_config in collection.fields.items(): - if not isinstance(raw_config, dict): - continue - if raw_config.get("type") == "id": - continue - if field_name == "_auth" or field_name in collection.files: - continue - + for field_name, raw_config in collection.writable_fields.items(): annotation = _field_annotation(raw_config) required = bool(raw_config.get("required", False)) has_default = "default" in raw_config diff --git a/tests/integration/test_generated_mongo_access.py b/tests/integration/test_generated_mongo_access.py new file mode 100644 index 00000000..677f67f8 --- /dev/null +++ b/tests/integration/test_generated_mongo_access.py @@ -0,0 +1,19 @@ +"""The access HTTP suite, served by the generated MongoDB backends: raw ASGI and DRF.""" + +from uuid import uuid4 + +import pytest + +from tests.unit.support.live_apps import MONGO_SERVERS, mongo, serve, write_spec +from tests.unit.test_access_http import * # noqa: F403 +from tests.unit.test_access_http import _SPEC + +pytestmark = pytest.mark.integration + + +@pytest.fixture(scope="module", params=MONGO_SERVERS, ids=lambda server: server.name) +def client(request, tmp_path_factory, mongo_uri): + server = request.param + spec = write_spec(tmp_path_factory.mktemp(server.name), _SPEC, mongo(mongo_uri, f"{server.name}_{uuid4().hex}")) + with serve(server, spec) as client: + yield client diff --git a/tests/integration/test_generated_mongo_identity.py b/tests/integration/test_generated_mongo_identity.py new file mode 100644 index 00000000..56c8a59e --- /dev/null +++ b/tests/integration/test_generated_mongo_identity.py @@ -0,0 +1,19 @@ +"""The identity HTTP suite, served by the generated MongoDB backends: raw ASGI and DRF.""" + +from uuid import uuid4 + +import pytest + +from tests.unit.support.live_apps import MONGO_SERVERS, mongo, serve, write_spec +from tests.unit.test_identity_http import * # noqa: F403 +from tests.unit.test_identity_http import _SPEC + +pytestmark = pytest.mark.integration + + +@pytest.fixture(scope="module", params=MONGO_SERVERS, ids=lambda server: server.name) +def client(request, tmp_path_factory, mongo_uri): + server = request.param + spec = write_spec(tmp_path_factory.mktemp(server.name), _SPEC, mongo(mongo_uri, f"{server.name}_{uuid4().hex}")) + with serve(server, spec) as client: + yield client diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index 50029e37..407de7b8 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -409,7 +409,7 @@ async def audit(request): sys.modules["app.handlers"] = handlers_module async def verify(): - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 200, body @@ -430,27 +430,27 @@ async def verify(): assert status == 200, body assert body == {{"authorized": True}} - status, body = await routes.handle_get_reports( + status, body = await routes.handle_get_reports__organizationId___id_( {{"organizationId": "org-1", "id": "r1"}}, None, {{}}, db ) assert status == 200, body assert body["organizationId"] == "org-1" db["organizations"].documents = [{{"id": "org-1", "ownerId": "other"}}] - status, body = await routes.handle_get_reports( + status, body = await routes.handle_get_reports__organizationId___id_( {{"organizationId": "org-1", "id": "r1"}}, None, {{}}, db ) assert status == 403, body db["organizations"].documents = [] db["teams"].documents = [] - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 403, body db["organizations"].documents = [{{"id": "org-1", "ownerId": "u1"}}] - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 200, body @@ -582,7 +582,7 @@ async def audit(request): sys.modules["app.handlers"] = handlers_module async def verify(): - response = await views.ReportsListView().get( + response = await views.View_reports__organizationId_().get( request, organizationId="org-1" ) assert response.status_code == 200, response.data @@ -601,27 +601,27 @@ async def verify(): assert response.status_code == 200, response.data assert response.data == {{"authorized": True}} - response = await views.ReportsDetailView().get( + response = await views.View_reports__organizationId___id_().get( request, organizationId="org-1", id="r1" ) assert response.status_code == 200, response.data assert response.data["organizationId"] == "org-1" db["organizations"].documents = [{{"id": "org-1", "ownerId": "other"}}] - response = await views.ReportsDetailView().get( + response = await views.View_reports__organizationId___id_().get( request, organizationId="org-1", id="r1" ) assert response.status_code == 403, response.data db["organizations"].documents = [] db["teams"].documents = [] - response = await views.ReportsListView().get( + response = await views.View_reports__organizationId_().get( request, organizationId="org-1" ) assert response.status_code == 403, response.data db["organizations"].documents = [{{"id": "org-1", "ownerId": "u1"}}] - response = await views.ReportsListView().get( + response = await views.View_reports__organizationId_().get( request, organizationId="org-1" ) assert response.status_code == 200, response.data diff --git a/tests/unit/support/live_apps.py b/tests/unit/support/live_apps.py new file mode 100644 index 00000000..98e0bd38 --- /dev/null +++ b/tests/unit/support/live_apps.py @@ -0,0 +1,116 @@ +"""Serve a spec over real HTTP from the development server and from the backends it generates.""" + +import os +import subprocess +import sys +import textwrap +from abc import ABC, abstractmethod +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import httpx +import pytest + +from tests.integration.support.network import free_port, wait_until +from yaml_api_generator.cli.compiler import compile_project + +STACKFORM = str(Path(sys.executable).with_name("stackform")) + + +class Server(ABC): + """How one consumer turns a spec into a process listening on `port`.""" + + name: str + environment: dict[str, str] = {} + + @abstractmethod + def command(self, spec: Path, port: int) -> tuple[list[str], Path]: ... + + +class DevServer(Server): + name = "dev" + + def command(self, spec, port): + return [STACKFORM, "run", str(spec), "--port", str(port)], spec.parent + + +class GeneratedApp(Server): + """The backend generated for one target, served by its own process.""" + + def __init__(self, target: str) -> None: + self.name = target + + def command(self, spec, port): + output = spec.parent / "generated" + compile_project(yaml_file=spec, output_dir=output, target=self.name) + return self.process(port), output / "backend" + + def process(self, port: int) -> list[str]: + return [sys.executable, "-m", "uvicorn", "app:app", "--port", str(port)] + + +class DjangoApp(GeneratedApp): + """The generated DRF backend under gunicorn; its tests skip where Django REST framework is not installed.""" + + environment = {"DJANGO_SECRET_KEY": "test", "JWT_SECRET": "test"} + + def __init__(self) -> None: + super().__init__("drf") + + def process(self, port): + for module in ("rest_framework", "gunicorn"): + pytest.importorskip(module) + return [sys.executable, "-m", "gunicorn", "--bind", f"127.0.0.1:{port}", "project.wsgi:application"] + + +SERVERS = (DevServer(), GeneratedApp("fastapi")) +MONGO_SERVERS = ( + GeneratedApp("raw"), + pytest.param( + DjangoApp(), + marks=pytest.mark.xfail(reason="generated DRF views define async handlers on the synchronous APIView", strict=True), + ), +) + + +def sqlite(directory: Path) -> str: + return f'datastores:\n store: {{ type: postgres, dsn: "sqlite+aiosqlite:///{directory / "store.sqlite"}", default: true }}' + + +def mongo(uri: str, database: str) -> str: + return f'mongo: {{ uri: "{uri}", database: {database} }}' + + +def write_spec(directory: Path, spec: str, datastore: str | None = None) -> Path: + """Write `spec`, whose `{datastore}` line becomes `datastore`, a SQLite file beside it by default.""" + path = directory / "spec.yaml" + path.write_text(textwrap.dedent(spec).replace("{datastore}", datastore or sqlite(directory))) + return path + + +def _serving(url: str) -> bool: + try: + httpx.get(url, timeout=1) + return True + except httpx.HTTPError: + return False + + +@contextmanager +def serve(server: Server, spec: Path, env: dict[str, str] | None = None) -> Iterator[httpx.Client]: + port = free_port() + command, cwd = server.command(spec, port) + url, log = f"http://127.0.0.1:{port}", spec.parent / f"{server.name}.log" + with log.open("wb") as output: + process = subprocess.Popen( + command, cwd=cwd, env={**os.environ, **server.environment, **(env or {})}, stdout=output, stderr=subprocess.STDOUT + ) + try: + if not wait_until(lambda: process.poll() is not None or _serving(url)) or process.poll() is not None: + raise RuntimeError(f"{server.name} server did not start:\n{log.read_text()}") + with httpx.Client(base_url=url, timeout=30) as client: + yield client + finally: + process.terminate() + process.wait(timeout=10) diff --git a/tests/unit/test_access_http.py b/tests/unit/test_access_http.py new file mode 100644 index 00000000..55e32468 --- /dev/null +++ b/tests/unit/test_access_http.py @@ -0,0 +1,147 @@ +"""Access rules over real HTTP, in the dev server and the generated FastAPI app.""" + +import pytest + +from tests.unit.support.live_apps import SERVERS, serve, write_spec + +_SPEC = """ +version: "0.3" +api: + basePath: /api + operations: + auth: { register: true } +{datastore} +entities: + user: + type: auth + auth: { identifier: email, via: password } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } + post: + fields: + id: { type: id } + title: { type: string } + ownerId: { type: string } + archivedAt: { type: string } +repositories: + users: { entity: user } + posts: { entity: post } +services: + users: { repository: users } + posts: { repository: posts } +controllers: + users: + service: users + base_path: /users + access: public + posts: + base_path: /posts + routes: + create: + service: posts + method: POST + path: "" + access: + - given: { principal: "identity:user" } + list: + service: posts + method: GET + path: "" + access: + - given: { principal: "identity:user", post: posts } + when: post.ownerId equals principal.sub + get: + service: posts + method: GET + path: /{id} + access: + - given: { document: posts } + when: document.archivedAt is not null + then: deny + reason: Archived posts are hidden. + - then: allow + live: + base_path: /live + routes: + get: + service: posts + method: GET + path: /{id} + access: + - given: { document: posts } + when: document.archivedAt is null + owned: + base_path: /owned + routes: + get: + service: posts + method: GET + path: /{ownerId}/{id} + access: + - given: { principal: "identity:user", owner: users by ownerId } + when: owner.id equals principal.sub +""" + + +@pytest.fixture(scope="module", params=SERVERS, ids=lambda server: server.name) +def client(request, tmp_path_factory): + with serve(request.param, write_spec(tmp_path_factory.mktemp(request.param.name), _SPEC)) as client: + yield client + + +@pytest.fixture(scope="module") +def accounts(client): + """Two users, each owning one post, as ``{email: (basic auth, user id, post)}``.""" + def account(email): + user = client.post("/api/users/register", json={"email": email, "password": "pw"}).json() + basic = (email, "pw") + post = client.post("/api/posts", json={"title": email, "ownerId": user["id"]}, auth=basic).json() + return basic, user["id"], post + + return {email: account(email) for email in ("ann@x.io", "bob@x.io")} + + +ARCHIVED = {"archivedAt": "2026-01-01"} + + +def _get(client, accounts, path, **fields): + """As ann, GET ``path`` followed by the id of a new post of hers holding ``fields``.""" + basic = accounts["ann@x.io"][0] + post = client.post("/api/posts", json={"title": "p", **fields}, auth=basic).json() + return client.get(f"{path}/{post['id']}", auth=basic) + + +def test_list_returns_only_the_rows_the_rule_admits(client, accounts): + basic, user_id, _ = accounts["ann@x.io"] + + rows = client.get("/api/posts", auth=basic).json() + + assert {row["ownerId"] for row in rows} == {user_id} + + +def test_document_binding_by_parameter_loads_the_named_record(client, accounts): + ann, ann_id, ann_post = accounts["ann@x.io"] + _, bob_id, bob_post = accounts["bob@x.io"] + + assert client.get(f"/api/owned/{ann_id}/{ann_post['id']}", auth=ann).status_code == 200 + assert client.get(f"/api/owned/{bob_id}/{bob_post['id']}", auth=ann).status_code == 403 + + +@pytest.mark.parametrize( + ("fields", "status"), + [({"archivedAt": None}, 200), ({}, 200), (ARCHIVED, 403)], + ids=["explicit-null", "absent", "set"], +) +def test_is_null_admits_null_and_absent_values_only(client, accounts, fields, status): + assert _get(client, accounts, "/api/live", **fields).status_code == status + + +def test_is_not_null_admits_null_values(client, accounts): + assert _get(client, accounts, "/api/posts", archivedAt=None).status_code == 200 + + +def test_is_not_null_denies_set_values_with_the_rule_reason(client, accounts): + response = _get(client, accounts, "/api/posts", **ARCHIVED) + + assert (response.status_code, response.json()["detail"]) == (403, "Archived posts are hidden.") diff --git a/tests/unit/test_access_reason.py b/tests/unit/test_access_reason.py index fbe16a89..40b57aed 100644 --- a/tests/unit/test_access_reason.py +++ b/tests/unit/test_access_reason.py @@ -279,7 +279,7 @@ async def principal(headers, db): posts.resolve_principal = principal database = Database() - result = asyncio.run(posts.handle_create_posts( + result = asyncio.run(posts.handle_post_posts_( {{}}, {{"title": "Post", "ownerId": "blocked"}}, {{}}, @@ -288,13 +288,13 @@ async def principal(headers, db): assert result == (403, {{"detail": "Blocked accounts cannot create posts."}}) current["sub"] = "u1" - result = asyncio.run(posts.handle_get_posts( + result = asyncio.run(posts.handle_get_posts__id_( {{"id": "p1"}}, None, {{}}, database )) assert result == (403, {{"detail": "Only the owner may read this post."}}) current["sub"] = "u2" - status, document = asyncio.run(posts.handle_get_posts( + status, document = asyncio.run(posts.handle_get_posts__id_( {{"id": "p1"}}, None, {{}}, database )) assert status == 200 diff --git a/tests/unit/test_drf_path_views.py b/tests/unit/test_drf_path_views.py new file mode 100644 index 00000000..bba394e2 --- /dev/null +++ b/tests/unit/test_drf_path_views.py @@ -0,0 +1,53 @@ +"""The generated DRF backend serves each path from its own view, so routes sharing a collection keep their own access.""" + +import ast +import re + +import pytest + +from tests.unit.support.live_apps import mongo, write_spec +from tests.unit.test_identity_http import _SPEC +from yaml_api_generator.cli.compiler import compile_project + + +@pytest.fixture(scope="module") +def backend(tmp_path_factory): + directory = tmp_path_factory.mktemp("drf") + compile_project(yaml_file=write_spec(directory, _SPEC, mongo("mongodb://localhost", "drf")), output_dir=directory / "generated", target="drf") + return directory / "generated" / "backend" / "api" + + +def _patterns(backend): + return re.findall(r'path\("([^"]*)", (\w+)\.as_view\(\)', (backend / "urls.py").read_text()) + + +def _grants(backend, view): + """The ``assignable`` flag of every role grant in ``view`` of the users module.""" + module = ast.parse((backend / "views" / "users.py").read_text()) + cls = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == view) + return [ + keyword.value.value + for call in ast.walk(cls) + if isinstance(call, ast.Call) and getattr(call.func, "attr", None) == "granted" + for keyword in call.keywords + ] + + +def test_every_path_has_its_own_view(backend): + patterns = _patterns(backend) + + assert len({view for _, view in patterns}) == len(patterns) + + +def test_literal_paths_precede_parameterised_siblings(backend): + paths = [path for path, _ in _patterns(backend)] + + assert paths.index("users/register") < paths.index("users/") + + +def test_register_grants_the_default_and_authenticated_create_the_requested_role(backend): + assert (_grants(backend, "View_users_register"), _grants(backend, "View_users")) == ([False], [True]) + + +def test_disabled_login_serves_no_login_route(backend): + assert not any(path.endswith("/login") for path, _ in _patterns(backend)) diff --git a/tests/unit/test_fastapi_authz_generation.py b/tests/unit/test_fastapi_authz_generation.py index f3ad5105..645fb9ce 100644 --- a/tests/unit/test_fastapi_authz_generation.py +++ b/tests/unit/test_fastapi_authz_generation.py @@ -58,16 +58,14 @@ def add_hierarchy(ctx): def test_principal_includes_roles_extraction(tmp_path): _generate("roles-access.yaml", tmp_path) auth_module = (tmp_path / "dependencies" / "auth.py").read_text() - assert "_extract_roles" in auth_module - assert '"roles": _extract_roles(' in auth_module - assert "_extract_roles(doc, 'role', True)" in auth_module + assert "\"roles\": Roles(field='role', default='user').of(doc)" in auth_module def test_roles_field_default_emits_python_none(tmp_path): import ast _generate("auth-password.yaml", tmp_path) auth_module = (tmp_path / "dependencies" / "auth.py").read_text() - assert "_extract_roles(doc, None, False)" in auth_module + assert "Roles(field=None, default=None).of(doc)" in auth_module names = {n.id for n in ast.walk(ast.parse(auth_module)) if isinstance(n, ast.Name)} assert "null" not in names diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index 694657f7..fe5afb6e 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -179,7 +179,7 @@ def test_drf_views_per_collection(tmp_path): gen.generate(ctx, tmp_path) assert (tmp_path / "api" / "views" / "posts.py").exists() content = (tmp_path / "api" / "views" / "posts.py").read_text() - assert "ListView" in content or "DetailView" in content + assert "class View_posts(APIView)" in content def test_drf_access_rule_enforcement_in_views(tmp_path): diff --git a/tests/unit/test_identity_http.py b/tests/unit/test_identity_http.py new file mode 100644 index 00000000..4ae52d6f --- /dev/null +++ b/tests/unit/test_identity_http.py @@ -0,0 +1,140 @@ +"""Role assignment, login exposure and readonly fields over real HTTP, in the dev server and the generated FastAPI app.""" + +import pytest + +from tests.unit.support.live_apps import SERVERS, serve, write_spec + +_SPEC = """ +version: "0.3" +api: + basePath: /api + operations: + auth: { register: true, login: false } +{datastore} +entities: + user: + type: auth + auth: { identifier: email, via: password, roles: role } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } + role: { type: enum, values: [admin, member], default: member } + account: + type: auth + auth: { identifier: email, via: password, roles: role } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } + role: { type: enum, values: [admin, member], default: member } + member: + type: auth + auth: { identifier: email, via: password } + roles: { values: [boss, peon], default: peon } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } + note: + fields: + id: { type: id } + title: { type: string } + role: { type: string } + roles: { type: string } + stamp: { type: string, readonly: true } +repositories: + users: { entity: user } + accounts: { entity: account } + members: { entity: member } + notes: { entity: note } +services: + users: { repository: users } + accounts: { repository: accounts } + members: { repository: members } + notes: { repository: notes } +controllers: + users: + service: users + base_path: /users + access: user:admin + operations: + register: { access: public } + create: + access: + - given: { principal: "identity:user" } + accounts: + service: accounts + base_path: /accounts + access: account:admin + operations: + create: { access: public } + members: + service: members + base_path: /members + access: member:peon + operations: + register: { access: public } + notes: + service: notes + base_path: /notes + access: public +""" + +ADMIN = {"email": "admin@x.io", "password": "pw", "role": "admin"} + + +@pytest.fixture(scope="module", params=SERVERS, ids=lambda server: server.name) +def client(request, tmp_path_factory): + with serve(request.param, write_spec(tmp_path_factory.mktemp(request.param.name), _SPEC)) as client: + yield client + + +def _credentials(account): + return account["email"], account["password"] + + +def test_anonymous_register_gets_the_default_role(client): + response = client.post("/api/users/register", json={"email": "a@x.io", "password": "pw", "role": "admin"}) + + assert response.status_code == 200 + assert response.json()["role"] == "member" + assert client.get("/api/users", auth=("a@x.io", "pw")).status_code == 403 + + +def test_anonymous_create_gets_the_default_role(client): + response = client.post("/api/accounts", json={"email": "c@x.io", "password": "pw", "role": "admin"}) + + assert response.json()["role"] == "member" + assert client.get("/api/accounts", auth=("c@x.io", "pw")).status_code == 403 + + +def test_authenticated_create_assigns_the_requested_role(client): + client.post("/api/users/register", json={"email": "d@x.io", "password": "pw"}) + + created = client.post("/api/users", json=ADMIN, auth=("d@x.io", "pw")) + + assert created.json()["role"] == "admin" + assert client.get("/api/users", auth=_credentials(ADMIN)).status_code == 200 + + +def test_managed_roles_start_at_the_default(client): + client.post("/api/members/register", json={"email": "m@x.io", "password": "pw"}) + + assert client.get("/api/members", auth=("m@x.io", "pw")).status_code == 200 + + +def test_non_identity_role_fields_are_stored(client): + note = client.post("/api/notes", json={"title": "t", "role": "editor", "roles": "all"}).json() + + stored = client.get(f"/api/notes/{note['id']}").json() + assert (stored["role"], stored["roles"]) == ("editor", "all") + + +def test_readonly_fields_are_never_written_by_clients(client): + client.post("/api/notes", json={"title": "forged", "stamp": "forged"}) + + assert all(note.get("stamp") != "forged" for note in client.get("/api/notes").json()) + + +def test_disabled_login_serves_no_login_route(client): + client.post("/api/users/register", json={"email": "l@x.io", "password": "pw"}) + + assert client.post("/api/users/login", json={"email": "l@x.io", "password": "pw"}).status_code in (404, 405) diff --git a/tests/unit/test_policy_roles_from_claims.py b/tests/unit/test_policy_roles_from_claims.py index b5b02da9..05a2c1d8 100644 --- a/tests/unit/test_policy_roles_from_claims.py +++ b/tests/unit/test_policy_roles_from_claims.py @@ -18,6 +18,7 @@ from yaml_api_generator.generators.backend.fastapi import access_filters from yaml_api_generator.server.common.steps.authorization import _principal_has_role from yaml_api_generator.ir import IRProviderBinding +from yaml_api_generator.runtime.access import role_names from yaml_api_generator.ir.api import IROIDCProvider from yaml_api_generator.server.common.auth import idp from yaml_api_generator.server.common.repositories import RepositoryRegistry @@ -169,7 +170,7 @@ def _exec_roles_from_claims(source): fragment = source[source.index("def _roles_from_claims"):] fragment = fragment[:fragment.index("\n\n")] module = types.ModuleType("auth_fragment") - module.__dict__.update({"resolve_claim_path": resolve_claim_path, "Any": object}) + module.__dict__.update({"resolve_claim_path": resolve_claim_path, "role_names": role_names, "Any": object}) exec(compile(fragment, "", "exec"), module.__dict__) return module._roles_from_claims From a18878998d5d83d31cbbbfad872663af748a75df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 19:37:15 +0100 Subject: [PATCH 12/89] Register the 0.1 mongo block as the default datastore --- .../parsers/transformers/v01/__init__.py | 11 +++++------ tests/unit/test_v01_transformer.py | 8 ++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/yaml_api_generator/parsers/transformers/v01/__init__.py b/src/yaml_api_generator/parsers/transformers/v01/__init__.py index 01ab648a..92966ac6 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v01/__init__.py @@ -9,12 +9,12 @@ from yaml_api_generator.ir import ( IRApi, IRDocument, - IRMongo, IRRepository, IRService, ) from yaml_api_generator.parsers.yaml_parser import ConfigErrors +from ..v02.datastore_transformer import parse_datastores from .collection_transformer import CollectionTransformer from .identity_transformer import IdentityTransformer from .route_transformer import RouteTransformer @@ -96,7 +96,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: for route in routes ] - mongo_data = resolved.get("mongo", {}) + datastores, default_datastore, _ = parse_datastores(resolved) return IRDocument( version=str(resolved["version"]), api=api_ir, @@ -105,8 +105,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: services=services, routes=routes, identityCollections=identity_collections, - mongo=IRMongo( - uri=mongo_data.get("uri", "mongodb://localhost:27017"), - database=mongo_data.get("database", "yaml_api"), - ), + mongo=datastores[default_datastore], + datastores=datastores, + default_datastore=default_datastore, ) diff --git a/tests/unit/test_v01_transformer.py b/tests/unit/test_v01_transformer.py index 055dc316..25176802 100644 --- a/tests/unit/test_v01_transformer.py +++ b/tests/unit/test_v01_transformer.py @@ -3,6 +3,8 @@ import pytest from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir +from yaml_api_generator.server.common.loader import build_registry +from yaml_api_generator.server.common.repositories.mongodb import MongoRepository from tests.conftest import INVALID_DIR, VALID_DIR EXAMPLES_DIR = pathlib.Path(__file__).parent.parent.parent / "examples" / "valid" @@ -30,6 +32,12 @@ def test_v01_blog_collections_parsed() -> None: assert "delete" in users_ops +def test_v01_mongo_block_is_the_default_datastore() -> None: + ir = load_and_generate_ir(str(EXAMPLES_DIR / "blog.yaml")).ir + assert ir.datastores == {ir.default_datastore: ir.mongo} + assert isinstance(build_registry(ir).for_collection("posts"), MongoRepository) + + # --- Test 126: ID field extraction edge cases --- From 7850b8004b92dd58eccadffab274587e0d3923c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 19:42:28 +0100 Subject: [PATCH 13/89] Keep non-identity role fields, keep record ids on raw replace and return SQL numbers as floats --- .../generators/backend/fastapi/sql_filters.py | 19 ++----- .../shared/templates/repository.py.jinja | 11 +--- .../server/common/pipeline_builder.py | 18 ++----- .../server/common/repositories/sql_schema.py | 6 +-- .../server/common/steps/repositories.py | 42 +++++++-------- tests/unit/support/payload_requests.py | 32 ++++++++++-- tests/unit/test_free_slug_fetch.py | 12 ++--- tests/unit/test_repository_steps.py | 51 ++++++++++++------- 8 files changed, 98 insertions(+), 93 deletions(-) diff --git a/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py b/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py index d35123dd..b1fc74ad 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py +++ b/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py @@ -2,12 +2,14 @@ from typing import Any +from yaml_api_generator.server.common.repositories.sql_schema import column_type + def sql_column(name: str, cfg: dict[str, Any], id_field: str) -> str: field_type = cfg.get("type", "string") is_pk = name == id_field or field_type == "id" - parts = [f'"{name}"', _column_type(cfg, field_type)] + parts = [f'"{name}"', f"sa.{column_type(cfg)!r}"] if is_pk: parts.append("primary_key=True") elif cfg.get("required"): @@ -15,18 +17,3 @@ def sql_column(name: str, cfg: dict[str, Any], id_field: str) -> str: if cfg.get("unique"): parts.append("unique=True") return f"sa.Column({', '.join(parts)})" - - -def _column_type(cfg: dict[str, Any], field_type: str) -> str: - if field_type in ("array", "object", "file"): - return "sa.JSON()" - if field_type == "number": - return "sa.Numeric()" - if field_type == "boolean": - return "sa.Boolean()" - if field_type == "datetime": - return "sa.DateTime(timezone=True)" - max_length = cfg.get("maxLength") - if field_type == "string" and isinstance(max_length, int): - return f"sa.String({max_length})" - return "sa.Text()" \ No newline at end of file diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index aa5ae0c5..c2bc6198 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -166,20 +166,13 @@ class {{ class_name }}Repository: ) -> dict[str, Any] | None: return await self.update_by(self._match_filter(document_id), data) - @_translate_conflict async def replace( self, document_id: str, data: dict[str, Any] ) -> dict[str, Any] | None: - doc = await self._col.find_one_and_replace( - self._match_filter(document_id), - data, - return_document=ReturnDocument.AFTER, - ) - return self._normalize(doc) + return await self.replace_by(self._match_filter(document_id), data) async def delete(self, document_id: str) -> dict[str, Any] | None: - doc = await self._col.find_one_and_delete(self._match_filter(document_id)) - return self._normalize(doc) + return await self.delete_by(self._match_filter(document_id)) {% for op in custom_operations | default([]) %} {% set q = op.query %} {% set col_expr = "self._db[\"" + q["collection"] + "\"]" if q and q.get("collection") else "self._col" %} diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 1f0e4dad..e9c9910a 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -284,31 +284,21 @@ def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: operation = route.operation collection = self._ctx.collections[collection_name] id_field = collection.id_field - identity_collection = self._ctx.auth.identity_collections.get(collection_name) - uses_password = ( - identity_collection is not None and identity_collection.has_password - ) + identity = self._ctx.auth.identity_collections.get(collection_name) if operation == "list": return RepositoryListStep(collection_name, id_field, route.pagination) if operation == "get": return RepositoryGetStep(collection_name, id_field, self._locator(route)) if operation == "create": - roles_config = identity_collection.roles if identity_collection is not None else None - return RepositoryCreateStep( - collection_name, - id_field, - collection.id_format, - uses_password, - roles_config=roles_config, - ) + return RepositoryCreateStep(collection_name, id_field, collection.id_format, identity) if operation == "update": return RepositoryUpdateStep( - collection_name, id_field, uses_password, self._locator(route) + collection_name, id_field, self._locator(route), identity=identity ) if operation == "replace": return RepositoryReplaceStep( - collection_name, id_field, uses_password, self._locator(route) + collection_name, id_field, self._locator(route), identity=identity ) if operation == "delete": return RepositoryDeleteStep(collection_name, id_field, self._locator(route)) diff --git a/src/yaml_api_generator/server/common/repositories/sql_schema.py b/src/yaml_api_generator/server/common/repositories/sql_schema.py index fb4ce875..7001609c 100644 --- a/src/yaml_api_generator/server/common/repositories/sql_schema.py +++ b/src/yaml_api_generator/server/common/repositories/sql_schema.py @@ -35,19 +35,19 @@ def _column(name: str, cfg: dict[str, Any], is_pk: bool) -> sa.Column: nullable = not (is_pk or bool(cfg.get("required"))) return sa.Column( name, - _column_type(cfg), + column_type(cfg), primary_key=is_pk, nullable=nullable, unique=True if cfg.get("unique") else None, ) -def _column_type(cfg: dict[str, Any]) -> sa.types.TypeEngine: +def column_type(cfg: dict[str, Any]) -> sa.types.TypeEngine: field_type = cfg.get("type", "string") if field_type in _JSON_TYPES: return sa.JSON() if field_type == "number": - return sa.Numeric() + return sa.Numeric(asdecimal=False) if field_type == "boolean": return sa.Boolean() if field_type == "datetime": diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 2fcc7aa9..bbd45be9 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -4,7 +4,8 @@ from ..auth import hash_password from ..request_context import RequestContext -from yaml_api_generator.ir.api import IRPagination, IRRolesConfig +from ..server_context import RuntimeIdentityDescriptor +from yaml_api_generator.ir.api import IRPagination from ..pagination import Page, build_response, map_request from ..repositories.base import BaseRepository from ..utils.ids import generate_id as _generate_id @@ -14,6 +15,7 @@ _PATH_TOKEN = re.compile(r"\{(\w+)\}") _REQUEST_BODY_MESSAGE = "Request body must be a JSON object" _NOT_FOUND_MESSAGE = "Resource not found" +_ROLE_FIELDS = frozenset({"role", "roles"}) def _repository(ctx: RequestContext) -> BaseRepository: @@ -108,18 +110,16 @@ def _inject_canonical_id_list( def _prepare_payload( payload: dict[str, Any], + identity: RuntimeIdentityDescriptor | None, *, - password_via: bool, - roles_config: IRRolesConfig | None = None, is_create: bool = False, ) -> dict[str, Any]: - normalized = dict(payload) + """Identity records keep roles in the _auth namespace, never writable through CRUD.""" + if identity is None: + return dict(payload) + normalized = {key: value for key, value in payload.items() if key not in _ROLE_FIELDS} - # Roles are managed exclusively through _auth namespace — never writable via CRUD - normalized.pop("roles", None) - normalized.pop("role", None) - - if password_via: + if identity.has_password: raw_password = normalized.pop("password", None) if raw_password is not None: if not isinstance(raw_password, str) or not raw_password.strip(): @@ -130,11 +130,11 @@ def _prepare_payload( auth_data["passwordHash"] = hash_password(raw_password) normalized["_auth"] = auth_data - if is_create and roles_config is not None: + if is_create and identity.roles is not None: auth_data = normalized.get("_auth", {}) if not isinstance(auth_data, dict): auth_data = {} - auth_data["roles"] = roles_config.default + auth_data["roles"] = identity.roles.default normalized["_auth"] = auth_data return normalized @@ -182,16 +182,10 @@ def RepositoryCreateStep( collection_name: str, id_field: str, id_format: str, - password_via: bool, - roles_config: IRRolesConfig | None = None, + identity: RuntimeIdentityDescriptor | None = None, ) -> Step: async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload( - _payload(ctx), - password_via=password_via, - roles_config=roles_config, - is_create=True, - ) + payload = _prepare_payload(_payload(ctx), identity, is_create=True) if id_format != "natural": payload[id_field] = _generate_id(id_format) result = await _repository(ctx).create(collection_name, payload) @@ -204,13 +198,14 @@ async def step(ctx: RequestContext) -> RequestContext: def RepositoryUpdateStep( collection_name: str, id_field: str, - password_via: bool, locator: RecordLocator | None = None, + *, + identity: RuntimeIdentityDescriptor | None = None, ) -> Step: locator = locator or RecordLocator(id_field) async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload(_payload(ctx), password_via=password_via) + payload = _prepare_payload(_payload(ctx), identity) if locator.is_scoped: result = await _repository(ctx).update_by( collection_name, locator.criteria(ctx.url_params), payload @@ -230,13 +225,14 @@ async def step(ctx: RequestContext) -> RequestContext: def RepositoryReplaceStep( collection_name: str, id_field: str, - password_via: bool, locator: RecordLocator | None = None, + *, + identity: RuntimeIdentityDescriptor | None = None, ) -> Step: locator = locator or RecordLocator(id_field) async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload(_payload(ctx), password_via=password_via) + payload = _prepare_payload(_payload(ctx), identity) if locator.is_scoped: result = await _repository(ctx).replace_by( collection_name, locator.criteria(ctx.url_params), payload diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py index fdce2776..e22830e3 100644 --- a/tests/unit/support/payload_requests.py +++ b/tests/unit/support/payload_requests.py @@ -2,7 +2,8 @@ The spec covers formats, array items, closed nested objects, defaults, readonly fields, strict scalars, partial updates and a DTO-typed body, which an access -rule binds for any signed-in user. +rule binds for any signed-in user. Round trips check what the apps store and +return besides the status codes. """ import textwrap @@ -34,7 +35,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload api: basePath: "" operations: - defaults: { list: true, get: true, create: true, update: true, delete: true } + defaults: { list: true, get: true, create: true, update: true, replace: true, delete: true } auth: { register: true, login: true } """ @@ -142,10 +143,19 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload ({"role": "admin", "unknown": 1}, 422), ({}, 422), ] +ROUND_TRIPS = [ + ["created number", 3], + ["replaced id kept", True], + ["replaced number", 1.5], + ["replaced item found", 200], + ["replaced item deleted", 200], + ["note role kept", "admin"], +] EXPECTED = [ *(["POST /items", body, status] for body, status in CREATES), *(["PATCH /items", body, status] for body, status in UPDATES), *(["POST /notes", body, status] for body, status in NOTES), + *ROUND_TRIPS, ] @@ -160,8 +170,23 @@ def _bearer(client: TestClient) -> dict[str, str]: return {"Authorization": f"Bearer {token}"} +def _round_trips(client: TestClient, headers: dict[str, str]) -> list[list]: + item = client.post("/items", json={"name": "ab", "rank": 3}).json() + path = f"/items/{item['id']}" + replaced = client.put(path, json={"name": "cd", "rank": 1.5}).json() + note = client.post("/notes", json={"role": "admin"}, headers=headers).json() + return [ + ["created number", item["rank"]], + ["replaced id kept", replaced.get("id") == item["id"]], + ["replaced number", replaced["rank"]], + ["replaced item found", client.get(path).status_code], + ["replaced item deleted", client.delete(path).status_code], + ["note role kept", note.get("role")], + ] + + def scenario(client: TestClient) -> list[list]: - """Each payload with the status it got, after one valid item is created for the updates.""" + """Each payload with the status it got, after one valid item is created for the updates, then the round trips.""" headers = _bearer(client) created = client.post("/items", json={"name": "ab"}) assert created.status_code == 200, created.text @@ -177,4 +202,5 @@ def scenario(client: TestClient) -> list[list]: ["POST /notes", body, client.post("/notes", json=body, headers=headers).status_code] for body, _ in NOTES ), + *_round_trips(client, headers), ] diff --git a/tests/unit/test_free_slug_fetch.py b/tests/unit/test_free_slug_fetch.py index 9eda9ab8..6928c3b5 100644 --- a/tests/unit/test_free_slug_fetch.py +++ b/tests/unit/test_free_slug_fetch.py @@ -152,7 +152,7 @@ async def test_update_step_multi_token_scoped_mutates_matching_record(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryUpdateStep( - "members", "id", False, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") ) result = await step( _ctx( @@ -171,7 +171,7 @@ async def test_replace_step_multi_token_wrong_parent_is_404(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryReplaceStep( - "members", "id", False, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") ) with pytest.raises(NotFoundError): await step( @@ -192,7 +192,7 @@ async def test_replace_step_multi_token_wrong_parent_is_404(): async def test_update_step_v02_id_token_mutates_record(): repo = InMemoryRepository({"items": [{"id": "i1", "title": "old"}]}) repo.register_id_field("items", "id") - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") result = await step( _ctx(repo, {"id": "i1"}, collection="items", operation="update", body={"title": "new"}) ) @@ -204,7 +204,7 @@ async def test_update_step_renamed_token_mutates_record(): repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "old"}]}) repo.register_id_field("assetCategories", "id") step = RepositoryUpdateStep( - "assetCategories", "id", False, record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") ) result = await step( _ctx( @@ -222,7 +222,7 @@ async def test_update_step_renamed_token_mutates_record(): async def test_replace_step_v02_id_token_replaces_record(): repo = InMemoryRepository({"items": [{"id": "i1", "title": "old", "extra": "drop"}]}) repo.register_id_field("items", "id") - step = RepositoryReplaceStep("items", "id", False) + step = RepositoryReplaceStep("items", "id") result = await step( _ctx( repo, @@ -243,7 +243,7 @@ async def test_replace_step_renamed_token_replaces_record(): ) repo.register_id_field("assetCategories", "id") step = RepositoryReplaceStep( - "assetCategories", "id", False, record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") ) result = await step( _ctx( diff --git a/tests/unit/test_repository_steps.py b/tests/unit/test_repository_steps.py index 26f80e14..09c3bc5e 100644 --- a/tests/unit/test_repository_steps.py +++ b/tests/unit/test_repository_steps.py @@ -3,7 +3,9 @@ import pytest from tests.unit.support.repositories import InMemoryRepository +from yaml_api_generator.ir.api import IRProviderBinding from yaml_api_generator.server.common.request_context import RequestContext +from yaml_api_generator.server.common.server_context import RuntimeIdentityDescriptor from yaml_api_generator.server.common.steps import NotFoundError from yaml_api_generator.server.common.steps.repositories import ( RepositoryCreateStep, @@ -59,40 +61,51 @@ def test_generate_id_unknown_format_raises(): # --- Payload Preparation (244-249) --- -def test_prepare_payload_password_via_false(): - payload = {"title": "hello", "password": "secret"} - result = _prepare_payload(payload, password_via=False) - assert result == {"title": "hello", "password": "secret"} +_PASSWORD_IDENTITY = RuntimeIdentityDescriptor( + name="users", + bindings=(IRProviderBinding(provider="password", identifier="email"),), + id_field="id", +) + + +def test_prepare_payload_keeps_non_identity_fields(): + payload = {"title": "hello", "password": "secret", "role": "admin", "roles": ["a"]} + assert _prepare_payload(payload, None) == payload + + +def test_prepare_payload_strips_identity_roles(): + result = _prepare_payload({"email": "a@b.co", "role": "admin", "roles": ["a"]}, _PASSWORD_IDENTITY) + assert result == {"email": "a@b.co"} -def test_prepare_payload_password_via_true_no_password_key(): +def test_prepare_payload_password_identity_no_password_key(): payload = {"title": "hello"} - result = _prepare_payload(payload, password_via=True) + result = _prepare_payload(payload, _PASSWORD_IDENTITY) assert result == {"title": "hello"} -def test_prepare_payload_password_via_true_valid_password(): +def test_prepare_payload_password_identity_valid_password(): payload = {"title": "hello", "password": "secret123"} - result = _prepare_payload(payload, password_via=True) + result = _prepare_payload(payload, _PASSWORD_IDENTITY) assert "password" not in result assert "_auth" in result assert "passwordHash" in result["_auth"] assert result["_auth"]["passwordHash"].startswith("pbkdf2$sha256$") -def test_prepare_payload_password_via_true_empty_password(): +def test_prepare_payload_password_identity_empty_password(): with pytest.raises(ValueError, match="non-empty string"): - _prepare_payload({"password": ""}, password_via=True) + _prepare_payload({"password": ""}, _PASSWORD_IDENTITY) -def test_prepare_payload_password_via_true_non_string_password(): +def test_prepare_payload_password_identity_non_string_password(): with pytest.raises(ValueError, match="non-empty string"): - _prepare_payload({"password": 123}, password_via=True) + _prepare_payload({"password": 123}, _PASSWORD_IDENTITY) def test_prepare_payload_existing_auth_dict_merged(): payload = {"_auth": {"apiKey": "abc123"}, "password": "secret"} - result = _prepare_payload(payload, password_via=True) + result = _prepare_payload(payload, _PASSWORD_IDENTITY) assert "apiKey" in result["_auth"] assert "passwordHash" in result["_auth"] @@ -176,7 +189,7 @@ async def test_repository_create_step_generates_id(): repo = InMemoryRepository() repo.register_id_field("items", "id") ctx = _make_ctx(repository=repo, body={"title": "new"}) - step = RepositoryCreateStep("items", "id", "uuidv4", False) + step = RepositoryCreateStep("items", "id", "uuidv4") result = await step(ctx) assert result.response["id"] is not None assert uuid.UUID(result.response["id"]).version == 4 @@ -191,7 +204,7 @@ async def test_repository_create_step_natural_pk_no_id_generated(): collection="countries", body={"code": "US", "name": "United States"}, ) - step = RepositoryCreateStep("countries", "code", "natural", False) + step = RepositoryCreateStep("countries", "code", "natural") result = await step(ctx) assert result.response["code"] == "US" assert "id" not in result.response or result.response.get("id") is None @@ -208,7 +221,7 @@ async def test_repository_update_step(): url_params={"id": "abc"}, body={"title": "new"}, ) - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") result = await step(ctx) assert result.response["title"] == "new" @@ -224,7 +237,7 @@ async def test_repository_update_step_not_found(): url_params={"id": "missing"}, body={"title": "x"}, ) - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") with pytest.raises(NotFoundError): await step(ctx) @@ -242,7 +255,7 @@ async def test_repository_replace_step(): url_params={"id": "abc"}, body={"id": "abc", "title": "replaced"}, ) - step = RepositoryReplaceStep("items", "id", False) + step = RepositoryReplaceStep("items", "id") result = await step(ctx) assert result.response["title"] == "replaced" @@ -289,6 +302,6 @@ async def test_repository_step_non_dict_body(): repo = InMemoryRepository() repo.register_id_field("items", "id") ctx = _make_ctx(repository=repo, body="not a dict") - step = RepositoryCreateStep("items", "id", "uuidv4", False) + step = RepositoryCreateStep("items", "id", "uuidv4") with pytest.raises(ValueError, match="Request body must be a JSON object"): await step(ctx) From b7eccdf37ce3c5d0c3679612cd44b6381630eede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 19:50:17 +0100 Subject: [PATCH 14/89] Serve declared login routes, honour api.port in DRF and UI, echo requested CORS headers and reject inconsistent mongo settings Order DRF routes so literal segments precede parameters, keep blank query values for limit validation, report schema errorMessage annotations, and add regression tests for each target. --- pyproject.toml | 5 + src/yaml_api_generator/cli/__init__.py | 5 +- src/yaml_api_generator/cli/compiler.py | 17 +- .../generators/backend/base.py | 5 +- .../generators/backend/builders/__init__.py | 11 +- .../backend/builders/auth_builder.py | 7 +- .../backend/builders/route_builder.py | 3 +- .../generators/backend/compile_context.py | 14 ++ .../generators/backend/drf/generator.py | 20 +- .../backend/drf/templates/api_urls.py.jinja | 11 +- .../backend/drf/templates/apps.py.jinja | 8 - .../backend/drf/templates/asgi.py.jinja | 23 ++- .../backend/drf/templates/auth.py.jinja | 172 +----------------- .../backend/drf/templates/cors.py.jinja | 12 ++ .../drf/templates/handler_views.py.jinja | 4 +- .../backend/drf/templates/jwks_views.py.jinja | 2 +- .../drf/templates/login_views.py.jinja | 49 ++--- .../drf/templates/requirements.txt.jinja | 3 +- .../backend/drf/templates/runserver.py.jinja | 13 ++ .../backend/drf/templates/settings.py.jinja | 8 +- .../backend/drf/templates/views.py.jinja | 6 +- .../backend/drf/templates/wsgi.py.jinja | 11 +- .../generators/backend/fastapi/generator.py | 35 +--- .../backend/fastapi/templates/auth.py.jinja | 7 +- .../fastapi/templates/login_routes.py.jinja | 6 +- .../backend/fastapi/templates/routes.py.jinja | 8 +- .../fastapi/templates/service.py.jinja | 5 +- .../fastapi/templates/service_ops.py.jinja | 42 +++-- .../generators/backend/raw/generator.py | 10 +- .../backend/raw/templates/app.py.jinja | 4 +- .../backend/raw/templates/auth.py.jinja | 137 +------------- .../raw/templates/login_routes.py.jinja | 33 +--- .../shared/templates/principal.py.jinja | 169 +++++++++++++++++ .../shared/templates/repository.py.jinja | 7 +- .../backend/shared/templates/service.py.jinja | 68 ++++--- .../generators/backend/utils.py | 12 ++ .../generators/deploy/context.py | 11 +- .../generators/deploy/context_builder.py | 14 +- .../generators/deploy/docker/generator.py | 7 +- .../deploy/docker/templates/Dockerfile.jinja | 3 +- .../deploy/docker/templates/README.md.jinja | 45 ++--- .../docker/templates/playbook.yml.jinja | 4 +- .../{mongodb => docker}/tasks/main.yml.jinja | 0 .../generators/deploy/generate.py | 3 +- .../generators/deploy/systemd/generator.py | 2 +- .../systemd/templates/files/app.service.jinja | 5 +- .../systemd/templates/playbook.yml.jinja | 2 +- .../templates/scaffold/.env.example.jinja | 2 +- .../templates/scaffold/vite.config.ts.jinja | 2 +- .../generators/frontend/renderer.py | 1 + src/yaml_api_generator/ir/api.py | 3 +- src/yaml_api_generator/ir/ui.py | 1 + .../schemas/yaml-spec-v0.1.schema.json | 11 +- .../schemas/yaml-spec-v0.2.schema.json | 125 ++++++++++--- .../schemas/yaml-spec-v0.3.schema.json | 145 ++++++++++++--- .../parsers/transformers/roles_config.py | 56 +++--- .../transformers/v02/route_transformer.py | 6 +- .../parsers/transformers/v03/pagination.py | 41 +---- src/yaml_api_generator/parsers/ui_parser.py | 1 + src/yaml_api_generator/parsers/yaml_parser.py | 10 +- src/yaml_api_generator/runtime/pagination.py | 4 +- src/yaml_api_generator/server/asgi/runner.py | 2 +- .../server/common/auth/__init__.py | 3 +- .../server/common/auth/api_key.py | 6 + .../server/common/pipeline_builder.py | 14 +- .../server/common/steps/authentication.py | 4 +- .../server/common/steps/repositories.py | 65 +++---- tests/fixtures/declared-routes/spec.yaml | 24 +++ tests/fixtures/valid/v03-service-input.yaml | 2 + tests/integration/support/declared_routes.py | 44 +++++ tests/integration/support/v01_platform.py | 37 +++- tests/integration/test_anonymous_access.py | 80 ++++++++ tests/integration/test_declared_routes.py | 34 ++++ .../test_generated_where_authorization.py | 18 +- tests/integration/test_v01_platform.py | 16 +- tests/unit/support/generated_apps.py | 9 +- tests/unit/test_auth_steps.py | 8 +- tests/unit/test_cli.py | 25 ++- tests/unit/test_cors.py | 2 +- tests/unit/test_deploy_build_context.py | 9 +- tests/unit/test_deploy_defaults.py | 120 ++++++++++-- tests/unit/test_deploy_file_datastores.py | 4 +- tests/unit/test_free_slug_fetch.py | 12 +- tests/unit/test_generated_entry_points.py | 67 +++++++ tests/unit/test_mongo_graduation.py | 4 +- tests/unit/test_pagination.py | 5 +- tests/unit/test_repository_steps.py | 68 ++++--- tests/unit/test_schema_rejection.py | 16 ++ tests/unit/test_v01_transformer.py | 5 + uv.lock | 150 +++++++++++++++ 90 files changed, 1489 insertions(+), 820 deletions(-) create mode 100644 src/yaml_api_generator/generators/backend/drf/templates/cors.py.jinja create mode 100644 src/yaml_api_generator/generators/backend/drf/templates/runserver.py.jinja create mode 100644 src/yaml_api_generator/generators/backend/shared/templates/principal.py.jinja rename src/yaml_api_generator/generators/deploy/docker/templates/roles/{mongodb => docker}/tasks/main.yml.jinja (100%) create mode 100644 src/yaml_api_generator/server/common/auth/api_key.py create mode 100644 tests/fixtures/declared-routes/spec.yaml create mode 100644 tests/integration/support/declared_routes.py create mode 100644 tests/integration/test_anonymous_access.py create mode 100644 tests/integration/test_declared_routes.py create mode 100644 tests/unit/test_generated_entry_points.py diff --git a/pyproject.toml b/pyproject.toml index 772b7d24..760e745d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,12 @@ markers = [ [dependency-groups] dev = [ + "adrf>=0.1.14", "aiosqlite>=0.22.1", + "django>=5.2.18", + "django-cors-headers>=4.9.0", + "djangorestframework>=3.18.3", + "drf-spectacular>=0.30.0", "pytest>=9.0.3", "pytest-asyncio>=1.3.0", ] diff --git a/src/yaml_api_generator/cli/__init__.py b/src/yaml_api_generator/cli/__init__.py index e0f1c818..07de7e56 100644 --- a/src/yaml_api_generator/cli/__init__.py +++ b/src/yaml_api_generator/cli/__init__.py @@ -23,7 +23,7 @@ def cli(): def build(yaml_file: str, what: str, schema_file: str | None, output_dir: str | None, target: str, output_json: bool): """Build backend, frontend, deploy, or all.""" from pathlib import Path - from .compiler import compile_project + from .compiler import backend_generator, compile_project from ..parsers.yaml_parser import ConfigErrors, load_and_generate_ir from ..generators.frontend.generator import generate_frontend from ..generators.frontend.packs.loader import load_pack @@ -86,6 +86,7 @@ def build(yaml_file: str, what: str, schema_file: str | None, output_dir: str | else: generate_deploy( ir=result.ir, + application=backend_generator(target).asgi_application, host="localhost", ssh_user="ubuntu", ssh_key="~/.ssh/id_rsa", @@ -104,7 +105,7 @@ def build(yaml_file: str, what: str, schema_file: str | None, output_dir: str | @click.argument("yaml_file", type=click.Path(exists=True)) @click.option("-s", "--schema", "schema_file", type=click.Path(exists=True), help="JSON schema file") @click.option("--host", default="127.0.0.1", help="Host to bind") -@click.option("-p", "--port", type=int, help="Port to bind (default: api.port)") +@click.option("-p", "--port", type=click.IntRange(1, 65535), help="Port to bind (default: api.port)") @click.option("--raw", is_flag=True, help="Use raw ASGI runner") @click.option("-r", "--reload", is_flag=True, help="Reload the app when the spec changes") @click.option("--json", "output_json", is_flag=True, help="Output errors in JSON") diff --git a/src/yaml_api_generator/cli/compiler.py b/src/yaml_api_generator/cli/compiler.py index ed3905a3..c40cb478 100644 --- a/src/yaml_api_generator/cli/compiler.py +++ b/src/yaml_api_generator/cli/compiler.py @@ -20,6 +20,17 @@ } +def backend_generator(target: str) -> type[CodeGenerator]: + """Return the generator class of a backend target. + + Raises: + ValueError: If target is unsupported + """ + if target not in _GENERATORS: + raise ValueError(f"Unknown target '{target}'. Supported: {', '.join(_GENERATORS)}") + return _GENERATORS[target] + + def compile_project( yaml_file: str | Path, schema_file: str | Path | None = None, @@ -40,11 +51,7 @@ def compile_project( Raises: ValueError: If target is unsupported """ - generator_cls = _GENERATORS.get(target) - if generator_cls is None: - supported = ", ".join(_GENERATORS) - raise ValueError(f"Unknown target '{target}'. Supported: {supported}") - + generator_cls = backend_generator(target) result = load_and_generate_ir(yaml_file, schema_file) out = Path(output_dir) context = build_context(result.ir, SchemaManagement.for_spec(yaml_file)) diff --git a/src/yaml_api_generator/generators/backend/base.py b/src/yaml_api_generator/generators/backend/base.py index 30e84133..fbafd38a 100644 --- a/src/yaml_api_generator/generators/backend/base.py +++ b/src/yaml_api_generator/generators/backend/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Any +from typing import Any, Callable, ClassVar from jinja2 import ChoiceLoader, Environment, FileSystemLoader, StrictUndefined @@ -51,6 +51,9 @@ def template_environment(*template_dirs: Path) -> Environment: class CodeGenerator(ABC): + asgi_application: ClassVar[str] = "app:app" + """Import path an ASGI server loads to serve the generated backend.""" + @abstractmethod def generate(self, context: CompileContext, output_dir: Path) -> None: ... diff --git a/src/yaml_api_generator/generators/backend/builders/__init__.py b/src/yaml_api_generator/generators/backend/builders/__init__.py index 76ba7da9..d8529b7a 100644 --- a/src/yaml_api_generator/generators/backend/builders/__init__.py +++ b/src/yaml_api_generator/generators/backend/builders/__init__.py @@ -76,7 +76,16 @@ def build_context( datastore_types = {n: c["datastore_type"] for n, c in raw_collections.items()} auth = self._auth.build_auth_config( - identity_map, document.collections, document.providers, datastore_types + identity_map, + document.collections, + document.providers, + datastore_types, + requires_principal=any(route.needs_auth for route in routes), + login_paths={ + route.collection: route.relative_path + for route in routes + if route.operation == "login" + }, ) repository_operations = self._repository.build_repository_operations(document.repositories) diff --git a/src/yaml_api_generator/generators/backend/builders/auth_builder.py b/src/yaml_api_generator/generators/backend/builders/auth_builder.py index 9913c39c..62855019 100644 --- a/src/yaml_api_generator/generators/backend/builders/auth_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/auth_builder.py @@ -32,6 +32,8 @@ def build_auth_config( collections: dict[str, IRCollection], providers: dict[str, Any], datastore_types: dict[str, str], + requires_principal: bool, + login_paths: dict[str, str], ) -> AuthConfig: """Build auth configuration descriptor. @@ -40,13 +42,15 @@ def build_auth_config( collections: Collection IR objects providers: Provider configurations datastore_types: Collection name -> resolved datastore type + requires_principal: Whether any route is closed to anonymous callers + login_paths: Identity collection name -> relative path of its declared login route Returns: Auth configuration descriptor """ if not identity_map: return AuthConfig( - enabled=False, + enabled=requires_principal, identity_collections=[], has_password_auth=False, has_api_key_auth=False, @@ -86,6 +90,7 @@ def build_auth_config( roles_field=identity.roles_field, issues_jwt=issues_jwt, datastore_type=datastore_types.get(name, "mongodb"), + declared_login_path=login_paths.get(name), ) ) if identity.has_password: diff --git a/src/yaml_api_generator/generators/backend/builders/route_builder.py b/src/yaml_api_generator/generators/backend/builders/route_builder.py index 41b1911a..fac1ffed 100644 --- a/src/yaml_api_generator/generators/backend/builders/route_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/route_builder.py @@ -80,11 +80,10 @@ def build_routes( dtos: dict[str, IRDto], ) -> list[RouteDescriptor]: routes: list[RouteDescriptor] = [] - has_identity = bool(document.identityCollections) prefix = _base_prefix(document) for ir_route in document.routes: - needs_auth = not ir_route.is_public and has_identity + needs_auth = not ir_route.is_public relative_path = ir_route.path.removeprefix(prefix) or "/" path_params = _PATH_TOKEN.findall(ir_route.path) record_id_param, record_parent_params = self._record_params( diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index 902a93df..82fc9f72 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -288,6 +288,11 @@ def django_path(self) -> str: path = self.relative_path.lstrip("/") return re.sub(r"\{(\w+)\}", r"", path) + @property + def match_precedence(self) -> tuple[bool, ...]: + """Sort key for first-match routers: a literal segment outranks a parameter at the same depth.""" + return tuple(segment.startswith("{") for segment in self.relative_path.strip("/").split("/")) + @dataclass(frozen=True) class FileRouteDescriptor: @@ -317,6 +322,15 @@ class IdentityCollectionDescriptor: issues_jwt: bool = False datastore_type: str = "mongodb" id_format: str = "uuidv7" + declared_login_path: str | None = None + + @property + def login_path(self) -> str: + return self.declared_login_path or f"/{self.name}/login" + + @property + def refresh_path(self) -> str: + return f"{self.login_path.removesuffix('/login')}/refresh" @property def is_sql(self) -> bool: diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index c2618ef3..52869b85 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -6,9 +6,8 @@ from yaml_api_generator.utils.strings import to_pascal_case from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment from ..utils import ( - extract_identity_fields, - find_identity, group_by_collection, + identity_service_context, pagination_config, runtime_module_source, ) @@ -45,6 +44,8 @@ _TEMPLATES_DIR = _BACKEND_GENERATORS / "drf" / "templates" _SHARED_TEMPLATES_DIR = _BACKEND_GENERATORS / "shared" / "templates" class DRFGenerator(TemplateBasedGenerator): + asgi_application = "project.asgi:application" + def __init__(self): super().__init__() self._dir_prefix = "api" @@ -96,6 +97,8 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: ) if context.app.security_headers: (api_dir / "security_headers.py").write_text(runtime_module_source("security_headers")) + if context.app.cors and "*" in context.app.cors.headers: + (api_dir / "cors.py").write_text(env.get_template("cors.py.jinja").render()) self._render_views(env, context, collections_by_name, output_dir) self._render_repositories(env, context, collections_by_name, output_dir) self._render_authentication(env, context, output_dir) @@ -158,6 +161,12 @@ def _render_project_scaffold( template = env.get_template("manage.py.jinja") (output_dir / "manage.py").write_text(template.render()) + commands_dir = output_dir / self.dir_prefix / "management" / "commands" + commands_dir.mkdir(parents=True, exist_ok=True) + (commands_dir.parent / "__init__.py").write_text("") + (commands_dir / "__init__.py").write_text("") + (commands_dir / "runserver.py").write_text(env.get_template("runserver.py.jinja").render()) + def _render_serializers( self, env: Environment, @@ -196,15 +205,10 @@ def _render_services( ) def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: - identity = find_identity(context, col_name) - is_identity = identity is not None - return { "collection_name": col_name, "is_async": True, - "is_identity_collection": is_identity, - "password_via": is_identity and identity.has_password, - "identity_fields": extract_identity_fields(context, col_name), + **identity_service_context(context, col_name), "import_prefix": "api.repositories", "auth_import_prefix": "api.authentication.auth", "service_import_prefix": "api.services", diff --git a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja index bdf82d38..af2c1c48 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja @@ -30,18 +30,17 @@ urlpatterns = [ {% if auth and auth.has_password_auth %} {% for identity in auth.identity_collections %} {% if identity.has_password and identity.identifier %} - path("{{ identity.name }}/login", {{ identity.name | capitalize_collection }}LoginView.as_view(), name="{{ identity.name }}-login"), + path("{{ identity.login_path.lstrip('/') }}", {{ identity.name | capitalize_collection }}LoginView.as_view(), name="{{ identity.name }}-login"), {% endif %} {% endfor %} {% endif %} {% set seen_paths = [] %} -{% for route in routes if route.collection and route.operation in crud_op_names %} -{% if route.django_path not in seen_paths %} +{% for route in routes | selectattr("collection") | rejectattr("operation", "equalto", "login") | sort(attribute="match_precedence") %} +{% if route.operation not in crud_op_names %} + path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{{ route.operation | capitalize_collection }}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), +{% elif route.django_path not in seen_paths %} {% set _ = seen_paths.append(route.django_path) %} path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{% if route.has_id_param %}Detail{% else %}List{% endif %}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), {% endif %} {% endfor %} -{% for route in routes if route.collection and route.operation not in crud_op_names and route.operation != "login" %} - path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{{ route.operation | capitalize_collection }}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), -{% endfor %} ] diff --git a/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja index e061877c..e78249a6 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja @@ -1,7 +1,3 @@ -{% if ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} -import asyncio -{% endif %} - from django.apps import AppConfig @@ -12,7 +8,3 @@ class ApiConfig(AppConfig): def ready(self): from api.authentication.mongodb import connect_db connect_db() -{% if ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} - from api.authentication.mongodb import initialize_indexes - asyncio.run(initialize_indexes({{ ctx.mongo_unique_indexes | tojson }})) -{% endif %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/asgi.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/asgi.py.jinja index 924a8495..251fe26c 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/asgi.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/asgi.py.jinja @@ -1,7 +1,28 @@ +{% set create_mongo_indexes = ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} import os from django.core.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings") -application = get_asgi_application() +django_application = get_asgi_application() + +from api.authentication.mongodb import close_db{% if create_mongo_indexes %}, initialize_indexes{% endif %} # noqa: E402 + + +async def application(scope, receive, send): + """Django's ASGI handler rejects lifespan, so the database lifecycle is served here.""" + if scope["type"] != "lifespan": + await django_application(scope, receive, send) + return + while True: + message = await receive() + if message["type"] == "lifespan.startup": +{% if create_mongo_indexes %} + await initialize_indexes({{ ctx.mongo_unique_indexes | tojson }}) +{% endif %} + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await close_db() + await send({"type": "lifespan.shutdown.complete"}) + return diff --git a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja index 0ab63e2f..da256bea 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja @@ -1,170 +1,4 @@ -import base64 -import hashlib -import hmac -import secrets -from typing import Any - -from rest_framework.authentication import BaseAuthentication -from rest_framework.request import Request - -from api.authentication.mongodb import get_database -{% if auth.uses_jwt %} +{% extends "principal.py.jinja" %} +{% block jwt_import %} from api.authentication.jwt_utils import decode_jwt_token -{% endif %} -from {{ settings_module }} import settings - - -def hash_password(password: str, rounds: int = settings.PASSWORD_HASH_ROUNDS) -> str: - salt = secrets.token_bytes(16) - digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) - salt_b64 = base64.urlsafe_b64encode(salt).decode("ascii") - hash_b64 = base64.urlsafe_b64encode(digest).decode("ascii") - return f"pbkdf2$sha256${rounds}${salt_b64}${hash_b64}" - - -def verify_password(password: str, encoded_hash: str) -> bool: - parts = encoded_hash.split("$") - if len(parts) != 5: - return False - algorithm, digest_name, rounds_text, salt_b64, hash_b64 = parts - if algorithm != "pbkdf2" or digest_name != "sha256": - return False - try: - rounds = int(rounds_text) - salt = base64.urlsafe_b64decode(salt_b64.encode("ascii")) - expected = base64.urlsafe_b64decode(hash_b64.encode("ascii")) - except (ValueError, Exception): - return False - computed = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) - return hmac.compare_digest(computed, expected) - - -def _extract_api_key(headers: dict[str, str]) -> str | None: - api_key = headers.get("x-api-key") - if api_key: - token = api_key.strip() - if token: - return token - authorization = headers.get("authorization", "") - if authorization.startswith("Bearer "): - token = authorization.removeprefix("Bearer ").strip() - if token: - return token - return None - - -def _extract_basic_credentials(headers: dict[str, str]) -> tuple[str, str] | None: - authorization = headers.get("authorization", "") - if not authorization.startswith("Basic "): - return None - encoded = authorization.removeprefix("Basic ").strip() - try: - decoded = base64.b64decode(encoded).decode("utf-8") - except Exception: - return None - if ":" not in decoded: - return None - username, password = decoded.split(":", 1) - if not username or not password: - return None - return username, password - - -def _extract_bearer_token(headers: dict[str, str]) -> str | None: - authorization = headers.get("authorization", "") - if authorization.startswith("Bearer "): - token = authorization.removeprefix("Bearer ").strip() - if token: - return token - return None - - -ROLE_HIERARCHIES: dict[str, dict[str, set[str]]] = {{ auth.role_hierarchies | role_hierarchies_py }} - - -def _extract_roles(doc: dict[str, Any], roles_field: str | None) -> list[str]: - roles = doc.get(roles_field) if roles_field else (doc.get("roles") or doc.get("role")) - if isinstance(roles, list): - return [str(role) for role in roles] - if isinstance(roles, str): - return [roles] - return [] - - -def _principal_has_role(principal: dict | None, required_role: str) -> bool: - if principal is None: - return False - principal_roles = principal.get("roles", []) - if required_role in principal_roles: - return True - entity_hierarchy = ROLE_HIERARCHIES.get(principal.get("identity_collection", ""), {}) - return any(required_role in entity_hierarchy.get(role, set()) for role in principal_roles) - - -def resolve_principal(request: Request, db) -> dict[str, Any] | None: - headers = {k.lower(): v for k, v in request.META.items() if k.startswith("HTTP_")} - raw_headers: dict[str, str] = {} - for meta_key, value in headers.items(): - header_name = meta_key[5:].replace("_", "-").lower() - raw_headers[header_name] = value - -{% if auth.uses_jwt %} - bearer_token = _extract_bearer_token(raw_headers) - if bearer_token is not None: - claims = decode_jwt_token(bearer_token) - if claims is not None: - sub = claims.get("sub") - collection_name = claims.get("identity_collection") - if sub and collection_name: - col = db[collection_name] - for doc in col.find({}): -{% for identity in auth.identity_collections %} -{% if identity.issues_jwt %} - if collection_name == "{{ identity.name }}" and str(doc.get("{{ identity.id_field }}", "")) == sub: - return {"sub": sub, "identity_collection": collection_name, "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} -{% endif %} -{% endfor %} -{% endif %} - -{% for identity in auth.identity_collections %} - - # {{ identity.name }} identity resolution -{% if identity.has_api_key %} - token = _extract_api_key(raw_headers) - if token is not None: - col = db["{{ identity.name }}"] - for doc in col.find({}): - if str(doc.get("apiKey", "")) == token: - subject = doc.get("{{ identity.id_field }}") - if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} -{% endif %} -{% if identity.has_password and identity.identifier %} - credentials = _extract_basic_credentials(raw_headers) - if credentials is not None: - identifier_value, password = credentials - col = db["{{ identity.name }}"] - for doc in col.find({}): - if str(doc.get("{{ identity.identifier }}", "")) != identifier_value: - continue - stored_hash = doc["_auth"]["passwordHash"] - if not isinstance(stored_hash, str): - continue - if not verify_password(password, stored_hash): - continue - subject = doc.get("{{ identity.id_field }}") - if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc, "roles": _extract_roles(doc, {{ identity.roles_field | py_literal }})} -{% endif %} -{% endfor %} - - return None - - -class MongoDBAuthentication(BaseAuthentication): - def authenticate(self, request): - db = get_database() - principal = resolve_principal(request, db) - if principal is None: - return None - return (principal, None) +{% endblock %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/cors.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/cors.py.jinja new file mode 100644 index 00000000..d797282d --- /dev/null +++ b/src/yaml_api_generator/generators/backend/drf/templates/cors.py.jinja @@ -0,0 +1,12 @@ +from corsheaders.middleware import CorsMiddleware + + +class RequestedHeadersCorsMiddleware(CorsMiddleware): + """Allows every header a preflight requests, since a literal wildcard is void on credentialed requests.""" + + def add_response_headers(self, request, response): + response = super().add_response_headers(request, response) + requested = request.headers.get("access-control-request-headers") + if request.method == "OPTIONS" and requested and response.has_header("access-control-allow-origin"): + response["access-control-allow-headers"] = requested + return response diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index 108d3dba..bf2c8d23 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -2,7 +2,7 @@ from importlib import import_module from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from api.conflicts import ConflictError from api.service_inputs import prepare_service_inputs, resolve_access_bind @@ -23,7 +23,7 @@ from api.authentication.mongodb import get_database class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(APIView): async def {{ route.method | lower }}(self, request, **path_params): {% if route.needs_auth %} - principal = resolve_principal(request, get_database()) + principal = await resolve_principal(request.headers, get_database()) if principal is None: return Response( {"detail": "Unauthorized"}, diff --git a/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja index a01c2ca8..441754aa 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja @@ -1,6 +1,6 @@ from drf_spectacular.utils import extend_schema from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from api.authentication.jwt_utils import public_jwks diff --git a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja index a4fa75c2..24bbbcf7 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja @@ -1,9 +1,9 @@ from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from drf_spectacular.utils import extend_schema -from api.authentication.auth import verify_password +from api.authentication.auth import find_by_password from api.authentication.jwt_utils import create_jwt_token from api.authentication.mongodb import get_database @@ -16,42 +16,21 @@ class {{ class_name }}LoginView(APIView): permission_classes = [] @extend_schema(tags=["{{ identity.name | capitalize_collection }}"], auth=[]) - def post(self, request): + async def post(self, request): identifier_value = request.data.get("{{ identity.identifier }}") password = request.data.get("password") - - if not identifier_value or not password: - return Response( - {"detail": "Invalid credentials"}, - status=status.HTTP_401_UNAUTHORIZED, - ) - - db = get_database() - col = db["{{ identity.name }}"] - for doc in col.find({}): - if str(doc.get("{{ identity.identifier }}", "")) != str(identifier_value): - continue - stored_hash = doc["_auth"]["passwordHash"] - if not isinstance(stored_hash, str): - continue - if not verify_password(str(password), stored_hash): - continue - - subject = doc.get("{{ identity.id_field }}") - if subject is None: - continue - - token = create_jwt_token(str(subject), "{{ identity.name }}") - safe_user = { - k: v for k, v in doc.items() - if k not in ("password", "_id", "_auth") - } - return Response({"token": token, "user": safe_user}) - - return Response( - {"detail": "Invalid credentials"}, - status=status.HTTP_401_UNAUTHORIZED, + doc = ( + await find_by_password(get_database(), "{{ identity.name }}", str(identifier_value), str(password)) + if identifier_value and password + else None ) + subject = doc.get("{{ identity.id_field }}") if doc is not None else None + if subject is None: + return Response({"detail": "Invalid credentials"}, status=status.HTTP_401_UNAUTHORIZED) + + token = create_jwt_token(str(subject), "{{ identity.name }}") + safe_user = {k: v for k, v in doc.items() if k not in ("_id", "_auth", "password", "apiKey")} + return Response({"token": token, "user": safe_user}) {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja index e906d67d..0dedf9b1 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja @@ -1,10 +1,11 @@ django djangorestframework +adrf drf-spectacular {% if ctx.app.cors %} django-cors-headers {% endif %} -gunicorn +uvicorn pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} pydantic-settings diff --git a/src/yaml_api_generator/generators/backend/drf/templates/runserver.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/runserver.py.jinja new file mode 100644 index 00000000..ebab593e --- /dev/null +++ b/src/yaml_api_generator/generators/backend/drf/templates/runserver.py.jinja @@ -0,0 +1,13 @@ +import uvicorn +from django.core.management.commands.runserver import Command as DjangoRunserver + +from api.settings import settings + + +class Command(DjangoRunserver): + """Serves project.asgi:application with uvicorn: the async views share a loop-bound database client.""" + + default_port = str(settings.PORT) + + def run(self, **options): + uvicorn.run("project.asgi:application", host=self.addr, port=int(self.port), reload=options["use_reloader"]) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja index 8c302ab3..dc40375e 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja @@ -25,7 +25,9 @@ MIDDLEWARE = [ {% if ctx.app.security_headers %} "api.security_headers.DjangoSecurityHeadersMiddleware", {% endif %} -{% if ctx.app.cors %} +{% if ctx.app.cors and "*" in ctx.app.cors.headers %} + "api.cors.RequestedHeadersCorsMiddleware", +{% elif ctx.app.cors %} "corsheaders.middleware.CorsMiddleware", {% endif %} "django.middleware.common.CommonMiddleware", @@ -37,8 +39,12 @@ CORS_ALLOW_ALL_ORIGINS = True {% else %} CORS_ALLOWED_ORIGINS = {{ ctx.app.cors.origins }} {% endif %} +{% if "*" not in ctx.app.cors.methods %} CORS_ALLOW_METHODS = {{ ctx.app.cors.methods }} +{% endif %} +{% if "*" not in ctx.app.cors.headers %} CORS_ALLOW_HEADERS = {{ ctx.app.cors.headers }} +{% endif %} CORS_ALLOW_CREDENTIALS = {{ ctx.app.cors.credentials }} {% endif %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 3ac87bab..072cd9a5 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -1,6 +1,6 @@ from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from drf_spectacular.utils import extend_schema from api.conflicts import ConflictError {% set crud_op_names = ["list", "get", "create", "update", "replace", "delete"] %} @@ -76,7 +76,7 @@ from api.authentication.auth import resolve_principal, _principal_has_role {% macro pre_fetch_check(route) %} {% if route.needs_auth %} db = get_database() - _resolved_principal = resolve_principal(request, db) + _resolved_principal = await resolve_principal(request.headers, db) if _resolved_principal is None: return Response({"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED) {% if route.access_rules %} @@ -166,7 +166,7 @@ class {{ class_name }}ListView(APIView): serializer.is_valid(raise_exception=True) data = serializer.validated_data document = await service.create(data) - return Response(document, status=status.HTTP_201_CREATED) + return Response(document) {% endif %} except NotFoundError: return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/wsgi.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/wsgi.py.jinja index 72f9d10b..e1b00ddd 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/wsgi.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/wsgi.py.jinja @@ -1,13 +1,8 @@ """ -WSGI config for generated Django project. +WSGI entry point kept for Django tooling only. -Note: This project uses async views for optimal MongoDB performance. -ASGI (asgi.py) is recommended for deployment to maintain full async benefits. -WSGI will work but with reduced concurrency as async views are wrapped to sync. - -For production deployments, use: -- ASGI: uvicorn, daphne, or hypercorn (recommended) -- WSGI: gunicorn or uWSGI (compatibility mode) +The API views are async and share a loop-bound MongoDB client, so serve +project.asgi:application with an ASGI server such as uvicorn. """ import os diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 3f8c04c5..a565bce7 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -7,9 +7,8 @@ from yaml_api_generator.utils.strings import to_pascal_case, to_py_identifier from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment from ..utils import ( - extract_identity_fields, - find_identity, group_by_collection, + identity_service_context, pagination_config, runtime_module_source, write_environment_files, @@ -185,48 +184,18 @@ def _render_services( (output_dir / "services" / "_ops.py").write_text(ops_content) def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: - identity = find_identity(context, col_name) - is_identity = identity is not None - return { "collection_name": col_name, "operations": _crud_operations(col_data), - "is_identity_collection": is_identity, - "password_via": is_identity and identity.has_password, - "default_role": identity.roles.default if is_identity and identity.roles else None, - "identity_fields": extract_identity_fields(context, col_name), + **identity_service_context(context, col_name), } - @staticmethod - def _service_group(route_service: str | None, services: dict) -> str | None: - """Resolve the repository/service-group name backing a route's service. - - For expanded flat keys like 'list-posts', the IRService carries the - repository name directly. Falling back to the part after the first dash - handles edge cases where the service isn't in the index. - """ - if route_service is None: - return None - ir_svc = services.get(route_service) - if ir_svc is not None and ir_svc.repository: - return ir_svc.repository - # Fallback: strip the operation prefix (e.g. "list-posts" → "posts") - if "-" in route_service: - return route_service.split("-", 1)[1] - return route_service - def _build_routes_context( self, col_name: str, col_data: dict, ctx: CompileContext ) -> dict: - service_groups = list(dict.fromkeys( - g - for route in col_data["routes"] - if (g := self._service_group(route.service, ctx.services)) is not None - )) return { "ctx": ctx, "collection_name": col_name, - "service_groups": service_groups, "routes": col_data["routes"], "model_index": {m.name: m for m in col_data["models"]}, "repo_queries": { diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja index c36bac15..70536488 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/auth.py.jinja @@ -52,6 +52,11 @@ def verify_password(password: str, encoded_hash: str) -> bool: return hmac.compare_digest(computed, expected) +def hash_api_key(api_key: str) -> str: + """API keys are high-entropy, so an unsalted digest keeps them both secret and indexable.""" + return hashlib.sha256(api_key.encode("utf-8")).hexdigest() + + def _extract_api_key(headers: dict[str, str]) -> str | None: api_key = headers.get("x-api-key") if api_key: @@ -270,7 +275,7 @@ async def resolve_principal( {% if identity.has_api_key %} token = _extract_api_key(headers) if token is not None: - doc = await repositories["{{ identity.name }}"].find_by("apiKey", token) + doc = await repositories["{{ identity.name }}"].find_by("apiKey", hash_api_key(token)) if doc is not None: subject = doc.get("{{ identity.id_field }}") if subject is not None: diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja index a370a2c9..cabd691d 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/login_routes.py.jinja @@ -42,7 +42,7 @@ class {{ class_name }}LoginPayload(BaseModel): password: str = Field(..., min_length=1) -@router.post("/{{ identity.name }}/login", tags=["{{ identity.name | capitalize_collection }}"]) +@router.post("{{ identity.login_path }}", tags=["{{ identity.name | capitalize_collection }}"]) async def login_{{ identity.name }}(payload: {{ class_name }}LoginPayload{% if ctx.session.uses_cookie %}, request: Request{% endif %}, repository=Depends(get_{{ identity.name }}_repository)) -> Any: doc = await repository.find_by( "{{ pw_binding.identifier }}", payload.{{ pw_binding.identifier }} @@ -72,7 +72,7 @@ async def login_{{ identity.name }}(payload: {{ class_name }}LoginPayload{% if c {% if ctx.session.refresh %} {% if ctx.session.uses_cookie %} -@router.post("/{{ identity.name }}/refresh", tags=["{{ identity.name | capitalize_collection }}"]) +@router.post("{{ identity.refresh_path }}", tags=["{{ identity.name | capitalize_collection }}"]) async def refresh_{{ identity.name }}(request: Request) -> Any: presented = refresh_token_from_request(request) if presented is None: @@ -86,7 +86,7 @@ async def refresh_{{ identity.name }}(request: Request) -> Any: return response {% else %} -@router.post("/{{ identity.name }}/refresh", tags=["{{ identity.name | capitalize_collection }}"]) +@router.post("{{ identity.refresh_path }}", tags=["{{ identity.name | capitalize_collection }}"]) async def refresh_{{ identity.name }}(payload: RefreshPayload) -> Any: rotated = await rotate_session(payload.refresh_token) if rotated is None: diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 0365f2c8..3062a4e7 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -42,9 +42,9 @@ from dependencies.file_stores import record_files from fastapi.encoders import jsonable_encoder from dependencies.bindings import resolve_filter {% endif %} -{% for group in service_groups %} -from dependencies.services import get_{{ group }}_service -{% endfor %} +{% if crud_routes or file_routes %} +from dependencies.services import get_{{ collection_name }}_service +{% endif %} {% if repo_op_routes %} from dependencies.repositories import get_{{ collection_name }}_repository {% endif %} @@ -98,7 +98,7 @@ _files = record_files[{{ collection_name | tojson }}] {% macro openapi_extra(route, request_body=none) %}{% set security = route.needs_auth and security_schemes %}{% if security or request_body %}, openapi_extra={{ "{" }}{% if security %}"security": {{ security_schemes }}{% endif %}{% if security and request_body %}, {% endif %}{% if request_body %}"requestBody": {{ request_body }}{% endif %}{{ "}" }}{% endif %}{% endmacro %} {% macro auth_param(route) %}{% if auth.enabled and route.needs_auth %}, principal: dict[str, Any] = Depends(require_principal){% endif %}{% endmacro %} -{% macro svc_dep() %}svc=Depends(get_{{ service_groups | first }}_service){% endmacro %} +{% macro svc_dep() %}svc=Depends(get_{{ collection_name }}_service){% endmacro %} {% macro extra_path_params(route) %}{% for name in route.path_params if name != route.record_id_param %}, {{ name }}: str{% endfor %}{% endmacro %} {% macro record_criteria(route) %}{{ "{" }}{% for field, param in route.record_criteria %}"{{ field }}": {{ param }}{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} {% macro path_values(route) %}{{ "{" }}{% for name in route.path_params %}"{{ name }}": {{ name }}{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja index 51a7827c..5e1746ad 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja @@ -7,6 +7,9 @@ {% if password_via %} {% set _ = identity_mixins.append("PasswordMixin") %} {% endif %} +{% if api_key_via %} +{% set _ = identity_mixins.append("ApiKeyMixin") %} +{% endif %} from services._ops import ServiceBase{% for op in svc_mixins %}, {{ op }}Svc{% endfor %}{% for mixin in identity_mixins %}, {{ mixin }}{% endfor %} @@ -14,9 +17,7 @@ from services._ops import ServiceBase{% for op in svc_mixins %}, {{ op }}Svc{% e class {{ class_name }}Service({% for op in svc_mixins %}{{ op }}Svc, {% endfor %}{% for mixin in identity_mixins %}{{ mixin }}, {% endfor %}ServiceBase): {% if is_identity_collection %} IDENTITY_FIELDS = ({% for field_name in identity_fields %}"{{ field_name }}", {% endfor %}) -{% if password_via %} DEFAULT_ROLE = {{ default_role | py_literal }} -{% endif %} {% else %} pass {% endif %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja index 5af17ece..524ca41e 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service_ops.py.jinja @@ -1,8 +1,8 @@ from typing import Any from services.exceptions import NotFoundError, ValidationError -{% if auth.has_password_auth %} -from dependencies.auth import hash_password +{% if auth.has_password_auth or auth.has_api_key_auth %} +from dependencies.auth import {{ ([auth.has_password_auth and "hash_password", auth.has_api_key_auth and "hash_api_key"] | select | join(", ")) }} {% endif %} @@ -17,8 +17,8 @@ class ServiceBase: @staticmethod def _strip_internal_fields(document: dict) -> None: - document.pop("_auth", None) - document.pop("password", None) + for field_name in ("_auth", "password", "apiKey"): + document.pop(field_name, None) async def _before_create(self, data: dict) -> None: """Hook invoked before create; override to validate or enrich data.""" @@ -151,29 +151,37 @@ class DeleteSvc: class IdentityFieldsMixin: IDENTITY_FIELDS: tuple[str, ...] = () + DEFAULT_ROLE: str | None = None async def _before_create(self, data: dict) -> None: for field_name in self.IDENTITY_FIELDS: value = data.get(field_name) if not isinstance(value, str) or not value.strip(): raise ValidationError(field_name, f"Missing required field: {field_name}") + if self.DEFAULT_ROLE is not None: + data["_auth"] = {**data.get("_auth", {}), "roles": self.DEFAULT_ROLE} {% if auth.has_password_auth %} class PasswordMixin: - DEFAULT_ROLE: str | None = None - async def _before_write(self, data: dict) -> None: raw_password = data.pop("password", None) - if raw_password is None: - return - if not isinstance(raw_password, str) or not raw_password.strip(): - raise ValidationError("password", "Password must be a non-empty string") - auth_data = data.get("_auth", {}) - if not isinstance(auth_data, dict): - auth_data = {} - auth_data["passwordHash"] = hash_password(raw_password) - if self.DEFAULT_ROLE is not None: - auth_data.setdefault("roles", self.DEFAULT_ROLE) - data["_auth"] = auth_data + if raw_password is not None: + if not isinstance(raw_password, str) or not raw_password.strip(): + raise ValidationError("password", "Password must be a non-empty string") + auth_data = data.get("_auth", {}) + if not isinstance(auth_data, dict): + auth_data = {} + auth_data["passwordHash"] = hash_password(raw_password) + data["_auth"] = auth_data + await super()._before_write(data) +{% endif %} +{% if auth.has_api_key_auth %} + + +class ApiKeyMixin: + async def _before_write(self, data: dict) -> None: + if "apiKey" in data: + data["apiKey"] = hash_api_key(data["apiKey"]) + await super()._before_write(data) {% endif %} diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 46ac0a94..ecf27161 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -8,9 +8,8 @@ from yaml_api_generator.utils.strings import to_pascal_case from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment from ..utils import ( - extract_identity_fields, - find_identity, group_by_collection, + identity_service_context, pagination_config, runtime_module_source, write_environment_files, @@ -180,15 +179,10 @@ def _render_services( ) def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: - identity = find_identity(context, col_name) - is_identity = identity is not None - return { "collection_name": col_name, "is_async": True, - "is_identity_collection": is_identity, - "password_via": is_identity and identity.has_password, - "identity_fields": extract_identity_fields(context, col_name), + **identity_service_context(context, col_name), "import_prefix": "repositories", "auth_import_prefix": "dependencies.auth", "service_import_prefix": "services", diff --git a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja index 53790b78..ebaf2c58 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja @@ -50,7 +50,7 @@ ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + {% endfor %} {% for identity in ctx.auth.identity_collections %} {% if identity.has_password and identity.identifier %} -ROUTES.setdefault("POST", []).append((re.compile(r"^" + _PREFIX + r"/{{ identity.name }}/login$"), handle_login_{{ identity.name }})) +ROUTES.setdefault("POST", []).append((re.compile(r"^" + _PREFIX + r"{{ identity.login_path | path_to_regex }}$"), handle_login_{{ identity.name }})) {% endif %} {% endfor %} {% if ctx.session.uses_asymmetric %} @@ -99,7 +99,7 @@ class ASGIApp: k.decode("latin-1"): v.decode("latin-1") for k, v in scope.get("headers", []) } - parsed_query = parse_qs(scope.get("query_string", b"").decode("utf-8")) + parsed_query = parse_qs(scope.get("query_string", b"").decode("utf-8"), keep_blank_values=True) query = { key: values[0] if len(values) == 1 else values for key, values in parsed_query.items() diff --git a/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja index bfd986da..46d5eefa 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/auth.py.jinja @@ -1,135 +1,4 @@ -import base64 -import hashlib -import hmac -import secrets -from typing import Any - -from settings import settings -{% if auth.uses_jwt %} +{% extends "principal.py.jinja" %} +{% block jwt_import %} from dependencies.jwt_utils import decode_jwt_token -{% endif %} - - -def hash_password(password: str, rounds: int = settings.PASSWORD_HASH_ROUNDS) -> str: - salt = secrets.token_bytes(16) - digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) - salt_b64 = base64.urlsafe_b64encode(salt).decode("ascii") - hash_b64 = base64.urlsafe_b64encode(digest).decode("ascii") - return f"pbkdf2$sha256${rounds}${salt_b64}${hash_b64}" - - -def verify_password(password: str, encoded_hash: str) -> bool: - parts = encoded_hash.split("$") - if len(parts) != 5: - return False - algorithm, digest_name, rounds_text, salt_b64, hash_b64 = parts - if algorithm != "pbkdf2" or digest_name != "sha256": - return False - try: - rounds = int(rounds_text) - salt = base64.urlsafe_b64decode(salt_b64.encode("ascii")) - expected = base64.urlsafe_b64decode(hash_b64.encode("ascii")) - except (ValueError, Exception): - return False - computed = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) - return hmac.compare_digest(computed, expected) - - -def _extract_api_key(headers: dict[str, str]) -> str | None: - api_key = headers.get("x-api-key") - if api_key: - token = api_key.strip() - if token: - return token - authorization = headers.get("authorization", "") - if authorization.startswith("Bearer "): - token = authorization.removeprefix("Bearer ").strip() - if token: - return token - return None - - -def _extract_basic_credentials(headers: dict[str, str]) -> tuple[str, str] | None: - authorization = headers.get("authorization", "") - if not authorization.startswith("Basic "): - return None - encoded = authorization.removeprefix("Basic ").strip() - try: - decoded = base64.b64decode(encoded).decode("utf-8") - except Exception: - return None - if ":" not in decoded: - return None - username, password = decoded.split(":", 1) - if not username or not password: - return None - return username, password - - -def _extract_bearer_token(headers: dict[str, str]) -> str | None: - authorization = headers.get("authorization", "") - if authorization.startswith("Bearer "): - token = authorization.removeprefix("Bearer ").strip() - if token: - return token - return None - - -async def resolve_principal(headers: dict[str, str], db) -> dict[str, Any] | None: -{% if auth.uses_jwt %} - bearer_token = _extract_bearer_token(headers) - if bearer_token is not None: - claims = decode_jwt_token(bearer_token) - if claims is not None: - sub = claims.get("sub") - collection_name = claims.get("identity_collection") - if sub and collection_name: - col = db[collection_name] - cursor = col.find({}) - docs = await cursor.to_list(length=None) - for doc in docs: -{% for identity in auth.identity_collections %} -{% if identity.issues_jwt %} - if collection_name == "{{ identity.name }}" and str(doc.get("{{ identity.id_field }}", "")) == sub: - return {"sub": sub, "identity_collection": collection_name, "record": doc} -{% endif %} -{% endfor %} -{% endif %} - -{% for identity in auth.identity_collections %} - - # {{ identity.name }} identity resolution -{% if identity.has_api_key %} - token = _extract_api_key(headers) - if token is not None: - col = db["{{ identity.name }}"] - cursor = col.find({}) - docs = await cursor.to_list(length=None) - for doc in docs: - if str(doc.get("apiKey", "")) == token: - subject = doc.get("{{ identity.id_field }}") - if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc} -{% endif %} -{% if identity.has_password and identity.identifier %} - credentials = _extract_basic_credentials(headers) - if credentials is not None: - identifier_value, password = credentials - col = db["{{ identity.name }}"] - cursor = col.find({}) - docs = await cursor.to_list(length=None) - for doc in docs: - if str(doc.get("{{ identity.identifier }}", "")) != identifier_value: - continue - stored_hash = doc["_auth"]["passwordHash"] - if not isinstance(stored_hash, str): - continue - if not verify_password(password, stored_hash): - continue - subject = doc.get("{{ identity.id_field }}") - if subject is not None: - return {"sub": str(subject), "identity_collection": "{{ identity.name }}", "record": doc} -{% endif %} -{% endfor %} - - return None +{% endblock %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja index f535760b..aad00568 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja @@ -1,6 +1,6 @@ from typing import Any -from dependencies.auth import verify_password +from dependencies.auth import find_by_password from dependencies.jwt_utils import create_jwt_token import validation @@ -15,31 +15,14 @@ async def handle_login_{{ identity.name }}(url_params: dict, body: dict | None, except ValueError as exc: return 422, {"detail": str(exc)} - col = db["{{ identity.name }}"] - cursor = col.find({}) - docs = await cursor.to_list(length=None) + doc = await find_by_password(db, "{{ identity.name }}", data[{{ identity.identifier | tojson }}], data["password"]) + subject = doc.get("{{ identity.id_field }}") if doc is not None else None + if subject is None: + return 401, {"detail": "Invalid credentials"} - for doc in docs: - if str(doc.get("{{ identity.identifier }}", "")) != data[{{ identity.identifier | tojson }}]: - continue - stored_hash = doc["_auth"]["passwordHash"] - if not isinstance(stored_hash, str): - continue - if not verify_password(data["password"], stored_hash): - continue - - subject = doc.get("{{ identity.id_field }}") - if subject is None: - continue - - token = create_jwt_token(str(subject), "{{ identity.name }}") - safe_user = { - k: v for k, v in doc.items() - if k not in ("password", "_id", "_auth") - } - return 200, {"token": token, "user": safe_user} - - return 401, {"detail": "Invalid credentials"} + token = create_jwt_token(str(subject), "{{ identity.name }}") + safe_user = {k: v for k, v in doc.items() if k not in ("_id", "_auth", "password", "apiKey")} + return 200, {"token": token, "user": safe_user} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/principal.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/principal.py.jinja new file mode 100644 index 00000000..7159cce5 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared/templates/principal.py.jinja @@ -0,0 +1,169 @@ +import base64 +import hashlib +import hmac +import secrets +from collections.abc import Mapping +from typing import Any + +from {{ settings_module }} import settings +{% if auth.uses_jwt %} +{% block jwt_import %}{% endblock %} +{% endif %} +{% set api_key_identities = auth.identity_collections | selectattr("has_api_key") | map(attribute="name") | list %} +{% set password_identifiers = {} %} +{% for identity in auth.identity_collections if identity.has_password and identity.identifier %} +{% set _ = password_identifiers.update({identity.name: identity.identifier}) %} +{% endfor %} +{% set jwt_issuers = auth.identity_collections | selectattr("issues_jwt") | map(attribute="name") | list %} + + +def hash_password(password: str, rounds: int = settings.PASSWORD_HASH_ROUNDS) -> str: + salt = secrets.token_bytes(16) + digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) + salt_b64 = base64.urlsafe_b64encode(salt).decode("ascii") + hash_b64 = base64.urlsafe_b64encode(digest).decode("ascii") + return f"pbkdf2$sha256${rounds}${salt_b64}${hash_b64}" + + +def verify_password(password: str, encoded_hash: str) -> bool: + parts = encoded_hash.split("$") + if len(parts) != 5: + return False + algorithm, digest_name, rounds_text, salt_b64, hash_b64 = parts + if algorithm != "pbkdf2" or digest_name != "sha256": + return False + try: + rounds = int(rounds_text) + salt = base64.urlsafe_b64decode(salt_b64.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_b64.encode("ascii")) + except (ValueError, Exception): + return False + computed = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, rounds) + return hmac.compare_digest(computed, expected) + + +def hash_api_key(api_key: str) -> str: + """API keys are high-entropy, so an unsalted digest keeps them both secret and indexable.""" + return hashlib.sha256(api_key.encode("utf-8")).hexdigest() + + +def _extract_api_key(headers: Mapping[str, str]) -> str | None: + api_key = headers.get("x-api-key") + if api_key: + token = api_key.strip() + if token: + return token + return _extract_bearer_token(headers) + + +def _extract_basic_credentials(headers: Mapping[str, str]) -> tuple[str, str] | None: + authorization = headers.get("authorization", "") + if not authorization.startswith("Basic "): + return None + encoded = authorization.removeprefix("Basic ").strip() + try: + decoded = base64.b64decode(encoded).decode("utf-8") + except Exception: + return None + if ":" not in decoded: + return None + username, password = decoded.split(":", 1) + if not username or not password: + return None + return username, password + + +def _extract_bearer_token(headers: Mapping[str, str]) -> str | None: + authorization = headers.get("authorization", "") + if authorization.startswith("Bearer "): + token = authorization.removeprefix("Bearer ").strip() + if token: + return token + return None + + +ROLE_HIERARCHIES: dict[str, dict[str, set[str]]] = {{ auth.role_hierarchies | role_hierarchies_py }} + +# identity collection -> (id field, declared roles field, roles managed under _auth) +_IDENTITIES: dict[str, tuple[str, str | None, bool]] = { +{% for identity in auth.identity_collections %} + "{{ identity.name }}": ({{ identity.id_field | py_literal }}, {{ identity.roles_field | py_literal }}, {{ (identity.roles is not none) | py_literal }}), +{% endfor %} +} + + +def _extract_roles(doc: dict[str, Any], roles_field: str | None, managed_roles: bool) -> list[str]: + roles = doc.get(roles_field) if roles_field else (doc.get("roles") or doc.get("role")) + if roles is None and managed_roles: + auth_data = doc.get("_auth") + roles = auth_data.get("roles") if isinstance(auth_data, dict) else None + if isinstance(roles, list): + return [str(role) for role in roles] + if isinstance(roles, str): + return [roles] + return [] + + +def _principal_has_role(principal: dict | None, required_role: str) -> bool: + if principal is None: + return False + principal_roles = principal.get("roles", []) + if required_role in principal_roles: + return True + entity_hierarchy = ROLE_HIERARCHIES.get(principal.get("identity_collection", ""), {}) + return any(required_role in entity_hierarchy.get(role, set()) for role in principal_roles) + + +def _principal(identity_collection: str, doc: dict[str, Any] | None) -> dict[str, Any] | None: + if doc is None: + return None + id_field, roles_field, managed_roles = _IDENTITIES[identity_collection] + subject = doc.get(id_field) + if subject is None: + return None + return { + "sub": str(subject), + "identity_collection": identity_collection, + "record": doc, + "roles": _extract_roles(doc, roles_field, managed_roles), + } + + +_PASSWORD_IDENTIFIERS: dict[str, str] = {{ password_identifiers | py_literal }} + + +async def find_by_password(db, identity_collection: str, identifier_value: str, password: str) -> dict[str, Any] | None: + doc = await db[identity_collection].find_one({_PASSWORD_IDENTIFIERS[identity_collection]: identifier_value}) + auth_data = doc.get("_auth") if doc is not None else None + stored_hash = auth_data.get("passwordHash") if isinstance(auth_data, dict) else None + return doc if isinstance(stored_hash, str) and verify_password(password, stored_hash) else None + + +async def resolve_principal(headers: Mapping[str, str], db) -> dict[str, Any] | None: + headers = {name.lower(): value for name, value in headers.items()} +{% if jwt_issuers %} + bearer_token = _extract_bearer_token(headers) + claims = decode_jwt_token(bearer_token) if bearer_token is not None else None + if claims is not None and claims.get("sub") and claims.get("identity_collection") in {{ jwt_issuers | py_literal }}: + collection = claims["identity_collection"] + principal = _principal(collection, await db[collection].find_one({_IDENTITIES[collection][0]: claims["sub"]})) + if principal is not None: + return principal +{% endif %} +{% if api_key_identities %} + api_key = _extract_api_key(headers) + if api_key is not None: + for collection in {{ api_key_identities | py_literal }}: + principal = _principal(collection, await db[collection].find_one({"apiKey": hash_api_key(api_key)})) + if principal is not None: + return principal +{% endif %} +{% if password_identifiers %} + credentials = _extract_basic_credentials(headers) + if credentials is not None: + for collection in _PASSWORD_IDENTIFIERS: + principal = _principal(collection, await find_by_password(db, collection, *credentials)) + if principal is not None: + return principal +{% endif %} + return None diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index c784ee7d..d1c80486 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -76,6 +76,9 @@ def _translate_conflict(operation): {% set class_name = collection_name | capitalize_collection %} +_HIDDEN_FIELDS = frozenset({"_id", "_auth", "password", "apiKey"}) + + class {{ class_name }}Repository: COLLECTION = "{{ collection_name }}" ID_FIELD = "{{ id_field }}" @@ -88,10 +91,10 @@ class {{ class_name }}Repository: return {self.ID_FIELD: document_id} def _normalize(self, doc: dict[str, Any] | None) -> dict[str, Any] | None: - """Hide the datastore's own identifier; the service layer hides domain-internal fields.""" + """Hide the datastore's own identifier and every stored credential.""" if doc is None: return None - return {k: v for k, v in doc.items() if k != "_id"} + return {k: v for k, v in doc.items() if k not in _HIDDEN_FIELDS} async def list(self) -> list[dict[str, Any]]: cursor = self._col.find({}) diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja index b49d0ea4..a3094ef1 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja @@ -1,11 +1,16 @@ from {{ import_prefix }}.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository from {{ service_import_prefix }}.exceptions import NotFoundError, ValidationError -{% if is_identity_collection %} -from {{ auth_import_prefix }} import hash_password +{% set protects_credentials = password_via or api_key_via %} +{% if protects_credentials %} +from {{ auth_import_prefix }} import {{ ([password_via and "hash_password", api_key_via and "hash_api_key"] | select | join(", ")) }} {% endif %} class {{ collection_name | capitalize_collection }}Service: +{% if is_identity_collection %} + IDENTITY_FIELDS = ({% for field_name in identity_fields %}"{{ field_name }}", {% endfor %}) + +{% endif %} def __init__(self, repo: {{ collection_name | capitalize_collection }}Repository): self._repo = repo @@ -31,14 +36,17 @@ class {{ collection_name | capitalize_collection }}Service: {% if is_identity_collection %} self._validate_identity_fields(data) {% endif %} -{% if password_via %} - self._process_password(data) +{% if default_role is not none %} + data["_auth"] = {**data.get("_auth", {}), "roles": {{ default_role | py_literal }}} +{% endif %} +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} return await self._repo.create(data) async def update(self, resource_id: str, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = await self._repo.update(resource_id, data) if document is None: @@ -46,8 +54,8 @@ class {{ collection_name | capitalize_collection }}Service: return document async def update_by(self, criteria: dict, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = await self._repo.update_by(criteria, data) if document is None: @@ -55,8 +63,8 @@ class {{ collection_name | capitalize_collection }}Service: return document async def replace(self, resource_id: str, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = await self._repo.replace(resource_id, data) if document is None: @@ -64,8 +72,8 @@ class {{ collection_name | capitalize_collection }}Service: return document async def replace_by(self, criteria: dict, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = await self._repo.replace_by(criteria, data) if document is None: @@ -99,14 +107,17 @@ class {{ collection_name | capitalize_collection }}Service: {% if is_identity_collection %} self._validate_identity_fields(data) {% endif %} -{% if password_via %} - self._process_password(data) +{% if default_role is not none %} + data["_auth"] = {**data.get("_auth", {}), "roles": {{ default_role | py_literal }}} +{% endif %} +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} return self._repo.create(data) def update(self, resource_id: str, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = self._repo.update(resource_id, data) if document is None: @@ -114,8 +125,8 @@ class {{ collection_name | capitalize_collection }}Service: return document def replace(self, resource_id: str, data: dict) -> dict: -{% if password_via %} - self._process_password(data) +{% if protects_credentials %} + self._protect_credentials(data) {% endif %} document = self._repo.replace(resource_id, data) if document is None: @@ -128,10 +139,11 @@ class {{ collection_name | capitalize_collection }}Service: raise NotFoundError("{{ collection_name }}", resource_id) return document {% endif %} -{% if password_via %} +{% if protects_credentials %} @staticmethod - def _process_password(data: dict) -> None: + def _protect_credentials(data: dict) -> None: +{% if password_via %} raw_password = data.pop("password", None) if raw_password is not None: if not isinstance(raw_password, str) or not raw_password.strip(): @@ -142,12 +154,16 @@ class {{ collection_name | capitalize_collection }}Service: auth_data["passwordHash"] = hash_password(raw_password) data["_auth"] = auth_data {% endif %} +{% if api_key_via %} + if "apiKey" in data: + data["apiKey"] = hash_api_key(data["apiKey"]) +{% endif %} +{% endif %} {% if is_identity_collection %} - @staticmethod - def _validate_identity_fields(data: dict) -> None: -{% for field_name in identity_fields %} - if not isinstance(data.get("{{ field_name }}"), str) or not data.get("{{ field_name }}", "").strip(): - raise ValidationError("{{ field_name }}", "Missing required field: {{ field_name }}") -{% endfor %} + def _validate_identity_fields(self, data: dict) -> None: + for field_name in self.IDENTITY_FIELDS: + value = data.get(field_name) + if not isinstance(value, str) or not value.strip(): + raise ValidationError(field_name, f"Missing required field: {field_name}") {% endif %} diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 8c93fbe6..6b341d3c 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -106,3 +106,15 @@ def extract_identity_fields(context: CompileContext, col_name: str) -> list[str] seen.add(binding.identifier) return identity_fields + + +def identity_service_context(context: CompileContext, col_name: str) -> dict: + """What a collection's service needs to know about the identity it may back.""" + identity = find_identity(context, col_name) + return { + "is_identity_collection": identity is not None, + "password_via": identity is not None and identity.has_password, + "api_key_via": identity is not None and identity.has_api_key, + "default_role": identity.roles.default if identity is not None and identity.roles else None, + "identity_fields": extract_identity_fields(context, col_name), + } diff --git a/src/yaml_api_generator/generators/deploy/context.py b/src/yaml_api_generator/generators/deploy/context.py index 96d6d53b..b38beb12 100644 --- a/src/yaml_api_generator/generators/deploy/context.py +++ b/src/yaml_api_generator/generators/deploy/context.py @@ -25,8 +25,7 @@ class DeployContext: name: str port: int runtime: str - mongo_local: bool - mongo_db: str + application: str env_vars: dict[str, str] has_auth: bool has_frontend: bool @@ -35,6 +34,14 @@ class DeployContext: ssh_key: str datastores: tuple[DeployDatastore, ...] = () + @property + def local_datastores(self) -> tuple[DeployDatastore, ...]: + return tuple(ds for ds in self.datastores if ds.local) + + @property + def local_mongo(self) -> bool: + return any(ds.type == "mongodb" for ds in self.local_datastores) + @dataclass(frozen=True) class DockerDeployContext(DeployContext): diff --git a/src/yaml_api_generator/generators/deploy/context_builder.py b/src/yaml_api_generator/generators/deploy/context_builder.py index bd651b7d..5527679b 100644 --- a/src/yaml_api_generator/generators/deploy/context_builder.py +++ b/src/yaml_api_generator/generators/deploy/context_builder.py @@ -11,6 +11,7 @@ def build_deploy_context( ir: IRDocument, + application: str, host: str, ssh_user: str, ssh_key: str, @@ -20,7 +21,6 @@ def build_deploy_context( deploy = ir.deploy name = deploy.name or "app" - mongo_db = ir.mongo.database context = build_context(ir) secret_fields = [f for f in build_settings_fields(context) if f.secret] file_deployments = build_file_deployments(context.file_stores) @@ -34,8 +34,7 @@ def build_deploy_context( name=name, port=deploy.port, runtime=deploy.runtime, - mongo_local=deploy.mongo_local, - mongo_db=mongo_db, + application=application, env_vars=env_vars, has_auth=bool(ir.identityCollections), has_frontend=ir.has_ui, @@ -80,12 +79,10 @@ def build_deploy_context( def _build_datastores(ir: IRDocument) -> tuple[DeployDatastore, ...]: base_names = {"postgres": "postgres", "mongodb": "mongodb", "mongo": "mongodb"} type_counts = Counter(base_names.get(ds.type, ds.type) for ds in ir.datastores.values()) - seen: Counter = Counter() result: list[DeployDatastore] = [] for name, ds in ir.datastores.items(): base = base_names.get(ds.type, ds.type) - seen[base] += 1 service = base if type_counts[base] == 1 else f"{base}-{name}" result.append( _postgres_datastore(service, ds) if isinstance(ds, IRPostgres) @@ -105,20 +102,21 @@ def _postgres_datastore(service: str, ds: IRPostgres) -> DeployDatastore: type="postgres", image="postgres:16", local=local, - app_env={"DATABASE_URL": f"postgresql+asyncpg://{user}:{password}@{service}:5432/{db}"}, + app_env={"DATABASE_URL": f"postgresql+asyncpg://{user}:{password}@{service}:5432/{db}" if local else ds.dsn}, service_env={"POSTGRES_DB": db, "POSTGRES_USER": user, "POSTGRES_PASSWORD": password}, volume=f"{service}_data", volume_path="/var/lib/postgresql/data", ) -def _mongo_datastore(service: str, ds: IRMongo, local: bool) -> DeployDatastore: +def _mongo_datastore(service: str, ds: IRMongo, declared_local: bool | None) -> DeployDatastore: + local = ds.is_local if declared_local is None else declared_local return DeployDatastore( service=service, type="mongodb", image="mongo:8", local=local, - app_env={"MONGO_URI": f"mongodb://{service}:27017", "MONGO_DB": ds.database}, + app_env={"MONGO_URI": f"mongodb://{service}:27017" if local else ds.uri, "MONGO_DB": ds.database}, service_env={}, volume=f"{service}_data", volume_path="/data/db", diff --git a/src/yaml_api_generator/generators/deploy/docker/generator.py b/src/yaml_api_generator/generators/deploy/docker/generator.py index 7021eeec..164e1ec3 100644 --- a/src/yaml_api_generator/generators/deploy/docker/generator.py +++ b/src/yaml_api_generator/generators/deploy/docker/generator.py @@ -52,10 +52,9 @@ def generate(self, ctx: DeployContext, output_dir: Path, spec_dir: Path) -> None roles_app_templates.mkdir(parents=True, exist_ok=True) self._render(env, "roles/app/templates/env.j2.jinja", ctx, roles_app_templates / "env.j2") - if ctx.mongo_local: - roles_mongodb_tasks = output_dir / "roles" / "mongodb" / "tasks" - roles_mongodb_tasks.mkdir(parents=True, exist_ok=True) - self._render(env, "roles/mongodb/tasks/main.yml.jinja", ctx, roles_mongodb_tasks / "main.yml") + roles_docker_tasks = output_dir / "roles" / "docker" / "tasks" + roles_docker_tasks.mkdir(parents=True, exist_ok=True) + self._render(env, "roles/docker/tasks/main.yml.jinja", ctx, roles_docker_tasks / "main.yml") def _stage_build_contexts( self, diff --git a/src/yaml_api_generator/generators/deploy/docker/templates/Dockerfile.jinja b/src/yaml_api_generator/generators/deploy/docker/templates/Dockerfile.jinja index 247c0e3e..75ac93f9 100644 --- a/src/yaml_api_generator/generators/deploy/docker/templates/Dockerfile.jinja +++ b/src/yaml_api_generator/generators/deploy/docker/templates/Dockerfile.jinja @@ -3,5 +3,6 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . +ENV PORT={{ ctx.port }} EXPOSE {{ ctx.port }} -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "{{ ctx.port }}"] \ No newline at end of file +CMD exec uvicorn {{ ctx.application }} --host 0.0.0.0 --port "$PORT" diff --git a/src/yaml_api_generator/generators/deploy/docker/templates/README.md.jinja b/src/yaml_api_generator/generators/deploy/docker/templates/README.md.jinja index e838c96a..ab1ca635 100644 --- a/src/yaml_api_generator/generators/deploy/docker/templates/README.md.jinja +++ b/src/yaml_api_generator/generators/deploy/docker/templates/README.md.jinja @@ -31,10 +31,11 @@ Edit the `.env` file with your configuration: {% for name, value in ctx.env_vars.items() %} {{ name }}={{ value or "your_value_here" }} {% endfor %} -{% if ctx.mongo_local %} -MONGO_URI=mongodb://mongodb:27017 -MONGO_DB={{ ctx.mongo_db }} -{% endif %} +{% for ds in ctx.datastores %} +{% for name, value in ds.app_env.items() %} +{{ name }}={{ value }} +{% endfor %} +{% endfor %} ``` ### {% if ctx.has_frontend %}3{% else %}2{% endif %}. Start Services @@ -48,9 +49,9 @@ This will start: {% if ctx.has_frontend %} - Frontend on port 3000 {% endif %} -{% if ctx.mongo_local %} -- MongoDB on internal network (not exposed externally) -{% endif %} +{% for ds in ctx.local_datastores %} +- {{ ds.service }} on internal network (not exposed externally) +{% endfor %} ### {% if ctx.has_frontend %}4{% else %}3{% endif %}. Check Status @@ -61,7 +62,8 @@ docker compose logs -f ## Services -- **app**: Backend API service (FastAPI) +- **app**: Backend API service + - ASGI application: `{{ ctx.application }}` - Port: {{ ctx.port }} - Built from: `backend/` {% if ctx.has_auth %} @@ -74,11 +76,12 @@ docker compose logs -f - API proxy: `/api` -> `http://app:{{ ctx.port }}` {% endif %} -{% if ctx.mongo_local %} -- **mongodb**: MongoDB database - - Internal port: 27017 - - Data volume: `mongo_data` -{% endif %} +{% for ds in ctx.local_datastores %} +- **{{ ds.service }}**: {{ ds.type }} database (`{{ ds.image }}`) + {% if ds.volume %} + - Data volume: `{{ ds.volume }}` + {% endif %} +{% endfor %} ## Development @@ -158,9 +161,9 @@ docker compose logs app ``` Common issues: -{% if ctx.mongo_local %} -- MongoDB not ready: Wait a few seconds and restart -{% endif %} +{% for ds in ctx.local_datastores %} +- {{ ds.service }} not ready: Wait a few seconds and restart +{% endfor %} - Missing environment variables: Check `.env` file - Port already in use: Change port in docker-compose.yml @@ -174,10 +177,10 @@ Common issues: ### Database connection errors -{% if ctx.mongo_local %} -- MongoDB service should be running: `docker compose ps mongodb` -- Connection string should be: `mongodb://mongodb:27017` +{% for ds in ctx.datastores %} +{% if ds.local %} +- {{ ds.service }} should be running: `docker compose ps {{ ds.service }}` {% else %} -- Check your external MongoDB connection string in `.env` -- Ensure network connectivity from container to MongoDB +- Check the external {{ ds.type }} connection string in `.env` and network connectivity from the container {% endif %} +{% endfor %} diff --git a/src/yaml_api_generator/generators/deploy/docker/templates/playbook.yml.jinja b/src/yaml_api_generator/generators/deploy/docker/templates/playbook.yml.jinja index 86a6ca5a..5baf0043 100644 --- a/src/yaml_api_generator/generators/deploy/docker/templates/playbook.yml.jinja +++ b/src/yaml_api_generator/generators/deploy/docker/templates/playbook.yml.jinja @@ -7,7 +7,5 @@ # ansible-playbook -i inventory.yml playbook.yml --ask-vault-pass # group_vars/all is auto-loaded; see README.md ("Hardening") for the full flow. roles: - {% if ctx.mongo_local %} - - mongodb - {% endif %} + - docker - app \ No newline at end of file diff --git a/src/yaml_api_generator/generators/deploy/docker/templates/roles/mongodb/tasks/main.yml.jinja b/src/yaml_api_generator/generators/deploy/docker/templates/roles/docker/tasks/main.yml.jinja similarity index 100% rename from src/yaml_api_generator/generators/deploy/docker/templates/roles/mongodb/tasks/main.yml.jinja rename to src/yaml_api_generator/generators/deploy/docker/templates/roles/docker/tasks/main.yml.jinja diff --git a/src/yaml_api_generator/generators/deploy/generate.py b/src/yaml_api_generator/generators/deploy/generate.py index b50df98b..70f05a84 100644 --- a/src/yaml_api_generator/generators/deploy/generate.py +++ b/src/yaml_api_generator/generators/deploy/generate.py @@ -15,13 +15,14 @@ def generate_deploy( ir: IRDocument, + application: str, host: str, ssh_user: str, ssh_key: str, output_dir: str, spec_dir: str | Path, ) -> None: - ctx = build_deploy_context(ir, host, ssh_user, ssh_key) + ctx = build_deploy_context(ir, application, host, ssh_user, ssh_key) backend_cls = _BACKENDS.get(ctx.runtime) if backend_cls is None: raise ValueError(f"no backend for runtime: {ctx.runtime}") diff --git a/src/yaml_api_generator/generators/deploy/systemd/generator.py b/src/yaml_api_generator/generators/deploy/systemd/generator.py index e8fef3a4..57c29d2b 100644 --- a/src/yaml_api_generator/generators/deploy/systemd/generator.py +++ b/src/yaml_api_generator/generators/deploy/systemd/generator.py @@ -39,7 +39,7 @@ def generate(self, ctx: DeployContext, output_dir: Path, spec_dir: Path) -> None roles_py.mkdir(parents=True, exist_ok=True) self._render(env, "roles/python/tasks/main.yml.jinja", ctx, roles_py / "main.yml") - if ctx.mongo_local: + if ctx.local_mongo: roles_mongo = output_dir / "roles" / "mongodb" / "tasks" roles_mongo.mkdir(parents=True, exist_ok=True) self._render(env, "roles/mongodb/tasks/main.yml.jinja", ctx, roles_mongo / "main.yml") diff --git a/src/yaml_api_generator/generators/deploy/systemd/templates/files/app.service.jinja b/src/yaml_api_generator/generators/deploy/systemd/templates/files/app.service.jinja index 8ce45049..e2c9fdc5 100644 --- a/src/yaml_api_generator/generators/deploy/systemd/templates/files/app.service.jinja +++ b/src/yaml_api_generator/generators/deploy/systemd/templates/files/app.service.jinja @@ -5,9 +5,10 @@ After=network.target [Service] User={{ ctx.user }} WorkingDirectory=/opt/{{ ctx.name }} -ExecStart=/opt/{{ ctx.name }}/venv/bin/uvicorn app:app --host 0.0.0.0 --port {{ ctx.port }} -Restart={{ ctx.restart }} +Environment=PORT={{ ctx.port }} EnvironmentFile=/opt/{{ ctx.name }}/.env +ExecStart=/opt/{{ ctx.name }}/venv/bin/uvicorn {{ ctx.application }} --host 0.0.0.0 --port ${PORT} +Restart={{ ctx.restart }} [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/src/yaml_api_generator/generators/deploy/systemd/templates/playbook.yml.jinja b/src/yaml_api_generator/generators/deploy/systemd/templates/playbook.yml.jinja index e29d51ed..fc7255c7 100644 --- a/src/yaml_api_generator/generators/deploy/systemd/templates/playbook.yml.jinja +++ b/src/yaml_api_generator/generators/deploy/systemd/templates/playbook.yml.jinja @@ -2,7 +2,7 @@ hosts: app become: true roles: - {% if ctx.mongo_local %} + {% if ctx.local_mongo %} - mongodb {% endif %} - python diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/.env.example.jinja b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/.env.example.jinja index 3eeb4f4b..73c2a301 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/.env.example.jinja +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/.env.example.jinja @@ -1,3 +1,3 @@ # API Configuration # Override the backend API base URL (defaults to {{ context.base_url }} if not set) -# VITE_API_BASE_URL=http://localhost:8000/api/v1 +# VITE_API_BASE_URL=http://localhost:{{ context.api_port }}{{ context.base_url }} diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/vite.config.ts.jinja b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/vite.config.ts.jinja index 6ce773c9..eff80859 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/vite.config.ts.jinja +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/vite.config.ts.jinja @@ -7,7 +7,7 @@ export default defineConfig({ port: 3000, proxy: { '/api': { - target: 'http://localhost:8000', + target: 'http://localhost:{{ context.api_port }}', changeOrigin: true, }, }, diff --git a/src/yaml_api_generator/generators/frontend/renderer.py b/src/yaml_api_generator/generators/frontend/renderer.py index 4f60a0a9..d5c4915f 100644 --- a/src/yaml_api_generator/generators/frontend/renderer.py +++ b/src/yaml_api_generator/generators/frontend/renderer.py @@ -33,6 +33,7 @@ def _render_scaffold( "package.json", "index.html", "vite.config.ts", + ".env.example", "tsconfig.json", "tsconfig.node.json", "tailwind.config.js", diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index aabd5d21..bac0f598 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -75,7 +75,8 @@ class IRDeploy: runtime: str name: str | None = None port: int = 8000 - mongo_local: bool = False + mongo_local: bool | None = None + """Explicit ``deploy.mongo.local``; when unset each MongoDB datastore is local if its URI is.""" env_vars: tuple[str, ...] = field(default_factory=tuple) docker: IRDeployDocker | None = None systemd: IRDeploySystemd | None = None diff --git a/src/yaml_api_generator/ir/ui.py b/src/yaml_api_generator/ir/ui.py index 72be515a..a82fa371 100644 --- a/src/yaml_api_generator/ir/ui.py +++ b/src/yaml_api_generator/ir/ui.py @@ -98,6 +98,7 @@ class GeneratorContext: api_name: str api_version: str base_url: str + api_port: int collections: dict[str, UICollection] components: dict[str, UIComponent] views: dict[str, UIView] diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json index 9be55db0..34c0c25d 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json @@ -20,8 +20,8 @@ "type": "object", "additionalProperties": false, "properties": { - "defaultPageSize": { "type": "integer" }, - "maxPageSize": { "type": "integer" } + "defaultPageSize": { "type": "integer", "minimum": 1 }, + "maxPageSize": { "type": "integer", "minimum": 1 } } }, "operations": { @@ -131,7 +131,10 @@ }, "then": { "required": ["auth"] } } - ] + ], + "dependentSchemas": { + "roles": { "required": ["type"], "properties": { "type": { "const": "auth" } } } + } }, "AuthCollectionConfig": { "type": "object", @@ -142,7 +145,7 @@ "via": { "oneOf": [ { "type": "string" }, - { "$ref": "#/$defs/StringList" } + { "$ref": "#/$defs/StringList", "minItems": 1 } ] } } diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index e3cca1df..9260819a 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -8,38 +8,95 @@ "entities" ], "additionalProperties": false, - "if": { - "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", - "required": [ - "datastores" - ], - "properties": { - "datastores": { - "not": { - "additionalProperties": { - "required": [ - "type" - ], - "properties": { - "type": { - "enum": [ - "filesystem", - "s3" - ] + "allOf": [ + { + "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", + "if": { + "required": [ + "datastores" + ], + "properties": { + "datastores": { + "not": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } } } } } + }, + "then": { + "properties": { + "mongo": { + "not": {}, + "errorMessage": "mongo and a record datastore are mutually exclusive: declare MongoDB under datastores instead" + } + } } - } - }, - "then": { - "properties": { - "mongo": { - "not": {} + }, + { + "$comment": "deploy.mongo configures the MongoDB service, which a spec whose record datastores are all PostgreSQL does not have", + "if": { + "required": [ + "datastores", + "deploy" + ], + "properties": { + "datastores": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "not": { + "enum": [ + "mongodb", + "mongo" + ] + } + } + } + }, + "not": { + "additionalProperties": { + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } + } + } + } + } + }, + "then": { + "properties": { + "deploy": { + "properties": { + "mongo": { + "not": {}, + "errorMessage": "deploy.mongo requires a MongoDB datastore, and every record datastore is PostgreSQL" + } + } + } + } } } - }, + ], "properties": { "version": { "type": "string", @@ -319,7 +376,19 @@ ] } } - ] + ], + "dependentSchemas": { + "roles": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "auth" + } + } + } + } }, "EntityAuthConfig": { "type": "object", @@ -1064,7 +1133,9 @@ "type": "string" }, "port": { - "type": "integer" + "type": "integer", + "minimum": 1, + "maximum": 65535 }, "runtime": { "type": "string", diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 22dc2b63..321c0e30 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -9,38 +9,95 @@ "entities" ], "additionalProperties": false, - "if": { - "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", - "required": [ - "datastores" - ], - "properties": { - "datastores": { - "not": { - "additionalProperties": { - "required": [ - "type" - ], - "properties": { - "type": { - "enum": [ - "filesystem", - "s3" - ] + "allOf": [ + { + "$comment": "A record datastore replaces the mongo block, so declaring both is ambiguous", + "if": { + "required": [ + "datastores" + ], + "properties": { + "datastores": { + "not": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } } } } } + }, + "then": { + "properties": { + "mongo": { + "not": {}, + "errorMessage": "mongo and a record datastore are mutually exclusive: declare MongoDB under datastores instead" + } + } } - } - }, - "then": { - "properties": { - "mongo": { - "not": {} + }, + { + "$comment": "deploy.mongo configures the MongoDB service, which a spec whose record datastores are all PostgreSQL does not have", + "if": { + "required": [ + "datastores", + "deploy" + ], + "properties": { + "datastores": { + "additionalProperties": { + "required": [ + "type" + ], + "properties": { + "type": { + "not": { + "enum": [ + "mongodb", + "mongo" + ] + } + } + } + }, + "not": { + "additionalProperties": { + "properties": { + "type": { + "enum": [ + "filesystem", + "s3" + ] + } + } + } + } + } + } + }, + "then": { + "properties": { + "deploy": { + "properties": { + "mongo": { + "not": {}, + "errorMessage": "deploy.mongo requires a MongoDB datastore, and every record datastore is PostgreSQL" + } + } + } + } } } - }, + ], "properties": { "version": { "type": "string", @@ -325,7 +382,19 @@ ] } } - ] + ], + "dependentSchemas": { + "roles": { + "required": [ + "type" + ], + "properties": { + "type": { + "const": "auth" + } + } + } + } }, "ProviderConfig": { "type": "object", @@ -985,7 +1054,29 @@ } } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "strategy": { + "const": "keyset" + } + } + }, + "then": { + "properties": { + "size": { + "properties": { + "default": { + "minimum": 1 + } + } + } + } + } + } + ] }, "AccessConfig": { "oneOf": [ diff --git a/src/yaml_api_generator/parsers/transformers/roles_config.py b/src/yaml_api_generator/parsers/transformers/roles_config.py index 90ea7b2b..18a9fff0 100644 --- a/src/yaml_api_generator/parsers/transformers/roles_config.py +++ b/src/yaml_api_generator/parsers/transformers/roles_config.py @@ -7,38 +7,36 @@ def parse_roles_config( - section: str, owner: str, roles_data: Any + section: str, owner: str, roles_data: dict[str, Any] | None ) -> tuple[IRRolesConfig | None, list[ConfigError]]: - """Build the roles of ``
..roles``, validating its hierarchy.""" - if not isinstance(roles_data, dict): + """Build the roles of ``
..roles``, validating the roles it references.""" + if roles_data is None: return None, [] - - values = tuple(str(v) for v in roles_data.get("values", [])) - if not values: - return None, [] - - raw_hierarchy = roles_data.get("hierarchy", {}) - hierarchy = { - str(role): tuple(str(i) for i in (includes or [])) - for role, includes in raw_hierarchy.items() - } if isinstance(raw_hierarchy, dict) else {} - - errors = _hierarchy_errors(section, owner, values, hierarchy) - if errors: - return None, errors - - default = str(roles_data.get("default", values[0])) - return IRRolesConfig(values=values, default=default, hierarchy=hierarchy), [] + roles = IRRolesConfig( + values=tuple(roles_data["values"]), + default=roles_data["default"], + hierarchy={role: tuple(includes) for role, includes in roles_data.get("hierarchy", {}).items()}, + ) + errors = _default_errors(section, owner, roles) or _hierarchy_errors(section, owner, roles) + return (None, errors) if errors else (roles, []) + + +def _default_errors(section: str, owner: str, roles: IRRolesConfig) -> list[ConfigError]: + if roles.default in roles.values: + return [] + return [ + ConfigError( + code="ROLES_DEFAULT_UNKNOWN_ROLE", + message=f"Default role '{roles.default}' is not one of the declared roles", + path=f"{section}.{owner}.roles.default", + hint=f"Add '{roles.default}' to {section}.{owner}.roles.values", + ) + ] -def _hierarchy_errors( - section: str, - owner: str, - values: tuple[str, ...], - hierarchy: dict[str, tuple[str, ...]], -) -> list[ConfigError]: +def _hierarchy_errors(section: str, owner: str, roles: IRRolesConfig) -> list[ConfigError]: path = f"{section}.{owner}.roles.hierarchy" - referenced = {*hierarchy, *(i for includes in hierarchy.values() for i in includes)} + referenced = {*roles.hierarchy, *(i for includes in roles.hierarchy.values() for i in includes)} errors = [ ConfigError( code="ROLES_HIERARCHY_UNKNOWN_ROLE", @@ -46,12 +44,12 @@ def _hierarchy_errors( path=path, hint=f"Add '{role}' to {section}.{owner}.roles.values", ) - for role in sorted(referenced - set(values)) + for role in sorted(referenced - set(roles.values)) ] if errors: return errors try: - flatten_role_hierarchy(hierarchy) + flatten_role_hierarchy(roles.hierarchy) except RoleHierarchyCycleError as exc: return [ ConfigError( diff --git a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py index 0ff44c8c..0e5a15e3 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/route_transformer.py @@ -321,10 +321,6 @@ def build_deploy( else None ) - mongo_local = deploy_data.get("mongo", {}).get( - "local", any(isinstance(ds, IRMongo) and ds.is_local for ds in datastores.values()) - ) - env_vars = self._collect_env_vars(deploy_data, providers, datastores) extra_services = tuple( @@ -336,7 +332,7 @@ def build_deploy( runtime=str(runtime), name=deploy_data.get("name"), port=int(deploy_data.get("port", api.port)), - mongo_local=mongo_local, + mongo_local=deploy_data.get("mongo", {}).get("local"), env_vars=tuple(env_vars), docker=docker, systemd=systemd, diff --git a/src/yaml_api_generator/parsers/transformers/v03/pagination.py b/src/yaml_api_generator/parsers/transformers/v03/pagination.py index 46ef1f39..d7f35ff2 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/pagination.py +++ b/src/yaml_api_generator/parsers/transformers/v03/pagination.py @@ -89,7 +89,7 @@ def parse_pagination( strategy = raw.get("strategy") errors.extend(_STRATEGY.validate(strategy, f"{path}.strategy")) - size, size_errors = _parse_size(raw.get("size"), f"{path}.size") + size, size_errors = _parse_size(raw["size"], f"{path}.size") errors.extend(size_errors) count = raw.get("count", "none") @@ -103,12 +103,6 @@ def parse_pagination( if strategy == "keyset": errors.extend(_validate_keyset(sort, collection, path)) - if not size_errors and size.default == 0: - errors.append(ConfigError( - code="PAGINATION_INVALID_SIZE", - message="keyset pagination size.default must be positive", - path=f"{path}.size.default", - )) if errors: return None, errors @@ -123,35 +117,16 @@ def parse_pagination( ), [] -def _parse_size(raw: Any, path: str) -> tuple[IRPageSize, list[ConfigError]]: - if not isinstance(raw, dict): - return IRPageSize(default=0, max=0), [ConfigError( +def _parse_size(raw: dict[str, int], path: str) -> tuple[IRPageSize, list[ConfigError]]: + """The schema bounds each size; only their ordering is left to check.""" + size = IRPageSize(default=raw["default"], max=raw["max"]) + if size.default > size.max: + return size, [ConfigError( code="PAGINATION_INVALID_SIZE", - message="pagination.size must declare 'default' and 'max'", + message=f"Default page size {size.default} exceeds the maximum page size {size.max}", path=path, )] - default = raw.get("default") - maximum = raw.get("max") - errors: list[ConfigError] = [] - if not isinstance(default, int) or isinstance(default, bool) or default < 0: - errors.append(ConfigError( - code="PAGINATION_INVALID_SIZE", - message="pagination.size.default must be a non-negative integer", - path=f"{path}.default", - )) - if not isinstance(maximum, int) or isinstance(maximum, bool) or maximum < 1: - errors.append(ConfigError( - code="PAGINATION_INVALID_SIZE", - message="pagination.size.max must be a positive integer", - path=f"{path}.max", - )) - if not errors and default > maximum: - errors.append(ConfigError( - code="PAGINATION_INVALID_SIZE", - message="pagination.size.default must not exceed size.max", - path=path, - )) - return IRPageSize(default=default or 0, max=maximum or 0), errors + return size, [] def _parse_sort( diff --git a/src/yaml_api_generator/parsers/ui_parser.py b/src/yaml_api_generator/parsers/ui_parser.py index 5dd3506d..fafaa8f5 100644 --- a/src/yaml_api_generator/parsers/ui_parser.py +++ b/src/yaml_api_generator/parsers/ui_parser.py @@ -47,6 +47,7 @@ def build_generator_context( api_name=base_path.strip("/") or "api", api_version=version, base_url=base_url, + api_port=ir.api.port, collections=collections, components=components, views=views, diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 71b5c06d..5e1d4d02 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -28,7 +28,7 @@ import yaml from jsonschema import Draft202012Validator -from jsonschema.exceptions import best_match +from jsonschema.exceptions import ValidationError, best_match from referencing import Registry, Resource from ..ir import IRDocument @@ -162,13 +162,19 @@ def _validate_schema(config: Any, schema: dict[str, Any]) -> list[ConfigError]: return [ ConfigError( code="SCHEMA_VALIDATION_ERROR", - message=best_match([error]).message, + message=_schema_message(error), path=join_path(tuple(str(part) for part in error.path)), ) for error in errors ] +def _schema_message(error: ValidationError) -> str: + """A failing subschema's ``errorMessage`` annotation, else the validator's own message.""" + custom = error.schema.get("errorMessage") if isinstance(error.schema, dict) else None + return custom or best_match([error]).message + + _VERSION_SCHEMA_MAP: dict[str, str] = { "0.1": "yaml-spec-v0.1.schema.json", "0.2": "yaml-spec-v0.2.schema.json", diff --git a/src/yaml_api_generator/runtime/pagination.py b/src/yaml_api_generator/runtime/pagination.py index df1a5e3d..7780824e 100644 --- a/src/yaml_api_generator/runtime/pagination.py +++ b/src/yaml_api_generator/runtime/pagination.py @@ -102,6 +102,8 @@ def _limit(config: dict[str, Any], query: dict[str, Any]) -> int: requested = _integer(query, "limit") if requested is None: return config["size"]["default"] + if requested == 0: + raise InvalidPageRequestError("limit") maximum = config["size"]["max"] if requested > maximum: if config["on_overflow"] == "reject": @@ -152,8 +154,6 @@ def map_request(config: dict[str, Any], query: dict[str, Any]) -> Primitives: sort=sort, count=config["count"], ) - if limit == 0: - raise InvalidPageRequestError("limit") bounds: tuple[Bound, ...] = () after = _scalar(query.get("after")) if after: diff --git a/src/yaml_api_generator/server/asgi/runner.py b/src/yaml_api_generator/server/asgi/runner.py index 3da51178..f929ce5f 100644 --- a/src/yaml_api_generator/server/asgi/runner.py +++ b/src/yaml_api_generator/server/asgi/runner.py @@ -87,7 +87,7 @@ async def _handle_http( for k, v in scope.get("headers", []) } - query = parse_qs(scope.get("query_string", b"").decode("latin-1")) + query = parse_qs(scope.get("query_string", b"").decode("latin-1"), keep_blank_values=True) try: ctx = await pipeline.run( diff --git a/src/yaml_api_generator/server/common/auth/__init__.py b/src/yaml_api_generator/server/common/auth/__init__.py index 679e3f85..5475c3cf 100644 --- a/src/yaml_api_generator/server/common/auth/__init__.py +++ b/src/yaml_api_generator/server/common/auth/__init__.py @@ -1,4 +1,5 @@ +from .api_key import hash_api_key from .password import hash_password, verify_password from .jwt import SessionTokens, build_session_tokens -__all__ = ["hash_password", "verify_password", "SessionTokens", "build_session_tokens"] +__all__ = ["hash_api_key", "hash_password", "verify_password", "SessionTokens", "build_session_tokens"] diff --git a/src/yaml_api_generator/server/common/auth/api_key.py b/src/yaml_api_generator/server/common/auth/api_key.py new file mode 100644 index 00000000..f071b474 --- /dev/null +++ b/src/yaml_api_generator/server/common/auth/api_key.py @@ -0,0 +1,6 @@ +import hashlib + + +def hash_api_key(api_key: str) -> str: + """API keys are high-entropy, so an unsalted digest keeps them both secret and indexable.""" + return hashlib.sha256(api_key.encode("utf-8")).hexdigest() diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 7ac0cf88..957dec55 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -285,30 +285,22 @@ def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: collection = self._ctx.collections[collection_name] id_field = collection.id_field identity_collection = self._ctx.auth.identity_collections.get(collection_name) - uses_password = ( - identity_collection is not None and identity_collection.has_password - ) if operation == "list": return RepositoryListStep(collection_name, id_field, route.pagination) if operation == "get": return RepositoryGetStep(collection_name, id_field, self._locator(route)) if operation == "create": - roles_config = identity_collection.roles if identity_collection is not None else None return RepositoryCreateStep( - collection_name, - id_field, - collection.id_format, - uses_password, - roles_config=roles_config, + collection_name, id_field, collection.id_format, identity_collection ) if operation == "update": return RepositoryUpdateStep( - collection_name, id_field, uses_password, self._locator(route) + collection_name, id_field, identity_collection, self._locator(route) ) if operation == "replace": return RepositoryReplaceStep( - collection_name, id_field, uses_password, self._locator(route) + collection_name, id_field, identity_collection, self._locator(route) ) if operation == "delete": return RepositoryDeleteStep(collection_name, id_field, self._locator(route)) diff --git a/src/yaml_api_generator/server/common/steps/authentication.py b/src/yaml_api_generator/server/common/steps/authentication.py index 6b9eb518..00d14ac0 100644 --- a/src/yaml_api_generator/server/common/steps/authentication.py +++ b/src/yaml_api_generator/server/common/steps/authentication.py @@ -2,7 +2,7 @@ from typing import Any from yaml_api_generator.ir.api import IROIDCProvider, IRProviderBinding -from ..auth import idp, oidc, verify_password +from ..auth import hash_api_key, idp, oidc, verify_password from ..auth.jwt import SessionTokens from ..auth.utils import extract_bearer_token, extract_api_key, extract_basic_credentials from ..request_context import RequestContext @@ -228,7 +228,7 @@ async def _resolve_identity_for_collection( repository, collection_name=identity.name, field_name="apiKey", - field_value=token, + field_value=hash_api_key(token), id_field=id_field, identity=identity, ) diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 2fcc7aa9..9b452097 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -2,11 +2,12 @@ from dataclasses import dataclass from typing import Any, cast -from ..auth import hash_password +from ..auth import hash_api_key, hash_password from ..request_context import RequestContext -from yaml_api_generator.ir.api import IRPagination, IRRolesConfig +from yaml_api_generator.ir.api import IRPagination from ..pagination import Page, build_response, map_request from ..repositories.base import BaseRepository +from ..server_context import RuntimeIdentityDescriptor from ..utils.ids import generate_id as _generate_id from . import NotFoundError, Step @@ -14,6 +15,7 @@ _PATH_TOKEN = re.compile(r"\{(\w+)\}") _REQUEST_BODY_MESSAGE = "Request body must be a JSON object" _NOT_FOUND_MESSAGE = "Resource not found" +_HIDDEN_FIELDS = ("_auth", "password", "apiKey") def _repository(ctx: RequestContext) -> BaseRepository: @@ -91,8 +93,8 @@ def _inject_canonical_id( normalized[id_field] = str(normalized["_id"]) if id_field != "_id": normalized.pop("_id", None) - normalized.pop("_auth", None) - normalized.pop("password", None) + for field_name in _HIDDEN_FIELDS: + normalized.pop(field_name, None) return normalized @@ -108,9 +110,8 @@ def _inject_canonical_id_list( def _prepare_payload( payload: dict[str, Any], + identity: RuntimeIdentityDescriptor | None, *, - password_via: bool, - roles_config: IRRolesConfig | None = None, is_create: bool = False, ) -> dict[str, Any]: normalized = dict(payload) @@ -118,25 +119,23 @@ def _prepare_payload( # Roles are managed exclusively through _auth namespace — never writable via CRUD normalized.pop("roles", None) normalized.pop("role", None) - - if password_via: - raw_password = normalized.pop("password", None) - if raw_password is not None: - if not isinstance(raw_password, str) or not raw_password.strip(): - raise ValueError("Password must be a non-empty string") - auth_data = normalized.get("_auth", {}) - if not isinstance(auth_data, dict): - auth_data = {} - auth_data["passwordHash"] = hash_password(raw_password) - normalized["_auth"] = auth_data - - if is_create and roles_config is not None: - auth_data = normalized.get("_auth", {}) - if not isinstance(auth_data, dict): - auth_data = {} - auth_data["roles"] = roles_config.default + if identity is None: + return normalized + + auth_data = normalized.get("_auth", {}) + if not isinstance(auth_data, dict): + auth_data = {} + raw_password = normalized.pop("password", None) if identity.has_password else None + if raw_password is not None: + if not isinstance(raw_password, str) or not raw_password.strip(): + raise ValueError("Password must be a non-empty string") + auth_data["passwordHash"] = hash_password(raw_password) + if is_create and identity.roles is not None: + auth_data["roles"] = identity.roles.default + if auth_data: normalized["_auth"] = auth_data - + if identity.has_api_key and "apiKey" in normalized: + normalized["apiKey"] = hash_api_key(normalized["apiKey"]) return normalized @@ -182,16 +181,10 @@ def RepositoryCreateStep( collection_name: str, id_field: str, id_format: str, - password_via: bool, - roles_config: IRRolesConfig | None = None, + identity: RuntimeIdentityDescriptor | None = None, ) -> Step: async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload( - _payload(ctx), - password_via=password_via, - roles_config=roles_config, - is_create=True, - ) + payload = _prepare_payload(_payload(ctx), identity, is_create=True) if id_format != "natural": payload[id_field] = _generate_id(id_format) result = await _repository(ctx).create(collection_name, payload) @@ -204,13 +197,13 @@ async def step(ctx: RequestContext) -> RequestContext: def RepositoryUpdateStep( collection_name: str, id_field: str, - password_via: bool, + identity: RuntimeIdentityDescriptor | None = None, locator: RecordLocator | None = None, ) -> Step: locator = locator or RecordLocator(id_field) async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload(_payload(ctx), password_via=password_via) + payload = _prepare_payload(_payload(ctx), identity) if locator.is_scoped: result = await _repository(ctx).update_by( collection_name, locator.criteria(ctx.url_params), payload @@ -230,13 +223,13 @@ async def step(ctx: RequestContext) -> RequestContext: def RepositoryReplaceStep( collection_name: str, id_field: str, - password_via: bool, + identity: RuntimeIdentityDescriptor | None = None, locator: RecordLocator | None = None, ) -> Step: locator = locator or RecordLocator(id_field) async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload(_payload(ctx), password_via=password_via) + payload = _prepare_payload(_payload(ctx), identity) if locator.is_scoped: result = await _repository(ctx).replace_by( collection_name, locator.criteria(ctx.url_params), payload diff --git a/tests/fixtures/declared-routes/spec.yaml b/tests/fixtures/declared-routes/spec.yaml new file mode 100644 index 00000000..857e976a --- /dev/null +++ b/tests/fixtures/declared-routes/spec.yaml @@ -0,0 +1,24 @@ +version: "0.3" +api: + basePath: /api + operations: + auth: { register: true, login: true } +entities: + user: + type: auth + auth: { identifier: email, via: password } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } +repositories: + users: { entity: user } +services: + users: { repository: users } +controllers: + users: + service: users + base_path: /users + access: "*" + operations: + register: { access: public } + login: { access: public } diff --git a/tests/fixtures/valid/v03-service-input.yaml b/tests/fixtures/valid/v03-service-input.yaml index 31d3c292..b7d44416 100644 --- a/tests/fixtures/valid/v03-service-input.yaml +++ b/tests/fixtures/valid/v03-service-input.yaml @@ -41,6 +41,7 @@ controllers: service: archive method: POST path: /{postId}/archive + access: public notifications: base_path: /notifications routes: @@ -48,3 +49,4 @@ controllers: service: notify method: POST path: /{postId} + access: public diff --git a/tests/integration/support/declared_routes.py b/tests/integration/support/declared_routes.py new file mode 100644 index 00000000..ff0a4b32 --- /dev/null +++ b/tests/integration/support/declared_routes.py @@ -0,0 +1,44 @@ +"""Specs whose controllers declare the register and login routes, exercised identically on every server and target.""" + +import shutil +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +_REPOSITORY = Path(__file__).parents[3] + +PASSWORD = "correct-horse-battery" + + +@dataclass(frozen=True) +class Scenario: + spec: Path + prefix: str + profile: dict = field(default_factory=dict) + + def materialize(self, directory: Path, uri: str, database: str) -> Path: + """Copy the spec beside its imports into ``directory``, bound to the given MongoDB database.""" + shutil.copytree(self.spec.parent, directory, dirs_exist_ok=True) + document = yaml.safe_load(self.spec.read_text()) + target = directory / "spec.yaml" + target.write_text(yaml.safe_dump({**document, "mongo": {"uri": uri, "database": database}}, sort_keys=False)) + return target + + +SCENARIOS = { + "auth-users-proposed": Scenario(_REPOSITORY / "examples/comparison/auth-users-proposed.yaml", "/api/v1/auth", {"displayName": "Ada"}), + "auth-users-v0.2": Scenario(_REPOSITORY / "examples/comparison/auth-users/v0.2.yaml", "/api/v1/auth", {"displayName": "Ada"}), + "register-beside-record": Scenario(_REPOSITORY / "tests/fixtures/declared-routes/spec.yaml", "/api/users"), +} + + +def exercise(client, prefix: str, profile: dict) -> dict: + """Register, then sign in, through the declared routes.""" + identity = {"email": "ada@example.com", "password": PASSWORD} + registered = client.post(f"{prefix}/register", json={**identity, **profile}) + signed_in = client.post(f"{prefix}/login", json=identity) + return {"register": registered.status_code, "login": signed_in.status_code, "token": "token" in signed_in.json()} + + +EXPECTED = {"register": 200, "login": 200, "token": True} diff --git a/tests/integration/support/v01_platform.py b/tests/integration/support/v01_platform.py index 0dff66b4..9af160ed 100644 --- a/tests/integration/support/v01_platform.py +++ b/tests/integration/support/v01_platform.py @@ -8,6 +8,7 @@ version: "0.1" api: {{ pagination: {{ defaultPageSize: 2, maxPageSize: 3 }} }} mongo: {{ uri: "{uri}", database: {database} }} +security: {{ cors: {{ allowedOrigins: ["https://app.example"], allowCredentials: true }} }} collections: users: type: auth @@ -20,7 +21,7 @@ clients: type: auth auth: {{ identifier: clientId, via: [apiKey] }} - access: {{ create: public }} + access: {{ create: public, list: public }} fields: id: {{ type: id }} clientId: {{ type: string, required: true, unique: true }} @@ -34,17 +35,21 @@ def exercise(client, uri: str, database: str) -> dict: - """Drive pagination, API-key auth and the role hierarchy; return what each request observed.""" - client.post("/api/clients", json={"clientId": "importer", "apiKey": "key-1"}) + """Drive pagination, credentialed CORS, API-key auth, credential storage and the role hierarchy; return what each request observed.""" + registered_client = client.post("/api/clients", json={"clientId": "importer", "apiKey": "key-1"}).json() keyed = [ client.post("/api/posts", json={"title": f"post {n}"}, headers={"X-API-Key": "key-1"}).status_code for n in range(4) ] anonymous = client.post("/api/posts", json={"title": "anonymous"}).status_code - for email in ("admin@example.com", "reader@example.com"): - client.post("/api/users", json={"email": email, "password": PASSWORD}) + registered_users = [ + client.post("/api/users", json={"email": email, "password": PASSWORD}).json() + for email in ("admin@example.com", "reader@example.com") + ] with MongoClient(uri) as mongo: + stored_client = mongo[database]["clients"].find_one({"clientId": "importer"}) + stored_reader = mongo[database]["users"].find_one({"email": "reader@example.com"}) mongo[database]["users"].update_one({"email": "admin@example.com"}, {"$set": {"_auth.roles": ["admin"]}}) def users_listing(email: str) -> int: @@ -56,6 +61,22 @@ def users_listing(email: str) -> int: "anonymous": anonymous, "default_page": len(client.get("/api/posts").json()), "clamped_page": len(client.get("/api/posts", params={"limit": 50}).json()), + "zero_limit": client.get("/api/posts", params={"limit": 0}).status_code, + "blank_limit": client.get("/api/posts?limit=").status_code, + "preflight_allows": client.options( + "/api/posts", + headers={ + "Origin": "https://app.example", + "Access-Control-Request-Method": "POST", + "Access-Control-Request-Headers": "x-api-key, content-type", + }, + ).headers.get("access-control-allow-headers"), + "exposed_credentials": sorted( + {"_auth", "password", "apiKey"} + & {key for record in [registered_client, *registered_users, *client.get("/api/clients").json()] for key in record} + ), + "api_key_stored_in_plaintext": stored_client["apiKey"] == "key-1", + "default_role": stored_reader["_auth"]["roles"], "admin_lists_users": users_listing("admin@example.com"), "reader_lists_users": users_listing("reader@example.com"), } @@ -66,6 +87,12 @@ def users_listing(email: str) -> int: "anonymous": 401, "default_page": 2, "clamped_page": 3, + "zero_limit": 422, + "blank_limit": 422, + "preflight_allows": "x-api-key, content-type", + "exposed_credentials": [], + "api_key_stored_in_plaintext": False, + "default_role": "reader", "admin_lists_users": 200, "reader_lists_users": 403, } diff --git a/tests/integration/test_anonymous_access.py b/tests/integration/test_anonymous_access.py new file mode 100644 index 00000000..ef526d35 --- /dev/null +++ b/tests/integration/test_anonymous_access.py @@ -0,0 +1,80 @@ +"""A route closed to anonymous callers stays closed when the spec declares no identity, on every server and target.""" + +import pytest +from starlette.testclient import TestClient + +from tests.unit.support.generated_apps import run_generated, write_spec +from yaml_api_generator.server import build_app, build_raw_app + +pytestmark = pytest.mark.integration + +_V01 = """ +version: "0.1" +api: {{ basePath: /api }} +mongo: {{ uri: "{uri}", database: {database} }} +collections: + posts: + access: {{ list: public, create: "*" }} + fields: + id: {{ type: id }} + title: {{ type: string, required: true }} +""" + +_V0X = """ +version: "{version}" +api: {{ basePath: /api }} +mongo: {{ uri: "{uri}", database: {database} }} +entities: + post: + fields: + id: {{ type: id }} + title: {{ type: string, required: true }} +repositories: + posts: {{ entity: post }} +services: + list_posts: {{ repository: posts, operation: list }} + create_post: {{ repository: posts, operation: create }} +controllers: + posts: + base_path: /posts + routes: + list: {{ service: list_posts, method: GET, path: "", access: public }} + create: {{ service: create_post, method: POST, path: "", access: "*" }} +""" + +SPECS = {"0.1": _V01, "0.2": _V0X, "0.3": _V0X} +EXPECTED = {"list": 200, "create": 401} +_VERSIONS = pytest.mark.parametrize("version", SPECS) + + +def exercise(client) -> dict[str, int]: + return { + "list": client.get("/api/posts").status_code, + "create": client.post("/api/posts", json={"title": "anonymous"}).status_code, + } + + +def _spec(tmp_path, version, mongo_uri, mongo_database): + return write_spec(tmp_path, SPECS[version].format(version=version, uri=mongo_uri, database=mongo_database)) + + +@_VERSIONS +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev", "devraw"]) +def test_dev_server(build, version, tmp_path, monkeypatch, mongo_uri, mongo_database): + monkeypatch.setenv("YAML_API_MONGO_URI", mongo_uri) + monkeypatch.setenv("YAML_API_MONGO_DB", mongo_database) + with TestClient(build(str(_spec(tmp_path, version, mongo_uri, mongo_database)))) as client: + assert exercise(client) == EXPECTED + + +@_VERSIONS +@pytest.mark.parametrize("target", ["fastapi", "raw", "drf"]) +def test_generated(target, version, tmp_path, mongo_uri, mongo_database): + script = """ + from tests.integration.test_anonymous_access import exercise + + with client: + print(json.dumps(exercise(client))) + """ + env = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database, "DJANGO_SECRET_KEY": "test-secret"} + assert run_generated(_spec(tmp_path, version, mongo_uri, mongo_database), target, script, env) == EXPECTED diff --git a/tests/integration/test_declared_routes.py b/tests/integration/test_declared_routes.py new file mode 100644 index 00000000..d42806b5 --- /dev/null +++ b/tests/integration/test_declared_routes.py @@ -0,0 +1,34 @@ +"""Register and login routes declared by a controller are served at their declared paths on every server and target.""" + +import pytest +from starlette.testclient import TestClient + +from tests.integration.support.declared_routes import EXPECTED, SCENARIOS, exercise +from tests.unit.support.generated_apps import run_generated +from yaml_api_generator.server import build_app, build_raw_app + +pytestmark = pytest.mark.integration + + +@pytest.mark.parametrize("scenario", SCENARIOS.values(), ids=SCENARIOS.keys()) +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev", "devraw"]) +def test_dev_server(scenario, build, tmp_path, monkeypatch, mongo_uri, mongo_database): + monkeypatch.setenv("YAML_API_MONGO_URI", mongo_uri) + monkeypatch.setenv("YAML_API_MONGO_DB", mongo_database) + spec = scenario.materialize(tmp_path, mongo_uri, mongo_database) + with TestClient(build(str(spec))) as client: + assert exercise(client, scenario.prefix, scenario.profile) == EXPECTED + + +@pytest.mark.parametrize("scenario", SCENARIOS.values(), ids=SCENARIOS.keys()) +@pytest.mark.parametrize("target", ["fastapi", "raw", "drf"]) +def test_generated(scenario, target, tmp_path, mongo_uri, mongo_database): + spec = scenario.materialize(tmp_path, mongo_uri, mongo_database) + script = f""" + from tests.integration.support.declared_routes import exercise + + with client: + print(json.dumps(exercise(client, {scenario.prefix!r}, {scenario.profile!r}))) + """ + env = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database, "DJANGO_SECRET_KEY": "test-secret", "JWT_SECRET": "test-jwt-secret-0123456789abcdef0123456789"} + assert run_generated(spec, target, script, env) == EXPECTED diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index d6712202..601912d4 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -486,7 +486,8 @@ def test_generated_drf_enforces_where_per_rule(tmp_path): HTTP_422_UNPROCESSABLE_ENTITY=422, ) response_module = types.ModuleType("rest_framework.response") - views_module = types.ModuleType("rest_framework.views") + adrf_module = types.ModuleType("adrf") + views_module = types.ModuleType("adrf.views") spectacular_module = types.ModuleType("drf_spectacular") spectacular_utils = types.ModuleType("drf_spectacular.utils") auth_module = types.ModuleType("api.authentication.auth") @@ -508,13 +509,17 @@ def extend_schema(**kwargs): views_module.APIView = APIView spectacular_utils.extend_schema = extend_schema auth_module._principal_has_role = lambda principal, role: role in principal.get("roles", []) - auth_module.resolve_principal = lambda request, database: None + async def anonymous(headers, database): + return None + + auth_module.resolve_principal = anonymous auth_module.hash_password = lambda password: password mongodb_module.get_database = lambda: None sys.modules.update({{ "rest_framework": rest_framework, "rest_framework.response": response_module, - "rest_framework.views": views_module, + "adrf": adrf_module, + "adrf.views": views_module, "drf_spectacular": spectacular_module, "drf_spectacular.utils": spectacular_utils, "api.authentication.auth": auth_module, @@ -562,9 +567,10 @@ def find(self, criteria): "reports": Collection([{{"id": "r1", "organizationId": "org-1", "title": "visible"}}]), }} views.get_database = lambda: db - views.resolve_principal = lambda request, database: {{ - "sub": "u1", "identity_collection": "users", "roles": [] - }} + async def member(headers, database): + return {{"sub": "u1", "identity_collection": "users", "roles": []}} + + views.resolve_principal = member handler_views.get_database = lambda: db handler_views.resolve_principal = views.resolve_principal request = SimpleNamespace(data={{}}, query_params={{}}, headers={{}}, COOKIES={{}}) diff --git a/tests/integration/test_v01_platform.py b/tests/integration/test_v01_platform.py index 951f4b32..91046597 100644 --- a/tests/integration/test_v01_platform.py +++ b/tests/integration/test_v01_platform.py @@ -1,24 +1,26 @@ -"""0.1 pagination, API keys and role hierarchy over HTTP on the dev server and generated FastAPI.""" +"""0.1 pagination, credentialed CORS, API keys, credential storage and role hierarchy over HTTP on every server and target.""" import pytest from starlette.testclient import TestClient from tests.integration.support.v01_platform import EXPECTED, SPEC, exercise from tests.unit.support.generated_apps import run_generated, write_spec -from yaml_api_generator.server import build_app +from yaml_api_generator.server import build_app, build_raw_app pytestmark = pytest.mark.integration -def test_dev_server(tmp_path, monkeypatch, mongo_uri, mongo_database): +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["dev", "devraw"]) +def test_dev_server(build, tmp_path, monkeypatch, mongo_uri, mongo_database): monkeypatch.setenv("YAML_API_MONGO_URI", mongo_uri) monkeypatch.setenv("YAML_API_MONGO_DB", mongo_database) spec = write_spec(tmp_path, SPEC.format(uri=mongo_uri, database=mongo_database)) - with TestClient(build_app(str(spec))) as client: + with TestClient(build(str(spec))) as client: assert exercise(client, mongo_uri, mongo_database) == EXPECTED -def test_generated_fastapi(tmp_path, mongo_uri, mongo_database): +@pytest.mark.parametrize("target", ["fastapi", "raw", "drf"]) +def test_generated(target, tmp_path, mongo_uri, mongo_database): spec = write_spec(tmp_path, SPEC.format(uri=mongo_uri, database=mongo_database)) script = f""" from tests.integration.support.v01_platform import exercise @@ -26,5 +28,5 @@ def test_generated_fastapi(tmp_path, mongo_uri, mongo_database): with client: print(json.dumps(exercise(client, {mongo_uri!r}, {mongo_database!r}))) """ - env = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database} - assert run_generated(spec, "fastapi", script, env) == EXPECTED + env = {"MONGO_URI": mongo_uri, "MONGO_DB": mongo_database, "DJANGO_SECRET_KEY": "test-secret", "JWT_SECRET": "test-jwt-secret-0123456789abcdef0123456789"} + assert run_generated(spec, target, script, env) == EXPECTED diff --git a/tests/unit/support/generated_apps.py b/tests/unit/support/generated_apps.py index 75e7c6a0..bb43c9fe 100644 --- a/tests/unit/support/generated_apps.py +++ b/tests/unit/support/generated_apps.py @@ -8,16 +8,17 @@ from pathlib import Path from typing import Any -from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.cli.compiler import backend_generator, compile_project _REPOSITORY = Path(__file__).parents[3] _PRELUDE = """ +import importlib import json from starlette.testclient import TestClient -import app -client = TestClient(app.app) +module, attribute = {application!r}.split(":") +client = TestClient(getattr(importlib.import_module(module), attribute)) """ @@ -38,7 +39,7 @@ def run_generated(spec: Path, target: str, script: str, env: dict[str, str] | No The script can import the ``tests`` package, so probes are shared with in-process tests. """ result = subprocess.run( - [sys.executable, "-c", _PRELUDE + textwrap.dedent(script)], + [sys.executable, "-c", _PRELUDE.format(application=backend_generator(target).asgi_application) + textwrap.dedent(script)], cwd=generate(spec, target), env={**os.environ, "PYTHONPATH": str(_REPOSITORY), **(env or {})}, capture_output=True, diff --git a/tests/unit/test_auth_steps.py b/tests/unit/test_auth_steps.py index 83fbb1c4..b7d5394b 100644 --- a/tests/unit/test_auth_steps.py +++ b/tests/unit/test_auth_steps.py @@ -36,7 +36,7 @@ NotEquals, StringLiteral, ) -from yaml_api_generator.server.common.auth.password import hash_password +from yaml_api_generator.server.common.auth import hash_api_key, hash_password TOKENS = SymmetricSessionTokens("test-secret-at-least-thirty-two-bytes") @@ -241,7 +241,7 @@ async def test_204_api_key_auth_resolves_principal(self): { "id": "c1", "clientId": "myclient", - "apiKey": "ak123", + "apiKey": hash_api_key("ak123"), } ] } @@ -272,7 +272,7 @@ async def test_205_api_key_wrong_key_no_principal(self): { "id": "c1", "clientId": "myclient", - "apiKey": "ak123", + "apiKey": hash_api_key("ak123"), } ] } @@ -487,7 +487,7 @@ async def test_213_api_key_via_bearer_fallback(self): "clients": [ { "id": "c1", - "apiKey": "ak123", + "apiKey": hash_api_key("ak123"), } ] } diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 4179d87c..9d014ed2 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +import yaml from click.testing import CliRunner from yaml_api_generator.cli import cli @@ -116,6 +117,20 @@ def test_frontend_uses_spec_output_without_cli_override(tmp_path): assert (output / "src" / "pages" / "Articles.tsx").exists() +def test_frontend_targets_api_port(tmp_path): + output = tmp_path / "frontend" + spec = yaml.safe_load((V03_EXAMPLES / "crud-pagination" / "spec.yaml").read_text()) + spec["api"] = {**spec.get("api", {}), "port": 9100} + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec, sort_keys=False)) + + result = CliRunner().invoke(cli, ["build", str(path), "frontend", "--output", str(output)]) + + assert result.exit_code == 0, result.output + assert "target: 'http://localhost:9100'" in (output / "vite.config.ts").read_text() + assert "VITE_API_BASE_URL=http://localhost:9100/" in (output / ".env.example").read_text() + + def test_frontend_without_ui_block_exits1(): runner = CliRunner() result = runner.invoke( @@ -129,9 +144,13 @@ def test_frontend_without_ui_block_exits1(): assert result.exit_code == 1 -@pytest.mark.parametrize(("arguments", "port"), [([], 9100), (["-p", "9200"], 9200)], ids=["api-port", "flag"]) +@pytest.mark.parametrize( + ("arguments", "outcome"), + [([], (0, [9100])), (["-p", "9200"], (0, [9200])), (["-p", "0"], (2, []))], + ids=["api-port", "flag", "flag-out-of-range"], +) @pytest.mark.parametrize("raw", [False, True], ids=["fastapi", "raw"]) -def test_run_binds_api_port_unless_flag_given(tmp_path, monkeypatch, arguments, port, raw): +def test_run_binds_api_port_unless_flag_given(tmp_path, monkeypatch, arguments, outcome, raw): import uvicorn spec = write_spec(tmp_path, """ @@ -150,4 +169,4 @@ def test_run_binds_api_port_unless_flag_given(tmp_path, monkeypatch, arguments, bound = [] monkeypatch.setattr(uvicorn, "run", lambda app, **options: bound.append(options["port"])) result = CliRunner().invoke(cli, ["run", str(spec), *arguments, *(["--raw"] if raw else [])]) - assert (result.exit_code, bound) == (0, [port]), result.output + assert (result.exit_code, bound) == outcome, result.output diff --git a/tests/unit/test_cors.py b/tests/unit/test_cors.py index 9f0ff4a8..e7860e7f 100644 --- a/tests/unit/test_cors.py +++ b/tests/unit/test_cors.py @@ -86,7 +86,7 @@ def test_generated_drf_configures_cors_headers(tmp_path, version): backend = generate(write_spec(tmp_path, _spec(version)), "drf") settings = next(backend.glob("*/settings.py")).read_text() assert '"corsheaders"' in settings - assert "corsheaders.middleware.CorsMiddleware" in settings + assert "api.cors.RequestedHeadersCorsMiddleware" in settings assert f"CORS_ALLOWED_ORIGINS = ['{ORIGIN}']" in settings assert "CORS_ALLOW_METHODS = ['GET', 'POST']" in settings assert "CORS_ALLOW_CREDENTIALS = True" in settings diff --git a/tests/unit/test_deploy_build_context.py b/tests/unit/test_deploy_build_context.py index 12ff1375..cb4b6628 100644 --- a/tests/unit/test_deploy_build_context.py +++ b/tests/unit/test_deploy_build_context.py @@ -10,8 +10,7 @@ def _ctx(extra_services): name="app", port=8000, runtime="docker", - mongo_local=False, - mongo_db="testdb", + application="app:app", env_vars={}, has_auth=False, has_frontend=False, @@ -93,7 +92,7 @@ def test_required_secrets_are_added_to_env_vars(tmp_path): port: 8000 """) ir = load_and_generate_ir(str(spec)).ir - ctx = build_deploy_context(ir, "localhost", "ubuntu", "~/.ssh/id_rsa") + ctx = build_deploy_context(ir, "app:app", "localhost", "ubuntu", "~/.ssh/id_rsa") assert "JWT_SECRET" in ctx.secret_env_names assert "KEYCLOAK_CLIENT_SECRET" in ctx.secret_env_names assert ctx.env_vars["JWT_SECRET"] == "" @@ -114,7 +113,7 @@ def test_secrets_required_passthrough_in_app_environment(tmp_path): port: 8000 """) ir = load_and_generate_ir(str(spec)).ir - ctx = build_deploy_context(ir, "localhost", "ubuntu", "~/.ssh/id_rsa") + ctx = build_deploy_context(ir, "app:app", "localhost", "ubuntu", "~/.ssh/id_rsa") assert "JWT_SECRET=${JWT_SECRET}" in ctx.app_environment assert "KEYCLOAK_CLIENT_SECRET=${KEYCLOAK_CLIENT_SECRET}" in ctx.app_environment assert not any(":-" in v for v in ctx.app_environment if "JWT_SECRET" in v) @@ -135,7 +134,7 @@ def test_ansible_role_generates_secrets_and_renders_env(tmp_path): port: 8000 """) ir = load_and_generate_ir(str(spec)).ir - ctx = build_deploy_context(ir, "localhost", "ubuntu", "~/.ssh/id_rsa") + ctx = build_deploy_context(ir, "app:app", "localhost", "ubuntu", "~/.ssh/id_rsa") out = tmp_path / "out" DockerDeployGenerator().generate(ctx, out, tmp_path) diff --git a/tests/unit/test_deploy_defaults.py b/tests/unit/test_deploy_defaults.py index 4fc227fb..5769c616 100644 --- a/tests/unit/test_deploy_defaults.py +++ b/tests/unit/test_deploy_defaults.py @@ -1,8 +1,12 @@ -"""deploy.port and deploy.mongo.local default from the rest of the spec and honour explicit values.""" +"""deploy.port and deploy.mongo.local default from the rest of the spec, honour explicit values and reach every bundle file.""" import pytest +import yaml +from click.testing import CliRunner from tests.unit.support.generated_apps import write_spec +from yaml_api_generator.cli import cli +from yaml_api_generator.cli.compiler import backend_generator from yaml_api_generator.generators.deploy.context_builder import build_deploy_context from yaml_api_generator.parsers.yaml_parser import load_and_generate_ir @@ -27,7 +31,7 @@ routes: list: {{ service: posts, method: GET, path: "", access: public }} deploy: - runtime: docker + runtime: {runtime} {deploy} """ @@ -36,29 +40,50 @@ POSTGRES = 'datastores:\n main: { type: postgres, dsn: "postgresql+asyncpg://db.example.com/app", default: true }' _VERSIONS = pytest.mark.parametrize("version", ["0.2", "0.3"]) +_TARGETS = pytest.mark.parametrize("target", ["fastapi", "raw", "drf"]) + + +def _spec(tmp_path, version: str, store: str, api: str = "", deploy: str = "", runtime: str = "docker"): + return write_spec(tmp_path, _SPEC.format(version=version, api=api, store=store, deploy=deploy, runtime=runtime)) def _deploy_context(tmp_path, version: str, store: str, api: str = "", deploy: str = ""): - spec = write_spec(tmp_path, _SPEC.format(version=version, api=api, store=store, deploy=deploy)) - return build_deploy_context(load_and_generate_ir(str(spec)).ir, "localhost", "ubuntu", "~/.ssh/id_rsa") + ir = load_and_generate_ir(str(_spec(tmp_path, version, store, api, deploy))).ir + return build_deploy_context(ir, "app:app", "localhost", "ubuntu", "~/.ssh/deploy") + + +def _bundle(tmp_path, spec, target: str = "fastapi"): + output = tmp_path / "bundle" + result = CliRunner().invoke(cli, ["build", str(spec), "deploy", "-o", str(output), "-t", target]) + assert result.exit_code == 0, result.output + return output @_VERSIONS @pytest.mark.parametrize( - ("store", "deploy", "local"), + ("store", "deploy", "local", "mongo_uri"), [ - (LOCAL_MONGO, "", True), - (REMOTE_MONGO, "", False), - (POSTGRES, "", False), - (REMOTE_MONGO, "mongo: { local: true }", True), - (LOCAL_MONGO, "mongo: { local: false }", False), + (LOCAL_MONGO, "", True, "mongodb://mongodb:27017"), + (REMOTE_MONGO, "", False, "mongodb://db.example.com:27017"), + (POSTGRES, "", False, None), + (REMOTE_MONGO, "mongo: { local: true }", True, "mongodb://mongodb:27017"), + (LOCAL_MONGO, "mongo: { local: false }", False, "mongodb://localhost:27017"), ], ids=["local-uri", "remote-uri", "postgres-only", "explicit-local", "explicit-remote"], ) -def test_mongo_local(tmp_path, version, store, deploy, local): +def test_mongo_local(tmp_path, version, store, deploy, local, mongo_uri): context = _deploy_context(tmp_path, version, store, deploy=deploy) - assert context.mongo_local is local - assert [ds.local for ds in context.datastores if ds.type == "mongodb"] == ([local] if "mongo:" in store else []) + mongo = [ds for ds in context.datastores if ds.type == "mongodb"] + assert context.local_mongo is local + assert [(ds.local, ds.app_env["MONGO_URI"]) for ds in mongo] == ([(local, mongo_uri)] if mongo_uri else []) + assert (f"MONGO_URI=${{MONGO_URI:-{mongo_uri}}}" in context.app_environment) is bool(mongo_uri) + + +@_VERSIONS +def test_remote_postgres_keeps_its_dsn(tmp_path, version): + context = _deploy_context(tmp_path, version, POSTGRES) + assert context.local_datastores == () + assert "DATABASE_URL=${DATABASE_URL:-postgresql+asyncpg://db.example.com/app}" in context.app_environment @_VERSIONS @@ -71,3 +96,72 @@ def test_port(tmp_path, version, api, deploy, port): context = _deploy_context(tmp_path, version, LOCAL_MONGO, api=api, deploy=deploy) assert context.port == port assert f"PORT=${{PORT:-{port}}}" in context.app_environment + + +@_TARGETS +def test_docker_serves_target_application_on_port(tmp_path, target): + bundle = _bundle(tmp_path, _spec(tmp_path, "0.3", REMOTE_MONGO, api="port: 9100"), target) + dockerfile = (bundle / "backend" / "Dockerfile").read_text().splitlines() + application = backend_generator(target).asgi_application + assert "ENV PORT=9100" in dockerfile + assert f'CMD exec uvicorn {application} --host 0.0.0.0 --port "$PORT"' in dockerfile + + +@_TARGETS +def test_systemd_serves_target_application_on_port(tmp_path, target): + bundle = _bundle(tmp_path, _spec(tmp_path, "0.3", REMOTE_MONGO, api="port: 9100", runtime="systemd"), target) + service = (bundle / "files" / "app.service").read_text().splitlines() + application = backend_generator(target).asgi_application + assert "Environment=PORT=9100" in service + assert f"ExecStart=/opt/app/venv/bin/uvicorn {application} --host 0.0.0.0 --port ${{PORT}}" in service + + +@pytest.mark.parametrize( + ("store", "deploy", "services"), + [ + (LOCAL_MONGO, "", ["app", "mongodb"]), + (LOCAL_MONGO, "mongo: { local: false }", ["app"]), + (POSTGRES, "", ["app"]), + ], + ids=["local-mongo", "explicit-remote", "postgres-only"], +) +def test_docker_runs_local_stores_in_compose_and_installs_docker(tmp_path, store, deploy, services): + bundle = _bundle(tmp_path, _spec(tmp_path, "0.3", store, deploy=deploy)) + assert list(yaml.safe_load((bundle / "docker-compose.yml").read_text())["services"]) == services + assert yaml.safe_load((bundle / "playbook.yml").read_text())[0]["roles"] == ["docker", "app"] + assert (bundle / "roles" / "docker" / "tasks" / "main.yml").exists() + assert not (bundle / "roles" / "mongodb").exists() + + +@pytest.mark.parametrize( + ("store", "deploy", "roles"), + [ + (LOCAL_MONGO, "", ["mongodb", "python", "app"]), + (REMOTE_MONGO, "mongo: { local: true }", ["mongodb", "python", "app"]), + (LOCAL_MONGO, "mongo: { local: false }", ["python", "app"]), + (POSTGRES, "", ["python", "app"]), + ], + ids=["local-uri", "explicit-local", "explicit-remote", "postgres-only"], +) +def test_systemd_installs_mongodb_only_for_a_local_mongo_store(tmp_path, store, deploy, roles): + bundle = _bundle(tmp_path, _spec(tmp_path, "0.3", store, deploy=deploy, runtime="systemd")) + assert yaml.safe_load((bundle / "playbook.yml").read_text())[0]["roles"] == roles + assert (bundle / "roles" / "mongodb").exists() is ("mongodb" in roles) + + +def test_readme_describes_only_the_compose_services(tmp_path): + bundle = _bundle(tmp_path, _spec(tmp_path, "0.3", POSTGRES)) + readme = (bundle / "README.md").read_text() + assert "mongodb" not in readme.lower() + assert "postgresql+asyncpg://db.example.com/app" in readme + + +@_VERSIONS +@pytest.mark.parametrize("runtime", ["docker", "systemd"]) +@pytest.mark.parametrize("local", ["true", "false"]) +def test_mongo_settings_without_a_mongo_store_rejected(tmp_path, version, runtime, local): + spec = _spec(tmp_path, version, POSTGRES, deploy=f"mongo: {{ local: {local} }}", runtime=runtime) + result = CliRunner().invoke(cli, ["build", str(spec), "deploy", "-o", str(tmp_path / "bundle")]) + assert result.exit_code != 0 + assert "deploy.mongo requires a MongoDB datastore" in result.output + assert not (tmp_path / "bundle").exists() diff --git a/tests/unit/test_deploy_file_datastores.py b/tests/unit/test_deploy_file_datastores.py index 6762bd36..cc2b5e0a 100644 --- a/tests/unit/test_deploy_file_datastores.py +++ b/tests/unit/test_deploy_file_datastores.py @@ -54,7 +54,7 @@ def _media(name, endpoint): def _compose(tmp_path, ir): out = tmp_path / "deploy" - DockerDeployGenerator().generate(build_deploy_context(ir, "localhost", "ubuntu", "~/.ssh/id"), out, tmp_path) + DockerDeployGenerator().generate(build_deploy_context(ir, "app:app", "localhost", "ubuntu", "~/.ssh/id"), out, tmp_path) return yaml.safe_load((out / "docker-compose.yml").read_text()) @@ -96,7 +96,7 @@ def test_remote_s3_store_adds_no_service(tmp_path): def test_systemd_gets_filesystem_path_env(tmp_path): - ctx = build_deploy_context(_ir(tmp_path, runtime="systemd"), "localhost", "ubuntu", "~/.ssh/id") + ctx = build_deploy_context(_ir(tmp_path, runtime="systemd"), "app:app", "localhost", "ubuntu", "~/.ssh/id") assert ctx.env_vars["FILES_UPLOADS_PATH"] == "./data/uploads" diff --git a/tests/unit/test_free_slug_fetch.py b/tests/unit/test_free_slug_fetch.py index 9eda9ab8..49c52d3e 100644 --- a/tests/unit/test_free_slug_fetch.py +++ b/tests/unit/test_free_slug_fetch.py @@ -152,7 +152,7 @@ async def test_update_step_multi_token_scoped_mutates_matching_record(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryUpdateStep( - "members", "id", False, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", None, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") ) result = await step( _ctx( @@ -171,7 +171,7 @@ async def test_replace_step_multi_token_wrong_parent_is_404(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryReplaceStep( - "members", "id", False, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", None, record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") ) with pytest.raises(NotFoundError): await step( @@ -192,7 +192,7 @@ async def test_replace_step_multi_token_wrong_parent_is_404(): async def test_update_step_v02_id_token_mutates_record(): repo = InMemoryRepository({"items": [{"id": "i1", "title": "old"}]}) repo.register_id_field("items", "id") - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") result = await step( _ctx(repo, {"id": "i1"}, collection="items", operation="update", body={"title": "new"}) ) @@ -204,7 +204,7 @@ async def test_update_step_renamed_token_mutates_record(): repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "old"}]}) repo.register_id_field("assetCategories", "id") step = RepositoryUpdateStep( - "assetCategories", "id", False, record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", None, record_locator("/api/v1/categories/{categoryId}", "id") ) result = await step( _ctx( @@ -222,7 +222,7 @@ async def test_update_step_renamed_token_mutates_record(): async def test_replace_step_v02_id_token_replaces_record(): repo = InMemoryRepository({"items": [{"id": "i1", "title": "old", "extra": "drop"}]}) repo.register_id_field("items", "id") - step = RepositoryReplaceStep("items", "id", False) + step = RepositoryReplaceStep("items", "id") result = await step( _ctx( repo, @@ -243,7 +243,7 @@ async def test_replace_step_renamed_token_replaces_record(): ) repo.register_id_field("assetCategories", "id") step = RepositoryReplaceStep( - "assetCategories", "id", False, record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", None, record_locator("/api/v1/categories/{categoryId}", "id") ) result = await step( _ctx( diff --git a/tests/unit/test_generated_entry_points.py b/tests/unit/test_generated_entry_points.py new file mode 100644 index 00000000..e3931060 --- /dev/null +++ b/tests/unit/test_generated_entry_points.py @@ -0,0 +1,67 @@ +"""Each generated target started through its own entry point serves on api.port.""" + +import contextlib +import os +import signal +import socket +import subprocess +import sys +import time + +import httpx +import pytest + +from tests.unit.support.generated_apps import generate, write_spec + +ENTRY_POINTS = { + "fastapi": ["app.py"], + "raw": ["app.py"], + "drf": ["manage.py", "runserver", "--noreload"], +} + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _first_response(port: int, server: subprocess.Popen, timeout: float = 30) -> httpx.Response: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline and server.poll() is None: + with contextlib.suppress(httpx.TransportError): + return httpx.get(f"http://127.0.0.1:{port}/api/missing", timeout=2) + time.sleep(0.2) + raise AssertionError(f"nothing answered on port {port}") + + +@pytest.mark.parametrize("target", ENTRY_POINTS) +def test_entry_point_serves_api_port(tmp_path, target): + port = _free_port() + spec = write_spec(tmp_path, f""" + version: "0.3" + api: {{ basePath: /api, port: {port} }} + mongo: {{ uri: "mongodb://localhost:27017", database: app }} + entities: + post: {{ fields: {{ id: {{ type: id }}, title: {{ type: string }} }} }} + repositories: {{ posts: {{ entity: post }} }} + services: {{ posts: {{ repository: posts }} }} + controllers: + posts: + base_path: /posts + routes: {{ list: {{ service: posts, method: GET, path: "", access: public }} }} + """) + backend = generate(spec, target) + server = subprocess.Popen( + [sys.executable, *ENTRY_POINTS[target]], + cwd=backend, + env={**os.environ, "DJANGO_SECRET_KEY": "test-secret", "JWT_SECRET": "test-jwt-secret-0123456789abcdef0123456789"}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + try: + assert _first_response(port, server).status_code == 404 + finally: + os.killpg(server.pid, signal.SIGTERM) + server.wait(10) diff --git a/tests/unit/test_mongo_graduation.py b/tests/unit/test_mongo_graduation.py index b2ebde49..ec1ed09c 100644 --- a/tests/unit/test_mongo_graduation.py +++ b/tests/unit/test_mongo_graduation.py @@ -208,7 +208,7 @@ async def test_skips_indexes_when_graduated(self, tmp_path): @pytest.mark.parametrize( ("target", "startup_file"), - [("drf", "api/apps.py"), ("raw", "app.py")], + [("drf", "project/asgi.py"), ("raw", "app.py")], ) def test_generated_mongo_targets_initialize_indexes_before_graduation( tmp_path, target, startup_file @@ -231,7 +231,7 @@ def test_generated_drf_views_package_does_not_import_absent_route_classes(tmp_pa @pytest.mark.parametrize( ("target", "startup_file"), - [("drf", "api/apps.py"), ("raw", "app.py")], + [("drf", "project/asgi.py"), ("raw", "app.py")], ) def test_generated_mongo_targets_skip_indexes_after_graduation( tmp_path, target, startup_file diff --git a/tests/unit/test_pagination.py b/tests/unit/test_pagination.py index 4fceb12a..7558e625 100644 --- a/tests/unit/test_pagination.py +++ b/tests/unit/test_pagination.py @@ -311,6 +311,7 @@ def test_offset_reject_over_max_raises(): (_offset(), {"offset": "invalid"}, "offset"), (_offset(), {"limit": "-1"}, "limit"), (_offset(), {"offset": "-1"}, "offset"), + (_offset(), {"limit": "0"}, "limit"), (_keyset(), {"limit": "0"}, "limit"), ], ) @@ -485,13 +486,13 @@ def test_keyset_without_sort_is_coded_error(tmp_path): assert "PAGINATION_KEYSET_SORT_REQUIRED" in _error_codes(excinfo) -def test_keyset_zero_default_is_coded_error(tmp_path): +def test_keyset_zero_default_is_schema_error(tmp_path): with pytest.raises(ConfigErrors) as excinfo: _load( tmp_path, "strategy: keyset\nsize: { default: 0, max: 100 }\nsort:\n - { field: id }", ) - assert "PAGINATION_INVALID_SIZE" in _error_codes(excinfo) + assert "SCHEMA_VALIDATION_ERROR" in _error_codes(excinfo) def test_keyset_non_unique_tiebreaker_is_coded_error(tmp_path): diff --git a/tests/unit/test_repository_steps.py b/tests/unit/test_repository_steps.py index 26f80e14..5729618f 100644 --- a/tests/unit/test_repository_steps.py +++ b/tests/unit/test_repository_steps.py @@ -3,6 +3,8 @@ import pytest from tests.unit.support.repositories import InMemoryRepository +from yaml_api_generator.ir.api import IRProviderBinding, IRRolesConfig +from yaml_api_generator.server.common.auth import hash_api_key from yaml_api_generator.server.common.request_context import RequestContext from yaml_api_generator.server.common.steps import NotFoundError from yaml_api_generator.server.common.steps.repositories import ( @@ -16,6 +18,7 @@ _inject_canonical_id, _prepare_payload, ) +from yaml_api_generator.server.common.server_context import RuntimeIdentityDescriptor def _make_ctx(**overrides) -> RequestContext: @@ -59,44 +62,67 @@ def test_generate_id_unknown_format_raises(): # --- Payload Preparation (244-249) --- -def test_prepare_payload_password_via_false(): +def _identity(*via: str, roles: IRRolesConfig | None = None) -> RuntimeIdentityDescriptor: + return RuntimeIdentityDescriptor( + name="users", + bindings=tuple(IRProviderBinding(provider=provider, identifier="email") for provider in via), + id_field="id", + roles=roles, + ) + + +def test_prepare_payload_without_identity_keeps_password(): payload = {"title": "hello", "password": "secret"} - result = _prepare_payload(payload, password_via=False) + result = _prepare_payload(payload, None) assert result == {"title": "hello", "password": "secret"} -def test_prepare_payload_password_via_true_no_password_key(): +def test_prepare_payload_password_identity_without_password_key(): payload = {"title": "hello"} - result = _prepare_payload(payload, password_via=True) + result = _prepare_payload(payload, _identity("password")) assert result == {"title": "hello"} -def test_prepare_payload_password_via_true_valid_password(): +def test_prepare_payload_password_identity_hashes_password(): payload = {"title": "hello", "password": "secret123"} - result = _prepare_payload(payload, password_via=True) + result = _prepare_payload(payload, _identity("password")) assert "password" not in result - assert "_auth" in result - assert "passwordHash" in result["_auth"] assert result["_auth"]["passwordHash"].startswith("pbkdf2$sha256$") -def test_prepare_payload_password_via_true_empty_password(): +def test_prepare_payload_rejects_empty_password(): with pytest.raises(ValueError, match="non-empty string"): - _prepare_payload({"password": ""}, password_via=True) + _prepare_payload({"password": ""}, _identity("password")) -def test_prepare_payload_password_via_true_non_string_password(): +def test_prepare_payload_rejects_non_string_password(): with pytest.raises(ValueError, match="non-empty string"): - _prepare_payload({"password": 123}, password_via=True) + _prepare_payload({"password": 123}, _identity("password")) def test_prepare_payload_existing_auth_dict_merged(): - payload = {"_auth": {"apiKey": "abc123"}, "password": "secret"} - result = _prepare_payload(payload, password_via=True) - assert "apiKey" in result["_auth"] + payload = {"_auth": {"provider": "abc123"}, "password": "secret"} + result = _prepare_payload(payload, _identity("password")) + assert result["_auth"]["provider"] == "abc123" assert "passwordHash" in result["_auth"] +def test_prepare_payload_stores_api_key_digest(): + result = _prepare_payload({"apiKey": "key-1"}, _identity("apiKey")) + assert result["apiKey"] == hash_api_key("key-1") != "key-1" + + +def test_prepare_payload_assigns_default_role_on_create_only(): + identity = _identity("apiKey", roles=IRRolesConfig(values=("reader",), default="reader")) + assert _prepare_payload({}, identity, is_create=True)["_auth"] == {"roles": "reader"} + assert "_auth" not in _prepare_payload({}, identity) + + +def test_canonical_id_hides_credentials(): + document = {"id": "1", "_auth": {}, "password": "x", "apiKey": "digest", "name": "n"} + assert _inject_canonical_id(document, "id") == {"id": "1", "name": "n"} + + # --- Canonical ID Normalization (250-253) --- @@ -176,7 +202,7 @@ async def test_repository_create_step_generates_id(): repo = InMemoryRepository() repo.register_id_field("items", "id") ctx = _make_ctx(repository=repo, body={"title": "new"}) - step = RepositoryCreateStep("items", "id", "uuidv4", False) + step = RepositoryCreateStep("items", "id", "uuidv4") result = await step(ctx) assert result.response["id"] is not None assert uuid.UUID(result.response["id"]).version == 4 @@ -191,7 +217,7 @@ async def test_repository_create_step_natural_pk_no_id_generated(): collection="countries", body={"code": "US", "name": "United States"}, ) - step = RepositoryCreateStep("countries", "code", "natural", False) + step = RepositoryCreateStep("countries", "code", "natural") result = await step(ctx) assert result.response["code"] == "US" assert "id" not in result.response or result.response.get("id") is None @@ -208,7 +234,7 @@ async def test_repository_update_step(): url_params={"id": "abc"}, body={"title": "new"}, ) - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") result = await step(ctx) assert result.response["title"] == "new" @@ -224,7 +250,7 @@ async def test_repository_update_step_not_found(): url_params={"id": "missing"}, body={"title": "x"}, ) - step = RepositoryUpdateStep("items", "id", False) + step = RepositoryUpdateStep("items", "id") with pytest.raises(NotFoundError): await step(ctx) @@ -242,7 +268,7 @@ async def test_repository_replace_step(): url_params={"id": "abc"}, body={"id": "abc", "title": "replaced"}, ) - step = RepositoryReplaceStep("items", "id", False) + step = RepositoryReplaceStep("items", "id") result = await step(ctx) assert result.response["title"] == "replaced" @@ -289,6 +315,6 @@ async def test_repository_step_non_dict_body(): repo = InMemoryRepository() repo.register_id_field("items", "id") ctx = _make_ctx(repository=repo, body="not a dict") - step = RepositoryCreateStep("items", "id", "uuidv4", False) + step = RepositoryCreateStep("items", "id", "uuidv4") with pytest.raises(ValueError, match="Request body must be a JSON object"): await step(ctx) diff --git a/tests/unit/test_schema_rejection.py b/tests/unit/test_schema_rejection.py index 6f6aaa30..13d8d781 100644 --- a/tests/unit/test_schema_rejection.py +++ b/tests/unit/test_schema_rejection.py @@ -32,6 +32,7 @@ OIDC_V01 = {"type": "oidc", "client_id": "id", "client_secret": "secret"} SAML = {"type": "saml", "idp_metadata_url": "https://idp/metadata", "sp_entity_id": "sp"} ACCESS_RULE = ("controllers", "posts", "access", 0) +ROLES = {"values": ["reader"], "default": "reader"} TITLE = ("fields", "title") # (versions, path, value, message fragment) @@ -79,6 +80,20 @@ ), "typo-root": (("0.1", "0.2", "0.3"), ("colections",), {}, "'colections' was unexpected"), "typo-field": (("0.2", "0.3"), ("entities", "post", *TITLE, "requird"), True, "'requird' was unexpected"), + "v01-max-page-size-zero": (("0.1",), ("api", "pagination", "maxPageSize"), 0, "less than the minimum of 1"), + "v01-default-page-size-negative": ( + ("0.1",), ("api", "pagination", "defaultPageSize"), -1, "less than the minimum of 1" + ), + "v01-via-empty": ( + ("0.1",), ("collections", "posts", "auth"), {"identifier": "title", "via": []}, "is not valid under any" + ), + "v01-roles-on-base-collection": (("0.1",), ("collections", "posts", "roles"), ROLES, "'type' is a required property"), + "roles-on-base-entity": (("0.2", "0.3"), ("entities", "post", "roles"), ROLES, "'type' is a required property"), + "deploy-port-zero": (("0.2", "0.3"), ("deploy",), {"runtime": "docker", "port": 0}, "less than the minimum of 1"), + "deploy-port-negative": (("0.2", "0.3"), ("deploy",), {"runtime": "docker", "port": -5}, "less than the minimum of 1"), + "deploy-port-above-range": ( + ("0.2", "0.3"), ("deploy",), {"runtime": "docker", "port": 65536}, "greater than the maximum of 65535" + ), } CASES = [ @@ -140,6 +155,7 @@ def test_mongo_block_beside_record_datastores_rejected(tmp_path, version): spec = {**BASE_SPECS[version], "mongo": MONGO, "datastores": {"main": POSTGRES}} errors = _load_errors(tmp_path, spec) assert [(error.code, error.path) for error in errors] == [("SCHEMA_VALIDATION_ERROR", "mongo")] + assert "mutually exclusive" in errors[0].message def test_mongo_block_beside_file_datastores_accepted(tmp_path): diff --git a/tests/unit/test_v01_transformer.py b/tests/unit/test_v01_transformer.py index a5a8646d..f1909742 100644 --- a/tests/unit/test_v01_transformer.py +++ b/tests/unit/test_v01_transformer.py @@ -218,6 +218,11 @@ def test_v01_roles_hierarchy_unknown_role_rejected(tmp_path) -> None: ] +def test_v01_roles_default_must_be_declared(tmp_path) -> None: + roles = "roles: { values: [reader, editor], default: admin }" + assert _platform_errors(tmp_path, roles=roles) == [("ROLES_DEFAULT_UNKNOWN_ROLE", "collections.users.roles.default")] + + def test_v01_shared_validators_check_collections(tmp_path) -> None: assert _platform_errors(tmp_path, field="class") == [("INVALID_FIELD_NAME", "collections.posts.fields.class")] assert [code for code, _ in _platform_errors(tmp_path, min_length=-1)] == ["FIELD_CONSTRAINT_INVALID"] diff --git a/uv.lock b/uv.lock index a6738577..de78219e 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,21 @@ resolution-markers = [ "python_full_version < '3.12'", ] +[[package]] +name = "adrf" +version = "0.1.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-property" }, + { name = "django", version = "5.2.18", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "django", version = "6.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "djangorestframework" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/f3/2e4647d679c1c3cb8f7316eabc85d4fafe396318a5aa389f2ef14a2df103/adrf-0.1.14.tar.gz", hash = "sha256:c6ded6771a4a2a65c8dad3d3bf027cf0bb7b01025f8e9dff18c9a58920edeac6", size = 19256, upload-time = "2026-08-11T23:39:39.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/30/9c482ba6256b0c4b57a4ad6a5da918f57064689d0d3d9595515707222ff9/adrf-0.1.14-py3-none-any.whl", hash = "sha256:dcf03cb6fbeb5d37dcb819740c17dd40db36481bbbb049f9fa8f39675747607b", size = 22763, upload-time = "2026-08-11T23:39:38.412Z" }, +] + [[package]] name = "aiobotocore" version = "3.9.1" @@ -227,6 +242,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "asgiref" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/90/f5fce14e95fccf53b58e8e2c47cd84d40d105d64073ae1ac75166db150e0/asgiref-3.13.0.tar.gz", hash = "sha256:034cc8cbf72d00553b4be256703ac545120b4e50af13b49dec99da899aeeecb4", size = 52941, upload-time = "2026-10-23T08:13:14.715Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/b0/ec211e1f7966247cb6469158d856bac9aa4c2fe13621bd6a352e4f850453/asgiref-3.13.0-py3-none-any.whl", hash = "sha256:f9596a2434512686473d50f96ff6207fbba49ae1a785dfac614df4aeb085674d", size = 27593, upload-time = "2026-10-23T08:13:13.394Z" }, +] + +[[package]] +name = "async-property" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/12/900eb34b3af75c11b69d6b78b74ec0fd1ba489376eceb3785f787d1a0a1d/async_property-0.2.2.tar.gz", hash = "sha256:17d9bd6ca67e27915a75d92549df64b5c7174e9dc806b30a3934dc4ff0506380", size = 16523, upload-time = "2023-07-03T17:21:55.688Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/80/9f608d13b4b3afcebd1dd13baf9551c95fc424d6390e4b1cfd7b1810cd06/async_property-0.2.2-py2.py3-none-any.whl", hash = "sha256:8924d792b5843994537f8ed411165700b27b2bd966cefc4daeefc1253442a9d7", size = 9546, upload-time = "2023-07-03T17:21:54.293Z" }, +] + [[package]] name = "asyncpg" version = "0.31.0" @@ -454,6 +487,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "django" +version = "5.2.18" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12'", +] +dependencies = [ + { name = "asgiref", marker = "python_full_version < '3.12'" }, + { name = "sqlparse", marker = "python_full_version < '3.12'" }, + { name = "tzdata", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/fe/79e692b430c7721bf4b812c328bfe687e73e3f28f56611a6a18a97e887cc/django-5.2.18.tar.gz", hash = "sha256:461c5dd06d2ea16bd5ca37d3f46e4def1d6b0fe7588c6f4e2119517bb0af8b2d", size = 10910238, upload-time = "2026-10-06T13:01:09.371Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/01/6568ec52b26548ca1b83d33249f7274db3bf74698f42c0da42ec9db1a24a/django-5.2.18-py3-none-any.whl", hash = "sha256:92ed81d500be6408ecd704d7bd1366c534f30427bffcc63c5fefb129561aec7c", size = 8316071, upload-time = "2026-10-06T13:01:04.413Z" }, +] + +[[package]] +name = "django" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +dependencies = [ + { name = "asgiref", marker = "python_full_version >= '3.12'" }, + { name = "sqlparse", marker = "python_full_version >= '3.12'" }, + { name = "tzdata", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/24/20d187a22ac821288b53c53b8f696019dcb0c7b4f973f052ade19392c74d/django-6.1.2.tar.gz", hash = "sha256:a1e92451ccb8b514e91bbb3b6d186d20b4030558f116b5d9de6535455ff210b7", size = 11251425, upload-time = "2026-10-06T12:53:29.622Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/83/90ff2dbfac7b519ed77620bdbb827cd2d1a83faf00d076bf95056428bce7/django-6.1.2-py3-none-any.whl", hash = "sha256:141efee6ec64d1db6db90683bf734c550102450f444fb099063b0be1bd27d991", size = 8420684, upload-time = "2026-10-06T12:53:17.381Z" }, +] + +[[package]] +name = "django-cors-headers" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "django", version = "5.2.18", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "django", version = "6.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/39/55822b15b7ec87410f34cd16ce04065ff390e50f9e29f31d6d116fc80456/django_cors_headers-4.9.0.tar.gz", hash = "sha256:fe5d7cb59fdc2c8c646ce84b727ac2bca8912a247e6e68e1fb507372178e59e8", size = 21458, upload-time = "2025-09-18T10:40:52.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/d8/19ed1e47badf477d17fb177c1c19b5a21da0fd2d9f093f23be3fb86c5fab/django_cors_headers-4.9.0-py3-none-any.whl", hash = "sha256:15c7f20727f90044dcee2216a9fd7303741a864865f0c3657e28b7056f61b449", size = 12809, upload-time = "2025-09-18T10:40:50.843Z" }, +] + +[[package]] +name = "djangorestframework" +version = "3.18.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django", version = "5.2.18", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "django", version = "6.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8d/b2/1cee76b188301f7d03169ca77fab2b3d3ec5d01b3469189f944af2d7f11a/djangorestframework-3.18.3.tar.gz", hash = "sha256:446a9b352e7eff630421ab3f2328bd2401b109a9470afa4a31189994911ed030", size = 923563, upload-time = "2026-10-06T21:33:51.521Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/dd/de5f4543765bec94e036a5e9593a42f8fa7efec8de78eb56b9ca834385f3/djangorestframework-3.18.3-py3-none-any.whl", hash = "sha256:8544bb674846731b1e3c9b309236ee1dc412905a0aa725be2ec193ca950a7d12", size = 903986, upload-time = "2026-10-06T21:33:49.917Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -463,6 +557,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] +[[package]] +name = "drf-spectacular" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django", version = "5.2.18", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "django", version = "6.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "djangorestframework" }, + { name = "inflection" }, + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/43/41d25039a6a53545420ebc98eb9f877ec9fe30c7bd03fefabcaf9b953af7/drf_spectacular-0.30.0.tar.gz", hash = "sha256:53e79e7ba00e240441b63c32273754a5368e4c2ab44a19f2595277cc1cd559c9", size = 252311, upload-time = "2026-07-06T11:29:46.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/56/74dd7b45bbde6d24494220b98d6961cb1200b63a1800332b430daa2c4551/drf_spectacular-0.30.0-py3-none-any.whl", hash = "sha256:006cf5921ebe20a9bd24f7c846261ebbf78780be5961b0d6e87afaa82afd62ff", size = 111150, upload-time = "2026-07-06T11:29:45.12Z" }, +] + [[package]] name = "fastapi" version = "0.136.1" @@ -707,6 +819,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" }, ] +[[package]] +name = "inflection" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091, upload-time = "2020-08-22T08:16:29.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454, upload-time = "2020-08-22T08:16:27.816Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -1778,6 +1899,15 @@ asyncio = [ { name = "greenlet" }, ] +[[package]] +name = "sqlparse" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/d3/3f06a1006f2261d1342aefb3c71eed02f5d4ca5bdbecd86ebc12ad38306e/sqlparse-0.6.0.tar.gz", hash = "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9", size = 178477, upload-time = "2026-08-13T19:16:06.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/50/f00935da0ec7cbf325f8dc4f772ae46fbc7b672dd62876e73f0a94adda57/sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f", size = 50070, upload-time = "2026-08-13T19:16:04.062Z" }, +] + [[package]] name = "stackform" version = "1.0.0" @@ -1816,7 +1946,13 @@ dev = [ [package.dev-dependencies] dev = [ + { name = "adrf" }, { name = "aiosqlite" }, + { name = "django", version = "5.2.18", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "django", version = "6.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django-cors-headers" }, + { name = "djangorestframework" }, + { name = "drf-spectacular" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] @@ -1852,7 +1988,12 @@ provides-extras = ["dev"] [package.metadata.requires-dev] dev = [ + { name = "adrf", specifier = ">=0.1.14" }, { name = "aiosqlite", specifier = ">=0.22.1" }, + { name = "django", specifier = ">=5.2.18" }, + { name = "django-cors-headers", specifier = ">=4.9.0" }, + { name = "djangorestframework", specifier = ">=3.18.3" }, + { name = "drf-spectacular", specifier = ">=0.30.0" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, ] @@ -1900,6 +2041,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, +] + [[package]] name = "urllib3" version = "2.8.0" From 7acb78d7dc7dbd52c4b7e77a0445a5710e5744f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 20:00:07 +0100 Subject: [PATCH 15/89] Store datetimes as canonical UTC strings, fill and reject readonly fields, validate bodies after access checks and give DTOs a partial update payload SQL datetime columns bind ISO strings; raw payloads use the shared Model helper; raw and DRF responses hide stored credentials. --- .../backend/builders/model_builder.py | 46 ++++++---- .../backend/builders/route_builder.py | 5 +- .../generators/backend/fastapi/generator.py | 1 + .../generators/backend/fastapi/sql_filters.py | 10 ++- .../backend/fastapi/templates/models.py.jinja | 8 ++ .../fastapi/templates/repository_sql.py.jinja | 1 + .../generators/backend/fastapi/type_mapper.py | 14 ++- .../generators/backend/raw/generator.py | 3 +- .../raw/templates/login_routes.py.jinja | 10 ++- .../backend/raw/templates/routes.py.jinja | 6 +- .../backend/raw/templates/validation.py.jinja | 60 ++----------- .../shared/templates/repository.py.jinja | 6 +- .../generators/backend/utils.py | 2 +- src/yaml_api_generator/ir/__init__.py | 4 +- src/yaml_api_generator/ir/api.py | 8 +- src/yaml_api_generator/runtime/fields.py | 49 +++++++++-- src/yaml_api_generator/runtime/payloads.py | 59 +++++++++++-- src/yaml_api_generator/runtime/sql_types.py | 20 +++++ .../server/common/repositories/sql_schema.py | 3 +- .../server/common/steps/validation.py | 14 +-- .../server/fastapi/factory.py | 88 +++++-------------- tests/unit/support/payload_requests.py | 61 ++++++++++--- tests/unit/test_fastapi_factory.py | 20 ++--- tests/unit/test_generators.py | 2 +- tests/unit/test_payload_models.py | 4 +- tests/unit/test_request_access_generation.py | 2 +- tests/unit/test_validation_step.py | 6 +- 27 files changed, 305 insertions(+), 207 deletions(-) create mode 100644 src/yaml_api_generator/runtime/sql_types.py diff --git a/src/yaml_api_generator/generators/backend/builders/model_builder.py b/src/yaml_api_generator/generators/backend/builders/model_builder.py index e570a1ab..977da66c 100644 --- a/src/yaml_api_generator/generators/backend/builders/model_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/model_builder.py @@ -2,7 +2,8 @@ Each collection gets one ``Fields`` model holding its writable fields; the create and replace payloads extend it and the update payload is -its partial form, so no field is declared twice. +its partial form, so no field is declared twice. A DTO is its own base and +gets the same partial update form. """ from yaml_api_generator.ir import ( @@ -12,7 +13,7 @@ IRIdentityCollection, IRRepository, SECRET_FIELD, - writable_fields, + payload_fields, ) from yaml_api_generator.utils.strings import to_pascal_case from ..compile_context import FieldDescriptor, ModelDescriptor @@ -22,10 +23,15 @@ _PASSWORD = FieldDescriptor("password", SECRET_FIELD) -def payload_name(collection: str, operation: str) -> str: - """``Fields`` for the shared base, else ``Payload``.""" - prefix = "".join(part.capitalize() for part in collection.split("_")) - return f"{prefix}Fields" if operation == "fields" else f"{prefix}{operation.capitalize()}Payload" +def payload_name(owner: str, operation: str) -> str: + """``Fields`` for the shared base, else ``Payload``.""" + suffix = "Fields" if operation == "fields" else f"{operation.capitalize()}Payload" + return f"{to_pascal_case(owner)}{suffix}" + + +def dto_payload_name(dto: str, operation: str) -> str: + """The model a route typed by ``dto`` validates its ``operation`` body with.""" + return payload_name(dto, "update") if operation == "update" else to_pascal_case(dto) def _descriptors(fields: dict[str, dict]) -> list[FieldDescriptor]: @@ -44,20 +50,28 @@ def build_models( return [ model for name, col in collections.items() - for model in self._family( - col, identity_map.get(name), repositories.get(name) - ) + for model in self._family(col, identity_map.get(name), repositories.get(name)) ] def build_dto_models(self, dtos: dict[str, IRDto]) -> list[ModelDescriptor]: return [ - ModelDescriptor( - name=to_pascal_case(dto.name), - collection=dto.name, - operation="dto", - fields=_descriptors(writable_fields(dto.fields)), - ) + model for dto in dtos.values() + for model in ( + ModelDescriptor( + name=dto_payload_name(dto.name, "dto"), + collection=dto.name, + operation="dto", + fields=_descriptors(payload_fields(dto.fields)), + ), + ModelDescriptor( + name=dto_payload_name(dto.name, "update"), + collection=dto.name, + operation="update", + fields=[], + base=dto_payload_name(dto.name, "dto"), + ), + ) ] def _family( @@ -79,7 +93,7 @@ def _family( name=base, collection=col.name, operation="fields", - fields=_descriptors(writable_fields(col.fields, col.files.keys())), + fields=_descriptors(payload_fields(col.fields, col.files.keys())), ), *( ModelDescriptor( diff --git a/src/yaml_api_generator/generators/backend/builders/route_builder.py b/src/yaml_api_generator/generators/backend/builders/route_builder.py index d506468f..7f309882 100644 --- a/src/yaml_api_generator/generators/backend/builders/route_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/route_builder.py @@ -6,8 +6,7 @@ import re from yaml_api_generator.ir import IRDocument, IRDto, IRSymbolRef -from yaml_api_generator.utils.strings import to_pascal_case -from .model_builder import payload_name +from .model_builder import dto_payload_name, payload_name from ..compile_context import ( AccessRuleDescriptor, DocumentBindingDescriptor, @@ -115,7 +114,7 @@ def build_routes( needs_request = self._needs_request(access_rules) body_dto = ir_route.body_dto(dtos) - typed_body_model = to_pascal_case(body_dto.name) if body_dto else None + typed_body_model = dto_payload_name(body_dto.name, ir_route.operation) if body_dto else None if ir_route.handler is not None or ir_route.collection is None: routes.append( diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 4d702db7..14e5bb54 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -409,6 +409,7 @@ def _render_dependencies( (deps_dir / "postgres.py").write_text( env.get_template("postgres.py.jinja").render(ctx=context) ) + (deps_dir / "sql_types.py").write_text(runtime_module_source("sql_types")) if context.auth.has_app_oidc_auth: (deps_dir / "login_codes.py").write_text( env.get_template("login_codes.py.jinja").render(auth=context.auth) diff --git a/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py b/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py index b1fc74ad..ac98f959 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py +++ b/src/yaml_api_generator/generators/backend/fastapi/sql_filters.py @@ -2,6 +2,8 @@ from typing import Any +import sqlalchemy as sa + from yaml_api_generator.server.common.repositories.sql_schema import column_type @@ -9,7 +11,7 @@ def sql_column(name: str, cfg: dict[str, Any], id_field: str) -> str: field_type = cfg.get("type", "string") is_pk = name == id_field or field_type == "id" - parts = [f'"{name}"', f"sa.{column_type(cfg)!r}"] + parts = [f'"{name}"', _constructor(column_type(cfg))] if is_pk: parts.append("primary_key=True") elif cfg.get("required"): @@ -17,3 +19,9 @@ def sql_column(name: str, cfg: dict[str, Any], id_field: str) -> str: if cfg.get("unique"): parts.append("unique=True") return f"sa.Column({', '.join(parts)})" + + +def _constructor(column: sa.types.TypeEngine) -> str: + """The source building ``column``; SQLAlchemy's own types are reached through ``sa``.""" + namespace = "sa." if type(column).__module__.startswith("sqlalchemy.") else "" + return f"{namespace}{column!r}" diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja index 6c255c9b..2a5468f3 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/models.py.jinja @@ -6,11 +6,19 @@ {% if declaration.attributes is defined %} class {{ declaration.name }}({{ declaration.base }}): +{% if declaration.readonly %} + _readonly = frozenset({{ declaration.readonly | sort | list | py_literal }}) +{% endif %} +{% if declaration.server_defaults %} + _server_defaults = {{ declaration.server_defaults | py_literal }} +{% endif %} {% for attribute in declaration.attributes %} {{ attribute.name }}: {{ attribute.annotation }}{% if attribute.default is not none %} = {{ attribute.default }}{% endif %} {% else %} +{% if not declaration.readonly %} pass +{% endif %} {% endfor %} {% else %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja index 087ab480..d7683db3 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository_sql.py.jinja @@ -2,6 +2,7 @@ import sqlalchemy as sa from dependencies.postgres import metadata +from dependencies.sql_types import IsoDateTime from ._sql_ops import SQLRepositoryBase{% for op in operations %}, {{ op | capitalize }}Op{% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py b/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py index 310ad00f..b1ab4bcb 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py +++ b/src/yaml_api_generator/generators/backend/fastapi/type_mapper.py @@ -10,10 +10,9 @@ _IMPORTS: dict[str, tuple[str, ...]] = { - "datetime": ("datetime",), "typing": ("Annotated", "Any", "Literal"), "pydantic": ("Field",), - "dependencies.payloads": ("OpenPayload", "Payload", "formatted", "partial"), + "dependencies.payloads": ("OpenPayload", "Payload", "Timestamp", "formatted", "partial"), } @@ -26,9 +25,13 @@ class Attribute: @dataclass(frozen=True) class ClassDeclaration: + """A payload class; ``readonly`` fields are rejected and ``server_defaults`` added on dump.""" + name: str base: str attributes: tuple[Attribute, ...] + readonly: frozenset[str] = frozenset() + server_defaults: dict[str, Any] | None = None @dataclass(frozen=True) @@ -80,11 +83,16 @@ def _class( ) -> None: if any(declaration.name == name for declaration in self.declarations): raise ValueError(f"Two payload models are both named '{name}'") + readonly = {field: config for field, config in properties.items() if config.get("readonly")} attributes = tuple( self._attribute(f"{owner}{to_pascal_case(field)}", field, config) for field, config in properties.items() + if field not in readonly + ) + server_defaults = {field: config["default"] for field, config in readonly.items() if "default" in config} + self.declarations.append( + ClassDeclaration(name, base, attributes, frozenset(readonly), server_defaults or None) ) - self.declarations.append(ClassDeclaration(name, base, attributes)) def _attribute(self, nested_name: str, name: str, config: dict[str, Any]) -> Attribute: annotation = self._annotation(config, nested_name) diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index 206b949f..c70bdd0e 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -8,6 +8,7 @@ IdentityCollectionDescriptor, ) from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields +from yaml_api_generator.ir import SECRET_FIELD from yaml_api_generator.utils.strings import to_pascal_case from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment from ..utils import ( @@ -262,7 +263,7 @@ def _render_login_routes( routes_dir.mkdir(parents=True, exist_ok=True) template = env.get_template("login_routes.py.jinja") - content = template.render(ctx=context) + content = template.render(ctx=context, secret=SECRET_FIELD) (routes_dir / "login.py").write_text(content) def _render_dependencies( diff --git a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja index f535760b..d0931b9b 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/login_routes.py.jinja @@ -2,16 +2,18 @@ from typing import Any from dependencies.auth import verify_password from dependencies.jwt_utils import create_jwt_token -import validation +from fields import Model {% for identity in ctx.auth.identity_collections %} {% if identity.has_password and identity.identifier %} +{% set login_payload = identity.name | capitalize_collection ~ "LoginPayload" %} +{{ login_payload }} = Model({{ {identity.identifier: secret, "password": secret} | py_literal }}) + + async def handle_login_{{ identity.name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: - if body is None: - return 422, {"detail": "Request body is required"} try: - data = validation.validate_{{ identity.name }}_login(body) + data = {{ login_payload }}.conform(body) except ValueError as exc: return 422, {"detail": str(exc)} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 5fca388c..2a34725c 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -188,13 +188,13 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {% elif route.operation == "create" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) + data = validation.{{ route.typed_body_model or route.body_model }}.conform(body) document = await service.create(data) return 200, document {% elif route.operation == "update" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) + data = validation.{{ route.typed_body_model or route.body_model }}.conform(body) {% if route is needs_post_fetch %} {% if route.record_parent_params %} existing = await service.get_by({{ record_criteria(route) }}) @@ -212,7 +212,7 @@ async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[ {% elif route.operation == "replace" %} if body is None: return 422, {"detail": "Request body is required"} - data = validation.validate({{ (route.typed_body_model or route.body_model) | tojson }}, body) + data = validation.{{ route.typed_body_model or route.body_model }}.conform(body) {% if route is needs_post_fetch %} {% if route.record_parent_params %} existing = await service.get_by({{ record_criteria(route) }}) diff --git a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja index 06f8a2c4..e8352682 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/validation.py.jinja @@ -1,57 +1,13 @@ -from typing import Any, NamedTuple - -from fields import conform_object - - -class _Payload(NamedTuple): - base: str | None - properties: dict[str, dict[str, Any]] - partial: bool = False - - -_PAYLOADS = { +from fields import Model, partial {% for model in models %} - {{ model.name | tojson }}: _Payload( - {{ model.base | tojson if model.base else "None" }}, - {{ "{" }}{% for field in model.fields %}{{ field.name | tojson }}: {{ field.config | py_literal }}{{ ", " if not loop.last }}{% endfor %}{{ "}" }}, -{% if model.operation == "update" %} - partial=True, -{% endif %} - ), -{% endfor %} -} +{% set properties %}{{ "{" }}{% for field in model.fields %}{{ field.name | tojson }}: {{ field.config | py_literal }}{{ ", " if not loop.last }}{% endfor %}{{ "}" }}{% endset %} +{% if model.operation == "update" %} +{{ model.name }} = partial({{ model.base }}) +{% elif model.base %} +{{ model.name }} = {{ model.base }}{% if model.fields %}.extend({{ properties }}){% endif %} -def _properties(model: str) -> dict[str, dict[str, Any]]: - payload = _PAYLOADS[model] - return {**(_properties(payload.base) if payload.base else {}), **payload.properties} - - -def validate(model: str, body: Any) -> dict[str, Any]: - """``body`` checked against the payload model ``model``.""" - return conform_object(_properties(model), body, partial=_PAYLOADS[model].partial) - -{% for identity in auth.identity_collections %} -{% if identity.has_password and identity.identifier %} -def validate_{{ identity.name }}_login(body: dict) -> dict: - """Validate login payload for {{ identity.name }}.""" - if not isinstance(body, dict): - raise ValueError("Request body must be a JSON object") - - known_fields = { {{ identity.identifier | tojson }}, "password"} - unknown = set(body.keys()) - known_fields - if unknown: - raise ValueError(f"Unknown fields: {', '.join(sorted(unknown))}") - - if {{ identity.identifier | tojson }} not in body or not isinstance(body[{{ identity.identifier | tojson }}], str) or not body[{{ identity.identifier | tojson }}].strip(): - raise ValueError("Missing required field: {{ identity.identifier }}") - if "password" not in body or not isinstance(body["password"], str) or not body["password"].strip(): - raise ValueError("Missing required field: password") - - return { - {{ identity.identifier | tojson }}: body[{{ identity.identifier | tojson }}], - "password": body["password"], - } - +{% else %} +{{ model.name }} = Model({{ properties }}) {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index c2bc6198..771d4b78 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -19,6 +19,8 @@ from {{ conflict_import }} import ConflictError from {{ pagination_import }} import Page, Primitives, expand_bound +_HIDDEN_FIELDS = frozenset({"_id", "_auth"}) + _PAGINATION_COMPARATORS = { "gt": "$gt", "gte": "$gte", @@ -88,10 +90,10 @@ class {{ class_name }}Repository: return {self.ID_FIELD: document_id} def _normalize(self, doc: dict[str, Any] | None) -> dict[str, Any] | None: - """Hide the datastore's own identifier; the service layer hides domain-internal fields.""" + """Hide the datastore's own identifier and the stored credentials.""" if doc is None: return None - return {k: v for k, v in doc.items() if k != "_id"} + return {k: v for k, v in doc.items() if k not in _HIDDEN_FIELDS} async def list(self) -> list[dict[str, Any]]: cursor = self._col.find({}) diff --git a/src/yaml_api_generator/generators/backend/utils.py b/src/yaml_api_generator/generators/backend/utils.py index 10ab1474..edba1cad 100644 --- a/src/yaml_api_generator/generators/backend/utils.py +++ b/src/yaml_api_generator/generators/backend/utils.py @@ -12,7 +12,7 @@ _RUNTIME_MODULES = frozenset( - {"dpop", "pagination", "fields", "payloads", "files", "multipart", "file_requests", "filesystem_store", "s3_store"} + {"dpop", "pagination", "fields", "payloads", "files", "multipart", "file_requests", "filesystem_store", "s3_store", "sql_types"} ) diff --git a/src/yaml_api_generator/ir/__init__.py b/src/yaml_api_generator/ir/__init__.py index e84f8959..7f775a0a 100644 --- a/src/yaml_api_generator/ir/__init__.py +++ b/src/yaml_api_generator/ir/__init__.py @@ -18,7 +18,7 @@ IRWhereConjunct, WhereRhs, SECRET_FIELD, - writable_fields, + payload_fields, IRMongo, IRMongoDatastore, IRPolicy, @@ -110,7 +110,7 @@ "IRWhereConjunct", "WhereRhs", "SECRET_FIELD", - "writable_fields", + "payload_fields", "IRMongo", "IRMongoDatastore", "IRPolicy", diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index b59e7d70..e0d873d5 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -252,16 +252,18 @@ class IRCollection: """The config of a credential a request sends, such as a password.""" -def writable_fields( +def payload_fields( fields: dict[str, Any], excluded: Collection[str] = () ) -> dict[str, dict[str, Any]]: - """The field configs a client may send: no ids, readonly, auth or ``excluded`` fields.""" + """The field configs a request payload is checked against: no ids, auth or ``excluded`` fields. + + Readonly fields stay, since their keys are rejected and their defaults filled in. + """ return { name: config for name, config in fields.items() if isinstance(config, dict) and config.get("type") != "id" - and not config.get("readonly", False) and name != "_auth" and name not in excluded } diff --git a/src/yaml_api_generator/runtime/fields.py b/src/yaml_api_generator/runtime/fields.py index 228b4b41..06dbf281 100644 --- a/src/yaml_api_generator/runtime/fields.py +++ b/src/yaml_api_generator/runtime/fields.py @@ -3,13 +3,15 @@ Shared verbatim by the development server and every generated backend, so a field config means the same thing wherever a request lands. ``conform`` checks one value, ``conform_object`` a key/value map and ``conform_inputs`` a service's -inputs; each returns the value with the declared defaults filled in. +inputs; each returns the value with the declared defaults filled in. A +``Model`` names the properties of one request payload. """ import re from collections.abc import Callable from copy import deepcopy -from datetime import datetime +from dataclasses import dataclass, replace +from datetime import UTC, datetime from typing import Any from urllib.parse import urlparse from uuid import UUID @@ -88,14 +90,23 @@ def _boolean(config: dict[str, Any], value: Any, path: str) -> bool: return value +def iso_utc(moment: datetime) -> str: + """``moment`` as an ISO 8601 UTC timestamp; a naive one is taken as UTC.""" + return (moment if moment.tzinfo else moment.replace(tzinfo=UTC)).astimezone(UTC).isoformat() + + +def canonical_datetime(value: str) -> str: + """``value`` in the form ``iso_utc`` gives.""" + return iso_utc(datetime.fromisoformat(value)) + + def _datetime(config: dict[str, Any], value: Any, path: str) -> str: if not isinstance(value, str): raise FieldError(path, "must be a datetime string") try: - datetime.fromisoformat(value) + return canonical_datetime(value) except ValueError as exc: raise FieldError(path, "is not a valid ISO datetime") from exc - return value def _enum(config: dict[str, Any], value: Any, path: str) -> Any: @@ -158,7 +169,9 @@ def conform_object( ) -> dict[str, Any]: """Check a map against its declared properties. - ``partial`` accepts any subset of the properties and fills in no defaults. + A readonly property is never accepted; its default is filled in like any + other. ``partial`` accepts any subset of the properties and fills in no + defaults. """ if not isinstance(value, dict): raise FieldError(path, "must be an object") if path else ValueError(NOT_AN_OBJECT) @@ -166,11 +179,15 @@ def conform_object( unknown = [] if open else sorted(value.keys() - properties.keys()) if unknown: raise ValueError(f"Unknown fields: {', '.join(map(qualified, unknown))}") + readonly = sorted(name for name in value.keys() & properties.keys() if properties[name].get("readonly")) + if readonly: + raise ValueError(f"Readonly fields: {', '.join(map(qualified, readonly))}") expected = {} if partial else properties missing = [ name for name, config in expected.items() - if config.get("required") and "default" not in config and name not in value + if config.get("required") and not config.get("readonly") and "default" not in config + and name not in value ] if missing: raise ValueError(f"Missing required fields: {', '.join(map(qualified, missing))}") @@ -190,6 +207,26 @@ def conform_object( } +@dataclass(frozen=True) +class Model: + """A request payload: its properties, whether it keeps undeclared keys and whether it accepts any subset.""" + + properties: dict[str, dict[str, Any]] + open: bool = False + partial: bool = False + + def extend(self, properties: dict[str, dict[str, Any]]) -> "Model": + return replace(self, properties={**self.properties, **properties}) + + def conform(self, value: Any) -> dict[str, Any]: + return conform_object(self.properties, value, open=self.open, partial=self.partial) + + +def partial(model: Model) -> Model: + """``model`` accepting any subset of its properties, with no defaults filled in.""" + return replace(model, partial=True) + + def conform_inputs( inputs: dict[str, dict[str, Any]], values: dict[str, Any] ) -> dict[str, Any]: diff --git a/src/yaml_api_generator/runtime/payloads.py b/src/yaml_api_generator/runtime/payloads.py index bedea369..f0cd85b3 100644 --- a/src/yaml_api_generator/runtime/payloads.py +++ b/src/yaml_api_generator/runtime/payloads.py @@ -1,11 +1,35 @@ """Pydantic building blocks for request payload models.""" -from datetime import datetime -from typing import Annotated +from copy import deepcopy +from typing import Annotated, Any, ClassVar -from pydantic import AfterValidator, BaseModel, ConfigDict, Field, create_model +from pydantic import ( + AfterValidator, + BaseModel, + ConfigDict, + Field, + GetCoreSchemaHandler, + GetJsonSchemaHandler, + SerializerFunctionWrapHandler, + create_model, + model_serializer, + model_validator, +) +from pydantic_core import core_schema -from .fields import check_format +from .fields import canonical_datetime, check_format + + +class Timestamp(str): + """An ISO 8601 datetime string, stored as its canonical UTC form.""" + + @classmethod + def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function(canonical_datetime, core_schema.str_schema()) + + @classmethod + def __get_pydantic_json_schema__(cls, schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler) -> dict[str, Any]: + return {**handler(schema), "format": "date-time"} SCALAR_TYPES: dict[str, type] = { @@ -14,7 +38,7 @@ "ref": str, "number": float, "boolean": bool, - "datetime": datetime, + "datetime": Timestamp, } STRICT_TYPES = frozenset({float, bool}) """Types never coerced from another JSON type.""" @@ -23,12 +47,29 @@ class Payload(BaseModel): - """A closed object: undeclared keys are rejected.""" + """A closed object: undeclared and ``_readonly`` keys are rejected. + + ``_server_defaults`` are the defaults of readonly fields, added whenever the + payload is dumped. + """ model_config = ConfigDict(extra="forbid") + _readonly: ClassVar[frozenset[str]] = frozenset() + _server_defaults: ClassVar[dict[str, Any]] = {} + + @model_validator(mode="before") + @classmethod + def _reject_readonly(cls, data: Any) -> Any: + if isinstance(data, dict) and (sent := sorted(cls._readonly & data.keys())): + raise ValueError(f"Readonly fields: {', '.join(sent)}") + return data + + @model_serializer(mode="wrap") + def _add_server_defaults(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + return {**handler(self), **deepcopy(self._server_defaults)} -class OpenPayload(BaseModel): +class OpenPayload(Payload): """An object that keeps the keys it does not declare.""" model_config = ConfigDict(extra="allow") @@ -38,8 +79,8 @@ def formatted(format_name: str) -> AfterValidator: return AfterValidator(lambda value: check_format(format_name, value)) -def partial(model: type[BaseModel], name: str) -> type[BaseModel]: - """``model`` with every field omittable; types, constraints and nullability are kept.""" +def partial(model: type[Payload], name: str) -> type[Payload]: + """``model`` with every field omittable and no server defaults; types, constraints and nullability are kept.""" return create_model( name, __base__=Payload, diff --git a/src/yaml_api_generator/runtime/sql_types.py b/src/yaml_api_generator/runtime/sql_types.py new file mode 100644 index 00000000..295656e4 --- /dev/null +++ b/src/yaml_api_generator/runtime/sql_types.py @@ -0,0 +1,20 @@ +"""SQLAlchemy column types for field types whose stored form differs from their JSON form.""" + +from datetime import datetime + +import sqlalchemy as sa + +from .fields import iso_utc + + +class IsoDateTime(sa.TypeDecorator): + """A timestamp column read and written as the canonical ISO 8601 UTC string.""" + + impl = sa.DateTime(timezone=True) + cache_ok = True + + def process_bind_param(self, value: str | None, dialect: sa.Dialect) -> datetime | None: + return None if value is None else datetime.fromisoformat(value) + + def process_result_value(self, value: datetime | None, dialect: sa.Dialect) -> str | None: + return None if value is None else iso_utc(value) diff --git a/src/yaml_api_generator/server/common/repositories/sql_schema.py b/src/yaml_api_generator/server/common/repositories/sql_schema.py index 7001609c..53b3a454 100644 --- a/src/yaml_api_generator/server/common/repositories/sql_schema.py +++ b/src/yaml_api_generator/server/common/repositories/sql_schema.py @@ -10,6 +10,7 @@ import sqlalchemy as sa from yaml_api_generator.ir import IRCollection +from yaml_api_generator.runtime.sql_types import IsoDateTime _JSON_TYPES = frozenset({"array", "object", "file"}) @@ -51,7 +52,7 @@ def column_type(cfg: dict[str, Any]) -> sa.types.TypeEngine: if field_type == "boolean": return sa.Boolean() if field_type == "datetime": - return sa.DateTime(timezone=True) + return IsoDateTime() max_length = cfg.get("maxLength") if field_type == "string" and isinstance(max_length, int): return sa.String(max_length) diff --git a/src/yaml_api_generator/server/common/steps/validation.py b/src/yaml_api_generator/server/common/steps/validation.py index 8f23b7ff..afccf4db 100644 --- a/src/yaml_api_generator/server/common/steps/validation.py +++ b/src/yaml_api_generator/server/common/steps/validation.py @@ -2,8 +2,8 @@ from typing import Any -from yaml_api_generator.ir import SECRET_FIELD, IRDto, writable_fields -from yaml_api_generator.runtime.fields import NOT_AN_OBJECT, conform_object +from yaml_api_generator.ir import SECRET_FIELD, IRDto, payload_fields +from yaml_api_generator.runtime.fields import NOT_AN_OBJECT, Model from ..request_context import RequestContext from ..server_context import RuntimeCollectionDescriptor, RuntimeIdentityDescriptor @@ -40,7 +40,7 @@ def ValidateBodyStep( identity_collection: RuntimeIdentityDescriptor | None = None, dto: IRDto | None = None, ) -> Step: - """Check the body against ``dto`` when the route types it, else against the entity.""" + """Check the body against ``dto`` when the route types it, else against the entity; updates accept any subset.""" if operation in _NO_BODY_OPS: return lambda ctx: ctx if operation == "login": @@ -48,13 +48,13 @@ def ValidateBodyStep( raise ValueError("login operation requires an identity collection") properties = login_properties(identity_collection) elif dto is not None: - properties = writable_fields(dto.fields) + properties = payload_fields(dto.fields) else: properties = { **credential_properties(operation, identity_collection), - **writable_fields(collection.fields, collection.files), + **payload_fields(collection.fields, collection.files), } - partial = operation == "update" and dto is None + model = Model(properties, partial=operation == "update") files = set(collection.files) def step(ctx: RequestContext) -> RequestContext: @@ -63,7 +63,7 @@ def step(ctx: RequestContext) -> RequestContext: raise ValueError(NOT_AN_OBJECT) uploads = {name: value for name, value in body.items() if name in files} fields = {name: value for name, value in body.items() if name not in files} - ctx.body = {**conform_object(properties, fields, partial=partial), **uploads} + ctx.body = {**model.conform(fields), **uploads} return ctx return step diff --git a/src/yaml_api_generator/server/fastapi/factory.py b/src/yaml_api_generator/server/fastapi/factory.py index 34a3e1f6..ed800498 100644 --- a/src/yaml_api_generator/server/fastapi/factory.py +++ b/src/yaml_api_generator/server/fastapi/factory.py @@ -20,13 +20,14 @@ ) from yaml_api_generator.runtime.files import FileError, RecordFiles from yaml_api_generator.runtime.file_requests import ( + json_body, json_request_body, read_json, read_upload, upload_request_body, ) from yaml_api_generator.runtime.multipart import file_download_responses, file_upload_body -from yaml_api_generator.ir import writable_fields +from yaml_api_generator.ir import payload_fields from yaml_api_generator.ir.api import IRDto, IROIDCProvider, IRProviderBinding from yaml_api_generator.utils.strings import to_pascal_case, to_py_identifier from ..common.auth import oidc @@ -74,7 +75,11 @@ def _model_field(config: dict[str, Any]) -> tuple[Any, Any]: def create_pydantic_model_fields( properties: dict[str, dict[str, Any]], ) -> dict[str, tuple[Any, Any]]: - return {name: _model_field(config) for name, config in properties.items()} + return { + name: _model_field(config) + for name, config in properties.items() + if not config.get("readonly") + } def create_pydantic_model( @@ -83,7 +88,7 @@ def create_pydantic_model( identity_collection: RuntimeIdentityDescriptor | None = None, dto: IRDto | None = None, ) -> type[BaseModel] | None: - """The body model of ``operation``; nested objects and formats are left to ``ValidateBodyStep``.""" + """The documented body of ``operation``; ``ValidateBodyStep`` validates the body itself.""" model_name = to_pascal_case(collection.name) if operation == "login" and identity_collection is not None: return create_model( @@ -103,12 +108,12 @@ def create_pydantic_model( return create_model( to_pascal_case(dto.name), __base__=Payload, - **create_pydantic_model_fields(writable_fields(dto.fields)), + **create_pydantic_model_fields(payload_fields(dto.fields)), ) fields = create_model( f"{model_name}Fields", __base__=Payload, - **create_pydantic_model_fields(writable_fields(collection.fields, collection.files)), + **create_pydantic_model_fields(payload_fields(collection.fields, collection.files)), ) if operation == "update": return partial(fields, f"{model_name}UpdatePayload") @@ -159,11 +164,17 @@ def register_fastapi_routes( endpoint = _build_endpoint( pipeline, body_model, route, uses_cookie=ctx.session.uses_cookie ) + body_document = ( + {} if body_model is None + else {"requestBody": json_request_body(None, body_model, partial=route.operation == "update")} + ) app.add_api_route( route.path, endpoint, methods=[route.method], - openapi_extra=_openapi_extra(route, ctx.auth), + openapi_extra=_openapi_extra( + route, ctx.auth, parameters=_path_parameters(route.path), **body_document + ), ) for file_route in ctx.file_routes: @@ -308,7 +319,6 @@ async def endpoint(request: Request) -> Any: pipeline=pipeline, request=request, body=None, - url_params={}, uses_cookie=uses_cookie, operation=operation, upload=read(request), @@ -323,7 +333,6 @@ async def endpoint(request: Request) -> StreamingResponse: pipeline=pipeline, request=request, body=None, - url_params={}, uses_cookie=uses_cookie, ) return StreamingResponse(ctx.response.chunks, headers=ctx.response.headers) @@ -584,82 +593,30 @@ def _build_endpoint( *, uses_cookie: bool = False, ): - has_id_param = "{id}" in route.path - - if body_model is None: - if has_id_param: - - async def endpoint_no_body_with_id(request: Request, id: str) -> Any: - return await _run_pipeline( - pipeline=pipeline, - request=request, - body=None, - url_params={"id": id}, - uses_cookie=uses_cookie, - operation=route.operation, - ) - - return endpoint_no_body_with_id - - async def endpoint_no_body(request: Request) -> Any: - return await _run_pipeline( - pipeline=pipeline, - request=request, - body=None, - url_params={}, - uses_cookie=uses_cookie, - operation=route.operation, - ) - - return endpoint_no_body - - exclude_unset = route.operation == "update" - - if has_id_param: + """An endpoint passing the raw JSON body to ``pipeline``, which validates it after the access checks.""" - async def endpoint_with_body_and_id( - request: Request, - id: str, - payload: BaseModel, - ) -> Any: - return await _run_pipeline( - pipeline=pipeline, - request=request, - body=payload.model_dump(mode="json", exclude_unset=exclude_unset), - url_params={"id": id}, - uses_cookie=uses_cookie, - operation=route.operation, - ) - - endpoint_with_body_and_id.__annotations__["payload"] = body_model - return endpoint_with_body_and_id - - async def endpoint_with_body(request: Request, payload: BaseModel) -> Any: + async def endpoint(request: Request) -> Any: return await _run_pipeline( pipeline=pipeline, request=request, - body=payload.model_dump(mode="json", exclude_unset=exclude_unset), - url_params={}, + body=None if body_model is None else await json_body(request), uses_cookie=uses_cookie, operation=route.operation, ) - endpoint_with_body.__annotations__["payload"] = body_model - return endpoint_with_body + return endpoint async def _execute( pipeline: Pipeline, request: Request, body: dict[str, Any] | None, - url_params: dict[str, str], uses_cookie: bool = False, operation: str | None = None, upload: RequestBody | None = None, ) -> RequestContext: try: path_params = {key: str(value) for key, value in request.path_params.items()} - path_params.update(url_params) query_params = getattr(request, "query_params", None) query: dict[str, str | list[str]] = ( { @@ -706,12 +663,11 @@ async def _run_pipeline( pipeline: Pipeline, request: Request, body: dict[str, Any] | None, - url_params: dict[str, str], uses_cookie: bool = False, operation: str | None = None, upload: RequestBody | None = None, ) -> Any: - ctx = await _execute(pipeline, request, body, url_params, uses_cookie, operation, upload) + ctx = await _execute(pipeline, request, body, uses_cookie, operation, upload) response_payload = jsonable_encoder(ctx.response) if not uses_cookie or operation not in {"login", "refresh"}: return response_payload diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py index e22830e3..d47a9c7f 100644 --- a/tests/unit/support/payload_requests.py +++ b/tests/unit/support/payload_requests.py @@ -1,9 +1,9 @@ """Valid and invalid request payloads, sent alike to every app under test. The spec covers formats, array items, closed nested objects, defaults, readonly -fields, strict scalars, partial updates and a DTO-typed body, which an access -rule binds for any signed-in user. Round trips check what the apps store and -return besides the status codes. +fields, datetimes, strict scalars, partial updates and DTO-typed bodies, which +access rules bind for any signed-in user. Round trips check what the apps store +and return besides the status codes. """ import textwrap @@ -44,6 +44,8 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload noteInput: fields: role: { type: enum, values: [admin, member], required: true } + level: { type: number, default: 2 } + labels: { type: array, items: { type: string, maxLength: 8 } } entities: user: type: auth @@ -59,13 +61,15 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload status: { type: enum, values: [draft, live], default: draft } rank: { type: number } active: { type: boolean } - stamp: { type: string, readonly: true } + stamp: { type: string, readonly: true, default: S0 } + due: { type: datetime } tags: { type: array, items: { type: string, maxLength: 3 } } meta: type: object properties: color: { type: string, required: true } size: { type: number, min: 1 } + locked: { type: boolean, readonly: true, default: false } additionalProperties: false points: type: array @@ -78,6 +82,8 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload fields: id: { type: id, format: uuidv4 } role: { type: string } + level: { type: number } + labels: { type: array, items: { type: string } } repositories: users: { entity: user } items: { entity: item } @@ -105,6 +111,9 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload create: access: - given: { caller: "identity:user", note: "request.body:noteInput" } + update: + access: + - given: { caller: "identity:user", note: "request.body:noteInput" } """ CREATES = [ @@ -124,6 +133,10 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload ({"name": "ab", "active": "true"}, 422), ({"name": "ab", "active": 1}, 422), ({"name": "ab", "stamp": "x"}, 422), + ({"name": "ab", "meta": {"color": "r", "locked": True}}, 422), + ({"name": "ab", "due": 1704103200}, 422), + ({"name": "ab", "due": "2024-01-01T10:00:00Z"}, 200), + ({"name": "ab", "due": "soon"}, 422), ({"name": "ab", "unknown": 1}, 422), ({"name": "a"}, 422), ({}, 422), @@ -145,11 +158,21 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload ] ROUND_TRIPS = [ ["created number", 3], + ["readonly default stored", "S0"], + ["nested readonly default stored", False], + ["datetime canonical", "2024-01-01T10:00:00+00:00"], + ["date-only datetime canonical", "2024-01-01T00:00:00+00:00"], ["replaced id kept", True], ["replaced number", 1.5], ["replaced item found", 200], ["replaced item deleted", 200], ["note role kept", "admin"], + ["note dto default stored", 2], + ["note partial update", 200], + ["note value kept by partial update", 5], + ["note empty update", 200], + ["anonymous note rejected first", 401], + ["credentials hidden", [False, False]], ] EXPECTED = [ *(["POST /items", body, status] for body, status in CREATES), @@ -163,30 +186,48 @@ def spec(datastore: str) -> str: return _API + textwrap.dedent(datastore) + _ENTITIES +_LOGIN = {"email": "caller@example.com", "password": "caller-password"} + + def _bearer(client: TestClient) -> dict[str, str]: - login = {"email": "caller@example.com", "password": "caller-password"} - assert client.post("/users/register", json=login).status_code == 200 - token = client.post("/users/login", json=login).json()["token"] + token = client.post("/users/login", json=_LOGIN).json()["token"] return {"Authorization": f"Bearer {token}"} -def _round_trips(client: TestClient, headers: dict[str, str]) -> list[list]: - item = client.post("/items", json={"name": "ab", "rank": 3}).json() +def _round_trips(client: TestClient, headers: dict[str, str], user: dict) -> list[list]: + item = client.post("/items", json={"name": "ab", "rank": 3, "meta": {"color": "r"}}).json() path = f"/items/{item['id']}" + due = client.post("/items", json={"name": "ab", "due": "2024-01-01T11:00:00+01:00"}).json() + day = client.post("/items", json={"name": "ab", "due": "2024-01-01"}).json() replaced = client.put(path, json={"name": "cd", "rank": 1.5}).json() note = client.post("/notes", json={"role": "admin"}, headers=headers).json() + fives = client.post("/notes", json={"role": "admin", "level": 5}, headers=headers).json() + labelled = client.patch(f"/notes/{fives['id']}", json={"labels": ["a"]}, headers=headers) + patched_user = client.patch(f"/users/{user['id']}", json={"email": "c@example.com"}, headers=headers) return [ ["created number", item["rank"]], + ["readonly default stored", item.get("stamp")], + ["nested readonly default stored", item["meta"].get("locked")], + ["datetime canonical", due.get("due")], + ["date-only datetime canonical", day.get("due")], ["replaced id kept", replaced.get("id") == item["id"]], ["replaced number", replaced["rank"]], ["replaced item found", client.get(path).status_code], ["replaced item deleted", client.delete(path).status_code], ["note role kept", note.get("role")], + ["note dto default stored", note.get("level")], + ["note partial update", labelled.status_code], + ["note value kept by partial update", labelled.json().get("level")], + ["note empty update", client.patch(f"/notes/{fives['id']}", json={}, headers=headers).status_code], + ["anonymous note rejected first", client.post("/notes", json={"role": "owner"}).status_code], + ["credentials hidden", ["_auth" in user, "_auth" in patched_user.json()]], ] def scenario(client: TestClient) -> list[list]: """Each payload with the status it got, after one valid item is created for the updates, then the round trips.""" + registered = client.post("/users/register", json=_LOGIN) + assert registered.status_code == 200, registered.text headers = _bearer(client) created = client.post("/items", json={"name": "ab"}) assert created.status_code == 200, created.text @@ -202,5 +243,5 @@ def scenario(client: TestClient) -> list[list]: ["POST /notes", body, client.post("/notes", json=body, headers=headers).status_code] for body, _ in NOTES ), - *_round_trips(client, headers), + *_round_trips(client, headers, registered.json()), ] diff --git a/tests/unit/test_fastapi_factory.py b/tests/unit/test_fastapi_factory.py index d2f8d7e7..feaedc11 100644 --- a/tests/unit/test_fastapi_factory.py +++ b/tests/unit/test_fastapi_factory.py @@ -1,11 +1,11 @@ import types -from datetime import datetime from typing import Any, Literal import pytest from pydantic import TypeAdapter, ValidationError from yaml_api_generator.ir import IRProviderBinding +from yaml_api_generator.runtime.payloads import Timestamp from yaml_api_generator.server.fastapi.factory import ( _field_annotation, _security_requirements, @@ -134,7 +134,7 @@ def test_boolean_field_is_never_coerced(value): def test_field_type_mapping_datetime(): - assert _field_annotation({"type": "datetime"}) is datetime + assert _field_annotation({"type": "datetime"}) is Timestamp def test_field_type_mapping_enum(): @@ -330,7 +330,7 @@ class _FakeRequest: state = types.SimpleNamespace(principal=None) with pytest.raises(HTTPException) as exc_info: - await _run_pipeline(pipeline, _FakeRequest(), None, {}) + await _run_pipeline(pipeline, _FakeRequest(), None) assert exc_info.value.status_code == 401 @@ -352,7 +352,7 @@ class _FakeRequest: state = types.SimpleNamespace(principal=None) with pytest.raises(HTTPException) as exc_info: - await _run_pipeline(pipeline, _FakeRequest(), None, {}) + await _run_pipeline(pipeline, _FakeRequest(), None) assert exc_info.value.status_code == 403 @@ -374,7 +374,7 @@ class _FakeRequest: state = types.SimpleNamespace(principal=None) with pytest.raises(HTTPException) as exc_info: - await _run_pipeline(pipeline, _FakeRequest(), None, {}) + await _run_pipeline(pipeline, _FakeRequest(), None) assert exc_info.value.status_code == 404 @@ -395,7 +395,7 @@ class _FakeRequest: state = types.SimpleNamespace(principal=None) with pytest.raises(HTTPException) as exc_info: - await _run_pipeline(pipeline, _FakeRequest(), None, {}) + await _run_pipeline(pipeline, _FakeRequest(), None) assert exc_info.value.status_code == 422 @@ -477,10 +477,8 @@ def test_update_model_dump_exclude_unset_does_not_include_defaults(): # --- BUG 2: datetime model_dump with mode='json' produces ISO string --- -def test_create_model_dump_json_mode_serializes_datetime_to_iso_string(): +def test_create_model_dump_canonicalizes_datetime_to_utc(): col = _make_collection("events", {"starts_at": {"type": "datetime"}}) model = create_pydantic_model(col, "create") - instance = model(starts_at=datetime(2025, 1, 15, 10, 30, 0)) - dumped = instance.model_dump(mode="json") - assert isinstance(dumped["starts_at"], str) - assert dumped["starts_at"].startswith("2025-01-15") \ No newline at end of file + instance = model(starts_at="2025-01-15T11:30:00+01:00") + assert instance.model_dump(mode="json") == {"starts_at": "2025-01-15T10:30:00+00:00"} \ No newline at end of file diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index 25700014..b15c8de7 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -132,7 +132,7 @@ def test_fastapi_payload_module_maps_every_field_type(): "s": "str", "n": "Annotated[float, Field(strict=True)]", "b": "Annotated[bool, Field(strict=True)]", - "d": "datetime", + "d": "Timestamp", "e": "Literal['a', 'b']", "a": "list[str]", "o": "dict[str, Any]", diff --git a/tests/unit/test_payload_models.py b/tests/unit/test_payload_models.py index 61839638..03c634e6 100644 --- a/tests/unit/test_payload_models.py +++ b/tests/unit/test_payload_models.py @@ -91,7 +91,9 @@ def test_generated_drf_serializers_validate_through_the_shared_validator(mongo_s serializers = (api / "serializers" / "items.py").read_text() assert 'ItemsUpdatePayload = partial(ItemsFields, "ItemsUpdatePayload")' in serializers assert "class ItemsCreatePayload(ItemsFields):" in serializers - assert "NoteInput(data=request.data)" in (api / "views" / "notes.py").read_text() + notes = (api / "views" / "notes.py").read_text() + assert "NoteInput(data=request.data)" in notes + assert "NoteInputUpdatePayload(data=request.data)" in notes def _is_field_config(node: ast.expr) -> bool: diff --git a/tests/unit/test_request_access_generation.py b/tests/unit/test_request_access_generation.py index 5ccfec56..319de2b3 100644 --- a/tests/unit/test_request_access_generation.py +++ b/tests/unit/test_request_access_generation.py @@ -22,7 +22,7 @@ def _generate(tmp_path): def test_dto_model_emitted(tmp_path): ctx = _generate(tmp_path) - assert [m.name for m in ctx.dto_models] == ["CreateMembership"] + assert [m.name for m in ctx.dto_models] == ["CreateMembership", "CreateMembershipUpdatePayload"] dto_module = (tmp_path / "models" / "_dto.py").read_text() assert "class CreateMembership(Payload):" in dto_module assert "Literal['admin', 'member', 'viewer']" in dto_module diff --git a/tests/unit/test_validation_step.py b/tests/unit/test_validation_step.py index 6387ecd9..0be42483 100644 --- a/tests/unit/test_validation_step.py +++ b/tests/unit/test_validation_step.py @@ -192,7 +192,7 @@ def test_validate_body_datetime_valid_iso(): step = ValidateBodyStep(collection, "create") ctx = _make_ctx(body={"ts": "2025-01-15T10:30:00"}) result = step(ctx) - assert result.body["ts"] == "2025-01-15T10:30:00" + assert result.body["ts"] == "2025-01-15T10:30:00+00:00" def test_validate_body_datetime_invalid_iso(): @@ -338,7 +338,7 @@ def test_validate_body_datetime_iso_string_from_json_mode_passes(): step = ValidateBodyStep(collection, "create") ctx = _make_ctx(body={"starts_at": "2025-01-15T10:30:00"}) result = step(ctx) - assert result.body["starts_at"] == "2025-01-15T10:30:00" + assert result.body["starts_at"] == "2025-01-15T10:30:00+00:00" # --- Nullable non-required datetime regression (model_dump no exclude_unset) --- @@ -365,4 +365,4 @@ def test_validate_body_valid_iso_datetime_still_passes(): step = ValidateBodyStep(collection, "create") ctx = _make_ctx(body={"createdAt": "2025-06-27T12:00:00"}) result = step(ctx) - assert result.body["createdAt"] == "2025-06-27T12:00:00" + assert result.body["createdAt"] == "2025-06-27T12:00:00+00:00" From fe684a5cfabd91dce01d36d2121a9b45688b5eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 20:10:31 +0100 Subject: [PATCH 16/89] Serve generated DRF views on adrf with async principal lookups, a loop-safe Mongo client, spec-path routing and 200/422 statuses matching the other targets --- .../backend/drf/templates/api_urls.py.jinja | 2 +- .../backend/drf/templates/apps.py.jinja | 23 +++++-- .../backend/drf/templates/auth.py.jinja | 12 ++-- .../drf/templates/handler_views.py.jinja | 4 +- .../backend/drf/templates/jwks_views.py.jinja | 2 +- .../drf/templates/login_views.py.jinja | 6 +- .../backend/drf/templates/payloads.py.jinja | 10 ++- .../drf/templates/project_urls.py.jinja | 3 +- .../drf/templates/requirements.txt.jinja | 2 + .../backend/drf/templates/settings.py.jinja | 2 + .../backend/drf/templates/views.py.jinja | 6 +- .../test_generated_where_authorization.py | 15 +++-- tests/integration/test_mongo_payloads.py | 63 ++++++++++++++++--- tests/unit/test_payload_models.py | 17 ++++- 14 files changed, 127 insertions(+), 40 deletions(-) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja index bdf82d38..8cb1260a 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja @@ -35,7 +35,7 @@ urlpatterns = [ {% endfor %} {% endif %} {% set seen_paths = [] %} -{% for route in routes if route.collection and route.operation in crud_op_names %} +{% for route in routes | sort(attribute="has_id_param") if route.collection and route.operation in crud_op_names %} {% if route.django_path not in seen_paths %} {% set _ = seen_paths.append(route.django_path) %} path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{% if route.has_id_param %}Detail{% else %}List{% endif %}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), diff --git a/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja index e061877c..f97cbc31 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/apps.py.jinja @@ -1,18 +1,29 @@ -{% if ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} +{% set creates_indexes = ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} +{% if creates_indexes %} import asyncio -{% endif %} +{% endif %} from django.apps import AppConfig +from api.authentication.mongodb import {% if creates_indexes %}close_db, connect_db, initialize_indexes{% else %}connect_db{% endif %} + + class ApiConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "api" def ready(self): - from api.authentication.mongodb import connect_db +{% if creates_indexes %} + asyncio.run(self.create_indexes()) +{% endif %} + connect_db() +{% if creates_indexes %} + + @staticmethod + async def create_indexes(): + """Runs on a client of its own: an async client stays bound to the event loop it first ran on.""" connect_db() -{% if ctx.mongo_unique_indexes and not ctx.mongo_schema_managed %} - from api.authentication.mongodb import initialize_indexes - asyncio.run(initialize_indexes({{ ctx.mongo_unique_indexes | tojson }})) + await initialize_indexes({{ ctx.mongo_unique_indexes | tojson }}) + await close_db() {% endif %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja index 0ab63e2f..56051990 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/auth.py.jinja @@ -101,7 +101,7 @@ def _principal_has_role(principal: dict | None, required_role: str) -> bool: return any(required_role in entity_hierarchy.get(role, set()) for role in principal_roles) -def resolve_principal(request: Request, db) -> dict[str, Any] | None: +async def resolve_principal(request: Request, db) -> dict[str, Any] | None: headers = {k.lower(): v for k, v in request.META.items() if k.startswith("HTTP_")} raw_headers: dict[str, str] = {} for meta_key, value in headers.items(): @@ -117,7 +117,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: collection_name = claims.get("identity_collection") if sub and collection_name: col = db[collection_name] - for doc in col.find({}): + async for doc in col.find({}): {% for identity in auth.identity_collections %} {% if identity.issues_jwt %} if collection_name == "{{ identity.name }}" and str(doc.get("{{ identity.id_field }}", "")) == sub: @@ -133,7 +133,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: token = _extract_api_key(raw_headers) if token is not None: col = db["{{ identity.name }}"] - for doc in col.find({}): + async for doc in col.find({}): if str(doc.get("apiKey", "")) == token: subject = doc.get("{{ identity.id_field }}") if subject is not None: @@ -144,7 +144,7 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: if credentials is not None: identifier_value, password = credentials col = db["{{ identity.name }}"] - for doc in col.find({}): + async for doc in col.find({}): if str(doc.get("{{ identity.identifier }}", "")) != identifier_value: continue stored_hash = doc["_auth"]["passwordHash"] @@ -162,9 +162,9 @@ def resolve_principal(request: Request, db) -> dict[str, Any] | None: class MongoDBAuthentication(BaseAuthentication): - def authenticate(self, request): + async def authenticate(self, request): db = get_database() - principal = resolve_principal(request, db) + principal = await resolve_principal(request, db) if principal is None: return None return (principal, None) diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index 108d3dba..73078565 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -2,7 +2,7 @@ from importlib import import_module from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from api.conflicts import ConflictError from api.service_inputs import prepare_service_inputs, resolve_access_bind @@ -23,7 +23,7 @@ from api.authentication.mongodb import get_database class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(APIView): async def {{ route.method | lower }}(self, request, **path_params): {% if route.needs_auth %} - principal = resolve_principal(request, get_database()) + principal = await resolve_principal(request, get_database()) if principal is None: return Response( {"detail": "Unauthorized"}, diff --git a/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja index a01c2ca8..441754aa 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/jwks_views.py.jinja @@ -1,6 +1,6 @@ from drf_spectacular.utils import extend_schema from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from api.authentication.jwt_utils import public_jwks diff --git a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja index a4fa75c2..34ecc631 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/login_views.py.jinja @@ -1,6 +1,6 @@ from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from drf_spectacular.utils import extend_schema from api.authentication.auth import verify_password @@ -16,7 +16,7 @@ class {{ class_name }}LoginView(APIView): permission_classes = [] @extend_schema(tags=["{{ identity.name | capitalize_collection }}"], auth=[]) - def post(self, request): + async def post(self, request): identifier_value = request.data.get("{{ identity.identifier }}") password = request.data.get("password") @@ -28,7 +28,7 @@ class {{ class_name }}LoginView(APIView): db = get_database() col = db["{{ identity.name }}"] - for doc in col.find({}): + async for doc in col.find({}): if str(doc.get("{{ identity.identifier }}", "")) != str(identifier_value): continue stored_hash = doc["_auth"]["passwordHash"] diff --git a/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja index 674668cc..534ddc31 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/payloads.py.jinja @@ -2,7 +2,7 @@ from typing import Any -from rest_framework import serializers +from rest_framework import serializers, status, views from .fields import conform_object @@ -23,3 +23,11 @@ class PayloadSerializer(serializers.Serializer): def partial(serializer: type[PayloadSerializer], name: str) -> type[PayloadSerializer]: """``serializer`` accepting any subset of its properties.""" return type(name, (serializer,), {"omittable": True}) + + +def exception_handler(exc: Exception, context: dict[str, Any]): + """DRF's handler, answering an invalid payload with 422 as every other target does.""" + response = views.exception_handler(exc, context) + if isinstance(exc, serializers.ValidationError): + response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + return response diff --git a/src/yaml_api_generator/generators/backend/drf/templates/project_urls.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/project_urls.py.jinja index 16386820..15037b49 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/project_urls.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/project_urls.py.jinja @@ -8,7 +8,8 @@ urlpatterns = [ {% if ctx.session.uses_asymmetric %} path(".well-known/jwks.json", JWKSView.as_view(), name="jwks"), {% endif %} - path("{{ (ctx.app.base_path ~ ('/' ~ ctx.app.api_version if ctx.app.api_version else '')).lstrip('/') }}/", include("api.urls")), +{% set prefix = (ctx.app.base_path ~ ('/' ~ ctx.app.api_version if ctx.app.api_version else '')).strip('/') %} + path("{{ prefix ~ '/' if prefix }}", include("api.urls")), path("schema", SpectacularAPIView.as_view(), name="schema"), path("docs", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), ] diff --git a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja index 3fc3b6d5..6b499433 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja @@ -1,7 +1,9 @@ django djangorestframework +adrf drf-spectacular gunicorn +uvicorn pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} pydantic-settings diff --git a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja index 23543202..28d999f8 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/settings.py.jinja @@ -14,6 +14,7 @@ INSTALLED_APPS = [ "django.contrib.contenttypes", "django.contrib.auth", "rest_framework", + "adrf", "drf_spectacular", "api", ] @@ -49,6 +50,7 @@ REST_FRAMEWORK = { ], "DEFAULT_AUTHENTICATION_CLASSES": [], "UNAUTHENTICATED_USER": None, + "EXCEPTION_HANDLER": "api.payloads.exception_handler", } SPECTACULAR_SETTINGS = { diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index f37a95b8..99014718 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -1,6 +1,6 @@ from rest_framework import status from rest_framework.response import Response -from rest_framework.views import APIView +from adrf.views import APIView from drf_spectacular.utils import extend_schema from api.conflicts import ConflictError {% set crud_op_names = ["list", "get", "create", "update", "replace", "delete"] %} @@ -80,7 +80,7 @@ from api.authentication.auth import resolve_principal, _principal_has_role {% macro pre_fetch_check(route) %} {% if route.needs_auth %} db = get_database() - _resolved_principal = resolve_principal(request, db) + _resolved_principal = await resolve_principal(request, db) if _resolved_principal is None: return Response({"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED) {% if route.access_rules %} @@ -170,7 +170,7 @@ class {{ class_name }}ListView(APIView): serializer.is_valid(raise_exception=True) data = serializer.validated_data document = await service.create(data) - return Response(document, status=status.HTTP_201_CREATED) + return Response(document) {% endif %} except NotFoundError: return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index d6712202..002ebb89 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -486,7 +486,8 @@ def test_generated_drf_enforces_where_per_rule(tmp_path): HTTP_422_UNPROCESSABLE_ENTITY=422, ) response_module = types.ModuleType("rest_framework.response") - views_module = types.ModuleType("rest_framework.views") + adrf_module = types.ModuleType("adrf") + views_module = types.ModuleType("adrf.views") spectacular_module = types.ModuleType("drf_spectacular") spectacular_utils = types.ModuleType("drf_spectacular.utils") auth_module = types.ModuleType("api.authentication.auth") @@ -514,7 +515,8 @@ def extend_schema(**kwargs): sys.modules.update({{ "rest_framework": rest_framework, "rest_framework.response": response_module, - "rest_framework.views": views_module, + "adrf": adrf_module, + "adrf.views": views_module, "drf_spectacular": spectacular_module, "drf_spectacular.utils": spectacular_utils, "api.authentication.auth": auth_module, @@ -562,11 +564,12 @@ def find(self, criteria): "reports": Collection([{{"id": "r1", "organizationId": "org-1", "title": "visible"}}]), }} views.get_database = lambda: db - views.resolve_principal = lambda request, database: {{ - "sub": "u1", "identity_collection": "users", "roles": [] - }} + async def principal(request, database): + return {{"sub": "u1", "identity_collection": "users", "roles": []}} + + views.resolve_principal = principal handler_views.get_database = lambda: db - handler_views.resolve_principal = views.resolve_principal + handler_views.resolve_principal = principal request = SimpleNamespace(data={{}}, query_params={{}}, headers={{}}, COOKIES={{}}) application_module = types.ModuleType("app") diff --git a/tests/integration/test_mongo_payloads.py b/tests/integration/test_mongo_payloads.py index aca4ec74..078e5437 100644 --- a/tests/integration/test_mongo_payloads.py +++ b/tests/integration/test_mongo_payloads.py @@ -1,12 +1,15 @@ -"""Development servers and a generated raw-ASGI app validate the shared payload scenario against a real MongoDB.""" +"""Development servers and generated raw-ASGI and DRF apps validate the shared payload scenario against a real MongoDB.""" import json import os +import socket import subprocess import sys +import time from pathlib import Path from uuid import uuid4 +import httpx import pytest from starlette.testclient import TestClient @@ -29,6 +32,26 @@ """ +DRF_PYTHON = os.environ.get("YAML_API_TEST_DRF_PYTHON") +"""An interpreter with the generated DRF requirements installed.""" + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _scenario_env(mongo_uri: str) -> dict[str, str]: + return { + **os.environ, + "MONGO_URI": mongo_uri, + "MONGO_DB": f"payloads_{uuid4().hex}", + "JWT_SECRET": JWT_SECRET, + "PASSWORD_HASH_ROUNDS": "1000", + } + + @pytest.fixture def spec_file(tmp_path, mongo_uri): path = tmp_path / "spec.yaml" @@ -50,15 +73,39 @@ def test_generated_raw_validates_payloads(spec_file, tmp_path, mongo_uri): result = subprocess.run( [sys.executable, "-c", _RUNNER], cwd=tmp_path / "raw" / "backend", - env={ - **os.environ, - "MONGO_URI": mongo_uri, - "MONGO_DB": f"payloads_{uuid4().hex}", - "JWT_SECRET": JWT_SECRET, - "PASSWORD_HASH_ROUNDS": "1000", - }, + env=_scenario_env(mongo_uri), capture_output=True, text=True, ) assert result.returncode == 0, result.stderr assert json.loads(result.stdout.splitlines()[-1]) == EXPECTED + + +@pytest.mark.integration +@pytest.mark.skipif(DRF_PYTHON is None, reason="YAML_API_TEST_DRF_PYTHON is not set") +def test_generated_drf_validates_payloads(spec_file, tmp_path, mongo_uri): + compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "drf"), target="drf") + port = _free_port() + server = subprocess.Popen( + [DRF_PYTHON, "-m", "uvicorn", "project.asgi:application", "--port", str(port)], + cwd=tmp_path / "drf" / "backend", + env={**_scenario_env(mongo_uri), "DJANGO_SECRET_KEY": "payload-scenario-django-key"}, + ) + try: + with httpx.Client(base_url=f"http://127.0.0.1:{port}") as client: + deadline = time.monotonic() + 30 + while not _serves(client) and time.monotonic() < deadline: + assert server.poll() is None, "DRF server exited" + time.sleep(0.2) + assert scenario(client) == EXPECTED + finally: + server.terminate() + server.wait() + + +def _serves(client: httpx.Client) -> bool: + try: + client.get("/schema") + except httpx.TransportError: + return False + return True diff --git a/tests/unit/test_payload_models.py b/tests/unit/test_payload_models.py index 03c634e6..75a232d5 100644 --- a/tests/unit/test_payload_models.py +++ b/tests/unit/test_payload_models.py @@ -1,8 +1,9 @@ """Request payloads are validated by their field configs in every consumer. The shared payload scenario runs against both development servers and a -generated FastAPI app executed in a subprocess. DRF cannot run here, so its -generated serializers are checked to delegate to the shared validator. +generated FastAPI app executed in a subprocess. DRF needs its own interpreter, +so here its generated serializers are checked to delegate to the shared +validator; the integration suite runs the scenario against it over HTTP. """ import ast @@ -96,6 +97,18 @@ def test_generated_drf_serializers_validate_through_the_shared_validator(mongo_s assert "NoteInputUpdatePayload(data=request.data)" in notes +def test_generated_drf_serves_async_views_at_spec_paths(mongo_spec_file): + backend = _generate(mongo_spec_file, "drf") + views = [*(backend / "api" / "views").glob("*.py")] + assert all("from rest_framework.views import APIView" not in view.read_text() for view in views) + assert "from adrf.views import APIView" in (backend / "api" / "views" / "users.py").read_text() + assert '"adrf",' in (backend / "project" / "settings.py").read_text() + assert "adrf" in (backend / "requirements.txt").read_text().split() + assert 'path("", include("api.urls"))' in (backend / "project" / "urls.py").read_text() + urls = (backend / "api" / "urls.py").read_text() + assert urls.index('"users/register"') < urls.index('"users/"') + + def _is_field_config(node: ast.expr) -> bool: return isinstance(node, ast.Dict) and any( isinstance(key, ast.Constant) and key.value == "type" for key in node.keys From a514f05038d240ef1c4979a698c10333b22cb192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:13:29 +0100 Subject: [PATCH 17/89] Default explicit controller routes to public and let operations override their method and path --- .../parsers/desugar/controller_desugarer.py | 19 +++++++-------- tests/unit/test_controller_desugar_access.py | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/yaml_api_generator/parsers/desugar/controller_desugarer.py b/src/yaml_api_generator/parsers/desugar/controller_desugarer.py index ea0c0781..3e8d4cdb 100644 --- a/src/yaml_api_generator/parsers/desugar/controller_desugarer.py +++ b/src/yaml_api_generator/parsers/desugar/controller_desugarer.py @@ -26,10 +26,11 @@ class ControllerDesugarer: """Expand every controller into explicit routes. Each route is layered from least to most specific: the controller - ``access``, what the route exposes (for a minified controller, the - service operation and its method/path convention), the ``operations`` - override and the ``routes`` entry. ``operations.: false`` drops - the route. + ``access`` (public when absent), what the route exposes (the ``routes`` + entry, or for a service-bound controller the service operation and its + method/path convention), the ``operations`` override and, for a + service-bound controller, the ``routes`` refinement. + ``operations.: false`` drops the route. """ def desugar( @@ -53,16 +54,16 @@ def _routes( ) -> dict[str, Any]: routes = entry.get("routes") or {} overrides = entry.get("operations") or {} + base = {"access": entry.get("access", "public")} if "service" in entry: - base = {"access": entry.get("access", "public")} exposed = self._exposed(ctrl_name, entry["service"], expanded_services) + refinements = routes else: - base = {"access": entry["access"]} if "access" in entry else {} - exposed = {name: {} for name in routes} + exposed, refinements = routes, {} errors = [ self._unexposed(ctrl_name, section, name) - for section, names in (("operations", overrides), ("routes", routes)) + for section, names in (("operations", overrides), ("routes", refinements)) for name in names if name not in exposed ] @@ -71,7 +72,7 @@ def _routes( **base, **route, **(overrides.get(name) if isinstance(overrides.get(name), dict) else {}), - **routes.get(name, {}), + **refinements.get(name, {}), } for name, route in exposed.items() if overrides.get(name) is not False diff --git a/tests/unit/test_controller_desugar_access.py b/tests/unit/test_controller_desugar_access.py index 3baccab0..47f557a5 100644 --- a/tests/unit/test_controller_desugar_access.py +++ b/tests/unit/test_controller_desugar_access.py @@ -74,3 +74,26 @@ def test_routes_form_access_takes_priority_over_operations_form() -> None: ControllerDesugarer().desugar(controllers, _expanded_services()) routes = controllers["users"]["routes"] assert routes["get"]["access"] == ["role:admin"] + + +def test_explicit_controller_without_access_is_public() -> None: + controllers = { + "stats": { + "routes": {"count": {"service": "count-posts", "method": "GET", "path": "/stats"}}, + } + } + ControllerDesugarer().desugar(controllers, _expanded_services()) + assert controllers["stats"]["routes"]["count"]["access"] == "public" + + +def test_explicit_controller_operations_override_route_method_and_path() -> None: + controllers = { + "stats": { + "access": "authenticated", + "routes": {"count": {"service": "count-posts", "method": "GET", "path": "/stats"}}, + "operations": {"count": {"method": "POST", "path": "/stats/run"}}, + } + } + ControllerDesugarer().desugar(controllers, _expanded_services()) + route = controllers["stats"]["routes"]["count"] + assert (route["method"], route["path"], route["access"]) == ("POST", "/stats/run", "authenticated") From 507aba020e08e7c2b15db72b68f3fb53f3c6931f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:13:29 +0100 Subject: [PATCH 18/89] Reject services and routes bound to abstract repositories or services --- .../parsers/desugar/__init__.py | 17 +++++++++++++++-- .../parsers/desugar/inheritance_desugarer.py | 11 ++++++----- tests/unit/test_repo_service_inheritance.py | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/yaml_api_generator/parsers/desugar/__init__.py b/src/yaml_api_generator/parsers/desugar/__init__.py index 9666a38c..a211244b 100644 --- a/src/yaml_api_generator/parsers/desugar/__init__.py +++ b/src/yaml_api_generator/parsers/desugar/__init__.py @@ -10,6 +10,14 @@ _SVC_DESUGARER = InheritanceDesugarer("service") +def _bindings(**sections: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + f"{section}.{name}": cfg + for section, entries in sections.items() + for name, cfg in entries.items() if isinstance(cfg, dict) + } + + def desugar_v02(config: dict[str, Any]) -> dict[str, Any]: """Desugar v02 config: expand shorthand syntax to explicit definitions. @@ -35,8 +43,13 @@ def desugar_v02(config: dict[str, Any]) -> dict[str, Any]: controllers = config.get("controllers", {}) EntityDesugarer().desugar(entities, repositories) - _REPO_DESUGARER.desugar(repositories, controllers) - _SVC_DESUGARER.desugar(services, controllers) + routes = { + f"controllers.{name}.routes.{route_name}": route + for name, ctrl in controllers.items() if isinstance(ctrl, dict) + for route_name, route in (ctrl.get("routes") or {}).items() + } + _REPO_DESUGARER.desugar(repositories, _bindings(controllers=controllers, services=services)) + _SVC_DESUGARER.desugar(services, {**_bindings(controllers=controllers), **routes}) ServiceDesugarer().desugar(services, repositories, entities, defaults, auth_defaults) ControllerDesugarer().desugar(controllers, services) diff --git a/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py b/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py index 6486f5c4..f9876946 100644 --- a/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py +++ b/src/yaml_api_generator/parsers/desugar/inheritance_desugarer.py @@ -18,7 +18,7 @@ class InheritanceDesugarer: def __init__(self, layer: str) -> None: self._layer = layer - def desugar(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: + def desugar(self, entries: dict[str, Any], bindings: dict[str, dict[str, Any]]) -> None: """Validate inheritance and fold every ancestor into the entries extending it. Every ``extends`` target must exist and be abstract, chains must not @@ -30,7 +30,8 @@ def desugar(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: Args: entries: The layer's config dict (repositories or services). - bindings: The controllers, whose ```` key references entries. + bindings: Each config path whose ```` key references an entry, + mapped to the config at that path. Raises: ConfigErrors: On a missing or non-abstract target, a cycle or a bound abstract entry. @@ -49,7 +50,7 @@ def desugar(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: ]: del entries[name] - def _validate_not_bound(self, entries: dict[str, Any], bindings: dict[str, Any]) -> None: + def _validate_not_bound(self, entries: dict[str, Any], bindings: dict[str, dict[str, Any]]) -> None: abstract_names = { name for name, cfg in entries.items() if isinstance(cfg, dict) and cfg.get("abstract") @@ -58,10 +59,10 @@ def _validate_not_bound(self, entries: dict[str, Any], bindings: dict[str, Any]) ConfigError( code=f"DESUGAR_ABSTRACT_{self._layer.upper()}_BOUND", message=f"'{binding_name}' references abstract {self._layer} '{binding_cfg[self._layer]}'", - path=f"controllers.{binding_name}.{self._layer}", + path=f"{binding_name}.{self._layer}", ) for binding_name, binding_cfg in bindings.items() - if isinstance(binding_cfg, dict) and binding_cfg.get(self._layer) in abstract_names + if binding_cfg.get(self._layer) in abstract_names ] if errors: raise ConfigErrors(errors) diff --git a/tests/unit/test_repo_service_inheritance.py b/tests/unit/test_repo_service_inheritance.py index 8da3c55b..394ebcf6 100644 --- a/tests/unit/test_repo_service_inheritance.py +++ b/tests/unit/test_repo_service_inheritance.py @@ -232,3 +232,21 @@ def test_abstract_service_generates_a_base_class_its_services_extend(tmp_path): assert "async def countAll" in base_src and "async def summary" in base_src assert "from services.auditable import AuditableService" in service_src assert "async def countAll" not in service_src + + +def test_service_bound_to_abstract_repository_raises(): + config = _base_config() + config["repositories"]["base"] = {"abstract": True, "operations": {"countAll": {"query": {"command": "estimatedDocumentCount"}}}} + config["services"]["posts"]["repository"] = "base" + with pytest.raises(ConfigErrors) as exc_info: + desugar_v02(config) + assert [e.path for e in exc_info.value.errors if e.code == "DESUGAR_ABSTRACT_REPOSITORY_BOUND"] == ["services.posts.repository"] + + +def test_route_targeting_abstract_service_raises(): + config = _base_config() + config["services"]["base"] = {"abstract": True, "operations": {"delete": False}} + config["controllers"]["stats"] = {"routes": {"count": {"service": "base", "method": "GET", "path": "/stats"}}} + with pytest.raises(ConfigErrors) as exc_info: + desugar_v02(config) + assert [e.path for e in exc_info.value.errors if e.code == "DESUGAR_ABSTRACT_SERVICE_BOUND"] == ["controllers.stats.routes.count.service"] From 001399c3eb13817c0b406a4977fe6e2202c2b757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:13:29 +0100 Subject: [PATCH 19/89] Drop unimplemented pagination modes from the schema and their name-specific rejection --- .../schemas/yaml-spec-v0.3.schema.json | 9 +-- .../parsers/transformers/v03/pagination.py | 69 ++----------------- tests/unit/test_pagination.py | 13 ++-- 3 files changed, 14 insertions(+), 77 deletions(-) diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 147dff0f..04333d28 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -869,7 +869,7 @@ } }, "PaginationConfig": { - "description": "Controller pagination. The parser accepts the offset and keyset strategies and count none/exact; cursor, range and count: estimated are named so it can reject them by name (PAGINATION_UNSUPPORTED).", + "description": "Controller pagination.", "type": "object", "additionalProperties": false, "required": [ @@ -881,9 +881,7 @@ "type": "string", "enum": [ "offset", - "keyset", - "cursor", - "range" + "keyset" ] }, "size": { @@ -908,8 +906,7 @@ "type": "string", "enum": [ "none", - "exact", - "estimated" + "exact" ], "default": "none" }, diff --git a/src/yaml_api_generator/parsers/transformers/v03/pagination.py b/src/yaml_api_generator/parsers/transformers/v03/pagination.py index 46ef1f39..22365431 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/pagination.py +++ b/src/yaml_api_generator/parsers/transformers/v03/pagination.py @@ -5,13 +5,11 @@ This module turns that declaration into an ``IRPagination`` and validates it against the entity it pages. -``offset`` and ``keyset`` are the implemented strategies. The grammar names -``cursor`` and ``range`` -- and ``count: estimated`` -- but no runtime backs -them, so a spec that asks for one is rejected here rather than built into an -app that would ignore it. +The JSON schema closes every enumerated key (strategy, count, onOverflow, +sort direction); this module checks what depends on the paged entity or on +several keys at once. """ -from dataclasses import dataclass from typing import Any from yaml_api_generator.ir import IRCollection @@ -21,56 +19,6 @@ _DIRECTIONS = {"asc", "desc"} -@dataclass(frozen=True) -class Choice: - """A closed set of accepted values, split into implemented and unsupported.""" - - subject: str - supported: frozenset[str] - unknown_code: str - unsupported: frozenset[str] = frozenset() - - def validate(self, value: Any, path: str) -> list[ConfigError]: - if value in self.supported: - return [] - hint = f"Use one of: {', '.join(sorted(self.supported))}" - if value in self.unsupported: - return [ConfigError( - code="PAGINATION_UNSUPPORTED", - message=( - f"{self.subject} '{value}' is named by the v0.3 grammar but is " - "not supported yet" - ), - path=path, - hint=hint, - )] - return [ConfigError( - code=self.unknown_code, - message=f"Unknown {self.subject} '{value}'", - path=path, - hint=hint, - )] - - -_STRATEGY = Choice( - subject="pagination strategy", - supported=frozenset({"offset", "keyset"}), - unknown_code="PAGINATION_INVALID_STRATEGY", - unsupported=frozenset({"cursor", "range"}), -) -_COUNT = Choice( - subject="count mode", - supported=frozenset({"none", "exact"}), - unknown_code="PAGINATION_INVALID_COUNT", - unsupported=frozenset({"estimated"}), -) -_OVERFLOW = Choice( - subject="onOverflow mode", - supported=frozenset({"clamp", "reject"}), - unknown_code="PAGINATION_INVALID_OVERFLOW", -) - - def parse_pagination( raw: Any, collection: IRCollection | None, @@ -87,17 +35,10 @@ def parse_pagination( errors: list[ConfigError] = [] strategy = raw.get("strategy") - errors.extend(_STRATEGY.validate(strategy, f"{path}.strategy")) size, size_errors = _parse_size(raw.get("size"), f"{path}.size") errors.extend(size_errors) - count = raw.get("count", "none") - errors.extend(_COUNT.validate(count, f"{path}.count")) - - on_overflow = raw.get("onOverflow", "clamp") - errors.extend(_OVERFLOW.validate(on_overflow, f"{path}.onOverflow")) - sort, sort_errors = _parse_sort(raw.get("sort"), collection, f"{path}.sort") errors.extend(sort_errors) @@ -116,8 +57,8 @@ def parse_pagination( return IRPagination( strategy=strategy, size=size, - count=count, - on_overflow=on_overflow, + count=raw.get("count", "none"), + on_overflow=raw.get("onOverflow", "clamp"), envelope=bool(raw.get("envelope", False)), sort=sort, ), [] diff --git a/tests/unit/test_pagination.py b/tests/unit/test_pagination.py index 4fceb12a..382fea83 100644 --- a/tests/unit/test_pagination.py +++ b/tests/unit/test_pagination.py @@ -3,8 +3,7 @@ Covers the three layers the feature decomposes into -- repository primitives applied backend-agnostically (offset and keyset over SQL and Mongo), the implicit query-param -> primitive mapping, and the explicit controller grammar -with its build-time validation. offset and keyset are exercised end to end; -cursor, range and count: estimated are only checked to be accepted as reserved. +with its build-time validation. offset and keyset are exercised end to end. """ import textwrap @@ -468,11 +467,11 @@ def test_keyset_pagination_parsed_with_sort(tmp_path): assert pagination.sort == (IRSortKey("score", "desc"), IRSortKey("id", "asc")) -def test_reserved_strategies_rejected(tmp_path): - for reserved in ("cursor", "range"): - extra = "sort:\n - { field: id }" if reserved == "cursor" else '"on": score' - with pytest.raises(ConfigErrors): - _load(tmp_path, f"strategy: {reserved}\nsize: {{ default: 10, max: 50 }}\ncount: none\n{extra}") +@pytest.mark.parametrize("unimplemented", ["strategy: cursor", "strategy: range", "strategy: offset\ncount: estimated"]) +def test_unimplemented_modes_are_rejected_by_the_schema(tmp_path, unimplemented): + with pytest.raises(ConfigErrors) as excinfo: + _load(tmp_path, f"{unimplemented}\nsize: {{ default: 10, max: 50 }}") + assert _error_codes(excinfo) == {"SCHEMA_VALIDATION_ERROR"} def _error_codes(excinfo) -> set[str]: From 3632a77bec48713264454e922408ca6532064425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:24:27 +0100 Subject: [PATCH 20/89] Reject handlers naming undefined vendored functions and services declaring both a handler and an operation --- .../schemas/yaml-spec-v0.2.schema.json | 6 ++ .../schemas/yaml-spec-v0.3.schema.json | 6 ++ .../transformers/v03/vendored_imports.py | 68 ++++++++++++++----- tests/unit/test_vendored_imports.py | 21 ++++++ 4 files changed, 85 insertions(+), 16 deletions(-) diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index 2fc7356c..9fad9227 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -788,6 +788,12 @@ "ServiceConfig": { "type": "object", "additionalProperties": false, + "not": { + "required": [ + "handler", + "operation" + ] + }, "properties": { "abstract": { "type": "boolean" diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 04333d28..1ee6155e 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -712,6 +712,12 @@ "ServiceConfig": { "type": "object", "additionalProperties": false, + "not": { + "required": [ + "handler", + "operation" + ] + }, "properties": { "abstract": { "type": "boolean" diff --git a/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py b/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py index fa48205d..63d0e46e 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py +++ b/src/yaml_api_generator/parsers/transformers/v03/vendored_imports.py @@ -29,6 +29,7 @@ generator does not decide PEP 440 satisfiability in general. """ +import ast from pathlib import Path from typing import Any, Iterator @@ -79,7 +80,11 @@ def process_vendored_imports( requirements: tuple[str, ...] = () if not errors: - errors = _undeclared_handler_errors(resolved, import_map) + sources = {vi.alias: vi.source for vi in vendored} + errors = [ + error for handler, path in _handler_sites(resolved) + if (error := _handler_error(handler, path, sources)) is not None + ] if not errors: requirements, errors = _merge_requirements(vendored) @@ -144,26 +149,57 @@ def _escaping_symlinks(source: Path) -> Iterator[Path]: yield entry -def _undeclared_handler_errors( - resolved: dict[str, Any], import_map: dict[str, str] -) -> list[ConfigError]: - errors: list[ConfigError] = [] - for handler, path in _handler_sites(resolved): - alias = handler.split(".", 1)[0] - if alias not in import_map: - errors.append(ConfigError( - code="IMPORT_UNDECLARED_PACKAGE", - message=f"Handler '{handler}' references package '{alias}', which no imports entry vendors", - path=path, - hint=f"Add an imports entry aliased '{alias}' that vendors the package", - )) - return errors +def _handler_error(handler: str, path: str, sources: dict[str, Path]) -> ConfigError | None: + alias, _, target = handler.partition(".") + if alias not in sources: + return ConfigError( + code="IMPORT_UNDECLARED_PACKAGE", + message=f"Handler '{handler}' references package '{alias}', which no imports entry vendors", + path=path, + hint=f"Add an imports entry aliased '{alias}' that vendors the package", + ) + if not _defines(sources[alias], target): + return ConfigError( + code="IMPORT_HANDLER_NOT_FOUND", + message=f"Handler '{handler}' names '{target}', which package '{alias}' does not define", + path=path, + hint="Point the handler at a module-level function of the vendored package", + ) + return None + + +def _defines(package: Path, target: str) -> bool: + """Whether ``target`` (``[module.]*name``) is bound at the top level of a module in ``package``.""" + *modules, name = target.split(".") + module = package.joinpath(*modules) + source = module / "__init__.py" if module.is_dir() else module.with_suffix(".py") + return source.is_file() and any( + name in _bound_names(node) for node in ast.parse(source.read_text()).body + ) + + +def _bound_names(node: ast.stmt) -> set[str]: + match node: + case ast.FunctionDef(name=name) | ast.AsyncFunctionDef(name=name) | ast.ClassDef(name=name): + return {name} + case ast.Import(names=aliases) | ast.ImportFrom(names=aliases): + return {alias.asname or alias.name.split(".")[0] for alias in aliases} + case ast.Assign(targets=targets): + return {target.id for target in targets if isinstance(target, ast.Name)} + case ast.AnnAssign(target=ast.Name(id=name)): + return {name} + return set() def _handler_sites(resolved: dict[str, Any]): for name, cfg in resolved.get("services", {}).items(): - if isinstance(cfg, dict) and isinstance(cfg.get("handler"), str): + if not isinstance(cfg, dict): + continue + if isinstance(cfg.get("handler"), str): yield cfg["handler"], f"services.{name}.handler" + for op_name, op_cfg in (cfg.get("operations") or {}).items(): + if isinstance(op_cfg, dict) and isinstance(op_cfg.get("handler"), str): + yield op_cfg["handler"], f"services.{name}.operations.{op_name}.handler" for repo_name, repo_cfg in resolved.get("repositories", {}).items(): if not isinstance(repo_cfg, dict): continue diff --git a/tests/unit/test_vendored_imports.py b/tests/unit/test_vendored_imports.py index 21d2397f..a7923df8 100644 --- a/tests/unit/test_vendored_imports.py +++ b/tests/unit/test_vendored_imports.py @@ -274,6 +274,27 @@ def test_handler_referencing_undeclared_package_raises(tmp_path): assert _codes(spec) == ["IMPORT_UNDECLARED_PACKAGE"] +@pytest.mark.parametrize("handler", ["vend.nosuch", "vend.helper.nosuch", "vend.nomodule.build", "vend"]) +def test_handler_naming_an_undefined_function_raises(tmp_path, handler): + _write_package(tmp_path / "src", "vend_pkg") + spec = _minimal(tmp_path, "vend: { path: ./src/vend_pkg, package: vend_pkg }", handler=handler) + assert _codes(spec) == ["IMPORT_HANDLER_NOT_FOUND"] + + +@pytest.mark.parametrize("handler", ["vend.route", "vend.build", "vend.helper.build"]) +def test_handler_naming_a_bound_function_is_accepted(tmp_path, handler): + _write_package(tmp_path / "src", "vend_pkg") + spec = _minimal(tmp_path, "vend: { path: ./src/vend_pkg, package: vend_pkg }", handler=handler) + load_and_generate_ir(str(spec)) + + +def test_service_with_both_handler_and_operation_raises(tmp_path): + _write_package(tmp_path / "src", "vend_pkg") + spec = _minimal(tmp_path, "vend: { path: ./src/vend_pkg, package: vend_pkg }", handler="vend.route") + spec.write_text(spec.read_text().replace("handler: vend.route", "handler: vend.route, repository: posts, operation: list")) + assert _codes(spec) == ["SCHEMA_VALIDATION_ERROR"] + + def test_v03_bare_string_import_rejected(tmp_path): spec = _minimal(tmp_path, "vend: some.author.module", handler="vend.route") assert _codes(spec) == ["SCHEMA_VALIDATION_ERROR"] From 8950acd2234b0ad8c9ceb2e9dc9cf80b283fb528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:25:37 +0100 Subject: [PATCH 21/89] Reject repository queries targeting an undeclared collection --- .../parsers/transformers/query_validation.py | 32 +++++++++++++++++++ src/yaml_api_generator/parsers/yaml_parser.py | 5 +++ .../test_repository_operation_validation.py | 21 ++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 src/yaml_api_generator/parsers/transformers/query_validation.py diff --git a/src/yaml_api_generator/parsers/transformers/query_validation.py b/src/yaml_api_generator/parsers/transformers/query_validation.py new file mode 100644 index 00000000..cc3de70f --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/query_validation.py @@ -0,0 +1,32 @@ +"""Collection references inside repository operation queries. + +A query may target another collection through ``query.collection``. That name +must be a concrete repository of the desugared spec, so a typo fails generation +instead of every target querying an empty, implicitly created collection. +""" + +from typing import Any + +from yaml_api_generator.parsers.yaml_parser import ConfigError + + +def validate_spec_queries(spec: dict[str, Any]) -> list[ConfigError]: + repositories = { + name: repository for name, repository in (spec.get("repositories") or {}).items() + if isinstance(repository, dict) + } + collections = {name for name, repository in repositories.items() if not repository.get("abstract")} + return [ + ConfigError( + code="QUERY_COLLECTION_NOT_FOUND", + message=f"Operation '{op_name}' queries unknown collection '{operation['query']['collection']}'", + path=f"repositories.{repo_name}.operations.{op_name}.query.collection", + hint=f"Use one of: {', '.join(sorted(collections))}", + ) + for repo_name, repository in repositories.items() + for op_name, operation in (repository.get("operations") or {}).items() + if isinstance(operation, dict) + and isinstance(operation.get("query"), dict) + and "collection" in operation["query"] + and operation["query"]["collection"] not in collections + ] diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 4bbc61e0..bb9329df 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -273,6 +273,11 @@ def load_and_generate_ir( if symbol_errors: raise ConfigErrors(symbol_errors) + from .transformers.query_validation import validate_spec_queries + query_errors = validate_spec_queries(resolved) + if query_errors: + raise ConfigErrors(query_errors) + from .transformers.path_validation import validate_spec_paths path_errors = validate_spec_paths(resolved) if path_errors: diff --git a/tests/unit/test_repository_operation_validation.py b/tests/unit/test_repository_operation_validation.py index c857ee1e..c38c802c 100644 --- a/tests/unit/test_repository_operation_validation.py +++ b/tests/unit/test_repository_operation_validation.py @@ -93,3 +93,24 @@ def test_declared_repository_operation_emits_its_query(tmp_path, generator): sources = [path.read_text() for path in out.rglob("*.py")] assert any("'state': 'published'" in source for source in sources) + + +def test_query_targeting_an_unknown_collection_is_rejected(tmp_path): + spec = copy.deepcopy(_SPEC) + spec["repositories"]["posts"]["operations"] = { + "recent": {"query": {"command": "find", "collection": "postz", "filter": {}}} + } + [error] = _errors(tmp_path, spec) + assert (error.code, error.path) == ( + "QUERY_COLLECTION_NOT_FOUND", "repositories.posts.operations.recent.query.collection" + ) + + +def test_query_targeting_a_declared_collection_is_accepted(tmp_path): + spec = copy.deepcopy(_SPEC) + spec["repositories"]["posts"]["operations"] = { + "recent": {"query": {"command": "find", "collection": "posts", "filter": {}}} + } + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec)) + load_and_generate_ir(str(path)) From cad072ec7baa6c322ef115caea8667942c97b20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:44:09 +0100 Subject: [PATCH 22/89] Keep the FastAPI repository mixin import on its own line when the repository extends a base --- .../backend/fastapi/templates/repository.py.jinja | 2 +- tests/integration/specs/services_mongo/spec.yaml | 8 ++++++++ tests/integration/test_services_mongo.py | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja index f3dadc38..7278b050 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/repository.py.jinja @@ -6,7 +6,7 @@ from typing import Any {% endif %} {% if not is_abstract %} -from ._ops import RepositoryBase{% for op in operations %}, {{ op | capitalize }}Op{% endfor %} +from ._ops import RepositoryBase{% for op in operations %}, {{ op | capitalize }}Op{% endfor +%} {% endif %} {% if generated_base %} from {{ generated_base.module }} import {{ generated_base.class_name }} diff --git a/tests/integration/specs/services_mongo/spec.yaml b/tests/integration/specs/services_mongo/spec.yaml index 921bd9b8..bb8b92a0 100644 --- a/tests/integration/specs/services_mongo/spec.yaml +++ b/tests/integration/specs/services_mongo/spec.yaml @@ -21,8 +21,14 @@ entities: body: { type: string, required: true } repositories: + countable: + abstract: true + operations: + countAll: + query: { command: countDocuments, filter: {} } tasks: entity: task + extends: countable operations: byTitle: query: { command: find, filter: { title: $title } } @@ -44,6 +50,7 @@ services: byTitle: true notesByBody: true retitle: true + countAll: true echo: { handler: hooks.echo } notes: { repository: notes } @@ -57,6 +64,7 @@ controllers: notesByBody: { method: GET, path: "/notes/{text}" } retitle: { method: POST, path: "/retitle/{old}/{new}" } echo: { method: POST, path: "/echo/{id}" } + countAll: { method: GET, path: /stats/count } notes: service: notes base_path: /notes diff --git a/tests/integration/test_services_mongo.py b/tests/integration/test_services_mongo.py index 10e20d33..1737c3e3 100644 --- a/tests/integration/test_services_mongo.py +++ b/tests/integration/test_services_mongo.py @@ -47,6 +47,12 @@ def test_operation_reads_the_collection_its_query_names(api): assert [document["body"] for document in response.json()] == [body] +def test_operation_inherited_from_abstract_repository_runs(api): + before = api.get("/api/tasks/stats/count").json() + _create(api, "tasks", title=uuid4().hex) + assert api.get("/api/tasks/stats/count").json() == before + 1 + + def test_bulk_write_applies_its_requests(api): old, new = uuid4().hex, uuid4().hex _create(api, "tasks", title=old) From 838ec43b5758d8b65878f5e7863d79d3821e8df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:49:40 +0100 Subject: [PATCH 23/89] Route literal path segments ahead of parameters at the same position --- .../backend/fastapi/templates/routes.py.jinja | 35 +++++++++---------- src/yaml_api_generator/ir/api.py | 5 +++ .../parsers/transformers/v01/__init__.py | 3 +- .../parsers/transformers/v02/__init__.py | 3 +- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 80dbbd33..7609f4b1 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -234,12 +234,27 @@ _files = record_files[{{ collection_name | tojson }}] {% endif %} {% endmacro %} -{% for route in crud_routes %} +{% for route in routes if route.operation != "login" %} {% set body_model = route.typed_body_model or route.body_model %} {% set partial = route.operation == "update" %} {% set body_expr = "upload.payload" if files else 'payload.model_dump(mode="json"' ~ (", exclude_unset=True" if partial else "") ~ ")" %} {% set record_expr = "upload.record" if files else body_expr %} -{% if route.operation == "list" %} +{% if route.runs_operation %} +@router.{{ route.method | lower }}("{{ route.relative_path }}", tags=["{{ collection_name | capitalize_collection }}"]{{ openapi_extra(route) }}) +async def {{ route.operation }}_{{ collection_name }}(request: Request{% for name in route.path_params %}, {{ name }}: str{% endfor %}{{ auth_param(route) }}, {{ svc_dep() }}{{ where_repo_params(route) }}) -> Any: + _body: dict[str, Any] = {} + try: + _body = await request.json() + except (ValueError, UnicodeDecodeError): + pass +{{ request_locals(route, "_body") }} +{{ pre_fetch_guard(route) }} +{{ bind_inputs(route) }} +{{ prepare_inputs(route, "dict(request.path_params)", "_body") }} +{{ call_with_request("svc." ~ route.operation ~ "(_request)", "principal" if auth.enabled and route.needs_auth else "None", "_body", "_service_inputs") }} + + +{% elif route.operation == "list" %} @router.{{ route.method | lower }}("{{ route.relative_path }}"{{ openapi_extra(route) }}) {% set list_prefix = (extra_path_params(route) ~ request_param(route) ~ auth_param(route)) | trim(chars=", ") %} async def list_{{ collection_name }}({{ list_prefix }}{% if list_prefix %}, {% endif %}{{ svc_dep() }}{{ where_repo_params(route) }}{% if route.pagination %}, limit: int | None = None{% if route.pagination.strategy == "offset" %}, offset: int | None = None{% else %}, after: str | None = None{% endif %}{% endif %}) -> Any: @@ -333,19 +348,3 @@ async def {{ file_route.handler_name }}({{ source.record_id_param }}: str{{ extr {% endfor %} -{% for route in operation_routes %} - - -@router.{{ route.method | lower }}("{{ route.relative_path }}", tags=["{{ collection_name | capitalize_collection }}"]{{ openapi_extra(route) }}) -async def {{ route.operation }}_{{ collection_name }}(request: Request{% for name in route.path_params %}, {{ name }}: str{% endfor %}{{ auth_param(route) }}, {{ svc_dep() }}{{ where_repo_params(route) }}) -> Any: - _body: dict[str, Any] = {} - try: - _body = await request.json() - except (ValueError, UnicodeDecodeError): - pass -{{ request_locals(route, "_body") }} -{{ pre_fetch_guard(route) }} -{{ bind_inputs(route) }} -{{ prepare_inputs(route, "dict(request.path_params)", "_body") }} -{{ call_with_request("svc." ~ route.operation ~ "(_request)", "principal" if auth.enabled and route.needs_auth else "None", "_body", "_service_inputs") }} -{% endfor %} diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index e1d5c277..3ae9d2c4 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -371,6 +371,11 @@ class IRRoute: handler: str | None = None pagination: IRPagination | None = None + @property + def precedence(self) -> tuple[bool, ...]: + """Sort key under which a literal path segment outranks a parameter at the same position.""" + return tuple(segment.startswith("{") for segment in self.path.split("/")) + @dataclass(frozen=True) class IRFileRoute: diff --git a/src/yaml_api_generator/parsers/transformers/v01/__init__.py b/src/yaml_api_generator/parsers/transformers/v01/__init__.py index 01ab648a..44597463 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v01/__init__.py @@ -4,6 +4,7 @@ """ from dataclasses import replace +from operator import attrgetter from typing import Any from yaml_api_generator.ir import ( @@ -103,7 +104,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: collections=collections, repositories=repositories, services=services, - routes=routes, + routes=sorted(routes, key=attrgetter("precedence")), identityCollections=identity_collections, mongo=IRMongo( uri=mongo_data.get("uri", "mongodb://localhost:27017"), diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index 1bacc959..e80c5aef 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -4,6 +4,7 @@ """ from dataclasses import fields as dataclass_fields +from operator import attrgetter from typing import Any from yaml_api_generator.ir import IRDocument @@ -143,7 +144,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: version=str(resolved["version"]), api=self._route.build_api(api_data), collections=collections, - routes=routes, + routes=sorted(routes, key=attrgetter("precedence")), identityCollections=identity_collections, dtos=dtos, services=services, From 9c61d55d0b2fb075a8467490ed485701664fe8d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 21:49:40 +0100 Subject: [PATCH 24/89] Name raw ASGI route handlers by method and path so controllers sharing a service keep their own access --- .../generators/backend/compile_context.py | 5 ++++ .../backend/raw/templates/app.py.jinja | 11 ++----- .../raw/templates/handler_routes.py.jinja | 2 +- .../backend/raw/templates/routes.py.jinja | 3 +- .../specs/services_mongo/spec.yaml | 29 ++++++++++++++++++- .../test_generated_where_authorization.py | 14 ++++----- tests/integration/test_services_mongo.py | 29 +++++++++++++++++-- tests/unit/test_access_reason.py | 6 ++-- 8 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index 1c41229c..e2bb51a4 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -291,6 +291,11 @@ def runs_operation(self) -> bool: self.handler is not None or self.operation not in CRUD_OPERATIONS ) + @property + def handler_name(self) -> str: + """Function name unique to this method and path across the whole API.""" + return to_py_identifier(f"handle_{self.method.lower()}{self.relative_path}") + @property def django_path(self) -> str: path = self.relative_path.lstrip("/") diff --git a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja index 0fdd9a16..0e0fed0f 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja @@ -37,16 +37,9 @@ def _path_to_regex(path: str) -> str: return re.sub(r"\{(\w+)\}", _replace, path) -{% set crud_routes = routes | rejectattr("operation", "equalto", "login") | list %} ROUTES: dict[str, list[tuple[re.Pattern[str], Any]]] = {} -{% for route in crud_routes %} -{% if route.collection %} -{% set handler_name = "handle_" ~ route.operation ~ "_" ~ route.collection %} -ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ handler_name }})) -{% endif %} -{% endfor %} -{% for route in handler_routes %} -ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), handle_custom_{{ loop.index }})) +{% for route in routes if route.operation != "login" %} +ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ route.handler_name }})) {% endfor %} {% for identity in ctx.auth.identity_collections %} {% if identity.has_password and identity.identifier %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja index 9ee98ca7..47957b57 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja @@ -45,7 +45,7 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: {% for route in routes %} -async def handle_custom_{{ loop.index }}( +async def {{ route.handler_name }}( url_params: dict[str, str], body: dict[str, Any] | None, headers: dict[str, str], diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index 33e5ed4d..019b44fc 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -132,8 +132,7 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: {% endmacro %} {% for route in routes %} {% if route.operation != "login" %} -{% set handler_name = "handle_" ~ route.operation ~ "_" ~ collection_name %} -async def {{ handler_name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: +async def {{ route.handler_name }}(url_params: dict, body: dict | None, headers: dict[str, str], db: Any, query: dict[str, Any] | None = None, cookies: dict[str, str] | None = None) -> tuple[int, Any]: repo = {{ collection_name | capitalize_collection }}Repository(db) service = {{ collection_name | capitalize_collection }}Service(repo) principal = None diff --git a/tests/integration/specs/services_mongo/spec.yaml b/tests/integration/specs/services_mongo/spec.yaml index bb8b92a0..eb2ea1a4 100644 --- a/tests/integration/specs/services_mongo/spec.yaml +++ b/tests/integration/specs/services_mongo/spec.yaml @@ -11,6 +11,12 @@ mongo: database: ${MONGO_DB} entities: + user: + type: auth + auth: { identifier: email, via: password } + fields: + id: { type: id } + email: { type: string, required: true, unique: true } task: fields: id: { type: id } @@ -21,6 +27,7 @@ entities: body: { type: string, required: true } repositories: + users: { entity: user, crud: [create] } countable: abstract: true operations: @@ -44,6 +51,7 @@ repositories: crud: [create] services: + users: { repository: users } tasks: repository: tasks operations: @@ -55,6 +63,25 @@ services: notes: { repository: notes } controllers: + users: + service: users + base_path: /users + access: public + archive: + service: tasks + base_path: /archive + access: "*" + operations: + byTitle: { method: GET, path: "/by-title/{title}" } + notesByBody: false + retitle: false + countAll: false + echo: false + totals: + routes: + tasks: { service: countAll-tasks, method: GET, path: /tasks-total } + operations: + tasks: { path: /totals/tasks } tasks: service: tasks base_path: /tasks @@ -64,7 +91,7 @@ controllers: notesByBody: { method: GET, path: "/notes/{text}" } retitle: { method: POST, path: "/retitle/{old}/{new}" } echo: { method: POST, path: "/echo/{id}" } - countAll: { method: GET, path: /stats/count } + countAll: { method: GET, path: /stats } notes: service: notes base_path: /notes diff --git a/tests/integration/test_generated_where_authorization.py b/tests/integration/test_generated_where_authorization.py index d6712202..61eeb3c0 100644 --- a/tests/integration/test_generated_where_authorization.py +++ b/tests/integration/test_generated_where_authorization.py @@ -408,7 +408,7 @@ async def audit(request): sys.modules["app.handlers"] = handlers_module async def verify(): - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 200, body @@ -417,39 +417,39 @@ async def verify(): assert db["teams"].criteria == [{{"id": "org-1"}}] db["organizations"].documents = [] - status, body = await handlers.handle_custom_1( + status, body = await handlers.handle_post_audit__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 403, body db["organizations"].documents = [{{"id": "org-1", "ownerId": "u1"}}] - status, body = await handlers.handle_custom_1( + status, body = await handlers.handle_post_audit__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 200, body assert body == {{"authorized": True}} - status, body = await routes.handle_get_reports( + status, body = await routes.handle_get_reports__organizationId___id_( {{"organizationId": "org-1", "id": "r1"}}, None, {{}}, db ) assert status == 200, body assert body["organizationId"] == "org-1" db["organizations"].documents = [{{"id": "org-1", "ownerId": "other"}}] - status, body = await routes.handle_get_reports( + status, body = await routes.handle_get_reports__organizationId___id_( {{"organizationId": "org-1", "id": "r1"}}, None, {{}}, db ) assert status == 403, body db["organizations"].documents = [] db["teams"].documents = [] - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 403, body db["organizations"].documents = [{{"id": "org-1", "ownerId": "u1"}}] - status, body = await routes.handle_list_reports( + status, body = await routes.handle_get_reports__organizationId_( {{"organizationId": "org-1"}}, None, {{}}, db ) assert status == 200, body diff --git a/tests/integration/test_services_mongo.py b/tests/integration/test_services_mongo.py index 1737c3e3..ea2ee716 100644 --- a/tests/integration/test_services_mongo.py +++ b/tests/integration/test_services_mongo.py @@ -47,10 +47,10 @@ def test_operation_reads_the_collection_its_query_names(api): assert [document["body"] for document in response.json()] == [body] -def test_operation_inherited_from_abstract_repository_runs(api): - before = api.get("/api/tasks/stats/count").json() +def test_single_segment_operation_path_outranks_the_record_route(api): + before = api.get("/api/tasks/stats").json() _create(api, "tasks", title=uuid4().hex) - assert api.get("/api/tasks/stats/count").json() == before + 1 + assert api.get("/api/tasks/stats").json() == before + 1 def test_bulk_write_applies_its_requests(api): @@ -73,3 +73,26 @@ def test_handler_receives_the_request_it_documents(api): "principal": None, "inputs": {}, } + + +@pytest.fixture(scope="module") +def user(api): + login = {"email": f"{uuid4().hex}@example.com", "password": "correct-horse-battery"} + registered = api.post("/api/users", json=login).json() + token = api.post("/api/users/login", json=login).json()["token"] + return {"id": registered["id"], "headers": {"Authorization": f"Bearer {token}"}} + + +def test_controllers_sharing_a_service_keep_their_own_access(api, user): + title = uuid4().hex + _create(api, "tasks", title=title) + assert api.get("/api/archive").status_code == 401 + assert api.get("/api/tasks").status_code == 200 + assert _titles(api.get(f"/api/archive/by-title/{title}", headers=user["headers"])) == [title] + + +def test_explicit_controller_route_is_public_at_its_overridden_path(api): + response = api.get("/api/totals/tasks") + assert response.status_code == 200, response.text + assert isinstance(response.json(), int) + assert api.get("/api/tasks-total").status_code == 404 diff --git a/tests/unit/test_access_reason.py b/tests/unit/test_access_reason.py index e20b5068..2f8f389a 100644 --- a/tests/unit/test_access_reason.py +++ b/tests/unit/test_access_reason.py @@ -281,7 +281,7 @@ async def principal(headers, db): posts.resolve_principal = principal database = Database() - result = asyncio.run(posts.handle_create_posts( + result = asyncio.run(posts.handle_post_posts_( {{}}, {{"title": "Post", "ownerId": "blocked"}}, {{}}, @@ -290,13 +290,13 @@ async def principal(headers, db): assert result == (403, {{"detail": "Blocked accounts cannot create posts."}}) current["sub"] = "u1" - result = asyncio.run(posts.handle_get_posts( + result = asyncio.run(posts.handle_get_posts__id_( {{"id": "p1"}}, None, {{}}, database )) assert result == (403, {{"detail": "Only the owner may read this post."}}) current["sub"] = "u2" - status, document = asyncio.run(posts.handle_get_posts( + status, document = asyncio.run(posts.handle_get_posts__id_( {{"id": "p1"}}, None, {{}}, database )) assert status == 200 From 86c4d84b0bfdc5d48a147393585d2b14f8729d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sat, 24 Oct 2026 23:47:57 +0100 Subject: [PATCH 25/89] Type field keys per field type in every schema and check bounds, defaults and refs in every field map of every version --- .../schemas/yaml-spec-v0.1.schema.json | 72 ++-- .../schemas/yaml-spec-v0.2.schema.json | 204 +++++++++--- .../schemas/yaml-spec-v0.3.schema.json | 312 ++++++++++++++---- .../transformers/constraint_validation.py | 142 ++------ .../parsers/transformers/dto_validation.py | 54 +-- .../parsers/transformers/field_maps.py | 35 ++ .../parsers/transformers/file_validation.py | 11 +- .../parsers/transformers/ref_validation.py | 20 +- .../v01/collection_transformer.py | 13 +- .../v02/collection_transformer.py | 13 +- src/yaml_api_generator/parsers/yaml_parser.py | 17 +- tests/unit/support/payload_requests.py | 4 + tests/unit/test_constraint_validation.py | 27 +- tests/unit/test_field_grammar.py | 135 ++++++++ tests/unit/test_file_field_validation.py | 6 +- tests/unit/test_request_access_validation.py | 19 +- tests/unit/test_v02_transformer.py | 2 +- 17 files changed, 711 insertions(+), 375 deletions(-) create mode 100644 src/yaml_api_generator/parsers/transformers/field_maps.py create mode 100644 tests/unit/test_field_grammar.py diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json index a86ce97e..799f4bad 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.1.schema.json @@ -182,48 +182,76 @@ "sp_entity_id": { "type": "string" } } }, - "FieldConfig": { + "FieldShape": { + "$comment": "The keys each field type accepts; FieldConfig and NestedFieldConfig close it.", "type": "object", - "additionalProperties": false, "required": ["type"], "properties": { "type": { "enum": ["string", "number", "boolean", "datetime", "enum", "array", "object", "ref", "id"] }, "required": { "type": "boolean" }, - "default": {}, - "unique": { "type": "boolean" }, - "min": { "type": "number" }, - "max": { "type": "number" }, - "minLength": { "type": "integer" }, - "maxLength": { "type": "integer" }, - "format": { - "enum": ["email", "e164", "uri", "uuidv7", "uuidv4"] - }, - "values": { "$ref": "#/$defs/StringList" }, - "items": { "$ref": "#/$defs/FieldConfig" }, - "properties": { - "type": "object", - "additionalProperties": { "$ref": "#/$defs/FieldConfig" } - }, - "additionalProperties": { "type": "boolean" }, - "ref": { "type": "string" } + "default": {} }, "allOf": [ + { + "if": { "properties": { "type": { "const": "string" } }, "required": ["type"] }, + "then": { + "properties": { + "minLength": { "type": "integer", "minimum": 0 }, + "maxLength": { "type": "integer", "minimum": 0 }, + "format": { "enum": ["email", "e164", "uri", "uuidv7", "uuidv4"] } + } + } + }, + { + "if": { "properties": { "type": { "const": "number" } }, "required": ["type"] }, + "then": { "properties": { "min": { "type": "number" }, "max": { "type": "number" } } } + }, { "if": { "properties": { "type": { "const": "enum" } }, "required": ["type"] }, - "then": { "required": ["values"] } + "then": { + "required": ["values"], + "properties": { "values": { "$ref": "#/$defs/StringList", "minItems": 1 } } + } }, { "if": { "properties": { "type": { "const": "array" } }, "required": ["type"] }, - "then": { "required": ["items"] } + "then": { "required": ["items"], "properties": { "items": { "$ref": "#/$defs/NestedFieldConfig" } } } + }, + { + "if": { "properties": { "type": { "const": "object" } }, "required": ["type"] }, + "then": { + "properties": { + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/NestedFieldConfig" } + }, + "additionalProperties": { "type": "boolean" } + } + } }, { "if": { "properties": { "type": { "const": "ref" } }, "required": ["type"] }, - "then": { "required": ["ref"] } + "then": { "required": ["ref"], "properties": { "ref": { "type": "string" } } } + }, + { + "if": { "properties": { "type": { "const": "id" } }, "required": ["type"] }, + "then": { "properties": { "format": { "enum": ["uuidv7", "uuidv4"] } } } } ] }, + "FieldConfig": { + "$comment": "A top-level collection field.", + "$ref": "#/$defs/FieldShape", + "properties": { "unique": { "type": "boolean" } }, + "unevaluatedProperties": false + }, + "NestedFieldConfig": { + "$comment": "A field inside object properties or array items.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false + }, "RolesConfig": { "type": "object", "additionalProperties": false, diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index 71917c0d..2a8210a9 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -515,7 +515,7 @@ "fields": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/FieldConfig" + "$ref": "#/$defs/PayloadFieldConfig" } } } @@ -555,9 +555,9 @@ } ] }, - "FieldConfig": { + "FieldShape": { + "$comment": "The keys each field type accepts; FieldConfig, NestedFieldConfig and PayloadFieldConfig close it.", "type": "object", - "additionalProperties": false, "required": [ "type" ], @@ -578,54 +578,67 @@ "required": { "type": "boolean" }, - "default": {}, - "unique": { + "readonly": { "type": "boolean" }, - "min": { - "type": "number" - }, - "max": { - "type": "number" - }, - "minLength": { - "type": "integer" - }, - "maxLength": { - "type": "integer" - }, - "format": { - "enum": [ - "email", - "e164", - "uri", - "uuidv7", - "uuidv4" - ] - }, - "values": { - "$ref": "#/$defs/StringList" - }, - "items": { - "$ref": "#/$defs/FieldConfig" - }, - "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/FieldConfig" + "default": {} + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "string" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "minLength": { + "type": "integer", + "minimum": 0 + }, + "maxLength": { + "type": "integer", + "minimum": 0 + }, + "format": { + "enum": [ + "email", + "e164", + "uri", + "uuidv7", + "uuidv4" + ] + } + } } }, - "additionalProperties": { - "type": "boolean" - }, - "ref": { - "type": "string" + { + "if": { + "properties": { + "type": { + "const": "number" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + } }, - "readonly": { - "type": "boolean" - } - }, - "allOf": [ { "if": { "properties": { @@ -640,7 +653,13 @@ "then": { "required": [ "values" - ] + ], + "properties": { + "values": { + "$ref": "#/$defs/StringList", + "minItems": 1 + } + } } }, { @@ -657,7 +676,37 @@ "then": { "required": [ "items" + ], + "properties": { + "items": { + "$ref": "#/$defs/NestedFieldConfig" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "object" + } + }, + "required": [ + "type" ] + }, + "then": { + "properties": { + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/NestedFieldConfig" + } + }, + "additionalProperties": { + "type": "boolean" + } + } } }, { @@ -674,11 +723,72 @@ "then": { "required": [ "ref" + ], + "properties": { + "ref": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "id" + } + }, + "required": [ + "type" ] + }, + "then": { + "properties": { + "format": { + "enum": [ + "uuidv7", + "uuidv4" + ] + } + } } } ] }, + "FieldConfig": { + "$comment": "A top-level entity field.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false, + "properties": { + "unique": { + "type": "boolean" + } + } + }, + "NestedFieldConfig": { + "$comment": "A field inside object properties or array items.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false + }, + "PayloadFieldConfig": { + "$comment": "A DTO or service input field, which carries no persistence keys.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false, + "properties": { + "type": { + "enum": [ + "string", + "number", + "boolean", + "datetime", + "enum", + "array", + "object", + "ref" + ] + } + } + }, "OperationConfig": { "type": "object", "additionalProperties": false, diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 79c86eef..37dd496d 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -217,7 +217,7 @@ ] }, { - "$ref": "#/$defs/FieldConfig" + "$ref": "#/$defs/PayloadFieldConfig" } ] } @@ -412,14 +412,14 @@ "fields": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/FieldConfig" + "$ref": "#/$defs/PayloadFieldConfig" } } } }, - "FieldConfig": { + "FieldShape": { + "$comment": "The keys each field type accepts; FieldConfig, NestedFieldConfig and PayloadFieldConfig close it.", "type": "object", - "additionalProperties": false, "required": [ "type" ], @@ -441,70 +441,82 @@ "required": { "type": "boolean" }, - "default": {}, - "unique": { - "type": "boolean" - }, - "min": { - "type": "number" - }, - "max": { - "type": "number" - }, - "minLength": { - "type": "integer" - }, - "maxLength": { - "type": "integer" - }, - "format": { - "enum": [ - "email", - "e164", - "uri", - "uuidv7", - "uuidv4" - ] - }, - "values": { - "$ref": "yaml-spec-v0.2.schema.json#/$defs/StringList" - }, - "items": { - "$ref": "#/$defs/FieldConfig" - }, - "properties": { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/FieldConfig" - } - }, - "additionalProperties": { - "type": "boolean" - }, - "ref": { - "type": "string" - }, "readonly": { "type": "boolean" - }, - "maxSize": { - "$ref": "#/$defs/FileSize" - }, - "types": { - "$ref": "#/$defs/MimeTypes" - }, - "disposition": { - "enum": [ - "attachment", - "inline" - ] - }, - "maxItems": { - "type": "integer", - "minimum": 1 } }, "allOf": [ + { + "if": { + "properties": { + "type": { + "not": { + "const": "file" + } + } + } + }, + "then": { + "properties": { + "default": {} + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "string" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "minLength": { + "type": "integer", + "minimum": 0 + }, + "maxLength": { + "type": "integer", + "minimum": 0 + }, + "format": { + "enum": [ + "email", + "e164", + "uri", + "uuidv7", + "uuidv4" + ] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "number" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + } + } + }, { "if": { "properties": { @@ -519,7 +531,13 @@ "then": { "required": [ "values" - ] + ], + "properties": { + "values": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/StringList", + "minItems": 1 + } + } } }, { @@ -536,7 +554,37 @@ "then": { "required": [ "items" + ], + "properties": { + "items": { + "$ref": "#/$defs/NestedFieldConfig" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "object" + } + }, + "required": [ + "type" ] + }, + "then": { + "properties": { + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/NestedFieldConfig" + } + }, + "additionalProperties": { + "type": "boolean" + } + } } }, { @@ -553,11 +601,147 @@ "then": { "required": [ "ref" + ], + "properties": { + "ref": { + "type": "string" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "id" + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "format": { + "enum": [ + "uuidv7", + "uuidv4" + ] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "file" + } + }, + "required": [ + "type" ] + }, + "then": { + "properties": { + "maxSize": { + "$ref": "#/$defs/FileSize" + }, + "types": { + "$ref": "#/$defs/MimeTypes" + }, + "disposition": { + "enum": [ + "attachment", + "inline" + ] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "array" + }, + "items": { + "properties": { + "type": { + "const": "file" + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "type", + "items" + ] + }, + "then": { + "properties": { + "maxItems": { + "type": "integer", + "minimum": 1 + } + } } } ] }, + "FieldConfig": { + "$comment": "A top-level entity field.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false, + "allOf": [ + { + "if": { + "properties": { + "type": { + "not": { + "const": "file" + } + } + } + }, + "then": { + "properties": { + "unique": { + "type": "boolean" + } + } + } + } + ] + }, + "NestedFieldConfig": { + "$comment": "A field inside object properties or array items.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false + }, + "PayloadFieldConfig": { + "$comment": "A DTO or service input field, which carries no persistence keys.", + "$ref": "#/$defs/FieldShape", + "unevaluatedProperties": false, + "properties": { + "type": { + "enum": [ + "string", + "number", + "boolean", + "datetime", + "enum", + "array", + "object", + "ref", + "file" + ] + } + } + }, "DatastoreConfig": { "type": "object", "additionalProperties": false, diff --git a/src/yaml_api_generator/parsers/transformers/constraint_validation.py b/src/yaml_api_generator/parsers/transformers/constraint_validation.py index 16caedcc..dcaa20f9 100644 --- a/src/yaml_api_generator/parsers/transformers/constraint_validation.py +++ b/src/yaml_api_generator/parsers/transformers/constraint_validation.py @@ -1,30 +1,20 @@ -"""Semantic coherence checks for field constraints in a v0.2 spec. +"""Semantic coherence checks for field constraints, in every field map of a spec. -The generator copies ``min``/``max``/``minLength``/``maxLength``/``default`` and -``values`` straight into Pydantic and DRF models. The JSON schema only checks -each key's own type, so impossible or mismatched constraints (``min`` above -``max``, ``minLength`` on a number, a ``default`` outside an ``enum``) reach the -models silently and either crash at import time or produce models that reject -every input. These checks run on the pre-desugar spec, mirroring -``validate_spec_names``, so the rejection happens once for all targets. +The JSON schema decides which keys a field of each type may carry. What it +cannot express is the relation between values: a lower bound above its upper +bound, or a ``default`` its own field would reject. These checks run on the +pre-desugar spec, so the rejection happens once for all targets. """ +from collections.abc import Iterator from typing import Any +from yaml_api_generator.parsers.transformers.field_maps import spec_fields from yaml_api_generator.parsers.transformers.file_validation import file_config from yaml_api_generator.parsers.yaml_parser import ConfigError from yaml_api_generator.runtime.fields import conform -# Constraints only meaningful for a given field type. min/max belong to numbers; -# minLength/maxLength belong to strings. format is shared by string and id -# (where uuidv4/uuidv7 select the key generator), so it is checked separately. -_NUMERIC_ONLY = ("min", "max") -_STRING_ONLY = ("minLength", "maxLength") -_FORMAT_TYPES = ("string", "id") -# File limits apply to file nodes; a file's record value is upload metadata, so -# it can be neither unique nor defaulted. -_FILE_ONLY = ("maxSize", "types", "disposition") -_FILE_FORBIDDEN = ("unique", "default") +_BOUNDS = (("min", "max"), ("minLength", "maxLength")) def _error(message: str, path: str, hint: str) -> ConfigError: @@ -33,89 +23,24 @@ def _error(message: str, path: str, hint: str) -> ConfigError: ) -def _check_field(field: dict[str, Any], path: str, errors: list[ConfigError]) -> None: - field_type = field.get("type") - - minimum, maximum = field.get("min"), field.get("max") - if minimum is not None and maximum is not None and minimum > maximum: - errors.append(_error( - f"min ({minimum}) is greater than max ({maximum})", path, - "Ensure min is less than or equal to max", - )) - min_len, max_len = field.get("minLength"), field.get("maxLength") - if min_len is not None and max_len is not None and min_len > max_len: - errors.append(_error( - f"minLength ({min_len}) is greater than maxLength ({max_len})", path, - "Ensure minLength is less than or equal to maxLength", - )) - for key in ("minLength", "maxLength"): - value = field.get(key) - if value is not None and value < 0: - errors.append(_error( - f"{key} ({value}) must not be negative", path, - f"Set {key} to zero or a positive integer", - )) - - if field_type != "number": - for key in _NUMERIC_ONLY: - if field.get(key) is not None: - errors.append(_error( - f"{key} is only valid on number fields, not '{field_type}'", path, - "Use minLength/maxLength for strings or remove the constraint", - )) - if field_type != "string": - for key in _STRING_ONLY: - if field.get(key) is not None: - errors.append(_error( - f"{key} is only valid on string fields, not '{field_type}'", path, - "Use min/max for numbers or remove the constraint", - )) - if field.get("format") is not None and field_type not in _FORMAT_TYPES: - errors.append(_error( - f"format is only valid on string or id fields, not '{field_type}'", path, - "Remove format or change the field type", - )) - - if field_type != "file": - for key in _FILE_ONLY: - if field.get(key) is not None: - errors.append(_error( - f"{key} is only valid on file fields, not '{field_type}'", path, - "Remove the constraint or change the field type to file", - )) - else: - for key in _FILE_FORBIDDEN: - if key in field: - errors.append(_error( - f"{key} is not valid on file fields", path, - f"Remove {key} from the file field", - )) - if field.get("maxItems") is not None and (field_type == "file" or file_config(field) is None): - errors.append(_error( - "maxItems is only valid on arrays of files", path, - "Remove maxItems or declare the field as an array of files", - )) - - if field_type == "enum": - values = field.get("values") - if values == []: - errors.append(_error( - "enum field has no values", path, - "List at least one allowed value", - )) - +def _check_field(field: dict[str, Any], path: str) -> Iterator[ConfigError]: + for low, high in _BOUNDS: + lower, upper = field.get(low), field.get(high) + if lower is not None and upper is not None and lower > upper: + yield _error( + f"{low} ({lower}) is greater than {high} ({upper})", path, + f"Ensure {low} is less than or equal to {high}", + ) if "default" in field: - _check_default(field, path, errors) - - if field_type == "array" and isinstance(field.get("items"), dict): - _check_field(field["items"], f"{path}.items", errors) - if field_type == "object": - for name, prop in (field.get("properties") or {}).items(): - if isinstance(prop, dict): - _check_field(prop, f"{path}.properties.{name}", errors) + yield from _check_default(field, path) + if isinstance(field.get("items"), dict): + yield from _check_field(field["items"], f"{path}.items") + for name, prop in (field.get("properties") or {}).items(): + if isinstance(prop, dict): + yield from _check_field(prop, f"{path}.properties.{name}") -def _check_default(field: dict[str, Any], path: str, errors: list[ConfigError]) -> None: +def _check_default(field: dict[str, Any], path: str) -> Iterator[ConfigError]: """A default must satisfy its own field config, nested properties and items included. Interpolation has already run, so a numeric or boolean looking string is an @@ -129,10 +54,10 @@ def _check_default(field: dict[str, Any], path: str, errors: list[ConfigError]) try: conform(field, default, path) except ValueError as error: - errors.append(_error( + yield _error( f"default {default!r} is invalid: {error}", path, "Make the default satisfy the field's own constraints or remove it", - )) + ) def _coerces_to(field_type: str, value: str) -> bool: @@ -148,14 +73,9 @@ def _coerces_to(field_type: str, value: str) -> bool: def validate_spec_constraints(spec: dict[str, Any]) -> list[ConfigError]: - """Validate field-constraint coherence across every entity in the spec.""" - errors: list[ConfigError] = [] - for entity_name, entity in (spec.get("entities") or {}).items(): - if not isinstance(entity, dict): - continue - for field_name, field in (entity.get("fields") or {}).items(): - if isinstance(field, dict): - _check_field( - field, f"entities.{entity_name}.fields.{field_name}", errors - ) - return errors + """Validate field-constraint coherence across every field map in the spec.""" + return [ + error + for path, field in spec_fields(spec) + for error in _check_field(field, path) + ] diff --git a/src/yaml_api_generator/parsers/transformers/dto_validation.py b/src/yaml_api_generator/parsers/transformers/dto_validation.py index aade2147..21307499 100644 --- a/src/yaml_api_generator/parsers/transformers/dto_validation.py +++ b/src/yaml_api_generator/parsers/transformers/dto_validation.py @@ -1,12 +1,8 @@ -"""Semantic checks for top-level ``dto`` declarations in a v0.2 spec. +"""Semantic checks for top-level ``dto`` declarations. -A DTO carries fields and validators only; persistence-shaped attributes belong -to entities, never to request-bound payload types. The schema accepts a DTO -field as a plain ``FieldConfig``, so these checks reject the persistence subset -the spec forbids: +The schema types a DTO field as a payload field, so persistence attributes +(``type: id``, ``unique``) never reach these checks: -- DTO_FORBIDS_PERSISTENCE_ATTR: a DTO field declares ``type: id``, ``unique`` or - ``format``. - DTO_NOT_FOUND: a DTO field, at any nesting depth, references a type that is neither a declared entity nor a declared DTO. - FILE_FIELD_CONTEXT_INVALID: a DTO field is, or contains, a ``type: file`` node. @@ -14,46 +10,28 @@ from typing import Any +from yaml_api_generator.parsers.transformers.field_maps import field_maps, spec_fields from yaml_api_generator.parsers.transformers.file_validation import file_context_errors from yaml_api_generator.parsers.transformers.ref_validation import field_refs from yaml_api_generator.parsers.yaml_parser import ConfigError -_FORBIDDEN_ATTRS = ("unique", "format") - def validate_spec_dtos(spec: dict[str, Any]) -> list[ConfigError]: - errors: list[ConfigError] = [] - dtos = spec.get("dto") or {} - known_refs = frozenset((spec.get("entities") or {}).keys() | dtos.keys()) - for dto_name, dto in dtos.items(): - if not isinstance(dto, dict): - continue - for field_name, field in (dto.get("fields") or {}).items(): - if not isinstance(field, dict): - continue - base = f"dto.{dto_name}.fields.{field_name}" - errors.extend(file_context_errors(field, base, "DTOs")) - if field.get("type") == "id": - errors.append(_forbidden("type: id", base)) - for attr in _FORBIDDEN_ATTRS: - if attr in field: - errors.append(_forbidden(attr, f"{base}.{attr}")) - errors.extend( + known_refs = frozenset((spec.get("entities") or {}).keys() | (spec.get("dto") or {}).keys()) + return [ + *( + error + for path, field in spec_fields(spec, ("dto",)) + for error in file_context_errors(field, path, "DTOs") + ), + *( ConfigError( code="DTO_NOT_FOUND", message=f"DTO field references undeclared type '{target}'", path=path, ) - for path, target in field_refs(dto.get("fields") or {}, f"dto.{dto_name}.fields") + for base, fields in field_maps(spec, ("dto",)) + for path, target in field_refs(fields, base) if target not in known_refs - ) - return errors - - -def _forbidden(attr: str, path: str) -> ConfigError: - return ConfigError( - code="DTO_FORBIDS_PERSISTENCE_ATTR", - message=f"DTO field declares persistence attribute '{attr}'", - path=path, - hint="DTOs carry validation only; move persistence attrs to an entity", - ) + ), + ] diff --git a/src/yaml_api_generator/parsers/transformers/field_maps.py b/src/yaml_api_generator/parsers/transformers/field_maps.py new file mode 100644 index 00000000..8c796dcb --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/field_maps.py @@ -0,0 +1,35 @@ +"""The field maps a spec declares, in every version, keyed by their dotted path.""" + +from collections.abc import Iterable, Iterator +from typing import Any + +# Each section maps an owner name to a config whose field map sits under this key. +FIELD_MAP_KEYS = { + "collections": "fields", + "entities": "fields", + "dto": "fields", + "services": "input", +} +RECORD_SECTIONS = ("collections", "entities") + + +def field_maps( + spec: dict[str, Any], sections: Iterable[str] = FIELD_MAP_KEYS +) -> Iterator[tuple[str, dict[str, Any]]]: + """The ``(path, fields)`` of every field map declared in ``sections``.""" + for section in sections: + key = FIELD_MAP_KEYS[section] + for name, owner in (spec.get(section) or {}).items(): + fields = owner.get(key) if isinstance(owner, dict) else None + if isinstance(fields, dict): + yield f"{section}.{name}.{key}", fields + + +def spec_fields( + spec: dict[str, Any], sections: Iterable[str] = FIELD_MAP_KEYS +) -> Iterator[tuple[str, dict[str, Any]]]: + """The ``(path, field)`` of every top-level field config declared in ``sections``.""" + for base, fields in field_maps(spec, sections): + for name, field in fields.items(): + if isinstance(field, dict): + yield f"{base}.{name}", field diff --git a/src/yaml_api_generator/parsers/transformers/file_validation.py b/src/yaml_api_generator/parsers/transformers/file_validation.py index 8b380e34..2c4cb3ce 100644 --- a/src/yaml_api_generator/parsers/transformers/file_validation.py +++ b/src/yaml_api_generator/parsers/transformers/file_validation.py @@ -11,6 +11,7 @@ from typing import Any from yaml_api_generator.parsers.errors import ConfigError, ConfigErrors +from yaml_api_generator.parsers.transformers.field_maps import spec_fields def file_config(field: Any) -> dict[str, Any] | None: @@ -51,18 +52,10 @@ def file_context_errors(field: Any, path: str, context: str) -> list[ConfigError ] -def _entity_fields(spec: dict[str, Any]) -> Iterator[tuple[str, Any]]: - for entity_name, entity in (spec.get("entities") or {}).items(): - fields = entity.get("fields") if isinstance(entity, dict) else None - if isinstance(fields, dict): - for field_name, field in fields.items(): - yield f"entities.{entity_name}.fields.{field_name}", field - - def validate_spec_files(spec: dict[str, Any]) -> list[ConfigError]: """Reject file fields nested below an entity's top level.""" errors: list[ConfigError] = [] - for path, field in _entity_fields(spec): + for path, field in spec_fields(spec, ("entities",)): top = file_config(field) allowed = None if top is None else path if top is field else f"{path}.items" errors.extend( diff --git a/src/yaml_api_generator/parsers/transformers/ref_validation.py b/src/yaml_api_generator/parsers/transformers/ref_validation.py index 514b3c95..afa054bd 100644 --- a/src/yaml_api_generator/parsers/transformers/ref_validation.py +++ b/src/yaml_api_generator/parsers/transformers/ref_validation.py @@ -1,12 +1,13 @@ -"""Reference targets of ``type: ref`` fields in a v0.2 spec. +"""Reference targets of ``type: ref`` fields on records. -REF_TARGET_NOT_FOUND: an entity field, at any nesting depth, references a name -that is not a declared entity. +REF_TARGET_NOT_FOUND: an entity (or v0.1 collection) field, at any nesting +depth, references a name that is not a declared entity (or collection). """ from collections.abc import Iterator from typing import Any +from yaml_api_generator.parsers.transformers.field_maps import RECORD_SECTIONS, field_maps from yaml_api_generator.parsers.yaml_parser import ConfigError @@ -27,16 +28,15 @@ def _refs(field: dict[str, Any], path: str) -> Iterator[tuple[str, str]]: def validate_spec_refs(spec: dict[str, Any]) -> list[ConfigError]: - entities = spec.get("entities") or {} + records = sorted(name for section in RECORD_SECTIONS for name in spec.get(section) or {}) return [ ConfigError( code="REF_TARGET_NOT_FOUND", - message=f"Field references undeclared entity '{target}'", + message=f"Field references undeclared record type '{target}'", path=path, - hint=f"Declare an entity named '{target}' or reference one of: {', '.join(sorted(entities))}", + hint=f"Declare '{target}' or reference one of: {', '.join(records)}", ) - for entity_name, entity in entities.items() - if isinstance(entity, dict) - for path, target in field_refs(entity.get("fields") or {}, f"entities.{entity_name}.fields") - if target not in entities + for base, fields in field_maps(spec, RECORD_SECTIONS) + for path, target in field_refs(fields, base) + if target not in records ] diff --git a/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py b/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py index c28315d6..89b8ff69 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v01/collection_transformer.py @@ -108,18 +108,7 @@ def _resolve_id_format( path=f"collections.{collection_name}.fields.{id_field}", ) ] - raw_format = field_config.get("format") - if raw_format is None: - return "uuidv7", [] - if raw_format in {"uuidv7", "uuidv4"}: - return str(raw_format), [] - return None, [ - ConfigError( - code="ID_FORMAT_INVALID", - message="id field format must be uuidv7 or uuidv4", - path=f"collections.{collection_name}.fields.{id_field}.format", - ) - ] + return str(field_config.get("format", "uuidv7")), [] def _validate_field_names( self, collection_name: str, fields: dict[str, Any] diff --git a/src/yaml_api_generator/parsers/transformers/v02/collection_transformer.py b/src/yaml_api_generator/parsers/transformers/v02/collection_transformer.py index a6fa8c88..c6729502 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/collection_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v02/collection_transformer.py @@ -134,18 +134,7 @@ def _resolve_id_format( path=f"entities.{entity_name}.fields.{id_field}", ) ] - raw = cfg.get("format") - if raw is None: - return "uuidv7", [] - if raw in {"uuidv7", "uuidv4"}: - return str(raw), [] - return None, [ - ConfigError( - code="ID_FORMAT_INVALID", - message="id field format must be uuidv7 or uuidv4", - path=f"entities.{entity_name}.fields.{id_field}.format", - ) - ] + return str(cfg.get("format", "uuidv7")), [] def _validate_field_names( self, entity_name: str, fields: dict[str, Any] diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 7aed961f..51ed1b67 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -225,16 +225,17 @@ def load_and_generate_ir( if name_errors: raise ConfigErrors(name_errors) - from .transformers.ref_validation import validate_spec_refs - ref_errors = validate_spec_refs(resolved) - if ref_errors: - raise ConfigErrors(ref_errors) + from .transformers.ref_validation import validate_spec_refs + ref_errors = validate_spec_refs(resolved) + if ref_errors: + raise ConfigErrors(ref_errors) - from .transformers.constraint_validation import validate_spec_constraints - constraint_errors = validate_spec_constraints(resolved) - if constraint_errors: - raise ConfigErrors(constraint_errors) + from .transformers.constraint_validation import validate_spec_constraints + constraint_errors = validate_spec_constraints(resolved) + if constraint_errors: + raise ConfigErrors(constraint_errors) + if version in ("0.2", "0.3"): from .transformers.file_validation import validate_spec_files file_errors = validate_spec_files(resolved) if file_errors: diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py index d47a9c7f..c9891da1 100644 --- a/tests/unit/support/payload_requests.py +++ b/tests/unit/support/payload_requests.py @@ -46,6 +46,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload role: { type: enum, values: [admin, member], required: true } level: { type: number, default: 2 } labels: { type: array, items: { type: string, maxLength: 8 } } + contact: { type: string, format: email } entities: user: type: auth @@ -84,6 +85,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload role: { type: string } level: { type: number } labels: { type: array, items: { type: string } } + contact: { type: string } repositories: users: { entity: user } items: { entity: item } @@ -153,6 +155,8 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload NOTES = [ ({"role": "admin"}, 200), ({"role": "owner"}, 422), + ({"role": "admin", "contact": "a@b.co"}, 200), + ({"role": "admin", "contact": "x"}, 422), ({"role": "admin", "unknown": 1}, 422), ({}, 422), ] diff --git a/tests/unit/test_constraint_validation.py b/tests/unit/test_constraint_validation.py index 0b2c35b3..049db614 100644 --- a/tests/unit/test_constraint_validation.py +++ b/tests/unit/test_constraint_validation.py @@ -30,26 +30,6 @@ def test_string_min_length_greater_than_max_length_rejected(): ) -@pytest.mark.parametrize("key", ["minLength", "maxLength"]) -def test_negative_length_rejected(key): - assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": "string", key: -1}) - - -@pytest.mark.parametrize("key", ["min", "max"]) -def test_numeric_constraint_on_non_numeric_rejected(key): - assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": "string", key: 1}) - - -@pytest.mark.parametrize("key", ["minLength", "maxLength", "format"]) -def test_string_constraint_on_non_string_rejected(key): - value = "email" if key == "format" else 1 - assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": "number", key: value}) - - -def test_empty_enum_rejected(): - assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": "enum", "values": []}) - - def test_default_outside_enum_rejected(): assert "FIELD_CONSTRAINT_INVALID" in _codes( {"type": "enum", "values": ["a", "b"], "default": "c"} @@ -67,6 +47,11 @@ def test_default_type_mismatch_rejected(field_type, default): assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": field_type, "default": default}) +def test_dto_field_checked(): + errors = validate_spec_constraints({"dto": {"d": {"fields": {"n": {"type": "number", "max": 1, "default": 2}}}}}) + assert [error.path for error in errors] == ["dto.d.fields.n"] + + def test_error_path_points_at_field(): errors = _errors({"type": "number", "min": 10, "max": 1}) assert errors[0].path == "entities.post.fields.value" @@ -81,7 +66,7 @@ def test_nested_array_item_constraint_rejected(): def test_nested_object_property_constraint_rejected(): field = { "type": "object", - "properties": {"name": {"type": "string", "minLength": -1}}, + "properties": {"name": {"type": "string", "minLength": 3, "maxLength": 1}}, } errors = _errors(field) assert errors[0].path == "entities.post.fields.value.properties.name" diff --git a/tests/unit/test_field_grammar.py b/tests/unit/test_field_grammar.py new file mode 100644 index 00000000..01499ffb --- /dev/null +++ b/tests/unit/test_field_grammar.py @@ -0,0 +1,135 @@ +"""Field configs in every version and every field map: keys by type, bounds, defaults and refs.""" + +import pytest +import yaml + +from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir + +VERSIONS = ("0.1", "0.2", "0.3") + + +def _entity_spec(version: str, field: dict) -> dict: + if version == "0.1": + return {"version": "0.1", "collections": {"things": {"fields": {"id": {"type": "id"}, "f": field}}}} + return { + "version": version, + "entities": {"thing": {"fields": {"id": {"type": "id"}, "f": field}}}, + "repositories": {"things": {"entity": "thing"}}, + "services": {"things": {"repository": "things"}}, + "controllers": {"things": {"service": "things", "base_path": "/things", "access": "public"}}, + } + + +def _dto_spec(version: str, field: dict) -> dict: + spec = _entity_spec(version, {"type": "string"}) + spec["dto"] = {"thingInput": {"fields": {"f": field}}} + return spec + + +def _codes(tmp_path, spec: dict) -> set[str]: + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec)) + try: + load_and_generate_ir(path) + except ConfigErrors as raised: + return {error.code for error in raised.errors} + return set() + + +INAPPLICABLE = [ + {"type": "string", "items": {"type": "string"}}, + {"type": "string", "properties": {}}, + {"type": "string", "additionalProperties": False}, + {"type": "string", "values": ["a"]}, + {"type": "string", "ref": "thing"}, + {"type": "string", "min": 1}, + {"type": "number", "format": "email"}, + {"type": "number", "minLength": 1}, + {"type": "boolean", "maxLength": 1}, + {"type": "array", "items": {"type": "string"}, "minLength": 1}, + {"type": "string", "minLength": -1}, + {"type": "enum", "values": []}, + {"type": "string", "format": "uuidv4x"}, + {"type": "object", "properties": {"a": {"type": "string", "unique": True}}}, + {"type": "object", "properties": {"a": {"type": "number", "format": "email"}}}, + {"type": "array", "items": {"type": "string", "unique": True}}, +] +INCOHERENT = [ + {"type": "number", "min": 5, "max": 1}, + {"type": "string", "minLength": 5, "maxLength": 1}, + {"type": "enum", "values": ["a"], "default": "b"}, + {"type": "number", "max": 1, "default": 5}, + {"type": "string", "format": "email", "default": "x"}, + { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + "default": {"a": "x", "b": 1}, + }, + {"type": "object", "properties": {"a": {"type": "number", "min": 5, "max": 1}}}, +] +DANGLING = [ + {"type": "ref", "ref": "ghost"}, + {"type": "array", "items": {"type": "ref", "ref": "ghost"}}, + {"type": "object", "properties": {"a": {"type": "ref", "ref": "ghost"}}}, +] +VALID = [ + {"type": "string", "format": "email", "minLength": 1, "default": "a@b.co"}, + {"type": "number", "min": 1, "max": 1, "default": 1}, + {"type": "enum", "values": ["a"], "default": "a"}, + {"type": "array", "items": {"type": "object", "properties": {"x": {"type": "number"}}}}, + {"type": "object", "properties": {"a": {"type": "string", "format": "uri"}}, "additionalProperties": False}, +] + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("field", INAPPLICABLE) +def test_inapplicable_key_fails_schema_validation(tmp_path, version, field): + assert _codes(tmp_path, _entity_spec(version, field)) == {"SCHEMA_VALIDATION_ERROR"} + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("field", INCOHERENT) +def test_incoherent_constraint_is_rejected(tmp_path, version, field): + assert _codes(tmp_path, _entity_spec(version, field)) == {"FIELD_CONSTRAINT_INVALID"} + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("field", DANGLING) +def test_ref_to_undeclared_record_is_rejected(tmp_path, version, field): + assert _codes(tmp_path, _entity_spec(version, field)) == {"REF_TARGET_NOT_FOUND"} + + +@pytest.mark.parametrize("version", VERSIONS) +@pytest.mark.parametrize("field", VALID) +def test_coherent_field_is_accepted(tmp_path, version, field): + assert _codes(tmp_path, _entity_spec(version, field)) == set() + + +@pytest.mark.parametrize("version", VERSIONS[1:]) +@pytest.mark.parametrize("field", [ + *INAPPLICABLE, + {"type": "id"}, + {"type": "string", "unique": True}, + {"type": "string", "readonly": True, "foo": 1}, +]) +def test_inapplicable_dto_key_fails_schema_validation(tmp_path, version, field): + assert _codes(tmp_path, _dto_spec(version, field)) == {"SCHEMA_VALIDATION_ERROR"} + + +@pytest.mark.parametrize("version", VERSIONS[1:]) +@pytest.mark.parametrize("field", INCOHERENT) +def test_incoherent_dto_constraint_is_rejected(tmp_path, version, field): + assert _codes(tmp_path, _dto_spec(version, field)) == {"FIELD_CONSTRAINT_INVALID"} + + +@pytest.mark.parametrize("version", VERSIONS[1:]) +@pytest.mark.parametrize("field", VALID) +def test_coherent_dto_field_is_accepted(tmp_path, version, field): + assert _codes(tmp_path, _dto_spec(version, field)) == set() + + +def test_incoherent_service_input_default_is_rejected(tmp_path): + spec = _entity_spec("0.3", {"type": "string"}) + spec["services"]["things"]["input"] = {"n": {"type": "number", "max": 1, "default": 5}} + assert _codes(tmp_path, spec) == {"FIELD_CONSTRAINT_INVALID"} diff --git a/tests/unit/test_file_field_validation.py b/tests/unit/test_file_field_validation.py index c4e23a91..7487a063 100644 --- a/tests/unit/test_file_field_validation.py +++ b/tests/unit/test_file_field_validation.py @@ -13,9 +13,9 @@ def _errors(name: str): @pytest.mark.parametrize("fixture, expected", [ - ("files-limit-on-non-file.yaml", ("FIELD_CONSTRAINT_INVALID", "entities.document.fields.title")), - ("files-max-items-on-non-file-array.yaml", ("FIELD_CONSTRAINT_INVALID", "entities.document.fields.tags")), - ("files-unique-file.yaml", ("FIELD_CONSTRAINT_INVALID", "entities.document.fields.avatar")), + ("files-limit-on-non-file.yaml", ("SCHEMA_VALIDATION_ERROR", "entities.document.fields.title")), + ("files-max-items-on-non-file-array.yaml", ("SCHEMA_VALIDATION_ERROR", "entities.document.fields.tags")), + ("files-unique-file.yaml", ("SCHEMA_VALIDATION_ERROR", "entities.document.fields.avatar")), ("files-nested.yaml", ("FILE_FIELD_NESTED", "entities.document.fields.profile.properties.avatar")), ("files-in-dto.yaml", ("FILE_FIELD_CONTEXT_INVALID", "dto.upload.fields.blob")), ("files-in-service-input.yaml", ("FILE_FIELD_CONTEXT_INVALID", "services.notify.input.blob")), diff --git a/tests/unit/test_request_access_validation.py b/tests/unit/test_request_access_validation.py index 5c9b1f91..f6e84d93 100644 --- a/tests/unit/test_request_access_validation.py +++ b/tests/unit/test_request_access_validation.py @@ -95,29 +95,14 @@ def test_request_referencing_create_rule_valid(): assert _access_codes(access, op_name="create") == set() -# ---- DTO_FORBIDS_PERSISTENCE_ATTR ---- +# ---- DTO fields ---- def _dto_codes(dto) -> set[str]: return {e.code for e in validate_spec_dtos({"entities": _ENTITIES, "dto": dto})} -def test_dto_forbids_id_type(): - dto = {"d": {"fields": {"x": {"type": "id"}}}} - assert "DTO_FORBIDS_PERSISTENCE_ATTR" in _dto_codes(dto) - - -def test_dto_forbids_unique(): - dto = {"d": {"fields": {"x": {"type": "string", "unique": True}}}} - assert "DTO_FORBIDS_PERSISTENCE_ATTR" in _dto_codes(dto) - - -def test_dto_forbids_format(): - dto = {"d": {"fields": {"x": {"type": "string", "format": "email"}}}} - assert "DTO_FORBIDS_PERSISTENCE_ATTR" in _dto_codes(dto) - - def test_dto_clean_fields_valid(): - dto = {"d": {"fields": {"x": {"type": "string", "required": True}}}} + dto = {"d": {"fields": {"x": {"type": "string", "required": True, "format": "email"}}}} assert _dto_codes(dto) == set() diff --git a/tests/unit/test_v02_transformer.py b/tests/unit/test_v02_transformer.py index 2a3f7854..a5a11f60 100644 --- a/tests/unit/test_v02_transformer.py +++ b/tests/unit/test_v02_transformer.py @@ -144,7 +144,7 @@ def test_74_id_format_uuidv4_parsed_from_yaml(): def test_77_id_format_invalid_string(): with pytest.raises(ConfigErrors) as exc_info: load_and_generate_ir(str(INVALID_DIR / "invalid-id-format.yaml")) - assert any(e.code == "ID_FORMAT_INVALID" for e in exc_info.value.errors) + assert any(e.code == "SCHEMA_VALIDATION_ERROR" for e in exc_info.value.errors) def test_78_operations_normalization_repo_overrides_defaults(tmp_path): From 48d9c70771e67c6d56c960bd9672085bbcab1260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 00:00:17 +0100 Subject: [PATCH 26/89] Store every field default in the canonical form its own field gives a request value --- .../transformers/constraint_validation.py | 30 ++-------- .../parsers/transformers/field_defaults.py | 58 +++++++++++++++++++ src/yaml_api_generator/parsers/yaml_parser.py | 3 + tests/unit/test_constraint_validation.py | 23 ++++++++ 4 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 src/yaml_api_generator/parsers/transformers/field_defaults.py diff --git a/src/yaml_api_generator/parsers/transformers/constraint_validation.py b/src/yaml_api_generator/parsers/transformers/constraint_validation.py index dcaa20f9..66f04ffe 100644 --- a/src/yaml_api_generator/parsers/transformers/constraint_validation.py +++ b/src/yaml_api_generator/parsers/transformers/constraint_validation.py @@ -9,10 +9,9 @@ from collections.abc import Iterator from typing import Any +from yaml_api_generator.parsers.transformers.field_defaults import canonical_default from yaml_api_generator.parsers.transformers.field_maps import spec_fields -from yaml_api_generator.parsers.transformers.file_validation import file_config from yaml_api_generator.parsers.yaml_parser import ConfigError -from yaml_api_generator.runtime.fields import conform _BOUNDS = (("min", "max"), ("minLength", "maxLength")) @@ -41,37 +40,16 @@ def _check_field(field: dict[str, Any], path: str) -> Iterator[ConfigError]: def _check_default(field: dict[str, Any], path: str) -> Iterator[ConfigError]: - """A default must satisfy its own field config, nested properties and items included. - - Interpolation has already run, so a numeric or boolean looking string is an - env-driven default (``${PORT:-8080}``) and is accepted for its type. - """ - default = field["default"] - if default is None or file_config(field) is not None: - return - if isinstance(default, str) and _coerces_to(field.get("type"), default): - return + """A default must satisfy its own field config, nested properties and items included.""" try: - conform(field, default, path) + canonical_default(field, path) except ValueError as error: yield _error( - f"default {default!r} is invalid: {error}", path, + f"default {field['default']!r} is invalid: {error}", path, "Make the default satisfy the field's own constraints or remove it", ) -def _coerces_to(field_type: str, value: str) -> bool: - if field_type == "number": - try: - float(value) - except ValueError: - return False - return True - if field_type == "boolean": - return value.lower() in ("true", "false", "1", "0") - return False - - def validate_spec_constraints(spec: dict[str, Any]) -> list[ConfigError]: """Validate field-constraint coherence across every field map in the spec.""" return [ diff --git a/src/yaml_api_generator/parsers/transformers/field_defaults.py b/src/yaml_api_generator/parsers/transformers/field_defaults.py new file mode 100644 index 00000000..2ba62dc2 --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/field_defaults.py @@ -0,0 +1,58 @@ +"""Field defaults in the form their own field gives a request value. + +Every consumer stores a default as the spec declares it, so the spec carries +each default already conformed: a datetime in its canonical UTC form, an object +with its nested defaults filled in. Interpolation leaves an env-driven number +or boolean default (``${PORT:-8080}``) as a string, which is read as its type. +""" + +from typing import Any + +from yaml_api_generator.parsers.transformers.field_maps import field_maps +from yaml_api_generator.parsers.transformers.file_validation import file_config +from yaml_api_generator.runtime.fields import conform + +_BOOLEANS = {"true": True, "1": True, "false": False, "0": False} + + +def _typed(field_type: Any, value: Any) -> Any: + if not isinstance(value, str): + return value + if field_type == "number": + try: + return float(value) + except ValueError: + return value + if field_type == "boolean": + return _BOOLEANS.get(value.lower(), value) + return value + + +def canonical_default(field: dict[str, Any], path: str = "default") -> Any: + """``field``'s default as its field conforms it; a ``ValueError`` when the field rejects it.""" + default = field["default"] + if default is None or file_config(field) is not None: + return default + return conform(field, _typed(field.get("type"), default), path) + + +def _canonical_field(field: Any) -> Any: + if not isinstance(field, dict): + return field + nested = { + **({"items": _canonical_field(field["items"])} if "items" in field else {}), + **( + {"properties": {name: _canonical_field(prop) for name, prop in field["properties"].items()}} + if isinstance(field.get("properties"), dict) + else {} + ), + } + canonical = {**field, **nested} + return {**canonical, "default": canonical_default(canonical)} if "default" in canonical else canonical + + +def canonicalize_defaults(spec: dict[str, Any]) -> None: + """Replace every default in ``spec``'s field maps, nested ones included, with its canonical form.""" + for _, fields in field_maps(spec): + fields.update({name: _canonical_field(field) for name, field in fields.items()}) + diff --git a/src/yaml_api_generator/parsers/yaml_parser.py b/src/yaml_api_generator/parsers/yaml_parser.py index 51ed1b67..cd7eb4de 100644 --- a/src/yaml_api_generator/parsers/yaml_parser.py +++ b/src/yaml_api_generator/parsers/yaml_parser.py @@ -235,6 +235,9 @@ def load_and_generate_ir( if constraint_errors: raise ConfigErrors(constraint_errors) + from .transformers.field_defaults import canonicalize_defaults + canonicalize_defaults(resolved) + if version in ("0.2", "0.3"): from .transformers.file_validation import validate_spec_files file_errors = validate_spec_files(resolved) diff --git a/tests/unit/test_constraint_validation.py b/tests/unit/test_constraint_validation.py index 049db614..6096c7ca 100644 --- a/tests/unit/test_constraint_validation.py +++ b/tests/unit/test_constraint_validation.py @@ -3,6 +3,7 @@ from yaml_api_generator.parsers.transformers.constraint_validation import ( validate_spec_constraints, ) +from yaml_api_generator.parsers.transformers.field_defaults import canonicalize_defaults from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir @@ -170,3 +171,25 @@ def test_invalid_constraint_rejected_end_to_end(tmp_path): with pytest.raises(ConfigErrors) as exc_info: load_and_generate_ir(str(yaml_file)) assert "FIELD_CONSTRAINT_INVALID" in {e.code for e in exc_info.value.errors} + + +# ---- Defaults reach every consumer in canonical form ---- + +def test_defaults_are_canonicalized_at_every_depth(): + spec = _spec({ + "type": "object", + "properties": { + "at": {"type": "datetime", "default": "2024-01-01T01:00:00+01:00"}, + "port": {"type": "number", "default": "8080"}, + }, + "default": {"port": 1}, + }) + canonicalize_defaults(spec) + field = spec["entities"]["post"]["fields"]["value"] + assert field["properties"]["at"]["default"] == "2024-01-01T00:00:00+00:00" + assert field["properties"]["port"]["default"] == 8080.0 + assert field["default"] == {"port": 1, "at": "2024-01-01T00:00:00+00:00"} + + +def test_env_string_default_is_bounded_like_its_type(): + assert "FIELD_CONSTRAINT_INVALID" in _codes({"type": "number", "max": 1, "default": "42"}) From c076d69c62acaf5afbd7b3749fbb238527926c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 00:00:17 +0100 Subject: [PATCH 27/89] Run the payload scenario on generated FastAPI over MongoDB, on PostgreSQL and on v0.1 collections, with DTO datetimes --- tests/integration/conftest.py | 7 ++ tests/integration/support/docker.py | 5 +- tests/integration/support/postgres.py | 38 ++++++++++ tests/integration/test_mongo_payloads.py | 46 +++--------- tests/integration/test_postgres_payloads.py | 36 ++++++++++ tests/integration/test_v01_mongo_payloads.py | 35 +++++++++ tests/unit/support/payload_requests.py | 6 ++ tests/unit/support/scenarios.py | 32 +++++++++ tests/unit/support/v01_payload_requests.py | 74 ++++++++++++++++++++ tests/unit/test_payload_models.py | 34 +++------ 10 files changed, 248 insertions(+), 65 deletions(-) create mode 100644 tests/integration/support/postgres.py create mode 100644 tests/integration/test_postgres_payloads.py create mode 100644 tests/integration/test_v01_mongo_payloads.py create mode 100644 tests/unit/support/scenarios.py create mode 100644 tests/unit/support/v01_payload_requests.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 652224f8..d21aeb2c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ from tests.integration.support.minio import MINIO from tests.integration.support.mongo import MONGO from tests.integration.support.oidc import live_issuer +from tests.integration.support.postgres import POSTGRES @pytest.fixture(scope="session") @@ -18,6 +19,12 @@ def mongo_uri(): yield uri +@pytest.fixture(scope="session") +def postgres_uri(): + with POSTGRES.endpoint() as uri: + yield uri + + @pytest.fixture(scope="session") def issuer(): with live_issuer(CLIENT_ID) as issuer: diff --git a/tests/integration/support/docker.py b/tests/integration/support/docker.py index 9127ff6c..18fc57a1 100644 --- a/tests/integration/support/docker.py +++ b/tests/integration/support/docker.py @@ -21,7 +21,7 @@ def docker_available() -> bool: @dataclass(frozen=True) class Service: - """A dockerised service reachable at `{scheme}://127.0.0.1:{port}` once `ready` accepts that endpoint.""" + """A dockerised service reachable at `{scheme}://{userinfo}127.0.0.1:{port}` once `ready` accepts that endpoint.""" name: str image: str @@ -31,6 +31,7 @@ class Service: ready: Callable[[str], bool] env: dict[str, str] = field(default_factory=dict) command: tuple[str, ...] = () + userinfo: str = "" @contextmanager def endpoint(self) -> Iterator[str]: @@ -58,7 +59,7 @@ def _container(self) -> Iterator[str]: if started.returncode != 0: pytest.skip(f"could not start {self.image}: {started.stderr.strip()}") try: - endpoint = f"{self.scheme}://127.0.0.1:{port}" + endpoint = f"{self.scheme}://{self.userinfo}127.0.0.1:{port}" if not wait_until(lambda: self.ready(endpoint)): pytest.skip(f"{self.image} container did not become ready") yield endpoint diff --git a/tests/integration/support/postgres.py b/tests/integration/support/postgres.py new file mode 100644 index 00000000..91853676 --- /dev/null +++ b/tests/integration/support/postgres.py @@ -0,0 +1,38 @@ +"""A real PostgreSQL: YAML_API_TEST_POSTGRES_URI when set, otherwise a throwaway container.""" + +from urllib.parse import urlsplit +from uuid import uuid4 + +import psycopg + +from .docker import Service + +PASSWORD = "postgres" + + +def _connectable(uri: str) -> bool: + try: + with psycopg.connect(uri, connect_timeout=1): + return True + except psycopg.Error: + return False + + +def fresh_database(uri: str) -> str: + """The asyncpg DSN of a new empty database on the server at `uri`.""" + name = f"payloads_{uuid4().hex}" + with psycopg.connect(uri, autocommit=True) as connection: + connection.execute(f'CREATE DATABASE "{name}"') + return urlsplit(uri)._replace(scheme="postgresql+asyncpg", path=f"/{name}").geturl() + + +POSTGRES = Service( + name="postgres", + image="postgres:16-alpine", + port=5432, + scheme="postgresql", + override="YAML_API_TEST_POSTGRES_URI", + ready=_connectable, + env={"POSTGRES_PASSWORD": PASSWORD}, + userinfo=f"postgres:{PASSWORD}@", +) diff --git a/tests/integration/test_mongo_payloads.py b/tests/integration/test_mongo_payloads.py index 078e5437..aba9f048 100644 --- a/tests/integration/test_mongo_payloads.py +++ b/tests/integration/test_mongo_payloads.py @@ -1,47 +1,26 @@ -"""Development servers and generated raw-ASGI and DRF apps validate the shared payload scenario against a real MongoDB.""" +"""Development servers and generated FastAPI, raw-ASGI and DRF apps validate the shared payload scenario against a real MongoDB.""" -import json import os -import socket import subprocess -import sys import time -from pathlib import Path from uuid import uuid4 import httpx import pytest from starlette.testclient import TestClient +from tests.integration.support.network import free_port +from tests.unit.support.scenarios import run_generated +from tests.unit.support import payload_requests from tests.unit.support.payload_requests import EXPECTED, JWT_SECRET, mongo_store, scenario, spec from yaml_api_generator.cli.compiler import compile_project from yaml_api_generator.server import build_app, build_raw_app -ROOT = Path(__file__).parents[2] - -_RUNNER = f""" -import json, sys -sys.path.append({str(ROOT)!r}) -from starlette.testclient import TestClient -from tests.unit.support.payload_requests import scenario -import app - -with TestClient(app.app) as client: - print(json.dumps(scenario(client))) -""" - - DRF_PYTHON = os.environ.get("YAML_API_TEST_DRF_PYTHON") """An interpreter with the generated DRF requirements installed.""" -def _free_port() -> int: - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - return probe.getsockname()[1] - - def _scenario_env(mongo_uri: str) -> dict[str, str]: return { **os.environ, @@ -68,24 +47,17 @@ def test_dev_server_validates_payloads(spec_file, build, monkeypatch): @pytest.mark.integration -def test_generated_raw_validates_payloads(spec_file, tmp_path, mongo_uri): - compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "raw"), target="raw") - result = subprocess.run( - [sys.executable, "-c", _RUNNER], - cwd=tmp_path / "raw" / "backend", - env=_scenario_env(mongo_uri), - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - assert json.loads(result.stdout.splitlines()[-1]) == EXPECTED +@pytest.mark.parametrize("target", ["fastapi", "raw"]) +def test_generated_backend_validates_payloads(spec_file, tmp_path, mongo_uri, target): + compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / target), target=target) + assert run_generated(tmp_path / target / "backend", payload_requests.__name__, _scenario_env(mongo_uri)) == EXPECTED @pytest.mark.integration @pytest.mark.skipif(DRF_PYTHON is None, reason="YAML_API_TEST_DRF_PYTHON is not set") def test_generated_drf_validates_payloads(spec_file, tmp_path, mongo_uri): compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "drf"), target="drf") - port = _free_port() + port = free_port() server = subprocess.Popen( [DRF_PYTHON, "-m", "uvicorn", "project.asgi:application", "--port", str(port)], cwd=tmp_path / "drf" / "backend", diff --git a/tests/integration/test_postgres_payloads.py b/tests/integration/test_postgres_payloads.py new file mode 100644 index 00000000..d44a8012 --- /dev/null +++ b/tests/integration/test_postgres_payloads.py @@ -0,0 +1,36 @@ +"""Development servers and a generated FastAPI app validate the shared payload scenario against a real PostgreSQL.""" + +import os + +import pytest +from starlette.testclient import TestClient + +from tests.integration.support.postgres import fresh_database +from tests.unit.support.scenarios import run_generated +from tests.unit.support import payload_requests +from tests.unit.support.payload_requests import EXPECTED, JWT_SECRET, SQL_STORE, scenario, spec +from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.server import build_app, build_raw_app + + +@pytest.fixture +def spec_file(tmp_path): + path = tmp_path / "spec.yaml" + path.write_text(spec(SQL_STORE)) + return path + + +@pytest.mark.integration +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["fastapi", "raw"]) +def test_dev_server_validates_payloads(spec_file, build, postgres_uri, monkeypatch): + monkeypatch.setenv("YAML_API_DB_STORE_DSN", fresh_database(postgres_uri)) + monkeypatch.setenv("JWT_SECRET", JWT_SECRET) + with TestClient(build(str(spec_file))) as client: + assert scenario(client) == EXPECTED + + +@pytest.mark.integration +def test_generated_fastapi_validates_payloads(spec_file, tmp_path, postgres_uri): + compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / "fastapi"), target="fastapi") + env = {**os.environ, "DATABASE_URL": fresh_database(postgres_uri), "JWT_SECRET": JWT_SECRET} + assert run_generated(tmp_path / "fastapi" / "backend", payload_requests.__name__, env) == EXPECTED diff --git a/tests/integration/test_v01_mongo_payloads.py b/tests/integration/test_v01_mongo_payloads.py new file mode 100644 index 00000000..1d9341f9 --- /dev/null +++ b/tests/integration/test_v01_mongo_payloads.py @@ -0,0 +1,35 @@ +"""A v0.1 spec serves and validates its collections on the development servers and the generated FastAPI and raw-ASGI apps.""" + +import os +from uuid import uuid4 + +import pytest +from starlette.testclient import TestClient + +from tests.unit.support import v01_payload_requests +from tests.unit.support.scenarios import run_generated +from tests.unit.support.v01_payload_requests import EXPECTED, scenario, spec +from yaml_api_generator.cli.compiler import compile_project +from yaml_api_generator.server import build_app, build_raw_app + + +@pytest.fixture +def spec_file(tmp_path, mongo_uri): + path = tmp_path / "spec.yaml" + path.write_text(spec(mongo_uri, f"v01_{uuid4().hex}")) + return path + + +@pytest.mark.integration +@pytest.mark.parametrize("build", [build_app, build_raw_app], ids=["fastapi", "raw"]) +def test_dev_server_serves_v01_collections(spec_file, build): + with TestClient(build(str(spec_file))) as client: + assert scenario(client) == EXPECTED + + +@pytest.mark.integration +@pytest.mark.parametrize("target", ["fastapi", "raw"]) +def test_generated_backend_serves_v01_collections(spec_file, tmp_path, mongo_uri, target): + compile_project(yaml_file=str(spec_file), output_dir=str(tmp_path / target), target=target) + env = {**os.environ, "MONGO_URI": mongo_uri, "MONGO_DB": f"v01_{uuid4().hex}"} + assert run_generated(tmp_path / target / "backend", v01_payload_requests.__name__, env) == EXPECTED diff --git a/tests/unit/support/payload_requests.py b/tests/unit/support/payload_requests.py index c9891da1..27c7695a 100644 --- a/tests/unit/support/payload_requests.py +++ b/tests/unit/support/payload_requests.py @@ -47,6 +47,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload level: { type: number, default: 2 } labels: { type: array, items: { type: string, maxLength: 8 } } contact: { type: string, format: email } + seen: { type: datetime, default: "2024-01-01T00:00:00Z" } entities: user: type: auth @@ -86,6 +87,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload level: { type: number } labels: { type: array, items: { type: string } } contact: { type: string } + seen: { type: datetime } repositories: users: { entity: user } items: { entity: item } @@ -157,6 +159,8 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload ({"role": "owner"}, 422), ({"role": "admin", "contact": "a@b.co"}, 200), ({"role": "admin", "contact": "x"}, 422), + ({"role": "admin", "seen": "2024-01-01T00:00:00Z"}, 200), + ({"role": "admin", "seen": "soon"}, 422), ({"role": "admin", "unknown": 1}, 422), ({}, 422), ] @@ -172,6 +176,7 @@ def mongo_store(uri: str = "mongodb://localhost:27017", database: str = "payload ["replaced item deleted", 200], ["note role kept", "admin"], ["note dto default stored", 2], + ["note dto datetime default stored", "2024-01-01T00:00:00+00:00"], ["note partial update", 200], ["note value kept by partial update", 5], ["note empty update", 200], @@ -220,6 +225,7 @@ def _round_trips(client: TestClient, headers: dict[str, str], user: dict) -> lis ["replaced item deleted", client.delete(path).status_code], ["note role kept", note.get("role")], ["note dto default stored", note.get("level")], + ["note dto datetime default stored", client.get(f"/notes/{note['id']}", headers=headers).json().get("seen")], ["note partial update", labelled.status_code], ["note value kept by partial update", labelled.json().get("level")], ["note empty update", client.patch(f"/notes/{fives['id']}", json={}, headers=headers).status_code], diff --git a/tests/unit/support/scenarios.py b/tests/unit/support/scenarios.py new file mode 100644 index 00000000..62c5b06a --- /dev/null +++ b/tests/unit/support/scenarios.py @@ -0,0 +1,32 @@ +"""Runs a request scenario against a generated backend in its own interpreter, where its modules import.""" + +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).parents[3] + +_RUNNER = """ +import json, sys +sys.path.append({root!r}) +from starlette.testclient import TestClient +from {module} import scenario +import app + +with TestClient(app.app) as client: + print(json.dumps(scenario(client))) +""" + + +def run_generated(backend: Path, module: str, env: dict[str, str]) -> list: + """What `module.scenario` returns when run against the `app` in `backend`.""" + result = subprocess.run( + [sys.executable, "-c", _RUNNER.format(root=str(ROOT), module=module)], + cwd=backend, + env=env, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.splitlines()[-1]) diff --git a/tests/unit/support/v01_payload_requests.py b/tests/unit/support/v01_payload_requests.py new file mode 100644 index 00000000..bd1d6282 --- /dev/null +++ b/tests/unit/support/v01_payload_requests.py @@ -0,0 +1,74 @@ +"""Valid and invalid request payloads for a v0.1 collection, sent alike to every app under test. + +A v0.1 spec declares its model keys on collection fields; the apps start from +its ``mongo`` block and validate formats, items, closed nested objects, +defaults, number bounds and datetimes as v0.2 entities do. +""" + +from starlette.testclient import TestClient + + +def spec(uri: str, database: str) -> str: + return f""" +version: "0.1" +api: + basePath: "" + operations: + defaults: {{ list: true, get: true, create: true, update: true, replace: true, delete: true }} +mongo: + uri: {uri} + database: {database} +collections: + things: + fields: + id: {{ type: id }} + name: {{ type: string, required: true, minLength: 2 }} + email: {{ type: string, format: email }} + status: {{ type: enum, values: [draft, live], default: draft }} + count: {{ type: number, min: 0 }} + when: {{ type: datetime }} + tags: {{ type: array, items: {{ type: string, maxLength: 3 }} }} + meta: + type: object + properties: + color: {{ type: string, required: true }} + additionalProperties: false +""" + + +CREATES = [ + ({"name": "ab", "meta": {"color": "r"}, "tags": ["abc"], "count": 0}, 200), + ({"name": "a"}, 422), + ({"name": "ab", "email": "x"}, 422), + ({"name": "ab", "tags": ["abcd"]}, 422), + ({"name": "ab", "meta": {"color": "r", "x": 1}}, 422), + ({"name": "ab", "meta": {}}, 422), + ({"name": "ab", "count": -1}, 422), + ({"name": "ab", "count": "3"}, 422), + ({"name": "ab", "when": 1700000000}, 422), + ({"name": "ab", "status": "gone"}, 422), + ({"name": "ab", "unknown": 1}, 422), +] +ROUND_TRIPS = [ + ["default stored", "draft"], + ["date-only datetime canonical", "2024-01-01T00:00:00+00:00"], + ["partial update", [200, 2, "ab"]], + ["replaced", [200, "cd"]], + ["deleted", [200, 404]], +] +EXPECTED = [*(["POST /things", body, status] for body, status in CREATES), *ROUND_TRIPS] + + +def scenario(client: TestClient) -> list[list]: + created = client.post("/things", json={"name": "ab", "when": "2024-01-01"}).json() + path = f"/things/{created['id']}" + patched = client.patch(path, json={"count": 2}) + replaced = client.put(path, json={"name": "cd"}) + return [ + *(["POST /things", body, client.post("/things", json=body).status_code] for body, _ in CREATES), + ["default stored", client.get(path).json()["status"]], + ["date-only datetime canonical", created["when"]], + ["partial update", [patched.status_code, patched.json()["count"], patched.json()["name"]]], + ["replaced", [replaced.status_code, client.get(path).json()["name"]]], + ["deleted", [client.delete(path).status_code, client.get(path).status_code]], + ] diff --git a/tests/unit/test_payload_models.py b/tests/unit/test_payload_models.py index 75a232d5..1db619a0 100644 --- a/tests/unit/test_payload_models.py +++ b/tests/unit/test_payload_models.py @@ -7,16 +7,14 @@ """ import ast -import json import os -import subprocess -import sys from collections import Counter from pathlib import Path import pytest from fastapi.testclient import TestClient +from tests.unit.support import payload_requests from tests.unit.support.payload_requests import ( EXPECTED, JWT_SECRET, @@ -25,6 +23,7 @@ scenario, spec, ) +from tests.unit.support.scenarios import run_generated from yaml_api_generator.cli.compiler import compile_project from yaml_api_generator.server import build_app, build_raw_app @@ -53,29 +52,10 @@ def test_dev_server_validates_payloads(spec_file, build): assert scenario(client) == EXPECTED -_RUNNER = f""" -import json, sys -sys.path.append({str(ROOT)!r}) -from fastapi.testclient import TestClient -from tests.unit.support.payload_requests import scenario -import app - -with TestClient(app.app) as client: - print(json.dumps(scenario(client))) -""" - - def test_generated_fastapi_validates_payloads(spec_file): backend = _generate(spec_file, "fastapi") - result = subprocess.run( - [sys.executable, "-c", _RUNNER], - cwd=backend, - env={**os.environ, "DATABASE_URL": f"sqlite+aiosqlite:///{backend / 'generated.sqlite'}"}, - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - assert json.loads(result.stdout.splitlines()[-1]) == EXPECTED + env = {**os.environ, "DATABASE_URL": f"sqlite+aiosqlite:///{backend / 'generated.sqlite'}"} + assert run_generated(backend, payload_requests.__name__, env) == EXPECTED @pytest.fixture @@ -116,12 +96,14 @@ def _is_field_config(node: ast.expr) -> bool: def _field_declarations(source: str) -> Counter: - """Each field a module declares, as a class annotation or a field config map entry. + """Each field a module declares, as a class annotation or a payload's field config map entry. A declaration is its name with its type, so same-named fields of unrelated payloads stay distinct while a field copied into a sibling payload repeats. + Nested properties belong to their parent's declaration. """ nodes = list(ast.walk(ast.parse(source))) + payload_maps = (arg for node in nodes if isinstance(node, ast.Call) for arg in node.args) annotations = ( (node.target.id, ast.unparse(node.annotation)) for node in nodes @@ -130,7 +112,7 @@ def _field_declarations(source: str) -> Counter: ) configured = ( (key.value, ast.unparse(value)) - for node in nodes + for node in payload_maps if isinstance(node, ast.Dict) for key, value in zip(node.keys, node.values) if isinstance(key, ast.Constant) and _is_field_config(value) From 128456ff6ab6c8516705f87ae33d2be53a25b5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 00:06:37 +0100 Subject: [PATCH 28/89] Generate abstract service and repository bases on raw and DRF, route DRF by path and normalize the handler request Raw and DRF render services and repositories from shared layers, run operations through the service, serve one DRF view per path in route precedence, and hand handlers the last value of repeated query keys, an object body and a principal without credentials. --- .../generators/backend/compile_context.py | 5 + .../generators/backend/drf/generator.py | 75 +------- .../backend/drf/templates/api_urls.py.jinja | 32 +--- .../backend/drf/templates/handler_call.jinja | 8 +- .../drf/templates/handler_views.py.jinja | 11 +- .../backend/drf/templates/views.py.jinja | 171 +++++------------- .../generators/backend/fastapi/generator.py | 49 ++--- .../fastapi/templates/service.py.jinja | 13 +- .../templates/service_operations.jinja | 11 -- .../generators/backend/inheritance.py | 41 +++++ .../generators/backend/raw/generator.py | 65 +------ .../backend/raw/templates/app.py.jinja | 25 +-- .../backend/raw/templates/handler_call.jinja | 10 +- .../backend/raw/templates/routes.py.jinja | 2 +- .../shared/templates/repository.py.jinja | 5 +- .../shared/templates/repository_base.py.jinja | 11 ++ .../backend/shared/templates/service.py.jinja | 11 +- .../shared/templates/service_base.py.jinja | 8 + .../shared/templates/service_operations.jinja | 22 +++ .../generators/backend/shared_layers.py | 100 ++++++++++ .../runtime/handler_requests.py | 37 +++- .../server/common/steps/handlers.py | 7 +- .../specs/services_mongo/hooks/__init__.py | 2 +- .../specs/services_mongo/spec.yaml | 2 +- tests/integration/support/spec_apps.py | 15 +- tests/integration/test_services_mongo.py | 29 ++- tests/unit/test_generators.py | 3 +- tests/unit/test_repo_service_inheritance.py | 16 +- tests/unit/test_rs256_signing.py | 2 +- tests/unit/test_service_input_codegen.py | 6 +- tests/unit/test_services_spec_builds.py | 6 +- 31 files changed, 413 insertions(+), 387 deletions(-) delete mode 100644 src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja create mode 100644 src/yaml_api_generator/generators/backend/inheritance.py create mode 100644 src/yaml_api_generator/generators/backend/shared/templates/repository_base.py.jinja create mode 100644 src/yaml_api_generator/generators/backend/shared/templates/service_base.py.jinja create mode 100644 src/yaml_api_generator/generators/backend/shared/templates/service_operations.jinja create mode 100644 src/yaml_api_generator/generators/backend/shared_layers.py diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index 24475858..1f81c1fc 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -293,6 +293,11 @@ def handler_name(self) -> str: """Function name unique to this method and path across the whole API.""" return to_py_identifier(f"handle_{self.method.lower()}{self.relative_path}") + @property + def view_name(self) -> str: + """Class name of the view serving every method on this route's path.""" + return f"View{to_py_identifier(self.relative_path)}" + @property def django_path(self) -> str: path = self.relative_path.lstrip("/") diff --git a/src/yaml_api_generator/generators/backend/drf/generator.py b/src/yaml_api_generator/generators/backend/drf/generator.py index a7538abb..6eedd962 100644 --- a/src/yaml_api_generator/generators/backend/drf/generator.py +++ b/src/yaml_api_generator/generators/backend/drf/generator.py @@ -4,10 +4,9 @@ from yaml_api_generator.generators.backend.compile_context import CompileContext, IdentityCollectionDescriptor from yaml_api_generator.utils.strings import to_pascal_case -from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment +from ..base import RenderConfig, SingleFileRenderConfig, template_environment +from ..shared_layers import SharedLayersGenerator from ..utils import ( - extract_identity_fields, - find_identity, group_by_collection, pagination_config, MONGO_RUNTIME_MODULES, @@ -43,7 +42,9 @@ _BACKEND_GENERATORS = Path(__file__).parent.parent _TEMPLATES_DIR = _BACKEND_GENERATORS / "drf" / "templates" _SHARED_TEMPLATES_DIR = _BACKEND_GENERATORS / "shared" / "templates" -class DRFGenerator(TemplateBasedGenerator): +class DRFGenerator(SharedLayersGenerator): + auth_module = "api.authentication.auth" + def __init__(self): super().__init__() self._dir_prefix = "api" @@ -164,39 +165,6 @@ def _render_serializers( env.get_template("serializers.py.jinja").render(models=context.dto_models) ) - def _render_services( - self, - env: Environment, - context: CompileContext, - collections_by_name: dict[str, dict], - output_dir: Path, - ) -> None: - self._render_collection_based( - env, context, collections_by_name, output_dir, - RenderConfig( - template_name="service.py.jinja", - output_subdir="services", - import_class_suffix="Service", - context_builder=self._build_service_context, - additional_files=[("exceptions.py.jinja", "exceptions.py")], - ) - ) - - def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: - identity = find_identity(context, col_name) - is_identity = identity is not None - - return { - "collection_name": col_name, - "is_async": True, - "is_identity_collection": is_identity, - "password_via": is_identity and identity.has_password, - "identity_fields": extract_identity_fields(context, col_name), - "import_prefix": "api.repositories", - "auth_import_prefix": "api.authentication.auth", - "service_import_prefix": "api.services", - } - def _render_views( self, env: Environment, @@ -242,29 +210,6 @@ def _render_views( ) ) - def _render_repositories( - self, - env: Environment, - context: CompileContext, - collections_by_name: dict[str, dict], - output_dir: Path, - ) -> None: - self._render_collection_based( - env, context, collections_by_name, output_dir, - RenderConfig( - template_name="repository.py.jinja", - output_subdir="repositories", - import_class_suffix="Repository", - context_builder=lambda col_name, col_data, ctx: { - "collection_name": col_name, - "id_field": ctx.collections[col_name]["id_field"], - "id_field_format": ctx.collections[col_name]["id_format"], - "custom_operations": ctx.repository_operations.get(col_name, []), - "module_prefix": "api.", - }, - ) - ) - def _render_authentication( self, env: Environment, context: CompileContext, output_dir: Path ) -> None: @@ -292,16 +237,8 @@ def _render_urls( api_dir = output_dir / self.dir_prefix api_dir.mkdir(parents=True, exist_ok=True) - template = env.get_template("api_urls.py.jinja") - collections_with_routes = sorted( - {route.collection for route in context.routes if route.collection} - ) (api_dir / "urls.py").write_text( - template.render( - collections=collections_with_routes, - routes=context.routes, - auth=context.auth, - ) + env.get_template("api_urls.py.jinja").render(routes=context.routes, auth=context.auth) ) (api_dir / "__init__.py").write_text("") diff --git a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja index 8cb1260a..f665a6f8 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/api_urls.py.jinja @@ -1,32 +1,22 @@ from django.urls import path -{% set handler_routes = routes | selectattr("handler") | rejectattr("collection") | list %} +{% set served = routes | rejectattr("operation", "equalto", "login") | list %} +{% set handler_routes = served | rejectattr("collection") | list %} {% if handler_routes %} -from api.views._handlers import {% for route in handler_routes %}Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View{% if not loop.last %}, {% endif %}{% endfor %} +from api.views._handlers import {{ handler_routes | map(attribute="view_name") | unique | join(", ") }} {% endif %} - -{% set crud_op_names = ["list", "get", "create", "update", "replace", "delete"] %} -{% for col_name in collections %} -{% set collection_routes = routes | selectattr("collection", "equalto", col_name) | selectattr("operation", "in", crud_op_names) | list %} -{% set custom_routes = routes | selectattr("collection", "equalto", col_name) | rejectattr("operation", "in", crud_op_names) | rejectattr("operation", "equalto", "login") | list %} -{% set list_routes = collection_routes | selectattr("has_id_param", "false") | list %} -{% set detail_routes = collection_routes | selectattr("has_id_param") | list %} -from api.views.{{ col_name }} import {% if list_routes %}{{ col_name | capitalize_collection }}ListView{% if detail_routes or custom_routes %}, {% endif %}{% endif %}{% if detail_routes %}{{ col_name | capitalize_collection }}DetailView{% if custom_routes %}, {% endif %}{% endif %}{% for route in custom_routes %}{{ col_name | capitalize_collection }}{{ route.operation | capitalize_collection }}View{% if not loop.last %}, {% endif %}{% endfor %} - +{% for col_name in served | selectattr("collection") | map(attribute="collection") | unique %} +from api.views.{{ col_name }} import {{ served | selectattr("collection", "equalto", col_name) | map(attribute="view_name") | unique | join(", ") }} {% endfor %} {% if auth and auth.has_password_auth %} {% for identity in auth.identity_collections %} {% if identity.has_password and identity.identifier %} from api.views.login import {{ identity.name | capitalize_collection }}LoginView - {% endif %} {% endfor %} {% endif %} urlpatterns = [ -{% for route in handler_routes %} - path("{{ route.django_path }}", Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View.as_view(), name="handler-{{ route.operation }}-{{ loop.index }}"), -{% endfor %} {% if auth and auth.has_password_auth %} {% for identity in auth.identity_collections %} {% if identity.has_password and identity.identifier %} @@ -34,14 +24,8 @@ urlpatterns = [ {% endif %} {% endfor %} {% endif %} -{% set seen_paths = [] %} -{% for route in routes | sort(attribute="has_id_param") if route.collection and route.operation in crud_op_names %} -{% if route.django_path not in seen_paths %} -{% set _ = seen_paths.append(route.django_path) %} - path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{% if route.has_id_param %}Detail{% else %}List{% endif %}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), -{% endif %} -{% endfor %} -{% for route in routes if route.collection and route.operation not in crud_op_names and route.operation != "login" %} - path("{{ route.django_path }}", {{ route.collection | capitalize_collection }}{{ route.operation | capitalize_collection }}View.as_view(), name="{{ route.collection }}-{{ route.operation }}"), +{% for view_name in served | map(attribute="view_name") | unique %} +{% set route = served | selectattr("view_name", "equalto", view_name) | first %} + path("{{ route.django_path }}", {{ view_name }}.as_view(), name="{{ view_name }}"), {% endfor %} ] diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja index 05f04d9c..25199ce6 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_call.jinja @@ -1,4 +1,4 @@ -{# Runs the route's handler, or the repository operation it declares, over one HandlerRequest #} +{# Runs the route's service operation, or the handler of a route without a service, over one HandlerRequest #} {% macro run_operation(route) %} _request = HandlerRequest( params=path_params, @@ -9,9 +9,9 @@ principal=_resolved_principal, inputs=service_inputs, ) -{% if route.handler %} - return Response(await invoke({{ route.handler | py_literal }}, _request)) +{% if route.collection %} + return Response(await service.{{ route.operation }}(_request)) {% else %} - return Response(await repo.{{ route.operation }}(_request.bindings)) + return Response(await invoke({{ route.handler | py_literal }}, _request)) {% endif %} {% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index 20145bdf..2a71def5 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -4,7 +4,7 @@ from rest_framework.response import Response from adrf.views import APIView from api.conflicts import ConflictError -from api.handler_requests import HandlerRequest, invoke +from api.handler_requests import HandlerRequest, invoke, json_body from api.service_inputs import prepare_service_inputs, resolve_access_bind {% set ns_where = namespace(collections=[]) %} {% for route in routes %}{% for where_collection in route.where_collections %} @@ -19,8 +19,10 @@ from api.authentication.mongodb import get_database {% endif %} -{% for route in routes %} -class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(APIView): +{% for path, path_routes in routes | groupby("relative_path") %} +class {{ path_routes[0].view_name }}(APIView): +{% for route in path_routes %} + async def {{ route.method | lower }}(self, request, **path_params): {% if route.needs_auth %} _resolved_principal = await resolve_principal(request, get_database()) @@ -32,7 +34,7 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A {% else %} _resolved_principal = None {% endif %} - body = request.data if isinstance(request.data, dict) else {} + body = json_body(request.body) or {} {% if route.access_rules %} _request_body = body _request_path = path_params @@ -90,6 +92,7 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) except ValueError as exc: return Response({"detail": str(exc)}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) +{% endfor %} {% endfor %} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 4e1d995e..202e2ffb 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -34,7 +34,7 @@ from api.services.{{ collection_name }} import {{ collection_name | capitalize_c from api.services.exceptions import NotFoundError, ValidationError from api.service_inputs import prepare_service_inputs, resolve_access_bind {% if routes | selectattr("runs_operation") | list %} -from api.handler_requests import HandlerRequest, invoke +from api.handler_requests import HandlerRequest, json_body {% endif %} {% if routes | selectattr("pagination") | list %} from api.pagination import build_response, map_request @@ -46,9 +46,6 @@ from api.authentication.auth import resolve_principal, _principal_has_role {% endif %} {% set class_name = collection_name | capitalize_collection %} -{% set list_routes = crud_routes | selectattr("has_id_param", "false") | list %} -{% set detail_routes = crud_routes | selectattr("has_id_param") | list %} -{% set custom_routes = routes | rejectattr("operation", "equalto", "login") | rejectattr("operation", "in", crud_op_names) | list %} {% macro record_criteria(route) %}{{ "{" }}{% for field, param in route.record_criteria %}"{{ field }}": {{ param }}{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} {% macro security_decorator(route) %} @@ -143,9 +140,31 @@ from api.authentication.auth import resolve_principal, _principal_has_role {% endif %} {% endmacro %} -{% if list_routes %} -class {{ class_name }}ListView(APIView): -{% for route in list_routes %} +{% macro record_call(route, method) %} +{% if route.record_parent_params %} +await service.{{ method }}_by({{ record_criteria(route) }}{{ caller() }}) +{%- else %} +await service.{{ method }}({{ route.record_id_param }}{{ caller() }}) +{%- endif %} +{% endmacro %} + +{% macro prefetch(route) %} +{% if route is needs_post_fetch %} + prefetch = {% call record_call(route, "get") %}{% endcall %} + +{{ post_fetch_check(route, "prefetch") | indent(4, true) }} +{% endif %} +{% endmacro %} + +{% macro validated_data(route) %} + serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data +{% endmacro %} + +{% for path, path_routes in routes | rejectattr("operation", "equalto", "login") | groupby("relative_path") %} +class {{ path_routes[0].view_name }}(APIView): +{% for route in path_routes %} {{ security_decorator(route) }} async def {{ route.method | lower }}(self, request, **path_params): @@ -155,7 +174,11 @@ class {{ class_name }}ListView(APIView): db = get_database() repo = {{ class_name }}Repository(db) service = {{ class_name }}Service(repo) +{% if route.runs_operation %} + body = json_body(request.body) or {} +{% else %} body = request.data if isinstance(request.data, dict) else {} +{% endif %} {{ request_locals(route) }} {{ pre_fetch_check(route) }} {{ bind_inputs(route) }} @@ -170,131 +193,23 @@ class {{ class_name }}ListView(APIView): page = await service.list(map_request(pagination_config, pagination_query)) return Response(build_response(pagination_config, page, pagination_query)) {% else %} - documents = await service.list() - return Response(documents) + return Response(await service.list()) {% endif %} {% elif route.operation == "create" %} - serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) - serializer.is_valid(raise_exception=True) - data = serializer.validated_data - document = await service.create(data) - return Response(document) -{% endif %} - except NotFoundError: - return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) - except ConflictError as exc: - return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) - except (ValidationError, ValueError) as exc: - detail = exc.message if isinstance(exc, ValidationError) else str(exc) - return Response({"detail": detail}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) -{% endfor %} -{% endif %} - -{% for route in custom_routes %} -{% set custom_class = class_name ~ (route.operation | capitalize_collection) ~ "View" %} -class {{ custom_class }}(APIView): - -{{ security_decorator(route) }} - async def {{ route.method | lower }}(self, request, **path_params): - db = get_database() - repo = {{ class_name }}Repository(db) - body = request.data if isinstance(request.data, dict) else {} -{{ request_locals(route) }} -{{ pre_fetch_check(route) }} -{{ bind_inputs(route) }} - try: - service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{{ run_operation(route) }} - except NotFoundError: - return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) - except ConflictError as exc: - return Response({"detail": str(exc)}, status=status.HTTP_409_CONFLICT) - except (ValidationError, ValueError) as exc: - detail = exc.message if isinstance(exc, ValidationError) else str(exc) - return Response({"detail": detail}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) - - -{% endfor %} - -{% if detail_routes %} -class {{ class_name }}DetailView(APIView): -{% for route in detail_routes %} - -{{ security_decorator(route) }} - async def {{ route.method | lower }}(self, request, **path_params): -{% for name in route.path_params %} - {{ name }} = path_params["{{ name }}"] -{% endfor %} - db = get_database() - repo = {{ class_name }}Repository(db) - service = {{ class_name }}Service(repo) - body = request.data if isinstance(request.data, dict) else {} -{{ request_locals(route) }} -{{ pre_fetch_check(route) }} -{{ bind_inputs(route) }} - try: - service_inputs = prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) -{% if route.runs_operation %} -{{ run_operation(route) }} +{{ validated_data(route) }} + return Response(await service.create(data)) {% elif route.operation == "get" %} -{% if route.record_parent_params %} - result = await service.get_by({{ record_criteria(route) }}) -{% else %} - result = await service.get({{ route.record_id_param }}) -{% endif %} + result = {% call record_call(route, "get") %}{% endcall %} + {{ post_fetch_check(route, "result") | indent(4, true) }} return Response(result) -{% elif route.operation == "update" %} - serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) - serializer.is_valid(raise_exception=True) - data = serializer.validated_data -{% if route is needs_post_fetch %} -{% if route.record_parent_params %} - prefetch = await service.get_by({{ record_criteria(route) }}) -{% else %} - prefetch = await service.get({{ route.record_id_param }}) -{% endif %} -{{ post_fetch_check(route, "prefetch") | indent(4, true) }} -{% endif %} -{% if route.record_parent_params %} - result = await service.update_by({{ record_criteria(route) }}, data) -{% else %} - result = await service.update({{ route.record_id_param }}, data) -{% endif %} - return Response(result) -{% elif route.operation == "replace" %} - serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) - serializer.is_valid(raise_exception=True) - data = serializer.validated_data -{% if route is needs_post_fetch %} -{% if route.record_parent_params %} - prefetch = await service.get_by({{ record_criteria(route) }}) -{% else %} - prefetch = await service.get({{ route.record_id_param }}) -{% endif %} -{{ post_fetch_check(route, "prefetch") | indent(4, true) }} -{% endif %} -{% if route.record_parent_params %} - result = await service.replace_by({{ record_criteria(route) }}, data) -{% else %} - result = await service.replace({{ route.record_id_param }}, data) -{% endif %} - return Response(result) +{% elif route.operation in ("update", "replace") %} +{{ validated_data(route) }} +{{ prefetch(route) }} + return Response({% call record_call(route, route.operation) %}, data{% endcall %}) {% elif route.operation == "delete" %} -{% if route is needs_post_fetch %} -{% if route.record_parent_params %} - prefetch = await service.get_by({{ record_criteria(route) }}) -{% else %} - prefetch = await service.get({{ route.record_id_param }}) -{% endif %} -{{ post_fetch_check(route, "prefetch") | indent(4, true) }} -{% endif %} -{% if route.record_parent_params %} - result = await service.delete_by({{ record_criteria(route) }}) -{% else %} - result = await service.delete({{ route.record_id_param }}) -{% endif %} - return Response(result) +{{ prefetch(route) }} + return Response({% call record_call(route, "delete") %}{% endcall %}) {% endif %} except NotFoundError: return Response({"detail": "Not found"}, status=status.HTTP_404_NOT_FOUND) @@ -304,4 +219,6 @@ class {{ class_name }}DetailView(APIView): detail = exc.message if isinstance(exc, ValidationError) else str(exc) return Response({"detail": detail}, status=status.HTTP_422_UNPROCESSABLE_ENTITY) {% endfor %} -{% endif %} + + +{% endfor %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 710e3545..c915b4d8 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -3,10 +3,16 @@ from jinja2 import Environment from yaml_api_generator.generators.backend.compile_context import CompileContext, IdentityCollectionDescriptor -from yaml_api_generator.ir import IRServiceBase from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields from yaml_api_generator.utils.strings import to_pascal_case, to_py_identifier from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment +from ..inheritance import ( + own_repository_operations, + own_service_operations, + repository_base_class, + route_operations, + service_base_class, +) from ..utils import ( extract_identity_fields, find_identity, @@ -41,17 +47,6 @@ def _crud_operations(col_data: dict) -> list[str]: return [op for op in _CRUD_ORDER if op in present] -def _own_operations(operations: dict[str, str | None], base: IRServiceBase | None) -> dict[str, str | None]: - inherited = base.operations.items() if base else {}.items() - return {name: handler for name, handler in operations.items() if (name, handler) not in inherited} - - -def _generated_service_base(base: IRServiceBase | None) -> dict[str, str] | None: - if base is None: - return None - return {"module": f"services.{base.name}", "class_name": f"{to_pascal_case(base.name)}Service"} - - class FastAPIGenerator(TemplateBasedGenerator): def generate(self, context: CompileContext, output_dir: Path) -> None: validate_datastore_layout(context, "fastapi") @@ -182,21 +177,20 @@ def _render_services( operations=[], is_identity_collection=False, password_via=False, - custom_operations=_own_operations(base.operations, parent), - generated_base=_generated_service_base(parent), + custom_operations=own_service_operations(base.operations, parent), + generated_base=service_base_class(parent), )) def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: identity = find_identity(context, col_name) is_identity = identity is not None base = context.service_base(col_name) - operations = {route.operation: route.handler for route in col_data["routes"] if route.runs_operation} return { "collection_name": col_name, "operations": _crud_operations(col_data), - "custom_operations": _own_operations(operations, base), - "generated_base": _generated_service_base(base), + "custom_operations": own_service_operations(route_operations(col_data), base), + "generated_base": service_base_class(base), "is_identity_collection": is_identity, "password_via": is_identity and identity.has_password, "default_role": identity.roles.default if is_identity and identity.roles else None, @@ -351,19 +345,6 @@ def _render_repositories( init_imports: list[str] = [] uses_mongo = uses_sql = False - def generated_base(name: str) -> dict[str, str] | None: - base = context.repository_inherits.get(name) - if base is None: - return None - return {"module": f".{base}", "class_name": f"{to_pascal_case(base)}Repository"} - - def own_queries(name: str) -> list[dict]: - inherited = context.repository_operations.get(context.repository_inherits.get(name), []) - return [ - op for op in context.repository_operations.get(name, []) - if op["query"] is not None and op not in inherited - ] - # Abstract base repositories render as inheritable classes (no collection). for base_name, ds_type in context.repository_abstract.items(): is_sql = ds_type in ("postgres", "postgresql") @@ -372,8 +353,8 @@ def own_queries(name: str) -> list[dict]: abstract=True, collection_name=base_name, operations=[], - custom_operations=own_queries(base_name), - generated_base=generated_base(base_name), + custom_operations=own_repository_operations(context, base_name), + generated_base=repository_base_class(context, base_name), )) init_imports.append(f"from .{base_name} import {to_pascal_case(base_name)}Repository") uses_sql = uses_sql or is_sql @@ -389,8 +370,8 @@ def own_queries(name: str) -> list[dict]: "id_field_format": meta["id_format"], "collection_type": meta["type"], "fields": meta["fields"], - "custom_operations": own_queries(col_name), - "generated_base": generated_base(col_name), + "custom_operations": own_repository_operations(context, col_name), + "generated_base": repository_base_class(context, col_name), } template = sql_template if is_sql else mongo_template (repos_dir / f"{col_name}.py").write_text(template.render(**template_context)) diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja index fc63283b..517a9ef5 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/service.py.jinja @@ -1,22 +1,13 @@ -{% from "service_operations.jinja" import operation_methods %} +{% from "service_operations.jinja" import operation_imports, operation_methods %} {% set bases = [] %} {% for op in operations %}{% set _ = bases.append((op | capitalize) ~ "Svc") %}{% endfor %} {% if is_identity_collection %}{% set _ = bases.append("IdentityFieldsMixin") %}{% endif %} {% if password_via %}{% set _ = bases.append("PasswordMixin") %}{% endif %} {% set ops_bases = bases + ([] if generated_base else ["ServiceBase"]) %} -{% if custom_operations %} -from typing import Any - -from dependencies.handler_requests import HandlerRequest{% if custom_operations.values() | select | list %}, invoke{% endif %} - -{% endif %} {% if ops_bases %} from services._ops import {{ ops_bases | join(", ") }} {% endif %} -{% if generated_base %} -from {{ generated_base.module }} import {{ generated_base.class_name }} -{% endif %} - +{{ operation_imports(custom_operations, "dependencies.handler_requests", generated_base) }} class {{ collection_name | capitalize_collection }}Service({{ (ops_bases + ([generated_base.class_name] if generated_base else [])) | join(", ") }}): {% if is_identity_collection %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja deleted file mode 100644 index 3adc70a8..00000000 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/service_operations.jinja +++ /dev/null @@ -1,11 +0,0 @@ -{% macro operation_methods(operations) %} -{% for name, handler in operations.items() %} - - async def {{ name }}(self, request: HandlerRequest) -> Any: -{% if handler %} - return await invoke({{ handler | py_literal }}, request) -{% else %} - return await self._repo.{{ name }}(request.bindings) -{% endif %} -{% endfor %} -{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/inheritance.py b/src/yaml_api_generator/generators/backend/inheritance.py new file mode 100644 index 00000000..c8b37414 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/inheritance.py @@ -0,0 +1,41 @@ +"""What a generated service or repository class declares itself and what it inherits from the abstract one it extends.""" + +from yaml_api_generator.ir import IRServiceBase +from yaml_api_generator.utils.strings import to_pascal_case + +from .compile_context import CompileContext + + +def base_class(base: str | None, suffix: str) -> dict[str, str] | None: + """The sibling module and class a generated class extends, or None when it extends nothing.""" + if base is None: + return None + return {"module": f".{base}", "class_name": f"{to_pascal_case(base)}{suffix}"} + + +def service_base_class(base: IRServiceBase | None) -> dict[str, str] | None: + return base_class(base and base.name, "Service") + + +def own_service_operations(operations: dict[str, str | None], base: IRServiceBase | None) -> dict[str, str | None]: + """The operations a service implements itself, mapped to their handler or None for its repository's.""" + inherited = base.operations.items() if base else {}.items() + return {name: handler for name, handler in operations.items() if (name, handler) not in inherited} + + +def route_operations(col_data: dict) -> dict[str, str | None]: + """The non-generated operations the routes of one service run, mapped as ``own_service_operations`` expects.""" + return {route.operation: route.handler for route in col_data["routes"] if route.runs_operation} + + +def own_repository_operations(context: CompileContext, name: str) -> list[dict]: + """The query operations a repository declares itself rather than inherits.""" + inherited = context.repository_operations.get(context.repository_inherits.get(name), []) + return [ + op for op in context.repository_operations.get(name, []) + if op["query"] is not None and op not in inherited + ] + + +def repository_base_class(context: CompileContext, name: str) -> dict[str, str] | None: + return base_class(context.repository_inherits.get(name), "Repository") diff --git a/src/yaml_api_generator/generators/backend/raw/generator.py b/src/yaml_api_generator/generators/backend/raw/generator.py index c33476c5..8c887b55 100644 --- a/src/yaml_api_generator/generators/backend/raw/generator.py +++ b/src/yaml_api_generator/generators/backend/raw/generator.py @@ -10,10 +10,9 @@ from yaml_api_generator.generators.backend.builders.settings_builder import build_settings_fields from yaml_api_generator.ir import SECRET_FIELD from yaml_api_generator.utils.strings import to_pascal_case -from ..base import TemplateBasedGenerator, RenderConfig, SingleFileRenderConfig, template_environment +from ..base import SingleFileRenderConfig, template_environment +from ..shared_layers import SharedLayersGenerator from ..utils import ( - extract_identity_fields, - find_identity, group_by_collection, pagination_config, MONGO_RUNTIME_MODULES, @@ -43,7 +42,9 @@ def _replace(m: re.Match) -> str: return re.sub(r"\{(\w+)\}", _replace, path) -class RawASGIGenerator(TemplateBasedGenerator): +class RawASGIGenerator(SharedLayersGenerator): + auth_module = "dependencies.auth" + def generate(self, context: CompileContext, output_dir: Path) -> None: validate_datastore_layout(context, "raw") reject_cookie_delivery(context, "raw") @@ -153,62 +154,6 @@ def _render_validation( ) ) - def _render_services( - self, - env: Environment, - context: CompileContext, - collections_by_name: dict[str, dict], - output_dir: Path, - ) -> None: - self._render_collection_based( - env, context, collections_by_name, output_dir, - RenderConfig( - template_name="service.py.jinja", - output_subdir="services", - import_class_suffix="Service", - context_builder=self._build_service_context, - additional_files=[("exceptions.py.jinja", "exceptions.py")], - ) - ) - - def _build_service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: - identity = find_identity(context, col_name) - is_identity = identity is not None - - return { - "collection_name": col_name, - "is_async": True, - "is_identity_collection": is_identity, - "password_via": is_identity and identity.has_password, - "identity_fields": extract_identity_fields(context, col_name), - "import_prefix": "repositories", - "auth_import_prefix": "dependencies.auth", - "service_import_prefix": "services", - } - - def _render_repositories( - self, - env: Environment, - context: CompileContext, - collections_by_name: dict[str, dict], - output_dir: Path, - ) -> None: - self._render_collection_based( - env, context, collections_by_name, output_dir, - RenderConfig( - template_name="repository.py.jinja", - output_subdir="repositories", - import_class_suffix="Repository", - context_builder=lambda col_name, col_data, ctx: { - "collection_name": col_name, - "id_field": ctx.collections[col_name]["id_field"], - "id_field_format": ctx.collections[col_name]["id_format"], - "custom_operations": ctx.repository_operations.get(col_name, []), - "module_prefix": "", - }, - ) - ) - def _render_routes( self, env: Environment, diff --git a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja index 0e0fed0f..6ec340c6 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/app.py.jinja @@ -37,17 +37,18 @@ def _path_to_regex(path: str) -> str: return re.sub(r"\{(\w+)\}", _replace, path) -ROUTES: dict[str, list[tuple[re.Pattern[str], Any]]] = {} +# Each route: path pattern, handler, and whether its body is a record that must be JSON +ROUTES: dict[str, list[tuple[re.Pattern[str], Any, bool]]] = {} {% for route in routes if route.operation != "login" %} -ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ route.handler_name }})) +ROUTES.setdefault("{{ route.method }}", []).append((re.compile(r"^" + _PREFIX + r"{{ route.relative_path | path_to_regex }}$"), {{ route.handler_name }}, {{ (route.body_model is not none) | py_literal }})) {% endfor %} {% for identity in ctx.auth.identity_collections %} {% if identity.has_password and identity.identifier %} -ROUTES.setdefault("POST", []).append((re.compile(r"^" + _PREFIX + r"/{{ identity.name }}/login$"), handle_login_{{ identity.name }})) +ROUTES.setdefault("POST", []).append((re.compile(r"^" + _PREFIX + r"/{{ identity.name }}/login$"), handle_login_{{ identity.name }}, True)) {% endif %} {% endfor %} {% if ctx.session.uses_asymmetric %} -ROUTES.setdefault("GET", []).append((re.compile(r"^/\.well-known/jwks\.json$"), handle_jwks)) +ROUTES.setdefault("GET", []).append((re.compile(r"^/\.well-known/jwks\.json$"), handle_jwks, False)) {% endif %} @@ -78,15 +79,17 @@ class ASGIApp: method = scope["method"].upper() path = scope["path"] - handler, url_params = self._match_route(method, path) + handler, url_params, record_body = self._match_route(method, path) if handler is None: await self._send_json(send, 404, {"detail": "Not found"}) return body = await self._read_body(receive) if body is _INVALID_JSON: - await self._send_json(send, 400, {"detail": "Invalid JSON body"}) - return + if record_body: + await self._send_json(send, 400, {"detail": "Invalid JSON body"}) + return + body = None headers = { k.decode("latin-1"): v.decode("latin-1") @@ -117,13 +120,13 @@ class ASGIApp: await self._send_json(send, status, data) @staticmethod - def _match_route(method: str, path: str) -> tuple[Any, dict[str, str]]: + def _match_route(method: str, path: str) -> tuple[Any, dict[str, str], bool]: entries = ROUTES.get(method, []) - for pattern, handler in entries: + for pattern, handler, record_body in entries: match = pattern.match(path) if match: - return handler, match.groupdict() - return None, {} + return handler, match.groupdict(), record_body + return None, {}, False @staticmethod async def _read_body(receive: Any) -> dict[str, Any] | None: diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja index cecf7164..4f5590ac 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_call.jinja @@ -1,17 +1,17 @@ -{# Runs the route's handler, or the repository operation it declares, over one HandlerRequest #} +{# Runs the route's service operation, or the handler of a route without a service, over one HandlerRequest #} {% macro run_operation(route) %} _request = HandlerRequest( params=url_params, query=query or {}, headers=headers, cookies=cookies or {}, - body=body or {}, + body=body, principal=principal, inputs=service_inputs, ) -{% if route.handler %} - return 200, await invoke({{ route.handler | py_literal }}, _request) +{% if route.collection %} + return 200, await service.{{ route.operation }}(_request) {% else %} - return 200, await repo.{{ route.operation }}(_request.bindings) + return 200, await invoke({{ route.handler | py_literal }}, _request) {% endif %} {% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index f139878a..d5db8ece 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -12,7 +12,7 @@ from repositories.{{ where_collection }} import {{ where_collection | capitalize {% endfor %} from services.{{ collection_name }} import {{ collection_name | capitalize_collection }}Service from services.exceptions import NotFoundError, ValidationError -from handler_requests import HandlerRequest, invoke +from handler_requests import HandlerRequest from service_inputs import prepare_service_inputs, resolve_access_bind {% if routes | selectattr("pagination") | list %} from pagination import build_response, map_request diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja index 20cd3315..29eca709 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository.py.jinja @@ -21,6 +21,9 @@ from {{ module_prefix }}conflicts import ConflictError from {{ module_prefix }}mongo_operations import run_mongo from {{ module_prefix }}operations import present, resolve from {{ module_prefix }}pagination import Page, Primitives, expand_bound +{% if generated_base %} +from {{ generated_base.module }} import {{ generated_base.class_name }} +{% endif %} _HIDDEN_FIELDS = frozenset({"_id", "_auth"}) @@ -82,7 +85,7 @@ def _translate_conflict(operation): {% set class_name = collection_name | capitalize_collection %} -class {{ class_name }}Repository: +class {{ class_name }}Repository{% if generated_base %}({{ generated_base.class_name }}){% endif %}: COLLECTION = "{{ collection_name }}" ID_FIELD = "{{ id_field }}" diff --git a/src/yaml_api_generator/generators/backend/shared/templates/repository_base.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/repository_base.py.jinja new file mode 100644 index 00000000..8fbb6e54 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared/templates/repository_base.py.jinja @@ -0,0 +1,11 @@ +{% from "repository_operations.jinja" import operation_methods %} +from typing import Any +{% if generated_base %} + +from {{ generated_base.module }} import {{ generated_base.class_name }} +{% endif %} + + +class {{ collection_name | capitalize_collection }}Repository{% if generated_base %}({{ generated_base.class_name }}){% endif %}: + """Operations shared by the repositories extending it, run through their ``run_operation``.""" +{{ operation_methods(custom_operations) }} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja index b49d0ea4..930b390e 100644 --- a/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja +++ b/src/yaml_api_generator/generators/backend/shared/templates/service.py.jinja @@ -1,11 +1,13 @@ -from {{ import_prefix }}.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository -from {{ service_import_prefix }}.exceptions import NotFoundError, ValidationError +{% from "service_operations.jinja" import operation_imports, operation_methods %} +{{ operation_imports(custom_operations, module_prefix ~ "handler_requests", generated_base) }} +from {{ module_prefix }}repositories.{{ collection_name }} import {{ collection_name | capitalize_collection }}Repository +from {{ module_prefix }}services.exceptions import NotFoundError, ValidationError {% if is_identity_collection %} -from {{ auth_import_prefix }} import hash_password +from {{ auth_module }} import hash_password {% endif %} -class {{ collection_name | capitalize_collection }}Service: +class {{ collection_name | capitalize_collection }}Service{% if generated_base %}({{ generated_base.class_name }}){% endif %}: def __init__(self, repo: {{ collection_name | capitalize_collection }}Repository): self._repo = repo @@ -128,6 +130,7 @@ class {{ collection_name | capitalize_collection }}Service: raise NotFoundError("{{ collection_name }}", resource_id) return document {% endif %} +{{ operation_methods(custom_operations) }} {% if password_via %} @staticmethod diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service_base.py.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service_base.py.jinja new file mode 100644 index 00000000..87ebfd80 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared/templates/service_base.py.jinja @@ -0,0 +1,8 @@ +{% from "service_operations.jinja" import operation_imports, operation_methods %} +{{ operation_imports(custom_operations, module_prefix ~ "handler_requests", generated_base) }} + +class {{ collection_name | capitalize_collection }}Service{% if generated_base %}({{ generated_base.class_name }}){% endif %}: +{% if not custom_operations %} + pass +{% endif %} +{{ operation_methods(custom_operations) }} diff --git a/src/yaml_api_generator/generators/backend/shared/templates/service_operations.jinja b/src/yaml_api_generator/generators/backend/shared/templates/service_operations.jinja new file mode 100644 index 00000000..e92d1f21 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared/templates/service_operations.jinja @@ -0,0 +1,22 @@ +{% macro operation_imports(operations, handler_requests_module, generated_base) %} +{% if operations %} +from typing import Any + +from {{ handler_requests_module }} import HandlerRequest{% if operations.values() | select | list %}, invoke{% endif %} + +{% endif %} +{% if generated_base %} +from {{ generated_base.module }} import {{ generated_base.class_name }} +{% endif %} +{% endmacro %} +{% macro operation_methods(operations) %} +{% for name, handler in operations.items() %} + + async def {{ name }}(self, request: HandlerRequest) -> Any: +{% if handler %} + return await invoke({{ handler | py_literal }}, request) +{% else %} + return await self._repo.{{ name }}(request.bindings) +{% endif %} +{% endfor %} +{% endmacro %} diff --git a/src/yaml_api_generator/generators/backend/shared_layers.py b/src/yaml_api_generator/generators/backend/shared_layers.py new file mode 100644 index 00000000..4b4a01b4 --- /dev/null +++ b/src/yaml_api_generator/generators/backend/shared_layers.py @@ -0,0 +1,100 @@ +"""The service and repository layers a backend renders from the shared templates.""" + +from pathlib import Path + +from jinja2 import Environment + +from .base import RenderConfig, TemplateBasedGenerator +from .compile_context import CompileContext +from .inheritance import ( + own_repository_operations, + own_service_operations, + repository_base_class, + route_operations, + service_base_class, +) +from .utils import extract_identity_fields, find_identity + + +class SharedLayersGenerator(TemplateBasedGenerator): + """A generator whose services and repositories, abstract bases included, come from the shared templates.""" + + auth_module: str + """Module of the generated app that provides ``hash_password``.""" + + @property + def module_prefix(self) -> str: + return f"{self.dir_prefix}." if self.dir_prefix else "" + + def _render_services( + self, + env: Environment, + context: CompileContext, + collections_by_name: dict[str, dict], + output_dir: Path, + ) -> None: + self._render_collection_based( + env, context, collections_by_name, output_dir, + RenderConfig( + template_name="service.py.jinja", + output_subdir="services", + import_class_suffix="Service", + context_builder=self._service_context, + additional_files=[("exceptions.py.jinja", "exceptions.py")], + ) + ) + template = env.get_template("service_base.py.jinja") + for base in context.service_bases.values(): + parent = context.service_bases.get(base.inherits) + (output_dir / self.dir_prefix / "services" / f"{base.name}.py").write_text(template.render( + collection_name=base.name, + module_prefix=self.module_prefix, + custom_operations=own_service_operations(base.operations, parent), + generated_base=service_base_class(parent), + )) + + def _service_context(self, col_name: str, col_data: dict, context: CompileContext) -> dict: + identity = find_identity(context, col_name) + base = context.service_base(col_name) + return { + "collection_name": col_name, + "is_async": True, + "is_identity_collection": identity is not None, + "password_via": identity is not None and identity.has_password, + "identity_fields": extract_identity_fields(context, col_name), + "module_prefix": self.module_prefix, + "auth_module": self.auth_module, + "custom_operations": own_service_operations(route_operations(col_data), base), + "generated_base": service_base_class(base), + } + + def _render_repositories( + self, + env: Environment, + context: CompileContext, + collections_by_name: dict[str, dict], + output_dir: Path, + ) -> None: + self._render_collection_based( + env, context, collections_by_name, output_dir, + RenderConfig( + template_name="repository.py.jinja", + output_subdir="repositories", + import_class_suffix="Repository", + context_builder=lambda col_name, col_data, ctx: { + "collection_name": col_name, + "id_field": ctx.collections[col_name]["id_field"], + "id_field_format": ctx.collections[col_name]["id_format"], + "custom_operations": own_repository_operations(ctx, col_name), + "generated_base": repository_base_class(ctx, col_name), + "module_prefix": self.module_prefix, + }, + ) + ) + template = env.get_template("repository_base.py.jinja") + for base_name in context.repository_abstract: + (output_dir / self.dir_prefix / "repositories" / f"{base_name}.py").write_text(template.render( + collection_name=base_name, + custom_operations=own_repository_operations(context, base_name), + generated_base=repository_base_class(context, base_name), + )) diff --git a/src/yaml_api_generator/runtime/handler_requests.py b/src/yaml_api_generator/runtime/handler_requests.py index d4447de9..5c60e9f0 100644 --- a/src/yaml_api_generator/runtime/handler_requests.py +++ b/src/yaml_api_generator/runtime/handler_requests.py @@ -3,13 +3,35 @@ A handler is a sync or async callable that receives one ``HandlerRequest`` and returns the JSON response body. Raising ``ValueError`` answers 422. A repository operation resolves its query's ``$`` references against ``bindings``. + +Every backend hands over the same view: a repeated query key keeps its last +value, an empty or non-JSON body is ``{}``, and the principal's record never +carries the stored id or credentials. """ +import json from dataclasses import dataclass, field from importlib import import_module from inspect import isawaitable from typing import Any +_UNEXPOSED_RECORD_FIELDS = frozenset({"_id", "_auth", "password"}) + + +def _last_values(query: dict[str, Any]) -> dict[str, Any]: + return {key: value[-1] if isinstance(value, list) else value for key, value in query.items()} + + +def _principal_view(principal: dict[str, Any] | None) -> dict[str, Any] | None: + if principal is None: + return None + record = principal.get("record") + return { + **principal, + "roles": list(principal.get("roles", ())), + "record": record and {key: value for key, value in record.items() if key not in _UNEXPOSED_RECORD_FIELDS}, + } + @dataclass(frozen=True) class HandlerRequest: @@ -21,10 +43,23 @@ class HandlerRequest: principal: dict[str, Any] | None = None inputs: dict[str, Any] = field(default_factory=dict) + def __post_init__(self) -> None: + object.__setattr__(self, "query", _last_values(self.query)) + object.__setattr__(self, "body", self.body or {}) + object.__setattr__(self, "principal", _principal_view(self.principal)) + @property def bindings(self) -> dict[str, Any]: params = {**self.params, **self.inputs} - return {**params, "principal": self.principal or {}, "params": params, "body": self.body or {}} + return {**params, "principal": self.principal or {}, "params": params, "body": self.body} + + +def json_body(raw: bytes) -> Any: + """The JSON value a raw request body holds, or None when it holds none.""" + try: + return json.loads(raw) + except ValueError: + return None async def invoke(handler: str, request: HandlerRequest) -> Any: diff --git a/src/yaml_api_generator/server/common/steps/handlers.py b/src/yaml_api_generator/server/common/steps/handlers.py index f29ca71a..f836d4e0 100644 --- a/src/yaml_api_generator/server/common/steps/handlers.py +++ b/src/yaml_api_generator/server/common/steps/handlers.py @@ -17,13 +17,10 @@ async def step(ctx: RequestContext) -> RequestContext: def handler_request(ctx: RequestContext) -> HandlerRequest: return HandlerRequest( params=dict(ctx.url_params), - query={ - key: value[-1] if isinstance(value, list) else value - for key, value in (ctx.query or {}).items() - }, + query=dict(ctx.query or {}), headers=dict(ctx.headers), cookies=dict(ctx.cookies or {}), - body=ctx.body or {}, + body=ctx.body, principal=ctx.principal, inputs=dict(ctx.service_inputs or {}), ) diff --git a/tests/integration/specs/services_mongo/hooks/__init__.py b/tests/integration/specs/services_mongo/hooks/__init__.py index 5b4a46c2..f546032a 100644 --- a/tests/integration/specs/services_mongo/hooks/__init__.py +++ b/tests/integration/specs/services_mongo/hooks/__init__.py @@ -3,6 +3,6 @@ async def echo(request): "params": request.params, "query": request.query, "body": request.body, - "principal": request.principal and {"sub": request.principal["sub"]}, + "principal": request.principal, "inputs": request.inputs, } diff --git a/tests/integration/specs/services_mongo/spec.yaml b/tests/integration/specs/services_mongo/spec.yaml index eb2ea1a4..91fdeeb5 100644 --- a/tests/integration/specs/services_mongo/spec.yaml +++ b/tests/integration/specs/services_mongo/spec.yaml @@ -76,7 +76,7 @@ controllers: notesByBody: false retitle: false countAll: false - echo: false + echo: { method: POST, path: "/echo/{id}" } totals: routes: tasks: { service: countAll-tasks, method: GET, path: /tasks-total } diff --git a/tests/integration/support/spec_apps.py b/tests/integration/support/spec_apps.py index f07a939d..7f9f765b 100644 --- a/tests/integration/support/spec_apps.py +++ b/tests/integration/support/spec_apps.py @@ -9,6 +9,7 @@ from pathlib import Path import httpx +import pytest from .network import free_port, wait_until @@ -17,28 +18,34 @@ @dataclass(frozen=True) class Backend: - """`stackform run` when `target` is None, otherwise the app `stackform build` emits for `target`.""" + """`stackform run` when `target` is None, otherwise the app `stackform build` emits for `target`, served by `interpreter`.""" name: str target: str | None = None + app: str = "app:app" + interpreter: str | None = sys.executable def command(self, spec: Path, env: dict[str, str], port: int) -> tuple[list[str], Path]: if self.target is None: return [STACKFORM, "run", str(spec), "--port", str(port)], spec.parent + if self.interpreter is None: + pytest.skip(f"no interpreter with the {self.name} requirements installed") output = spec.parent / self.target subprocess.run( [STACKFORM, "build", str(spec), "backend", "--target", self.target, "--output", str(output)], env=env, check=True, capture_output=True, ) - return [sys.executable, "-m", "uvicorn", "app:app", "--port", str(port)], output / "backend" + return [self.interpreter, "-m", "uvicorn", self.app, "--port", str(port)], output / "backend" DEV = Backend("run") FASTAPI = Backend("fastapi", "fastapi") RAW = Backend("raw", "raw") +DRF = Backend("drf", "drf", "project.asgi:application", os.environ.get("YAML_API_TEST_DRF_PYTHON")) +"""Served by the interpreter YAML_API_TEST_DRF_PYTHON names, which has the generated DRF requirements installed.""" SQL_BACKENDS = (DEV, FASTAPI) -MONGO_BACKENDS = (DEV, FASTAPI, RAW) +MONGO_BACKENDS = (DEV, FASTAPI, RAW, DRF) def _answering(url: str) -> bool: @@ -52,7 +59,7 @@ def _answering(url: str) -> bool: @contextmanager def serve(backend: Backend, spec: Path, env: dict[str, str] | None = None) -> Iterator[httpx.Client]: """Serve `spec` with `backend` and yield a client bound to its root URL.""" - environment = {**os.environ, "JWT_SECRET": "spec-apps-secret", **(env or {})} + environment = {**os.environ, "JWT_SECRET": "spec-apps-secret", "DJANGO_SECRET_KEY": "spec-apps-django-key", **(env or {})} port = free_port() command, cwd = backend.command(spec, environment, port) url, log = f"http://127.0.0.1:{port}", spec.parent / f"{backend.name}.log" diff --git a/tests/integration/test_services_mongo.py b/tests/integration/test_services_mongo.py index ea2ee716..480e3205 100644 --- a/tests/integration/test_services_mongo.py +++ b/tests/integration/test_services_mongo.py @@ -75,12 +75,28 @@ def test_handler_receives_the_request_it_documents(api): } +@pytest.mark.parametrize( + "send", + [{}, {"json": None}, {"json": 0}, {"content": "plain", "headers": {"Content-Type": "text/plain"}}], + ids=["absent", "null", "zero", "text"], +) +def test_handler_receives_an_empty_body_for_any_non_object_body(api, send): + response = api.post("/api/tasks/echo/t1", **send) + assert response.status_code == 200, response.text + assert response.json()["body"] == {} + + +def test_handler_receives_the_last_value_of_a_repeated_query_key(api): + response = api.post("/api/tasks/echo/t1?page=1&page=2") + assert response.status_code == 200, response.text + assert response.json()["query"] == {"page": "2"} + @pytest.fixture(scope="module") def user(api): login = {"email": f"{uuid4().hex}@example.com", "password": "correct-horse-battery"} registered = api.post("/api/users", json=login).json() token = api.post("/api/users/login", json=login).json()["token"] - return {"id": registered["id"], "headers": {"Authorization": f"Bearer {token}"}} + return {"id": registered["id"], "email": login["email"], "headers": {"Authorization": f"Bearer {token}"}} def test_controllers_sharing_a_service_keep_their_own_access(api, user): @@ -91,6 +107,17 @@ def test_controllers_sharing_a_service_keep_their_own_access(api, user): assert _titles(api.get(f"/api/archive/by-title/{title}", headers=user["headers"])) == [title] +def test_handler_principal_carries_the_identity_without_its_credentials(api, user): + response = api.post("/api/archive/echo/t1", headers=user["headers"]) + assert response.status_code == 200, response.text + assert response.json()["principal"] == { + "sub": user["id"], + "identity_collection": "users", + "roles": [], + "record": {"id": user["id"], "email": user["email"]}, + } + + def test_explicit_controller_route_is_public_at_its_overridden_path(api): response = api.get("/api/totals/tasks") assert response.status_code == 200, response.text diff --git a/tests/unit/test_generators.py b/tests/unit/test_generators.py index b15c8de7..0d0729bd 100644 --- a/tests/unit/test_generators.py +++ b/tests/unit/test_generators.py @@ -172,7 +172,8 @@ def test_drf_views_per_collection(tmp_path): gen.generate(ctx, tmp_path) assert (tmp_path / "api" / "views" / "posts.py").exists() content = (tmp_path / "api" / "views" / "posts.py").read_text() - assert "ListView" in content or "DetailView" in content + assert "class View_posts(APIView)" in content + assert "class View_posts__id_(APIView)" in content def test_drf_when_expression_to_python(): diff --git a/tests/unit/test_repo_service_inheritance.py b/tests/unit/test_repo_service_inheritance.py index 394ebcf6..b576203d 100644 --- a/tests/unit/test_repo_service_inheritance.py +++ b/tests/unit/test_repo_service_inheritance.py @@ -11,7 +11,9 @@ from yaml_api_generator.parsers.desugar import desugar_v02 from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir +from yaml_api_generator.generators.backend.drf.generator import DRFGenerator from yaml_api_generator.generators.backend.fastapi.generator import FastAPIGenerator +from yaml_api_generator.generators.backend.raw.generator import RawASGIGenerator from yaml_api_generator.generators.backend.context_builder import build_context from tests.conftest import VALID_DIR @@ -230,10 +232,22 @@ def test_abstract_service_generates_a_base_class_its_services_extend(tmp_path): service_src = (tmp_path / "out" / "services" / "posts.py").read_text() assert "class AuditableService(ServiceBase)" in base_src assert "async def countAll" in base_src and "async def summary" in base_src - assert "from services.auditable import AuditableService" in service_src + assert "from .auditable import AuditableService" in service_src assert "async def countAll" not in service_src +@pytest.mark.parametrize(("generator", "layers"), [(RawASGIGenerator, "."), (DRFGenerator, "api")]) +def test_raw_and_drf_generate_the_abstract_bases_their_layers_extend(tmp_path, generator, layers): + generator().generate(build_context(_parse(_SERVICE_FIXTURE, tmp_path)), tmp_path / "services") + generator().generate(build_context(load_and_generate_ir(str(VALID_DIR / "repo-service-inheritance.yaml")).ir), tmp_path / "repositories") + services = tmp_path / "services" / layers / "services" + repositories = tmp_path / "repositories" / layers / "repositories" + assert "async def countAll" in (services / "auditable.py").read_text() + assert "class PostsService(AuditableService)" in (services / "posts.py").read_text() + assert "async def countAll" in (repositories / "countable.py").read_text() + assert "class PostsRepository(CountableRepository)" in (repositories / "posts.py").read_text() + + def test_service_bound_to_abstract_repository_raises(): config = _base_config() config["repositories"]["base"] = {"abstract": True, "operations": {"countAll": {"query": {"command": "estimatedDocumentCount"}}}} diff --git a/tests/unit/test_rs256_signing.py b/tests/unit/test_rs256_signing.py index 23392cb5..a2da2a80 100644 --- a/tests/unit/test_rs256_signing.py +++ b/tests/unit/test_rs256_signing.py @@ -210,7 +210,7 @@ def _run(output: Path, body: str, **env: str) -> dict: import app from dependencies.jwt_utils import create_jwt_token, decode_jwt_token -handler, params = app.ASGIApp._match_route("GET", "/.well-known/jwks.json") +handler, params, _ = app.ASGIApp._match_route("GET", "/.well-known/jwks.json") status, payload = asyncio.run(handler(params, None, {}, None)) token = create_jwt_token("user-1", "user") print(json.dumps({ diff --git a/tests/unit/test_service_input_codegen.py b/tests/unit/test_service_input_codegen.py index 8d0a2c44..e490fec4 100644 --- a/tests/unit/test_service_input_codegen.py +++ b/tests/unit/test_service_input_codegen.py @@ -142,7 +142,8 @@ def test_raw_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "routes/posts.py").read_text() handler_route = (tmp_path / "routes/handlers.py").read_text() assert "inputs=service_inputs" in repository_route - assert "repo.archive(_request.bindings)" in repository_route + assert "service.archive(_request)" in repository_route + assert "self._repo.archive(request.bindings)" in (tmp_path / "services/posts.py").read_text() assert "inputs=service_inputs" in handler_route @@ -152,5 +153,6 @@ def test_drf_emits_repository_and_handler_forwarding(tmp_path): repository_route = (tmp_path / "api/views/posts.py").read_text() handler_route = (tmp_path / "api/views/_handlers.py").read_text() assert "inputs=service_inputs" in repository_route - assert "repo.archive(_request.bindings)" in repository_route + assert "service.archive(_request)" in repository_route + assert "self._repo.archive(request.bindings)" in (tmp_path / "api/services/posts.py").read_text() assert "inputs=service_inputs" in handler_route diff --git a/tests/unit/test_services_spec_builds.py b/tests/unit/test_services_spec_builds.py index 1b4ea0a6..fc82ac3e 100644 --- a/tests/unit/test_services_spec_builds.py +++ b/tests/unit/test_services_spec_builds.py @@ -13,7 +13,7 @@ def test_services_spec_builds_for_drf(tmp_path, monkeypatch): monkeypatch.setenv("MONGO_DB", "services") compile_project(yaml_file=str(_SPEC), output_dir=str(tmp_path), target="drf") - views = tmp_path / "backend" / "api" / "views" + service = (tmp_path / "backend" / "api" / "services" / "tasks.py").read_text() assert compileall.compile_dir(str(tmp_path / "backend"), quiet=2) - assert "repo.retitle(_request.bindings)" in (views / "tasks.py").read_text() - assert "invoke('hooks.echo', _request)" in (views / "tasks.py").read_text() + assert "self._repo.retitle(request.bindings)" in service + assert "invoke('hooks.echo', request)" in service From e167b5d3f95bf2b751252be373f650dd22bc9bb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 00:06:45 +0100 Subject: [PATCH 29/89] Locate route records and path scopes in the IR and admit anonymous callers through rules without a principal Handler routes load their bound record through the record parameter, nested routes create, list and fetch within their parent scope on 0.2 and 0.3, parent bindings are accepted on create routes, request bodies seen by access rules and service inputs carry only the sent fields, and the raw and DRF backends require uuid_utils. --- .../backend/builders/route_builder.py | 34 +--- .../generators/backend/compile_context.py | 5 +- .../drf/templates/handler_views.py.jinja | 5 +- .../drf/templates/requirements.txt.jinja | 1 + .../backend/drf/templates/views.py.jinja | 18 +- .../backend/fastapi/access_filters.py | 30 +-- .../generators/backend/fastapi/generator.py | 1 + .../fastapi/templates/handler_routes.py.jinja | 9 +- .../backend/fastapi/templates/routes.py.jinja | 40 ++-- .../raw/templates/handler_routes.py.jinja | 5 +- .../raw/templates/requirements.txt.jinja | 1 + .../backend/raw/templates/routes.py.jinja | 13 +- src/yaml_api_generator/ir/api.py | 44 ++++- .../parsers/transformers/access_validation.py | 23 ++- .../parsers/transformers/v01/__init__.py | 3 +- .../transformers/v01/route_transformer.py | 3 +- .../parsers/transformers/v02/__init__.py | 9 +- .../transformers/v02/document_lookup.py | 47 +++-- .../transformers/v02/record_location.py | 37 ++++ .../parsers/transformers/v03/__init__.py | 3 + src/yaml_api_generator/runtime/access.py | 18 +- .../server/common/context_builder.py | 2 + .../server/common/pipeline_builder.py | 19 +- .../server/common/server_context.py | 6 +- .../server/common/steps/authorization.py | 29 ++- .../server/common/steps/repositories.py | 52 +++--- tests/integration/test_mongo_access_http.py | 16 +- tests/integration/test_mongo_identity_http.py | 2 +- tests/unit/support/live_apps.py | 17 +- tests/unit/test_access_http.py | 175 +++++++++++++++++- tests/unit/test_access_reason.py | 4 +- tests/unit/test_fastapi_files_codegen.py | 2 +- tests/unit/test_free_slug_fetch.py | 76 +++++--- tests/unit/test_pipeline.py | 2 + tests/unit/test_request_access_generation.py | 2 +- tests/unit/test_where_binding.py | 4 +- 36 files changed, 556 insertions(+), 201 deletions(-) create mode 100644 src/yaml_api_generator/parsers/transformers/v02/record_location.py diff --git a/src/yaml_api_generator/generators/backend/builders/route_builder.py b/src/yaml_api_generator/generators/backend/builders/route_builder.py index ddc47dd7..f0bad086 100644 --- a/src/yaml_api_generator/generators/backend/builders/route_builder.py +++ b/src/yaml_api_generator/generators/backend/builders/route_builder.py @@ -3,8 +3,6 @@ Converts IR routes into route descriptors with access control and model references. """ -import re - from yaml_api_generator.ir import IRDocument, IRDto, IRSymbolRef from .model_builder import dto_payload_name, payload_name from ..compile_context import ( @@ -17,9 +15,7 @@ WRITE_OPERATIONS = ("create", "update", "replace") -DETAIL_OPERATIONS = frozenset({"get", "update", "replace", "delete"}) _REQUEST_PARTS = frozenset({"body", "query", "header", "cookie"}) -_PATH_TOKEN = re.compile(r"\{(\w+)\}") def _base_prefix(document: IRDocument) -> str: @@ -84,11 +80,9 @@ def build_routes( for ir_route in document.routes: needs_auth = not ir_route.is_public and has_identity + admits_anonymous = needs_auth and ir_route.admits_anonymous relative_path = ir_route.path.removeprefix(prefix) or "/" - path_params = _PATH_TOKEN.findall(ir_route.path) - record_id_param, record_parent_params = self._record_params( - document.version, ir_route.operation, path_params - ) + path_params = ir_route.path_params service_input, service_forwarding = _service_config( document, ir_route.service ) @@ -125,9 +119,10 @@ def build_routes( collection=ir_route.collection, is_public=ir_route.is_public, needs_auth=needs_auth, + admits_anonymous=admits_anonymous, is_identity_create=False, body_model=None, - has_id_param=record_id_param is not None, + has_id_param=ir_route.record_param is not None, password_via=False, handler=ir_route.handler, access_rules=access_rules, @@ -135,8 +130,8 @@ def build_routes( needs_request=needs_request, typed_body_model=typed_body_model, service=ir_route.service, - record_id_param=record_id_param, - record_parent_params=record_parent_params, + record_id_param=ir_route.record_param, + record_parent_params=list(ir_route.scope), service_input=service_input, service_forwarding=service_forwarding, ) @@ -162,17 +157,18 @@ def build_routes( collection=ir_route.collection, is_public=ir_route.is_public, needs_auth=needs_auth, + admits_anonymous=admits_anonymous, is_identity_create=is_identity_create, body_model=body_model, - has_id_param=record_id_param is not None, + has_id_param=ir_route.record_param is not None, password_via=uses_password, access_rules=access_rules, path_params=path_params, needs_request=needs_request, typed_body_model=typed_body_model, service=ir_route.service, - record_id_param=record_id_param, - record_parent_params=record_parent_params, + record_id_param=ir_route.record_param, + record_parent_params=list(ir_route.scope), id_field=document.collections[ir_route.collection].idField, service_input=service_input, service_forwarding=service_forwarding, @@ -182,16 +178,6 @@ def build_routes( return routes - @staticmethod - def _record_params( - spec_version: str, operation: str, path_params: list[str] - ) -> tuple[str | None, list[str]]: - if operation not in DETAIL_OPERATIONS: - return None, [] - if spec_version == "0.3" and path_params: - return path_params[-1], path_params[:-1] - return ("id", []) if "id" in path_params else (None, []) - @staticmethod def _needs_request(rules: list[AccessRuleDescriptor] | None) -> bool: return any( diff --git a/src/yaml_api_generator/generators/backend/compile_context.py b/src/yaml_api_generator/generators/backend/compile_context.py index ebf585de..08923ccc 100644 --- a/src/yaml_api_generator/generators/backend/compile_context.py +++ b/src/yaml_api_generator/generators/backend/compile_context.py @@ -18,7 +18,7 @@ from dataclasses import dataclass, field from typing import Any, ClassVar -from ...ir.api import IRFileField, IRPagination, IRProvider, IRRolesConfig, IRService +from ...ir.api import CallerDemand, IRFileField, IRPagination, IRProvider, IRRolesConfig, IRService from ...ir.when import WhenExpr from ...runtime.access import Roles from ...utils.strings import to_py_identifier @@ -216,7 +216,7 @@ def binds(self, part: str) -> bool: @dataclass(frozen=True) -class AccessRuleDescriptor: +class AccessRuleDescriptor(CallerDemand): principal: str required_role: str | None = None identity_symbols: list[str] = field(default_factory=list) @@ -255,6 +255,7 @@ class RouteDescriptor: service_input: dict[str, Any] = field(default_factory=dict) service_forwarding: dict[str, dict[str, Any]] = field(default_factory=dict) pagination: IRPagination | None = None + admits_anonymous: bool = False @property def request_symbols(self) -> list[RequestSymbolDescriptor]: diff --git a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja index f80a134c..26ee4b1d 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/handler_views.py.jinja @@ -25,11 +25,13 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A async def {{ route.method | lower }}(self, request, **path_params): {% if route.needs_auth %} principal = await resolve_principal(request, get_database()) +{% if not route.admits_anonymous %} if principal is None: return Response( {"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED, ) +{% endif %} {% else %} principal = None {% endif %} @@ -68,7 +70,8 @@ class Handler{{ route.operation | capitalize_collection }}{{ loop.index }}View(A try: access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] except AccessDenied as exc: - return Response({"detail": exc.reason}, status=status.HTTP_403_FORBIDDEN) + code, detail = exc.refusal(principal) + return Response({"detail": detail}, status=code) bound_inputs = resolve_access_bind(access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja index 6b499433..ecc55edb 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/requirements.txt.jinja @@ -6,6 +6,7 @@ gunicorn uvicorn pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} +uuid_utils pydantic-settings {% for req in ctx.vendored_requirements %} {{ req }} diff --git a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja index 615766b5..f66a96d7 100644 --- a/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja +++ b/src/yaml_api_generator/generators/backend/drf/templates/views.py.jinja @@ -42,7 +42,9 @@ from api.authentication.mongodb import get_database {% set needs_auth = routes | selectattr("needs_auth") | list %} {% if needs_auth %} from api.authentication.auth import resolve_principal, _principal_has_role -from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at +from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, matching, negate, one_of, row_filter, value_at +{% elif crud_routes | selectattr("operation", "equalto", "list") | selectattr("record_parent_params") | list %} +from api.access import matching {% endif %} {% set class_name = collection_name | capitalize_collection %} @@ -83,8 +85,10 @@ from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, {% if route.needs_auth %} db = get_database() _resolved_principal = await resolve_principal(request, db) +{% if not route.admits_anonymous %} if _resolved_principal is None: return Response({"detail": "Unauthorized"}, status=status.HTTP_401_UNAUTHORIZED) +{% endif %} {% if route.access_rules %} principal = _resolved_principal {% set ns = namespace(identity_symbols=[]) %} @@ -129,7 +133,8 @@ from api.access import AccessDenied, Decision, Field, all_of, conjoin, constant, _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] {% endif %} except AccessDenied as exc: - return Response({"detail": exc.reason}, status=status.HTTP_403_FORBIDDEN) + code, detail = exc.refusal(principal) + return Response({"detail": detail}, status=code) {% endmacro %} {% macro post_fetch_check(route, doc_var) %} @@ -200,7 +205,11 @@ class {{ path_routes[0].view_name }}(APIView): try: prepare_service_inputs(path_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) {% if route.operation == "list" %} -{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} +{% set rules_filter = auth.enabled and route.needs_auth and route.access_rules %} +{% set row_filter_arg = ", _row_filter" if rules_filter or route.record_parent_params else "" %} +{% if route.record_parent_params %} + _row_filter = {{ "conjoin(_row_filter, " if rules_filter }}matching({{ record_criteria(route) }}){{ ")" if rules_filter }} +{% endif %} {% if route.pagination %} pagination_config = {{ route.pagination | pagination_config | py_literal }} pagination_query = dict(request.query_params) @@ -214,6 +223,9 @@ class {{ path_routes[0].view_name }}(APIView): serializer = {{ route.typed_body_model or route.body_model }}(data=request.data) serializer.is_valid(raise_exception=True) data = serializer.validated_data +{% if route.record_parent_params %} + data = {**data, **{{ record_criteria(route) }}} +{% endif %} {% if identity %} data = {{ identity.role_store | py_literal }}.granted(dict(data), assignable={{ route.needs_auth | py_literal }}) {% endif %} diff --git a/src/yaml_api_generator/generators/backend/fastapi/access_filters.py b/src/yaml_api_generator/generators/backend/fastapi/access_filters.py index a76c49cf..b680aea7 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/access_filters.py +++ b/src/yaml_api_generator/generators/backend/fastapi/access_filters.py @@ -76,6 +76,18 @@ def _operand(self, expr: WhenExpr) -> str: return f"value_at({expr.root}, {expr.path or ('sub',)!r})" +def _caller_conditions(rule: AccessRuleDescriptor) -> list[str]: + """The rule's demand on the caller: an identity of its collection, holding its role.""" + if not rule.principal.startswith("identity:"): + return [] + checks = [ + "principal is not None", + f'principal.get("identity_collection") == {rule.principal.removeprefix("identity:")!r}', + *([f"_principal_has_role(principal, {rule.required_role!r})"] if rule.required_role is not None else []), + ] + return [f"constant({' and '.join(checks)})"] + + def rule_to_predicate(rule: AccessRuleDescriptor) -> str: """Compile an access rule to a Python expression of the predicate it admits.""" compiler = _RuleCompiler( @@ -85,16 +97,7 @@ def rule_to_predicate(rule: AccessRuleDescriptor) -> str: ), ) conditions = [ - *( - [f'constant(principal.get("identity_collection") == {rule.principal.removeprefix("identity:")!r})'] - if rule.principal.startswith("identity:") - else [] - ), - *( - [f"constant(_principal_has_role(principal, {rule.required_role!r}))"] - if rule.required_role is not None - else [] - ), + *_caller_conditions(rule), *(f"constant({symbol} is not None)" for symbol in rule.document_symbols), *([compiler.compile(rule.when)] if rule.when is not None else []), ] @@ -141,8 +144,13 @@ def needs_post_fetch(route: RouteDescriptor) -> bool: ) +def principal_dependency(route: RouteDescriptor) -> str: + """The FastAPI dependency resolving the caller of ``route``.""" + return "get_optional_principal" if route.admits_anonymous else "require_principal" + + def is_unconditional(rule: AccessRuleDescriptor) -> bool: - return not rule.principal and rule.when is None and rule.required_role is None + return rule.admits_anonymous and rule.when is None def needs_document_rule(rule: AccessRuleDescriptor) -> bool: diff --git a/src/yaml_api_generator/generators/backend/fastapi/generator.py b/src/yaml_api_generator/generators/backend/fastapi/generator.py index 23e5b54d..da15ea54 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/generator.py +++ b/src/yaml_api_generator/generators/backend/fastapi/generator.py @@ -57,6 +57,7 @@ def generate(self, context: CompileContext, output_dir: Path) -> None: env.filters["rule_to_predicate"] = access_filters.rule_to_predicate env.filters["role_hierarchies_py"] = access_filters.role_hierarchies_py env.filters["request_symbol_binding"] = access_filters.request_symbol_binding + env.filters["principal_dependency"] = access_filters.principal_dependency env.tests["needs_post_fetch"] = access_filters.needs_post_fetch env.tests["needs_document_rule"] = access_filters.needs_document_rule env.tests["needs_current_document_rule"] = ( diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja index a846ce4b..5daaf6fb 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/handler_routes.py.jinja @@ -11,8 +11,9 @@ from dependencies.service_inputs import prepare_service_inputs, resolve_access_b {% for where_collection in ns_where.collections %} from dependencies.repositories import get_{{ where_collection }}_repository {% endfor %} -{% if routes | selectattr("needs_auth") | list %} -from dependencies.auth import require_principal +{% set principal_dependencies = routes | selectattr("needs_auth") | map("principal_dependency") | unique | sort | list %} +{% if principal_dependencies %} +from dependencies.auth import {{ principal_dependencies | join(", ") }} {% endif %} {% if routes | selectattr("access_rules") | list %} from dependencies.access import AccessDenied, Decision, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, value_at @@ -25,7 +26,7 @@ router = APIRouter(tags=["Handlers"]) {% for route in routes %} @router.{{ route.method | lower }}("{{ route.relative_path }}") async def {{ route.operation | py_identifier }}_{{ loop.index }}( - request: Request{% if route.needs_auth %}, principal: dict[str, Any] = Depends(require_principal){% endif %}{% for where_collection in route.where_collections %}, _where_{{ where_collection }}_repo=Depends(get_{{ where_collection }}_repository){% endfor %} + request: Request{% if route.needs_auth %}, principal: dict[str, Any]{{ " | None" if route.admits_anonymous }} = Depends({{ route | principal_dependency }}){% endif %}{% for where_collection in route.where_collections %}, _where_{{ where_collection }}_repo=Depends(get_{{ where_collection }}_repository){% endfor %} ) -> Any: body: dict[str, Any] = {} try: @@ -65,7 +66,7 @@ async def {{ route.operation | py_identifier }}_{{ loop.index }}( try: _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] except AccessDenied as exc: - raise HTTPException(status_code=403, detail=exc.reason) from exc + raise HTTPException(*exc.refusal(principal)) from exc bound_inputs = resolve_access_bind(_access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja index 52c84747..f446a820 100644 --- a/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/fastapi/templates/routes.py.jinja @@ -51,20 +51,19 @@ from dependencies.repositories import get_{{ collection_name }}_repository {% for where_collection in ns_where.collections %} from dependencies.repositories import get_{{ where_collection }}_repository {% endfor %} -{% set ns_auth = namespace(needs_require=false, needs_role=false) %} -{% for route in routes %} -{% if route.needs_auth %}{% set ns_auth.needs_require = true %}{% endif %} -{% if route.access_rules %}{% set ns_auth.needs_role = true %}{% endif %} -{% endfor %} -{% if auth.enabled and ns_auth.needs_require %} -from dependencies.auth import require_principal +{% set principal_dependencies = routes | selectattr("needs_auth") | map("principal_dependency") | unique | sort | list %} +{% if auth.enabled and principal_dependencies %} +from dependencies.auth import {{ principal_dependencies | join(", ") }} {% endif %} {% if identity %} from dependencies.access import Roles {% endif %} -{% if auth.enabled and ns_auth.needs_role %} -from dependencies.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at +{% set scoped_lists = crud_routes | selectattr("operation", "equalto", "list") | selectattr("record_parent_params") | list %} +{% if auth.enabled and routes | selectattr("access_rules") | list %} +from dependencies.access import AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, matching, negate, one_of, row_filter, value_at from dependencies.authorization import _principal_has_role +{% elif scoped_lists %} +from dependencies.access import matching {% endif %} {% set ns = namespace(model_imports=[]) %} {% for route in crud_routes %} @@ -101,7 +100,7 @@ _files = record_files[{{ collection_name | tojson }}] {% endif %} {% macro openapi_extra(route, request_body=none) %}{% set security = route.needs_auth and security_schemes %}{% if security or request_body %}, openapi_extra={{ "{" }}{% if security %}"security": {{ security_schemes }}{% endif %}{% if security and request_body %}, {% endif %}{% if request_body %}"requestBody": {{ request_body }}{% endif %}{{ "}" }}{% endif %}{% endmacro %} -{% macro auth_param(route) %}{% if auth.enabled and route.needs_auth %}, principal: dict[str, Any] = Depends(require_principal){% endif %}{% endmacro %} +{% macro auth_param(route) %}{% if auth.enabled and route.needs_auth %}, principal: dict[str, Any]{{ " | None" if route.admits_anonymous }} = Depends({{ route | principal_dependency }}){% endif %}{% endmacro %} {% macro svc_dep() %}svc=Depends(get_{{ service_groups | first }}_service){% endmacro %} {% macro extra_path_params(route) %}{% for name in route.path_params if name != route.record_id_param %}, {{ name }}: str{% endfor %}{% endmacro %} {% macro record_criteria(route) %}{{ "{" }}{% for field, param in route.record_criteria %}"{{ field }}": {{ param }}{% if not loop.last %}, {% endif %}{% endfor %}{{ "}" }}{% endmacro %} @@ -227,7 +226,7 @@ _files = record_files[{{ collection_name | tojson }}] _access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(_decisions)] {% endif %} except AccessDenied as exc: - raise HTTPException(status_code=403, detail=exc.reason) from exc + raise HTTPException(*exc.refusal(principal)) from exc {% endif %} {% endmacro %} @@ -246,10 +245,12 @@ _files = record_files[{{ collection_name | tojson }}] {% for route in crud_routes %} {% set body_model = route.typed_body_model or route.body_model %} {% set partial = route.operation == "update" %} -{% set body_expr = "upload.payload" if files else 'payload.model_dump(mode="json"' ~ (", exclude_unset=True" if partial else "") ~ ")" %} -{% set record_expr = "upload.record" if files else body_expr %} +{% set sent_expr = "upload.payload" if files else 'payload.model_dump(mode="json", exclude_unset=True)' %} +{% set record_expr = "upload.record" if files else 'payload.model_dump(mode="json"' ~ (", exclude_unset=True" if partial else "") ~ ")" %} +{% set record_expr = "{**" ~ record_expr ~ ", **" ~ record_criteria(route) ~ "}" if route.operation == "create" and route.record_parent_params else record_expr %} {% if route.operation == "list" %} -{% set row_filter_arg = ", _row_filter" if auth.enabled and route.needs_auth and route.access_rules else "" %} +{% set rules_filter = auth.enabled and route.needs_auth and route.access_rules %} +{% set row_filter_arg = ", _row_filter" if rules_filter or route.record_parent_params else "" %} @router.{{ route.method | lower }}("{{ route.relative_path }}"{{ openapi_extra(route) }}) {% set list_prefix = (extra_path_params(route) ~ request_param(route) ~ auth_param(route)) | trim(chars=", ") %} async def list_{{ collection_name }}({{ list_prefix }}{% if list_prefix %}, {% endif %}{{ svc_dep() }}{{ where_repo_params(route) }}{% if route.pagination %}, limit: int | None = None{% if route.pagination.strategy == "offset" %}, offset: int | None = None{% else %}, after: str | None = None{% endif %}{% endif %}) -> Any: @@ -257,6 +258,9 @@ async def list_{{ collection_name }}({{ list_prefix }}{% if list_prefix %}, {% e {{ authorize_rules(route, "Field(())", rows=true) }} {{ bind_inputs(route) }} {{ prepare_inputs(route, path_values(route), "None") }} +{% if route.record_parent_params %} + _row_filter = {{ "conjoin(_row_filter, " if rules_filter }}matching({{ record_criteria(route) }}){{ ")" if rules_filter }} +{% endif %} {% if route.pagination %} _pagination_config = {{ route.pagination | pagination_config | py_literal }} _pagination_query = {"limit": limit{% if route.pagination.strategy == "offset" %}, "offset": offset{% else %}, "after": after{% endif %}} @@ -291,10 +295,10 @@ async def get_{{ collection_name }}({{ route.record_id_param }}: str{{ extra_pat {% elif route.operation == "create" %} @router.{{ route.method | lower }}("{{ route.relative_path }}"{{ openapi_extra(route, upload_body(body_model, partial)) }}) async def create_{{ collection_name }}({{ (payload_param(body_model) ~ extra_path_params(route) ~ request_param(route) ~ auth_param(route) ~ upload_param(route, body_model, partial)) | trim(chars=", ") }}, {{ svc_dep() }}{{ where_repo_params(route) }}) -> Any: -{{ request_locals(route, body_expr) }} +{{ request_locals(route, sent_expr) }} {{ pre_fetch_guard(route) }} {{ bind_inputs(route) }} -{{ prepare_inputs(route, path_values(route), body_expr) }} +{{ prepare_inputs(route, path_values(route), sent_expr) }} {% if identity %} return await svc.create({{ identity.role_store | py_literal }}.granted({{ record_expr }}, assignable={{ route.needs_auth | py_literal }})) {% else %} @@ -306,14 +310,14 @@ async def create_{{ collection_name }}({{ (payload_param(body_model) ~ extra_pat {% set record_expr = "{**upload.payload, **carried_files(_files, existing)}" if files and route.operation == "replace" else record_expr %} @router.{{ route.method | lower }}("{{ route.relative_path }}"{{ openapi_extra(route, upload_body(body_model, partial) if partial else json_body(body_model)) }}) async def {{ route.operation }}_{{ collection_name }}({{ route.record_id_param }}: str{{ payload_param(body_model) }}{{ extra_path_params(route) }}{{ request_param(route) }}{{ auth_param(route) }}{{ upload_param(route, body_model, partial) }}, {{ svc_dep() }}{{ where_repo_params(route) }}) -> Any: -{{ request_locals(route, body_expr) }} +{{ request_locals(route, sent_expr) }} {{ pre_fetch_guard(route) }} {% if route is needs_post_fetch or files %} existing = await {{ record_call(route, "get") }} {{ post_fetch_guard(route, "existing") }} {% endif %} {{ bind_inputs(route) }} -{{ prepare_inputs(route, path_values(route), body_expr) }} +{{ prepare_inputs(route, path_values(route), sent_expr) }} {{ persist(record_call(route, route.operation, record_expr)) }} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja index 54393e85..3c0cea62 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/handler_routes.py.jinja @@ -37,8 +37,10 @@ async def handle_custom_{{ loop.index }}( ) -> tuple[int, Any]: {% if route.needs_auth %} principal = await resolve_principal(headers, db) +{% if not route.admits_anonymous %} if principal is None: return 401, {"detail": "Unauthorized"} +{% endif %} {% else %} principal = None {% endif %} @@ -75,7 +77,8 @@ async def handle_custom_{{ loop.index }}( try: access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] except AccessDenied as exc: - return 403, {"detail": exc.reason} + status, detail = exc.refusal(principal) + return status, {"detail": detail} bound_inputs = resolve_access_bind(access_bindings, locals()) {% else %} bound_inputs = {} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja b/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja index b8079569..f6919a03 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/requirements.txt.jinja @@ -1,5 +1,6 @@ pymongo pyjwt{{ "[crypto]" if ctx.session.uses_asymmetric else "" }} +uuid_utils uvicorn pydantic-settings {% for req in ctx.vendored_requirements %} diff --git a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja index b2eececa..af9798b4 100644 --- a/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja +++ b/src/yaml_api_generator/generators/backend/raw/templates/routes.py.jinja @@ -21,7 +21,7 @@ from pagination import build_response, map_request from dependencies.auth import resolve_principal {% endif %} import validation -from access import EVERY_ROW, AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, negate, one_of, row_filter, value_at +from access import EVERY_ROW, AccessDenied, Decision, Field, all_of, conjoin, constant, decide, disjoin, equals, has_path, is_null, matching, negate, one_of, row_filter, value_at ROLE_HIERARCHIES = {{ auth.role_hierarchies | role_hierarchies_py }} @@ -89,7 +89,8 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: access_bindings = {{ route.access_rules | map(attribute="bind") | list | py_literal }}[decide(decisions)] {% endif %} except AccessDenied as exc: - return 403, {"detail": exc.reason} + status, detail = exc.refusal(principal) + return status, {"detail": detail} {% elif rows %} rows = EVERY_ROW {% endif %} @@ -98,6 +99,9 @@ def _principal_has_role(principal: dict, required_role: str) -> bool: {% macro pre_fetch_authorize(route) %} {% if route.operation == "list" %} {{ authorize_rules(route, "Field(())", rows=true) }} +{% if route.record_parent_params %} + rows = conjoin(rows, matching({{ record_criteria(route) }})) +{% endif %} {% elif not (route is needs_post_fetch) %} {{ authorize_rules(route, "None") }} {% elif route.access_rules %} @@ -126,9 +130,11 @@ async def {{ route.handler_name }}(url_params: dict, body: dict | None, headers: principal = None {% if route.needs_auth %} principal = await resolve_principal(headers, db) +{% if not route.admits_anonymous %} if principal is None: return 401, {"detail": "Unauthorized"} {% endif %} +{% endif %} {{ pre_fetch_authorize(route) }} {{ bind_inputs(route) }} service_inputs = prepare_service_inputs(url_params, body, {{ route.service_input | py_literal }}, {{ route.service_forwarding | py_literal }}, bound_inputs) @@ -177,6 +183,9 @@ async def {{ route.handler_name }}(url_params: dict, body: dict | None, headers: if body is None: return 422, {"detail": "Request body is required"} data = validation.{{ route.typed_body_model or route.body_model }}.conform(body) +{% if route.record_parent_params %} + data = {**data, **{{ record_criteria(route) }}} +{% endif %} {% if identity %} data = {{ identity.role_store | py_literal }}.granted(data, assignable={{ route.needs_auth | py_literal }}) {% endif %} diff --git a/src/yaml_api_generator/ir/api.py b/src/yaml_api_generator/ir/api.py index 17de2b31..0533204a 100644 --- a/src/yaml_api_generator/ir/api.py +++ b/src/yaml_api_generator/ir/api.py @@ -19,6 +19,7 @@ eliminating repeated YAML parsing and validation. """ +import re from collections.abc import Collection, Iterable from dataclasses import asdict, dataclass, field from pathlib import Path @@ -28,6 +29,8 @@ from .when import WhenExpr from ..constants import CRUD_OPERATIONS +_PATH_PARAM = re.compile(r"\{(\w+)\}") + @dataclass(frozen=True) class IROIDCProvider: @@ -247,6 +250,10 @@ class IRCollection: files: dict[str, IRFileField] = field(default_factory=dict) file_store: str | None = None + def scope(self, params: Iterable[str], record_param: str | None) -> tuple[str, ...]: + """The path parameters, besides the record's own, that name fields of this collection.""" + return tuple(param for param in params if param != record_param and param in self.fields) + SECRET_FIELD: dict[str, Any] = {"type": "string", "required": True, "minLength": 1} """The config of a credential a request sends, such as a password.""" @@ -334,8 +341,25 @@ class IRVendoredImport: requires: tuple[str, ...] = () +AUTHENTICATED = "*" +"""The principal of a rule that any authenticated caller satisfies.""" + + +class CallerDemand: + """What an access rule asks of the caller: ``principal`` names the identity collection + (``identity:``), ``AUTHENTICATED`` or nothing; ``required_role`` a role it holds.""" + + principal: str + required_role: str | None + + @property + def admits_anonymous(self) -> bool: + """Whether the rule can hold for a caller who presents no credentials.""" + return not self.principal and self.required_role is None + + @dataclass(frozen=True) -class IRAccessRule: +class IRAccessRule(CallerDemand): principal: str required_role: str | None = None identity_symbols: dict[str, IRIdentityBinding] = field(default_factory=dict) @@ -379,6 +403,24 @@ class IRRoute: access_rules: list[IRAccessRule] | None = None handler: str | None = None pagination: IRPagination | None = None + # The path parameter carrying the id of the record the route acts on. + record_param: str | None = None + # Path parameters naming fields of the collection: the route reaches only rows holding their values. + scope: tuple[str, ...] = () + + @property + def path_params(self) -> list[str]: + return _PATH_PARAM.findall(self.path) + + @property + def admits_anonymous(self) -> bool: + """Whether an access rule of the route can admit a caller who presents no credentials.""" + return any(rule.admits_anonymous for rule in self.access_rules or ()) + + @property + def serves_crud(self) -> bool: + """Whether a repository CRUD operation serves the route, fetching its own record.""" + return self.handler is None and self.collection is not None and self.operation in CRUD_OPERATIONS def body_dto(self, dtos: dict[str, IRDto]) -> IRDto | None: """The DTO that the first DTO-typed request-body binding gives this route's body.""" diff --git a/src/yaml_api_generator/parsers/transformers/access_validation.py b/src/yaml_api_generator/parsers/transformers/access_validation.py index 852370d9..a74b57c2 100644 --- a/src/yaml_api_generator/parsers/transformers/access_validation.py +++ b/src/yaml_api_generator/parsers/transformers/access_validation.py @@ -19,8 +19,12 @@ from typing import Any -from yaml_api_generator.ir.when import collect_roots, needs_document -from yaml_api_generator.parsers.when_parser import parse_request_binding, parse_when_list +from yaml_api_generator.ir.when import collect_roots +from yaml_api_generator.parsers.when_parser import ( + parse_document_binding, + parse_request_binding, + parse_when_list, +) from yaml_api_generator.parsers.where_parser import is_where_binding from yaml_api_generator.parsers.yaml_parser import ConfigError @@ -44,20 +48,21 @@ def _is_request_value(value: Any) -> bool: def _is_current_document_value(value: Any, version: str) -> bool: - """Whether a ``given`` value binds the route's own record. + """Whether a ``given`` value binds the route's own record: a bare collection name. - A v0.3 ``where`` binding fetches a different record by predicate before the - rule runs, so it resolves on a create just as it does anywhere else. Only - the implicit ``{id}`` document binding depends on a record the create - operation has not produced yet. + `` by `` and a v0.3 ``where`` binding fetch a different + record before the rule runs, so they resolve on a create just as they do + anywhere else. """ if ( not isinstance(value, str) or value.startswith("identity:") or _is_request_value(value) + or (version == "0.3" and is_where_binding(value)) ): return False - return not (version == "0.3" and is_where_binding(value)) + binding, _ = parse_document_binding(value, "") + return binding is not None and binding[1] is None def _invalid(message: str, path: str, hint: str) -> ConfigError: @@ -175,7 +180,7 @@ def _check_when( hint=f"Bind '{root}' in this rule's 'given'", )) - if is_create and (needs_document(expr) or rule.get("document_symbols")): + if is_create and rule.get("document_symbols"): errors.append(_invalid( "document-dependent when on a create rule is short-circuited to allow", f"{path}.when", diff --git a/src/yaml_api_generator/parsers/transformers/v01/__init__.py b/src/yaml_api_generator/parsers/transformers/v01/__init__.py index 92966ac6..4ceb460f 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v01/__init__.py @@ -15,6 +15,7 @@ from yaml_api_generator.parsers.yaml_parser import ConfigErrors from ..v02.datastore_transformer import parse_datastores +from ..v02.record_location import id_param, locate_records from .collection_transformer import CollectionTransformer from .identity_transformer import IdentityTransformer from .route_transformer import RouteTransformer @@ -103,7 +104,7 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: collections=collections, repositories=repositories, services=services, - routes=routes, + routes=locate_records(routes, collections, id_param), identityCollections=identity_collections, mongo=datastores[default_datastore], datastores=datastores, diff --git a/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py b/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py index d35956c9..f4b78f20 100644 --- a/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py +++ b/src/yaml_api_generator/parsers/transformers/v01/route_transformer.py @@ -5,6 +5,7 @@ from typing import Any +from yaml_api_generator.ir.api import AUTHENTICATED from yaml_api_generator.ir import IRAccessRule, IRIdentityCollection, IROperations, IRRoute from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors from yaml_api_generator.parsers.when_parser import parse_when_expr @@ -179,7 +180,7 @@ def _parse_access_items( continue rules.append( IRAccessRule( - principal=resolved.principal or "", + principal=resolved.principal or AUTHENTICATED, required_role=resolved.required_role, when=when_expr, **extras, diff --git a/src/yaml_api_generator/parsers/transformers/v02/__init__.py b/src/yaml_api_generator/parsers/transformers/v02/__init__.py index ba282c1f..89ed75a5 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v02/__init__.py @@ -19,6 +19,7 @@ from .identity_transformer import IdentityTransformer from .policies_transformer import PoliciesTransformer from .policy_expansion import expand_rule_policies +from .record_location import id_param, locate_records from .repository_transformer import RepositoryTransformer from .route_transformer import RouteTransformer @@ -26,6 +27,8 @@ class V02Transformer: """Orchestrates v0.2 YAML to IR transformation.""" + _record_param = staticmethod(id_param) + def __init__(self): self._repository = RepositoryTransformer() self._collection = CollectionTransformer() @@ -129,7 +132,11 @@ def transform(self, resolved: dict[str, Any]) -> IRDocument: raise ConfigErrors(policy_errors) routes = desugar_document_lookups( - self._expand_route_policies(routes, policies, entity_to_collection), + locate_records( + self._expand_route_policies(routes, policies, entity_to_collection), + collections, + self._record_param, + ), collections, ) file_routes, file_route_errors = self._files.derive(collections, routes) diff --git a/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py b/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py index 97a97e5b..797213aa 100644 --- a/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py +++ b/src/yaml_api_generator/parsers/transformers/v02/document_lookup.py @@ -1,7 +1,11 @@ -"""Desugar `` by `` document bindings into where-bindings. +"""Desugar document bindings that name a record by route parameters into where-bindings. ``owner: users by ownerId`` loads the ``users`` record whose id field equals the ``ownerId`` route parameter, which is exactly ``users where = $ownerId``. + +A bare ``document: posts`` binds the route's own record. A CRUD route fetches that +record itself; any other route loads it through its record parameter, within the +scope its other path parameters set. """ from dataclasses import replace @@ -17,29 +21,44 @@ from yaml_api_generator.parsers.yaml_parser import ConfigError, ConfigErrors +def _invalid(message: str, path: str) -> ConfigErrors: + return ConfigErrors([ConfigError(code="DOCUMENT_BINDING_INVALID", message=message, path=path)]) + + +def _equal_to_params(fields_to_params: dict[str, str]) -> tuple[IRWhereConjunct, ...]: + return tuple( + IRWhereConjunct(field=field, rhs=IRSymbolRef(root=param)) + for field, param in fields_to_params.items() + ) + + +def _route_record(collection: IRCollection, route: IRRoute, path: str) -> dict[str, str]: + if route.record_param is None: + raise _invalid(f"'{collection.name}' binds the route's record, but the route path has no record parameter", path) + scope = collection.scope(route.path_params, route.record_param) + return {**{param: param for param in scope}, collection.idField: route.record_param} + + def _where_binding( - binding: IRDocumentBinding, collections: dict[str, IRCollection], path: str + binding: IRDocumentBinding, collections: dict[str, IRCollection], route: IRRoute, path: str ) -> IRDocumentBinding: - if binding.id_param is None: + if binding.where or (binding.id_param is None and route.serves_crud): return binding collection = collections.get(binding.collection) if collection is None: - raise ConfigErrors([ConfigError( - code="DOCUMENT_BINDING_INVALID", - message=f"document binding references unknown collection '{binding.collection}'", - path=path, - )]) - return IRDocumentBinding( - collection=binding.collection, - where=(IRWhereConjunct(field=collection.idField, rhs=IRSymbolRef(root=binding.id_param)),), + raise _invalid(f"document binding references unknown collection '{binding.collection}'", path) + lookup = ( + {collection.idField: binding.id_param} if binding.id_param is not None + else _route_record(collection, route, path) ) + return IRDocumentBinding(collection=binding.collection, where=_equal_to_params(lookup)) def _rule_lookups( - rule: IRAccessRule, collections: dict[str, IRCollection], path: str + rule: IRAccessRule, collections: dict[str, IRCollection], route: IRRoute, path: str ) -> IRAccessRule: return replace(rule, document_symbols={ - name: _where_binding(binding, collections, f"{path}.given.{name}") + name: _where_binding(binding, collections, route, f"{path}.given.{name}") for name, binding in rule.document_symbols.items() }) @@ -49,7 +68,7 @@ def desugar_document_lookups( ) -> list[IRRoute]: return [ replace(route, access_rules=[ - _rule_lookups(rule, collections, f"{route.method} {route.path} access[{index}]") + _rule_lookups(rule, collections, route, f"{route.method} {route.path} access[{index}]") for index, rule in enumerate(route.access_rules) ]) if route.access_rules else route for route in routes diff --git a/src/yaml_api_generator/parsers/transformers/v02/record_location.py b/src/yaml_api_generator/parsers/transformers/v02/record_location.py new file mode 100644 index 00000000..133e37b4 --- /dev/null +++ b/src/yaml_api_generator/parsers/transformers/v02/record_location.py @@ -0,0 +1,37 @@ +"""Locate the record each route acts on from its path parameters. + +A route acts on one record unless it lists or creates rows. The record +parameter carries that record's id; every other path parameter naming a field +of the route's collection scopes the route to the rows holding its value. +""" + +from collections.abc import Callable +from dataclasses import replace + +from yaml_api_generator.ir import IRCollection, IRRoute + +RecordParam = Callable[[list[str]], str | None] + + +def id_param(params: list[str]) -> str | None: + return "id" if "id" in params else None + + +def last_param(params: list[str]) -> str | None: + return params[-1] if params else None + + +def _acts_on_a_record(route: IRRoute) -> bool: + return not (route.serves_crud and route.operation in ("list", "create")) + + +def _located(route: IRRoute, collection: IRCollection | None, record_param: RecordParam) -> IRRoute: + own = record_param(route.path_params) if _acts_on_a_record(route) else None + scope = collection.scope(route.path_params, own) if collection is not None else () + return replace(route, record_param=own, scope=scope) + + +def locate_records( + routes: list[IRRoute], collections: dict[str, IRCollection], record_param: RecordParam +) -> list[IRRoute]: + return [_located(route, collections.get(route.collection), record_param) for route in routes] diff --git a/src/yaml_api_generator/parsers/transformers/v03/__init__.py b/src/yaml_api_generator/parsers/transformers/v03/__init__.py index 8a0c7397..53ee28cc 100644 --- a/src/yaml_api_generator/parsers/transformers/v03/__init__.py +++ b/src/yaml_api_generator/parsers/transformers/v03/__init__.py @@ -14,6 +14,7 @@ from ..v02 import V02Transformer from ..v02.access_transformer import AccessTransformer +from ..v02.record_location import last_param from ..v02.route_transformer import RouteTransformer from .pagination import parse_pagination from .vendored_imports import process_vendored_imports @@ -22,6 +23,8 @@ class V03Transformer(V02Transformer): """Orchestrates v0.3 YAML to IR transformation.""" + _record_param = staticmethod(last_param) + def __init__(self, spec_dir: Path) -> None: super().__init__() self._spec_dir = spec_dir diff --git a/src/yaml_api_generator/runtime/access.py b/src/yaml_api_generator/runtime/access.py index 1e61e3ac..8110b605 100644 --- a/src/yaml_api_generator/runtime/access.py +++ b/src/yaml_api_generator/runtime/access.py @@ -247,6 +247,11 @@ def all_of(*predicates: Predicate) -> Predicate: return reduce(conjoin, predicates, EVERY_ROW) +def matching(criteria: Mapping[str, Any]) -> Predicate: + """The rows whose top-level fields hold the values ``criteria`` maps them to.""" + return all_of(*(FieldEquals(Field((name,)), value) for name, value in criteria.items())) + + def negate(predicate: Predicate) -> Predicate: if isinstance(predicate, Constant): return constant(not predicate.value) @@ -276,6 +281,17 @@ def __init__(self, reason: str) -> None: super().__init__(reason) self.reason = reason + def refusal(self, principal: Any) -> tuple[int, str]: + """The status and detail refusing the caller ``principal`` names.""" + return 403, self.reason + + +class Unadmitted(AccessDenied): + """No rule admits the caller, whom authenticating may yet admit.""" + + def refusal(self, principal: Any) -> tuple[int, str]: + return (401, "Unauthorized") if principal is None else super().refusal(principal) + @dataclass(frozen=True) class Decision: @@ -310,7 +326,7 @@ def decide(decisions: list[Decision], holds: Callable[[Predicate], bool] = parti (None, None), ) if decision is None: - raise AccessDenied(refusal_reason(decisions)) + raise Unadmitted(refusal_reason(decisions)) if decision.effect == "deny": raise AccessDenied(decision.reason or "Forbidden") return index diff --git a/src/yaml_api_generator/server/common/context_builder.py b/src/yaml_api_generator/server/common/context_builder.py index f383793a..2d361572 100644 --- a/src/yaml_api_generator/server/common/context_builder.py +++ b/src/yaml_api_generator/server/common/context_builder.py @@ -166,6 +166,8 @@ def _runtime_route(route: IRRoute, document: IRDocument) -> RuntimeRouteDescript service_forwarding=service_forwarding, pagination=route.pagination, body_dto=route.body_dto(document.dtos), + record_param=route.record_param, + scope=route.scope, ) diff --git a/src/yaml_api_generator/server/common/pipeline_builder.py b/src/yaml_api_generator/server/common/pipeline_builder.py index 9185bbe2..88d1a16d 100644 --- a/src/yaml_api_generator/server/common/pipeline_builder.py +++ b/src/yaml_api_generator/server/common/pipeline_builder.py @@ -38,7 +38,7 @@ replace_file, ) from .steps.authorization import Admission, admit_record, admit_rows, screen_record -from .steps.repositories import RecordLocator, record_locator +from .steps.repositories import RecordLocator def _document_bindings(route: RuntimeRouteDescriptor) -> Iterator[IRDocumentBinding]: @@ -282,11 +282,9 @@ def _post_fetch_authorize_step(self, route: RuntimeRouteDescriptor) -> Step | No return None return PostFetchAuthorizeStep(route.access_rules, self._ctx.auth) - def _locator(self, route: RuntimeRouteDescriptor): - if getattr(self._ctx, "spec_version", "0.2") != "0.3": - return RecordLocator(self._ctx.collections[route.collection].id_field) - return record_locator( - route.path, self._ctx.collections[route.collection].id_field + def _locator(self, route: RuntimeRouteDescriptor) -> RecordLocator: + return RecordLocator( + self._ctx.collections[route.collection].id_field, route.record_param, route.scope ) def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: @@ -297,12 +295,17 @@ def _repository_step(self, route: RuntimeRouteDescriptor) -> Step: identity = self._ctx.auth.identity_collections.get(collection_name) if operation == "list": - return RepositoryListStep(collection_name, id_field, route.pagination) + return RepositoryListStep(collection_name, id_field, route.pagination, self._locator(route)) if operation == "get": return RepositoryGetStep(collection_name, id_field, self._locator(route)) if operation == "create": return RepositoryCreateStep( - collection_name, id_field, collection.id_format, identity, assignable=not route.is_public + collection_name, + id_field, + collection.id_format, + identity, + assignable=not route.is_public, + locator=self._locator(route), ) if operation == "update": return RepositoryUpdateStep( diff --git a/src/yaml_api_generator/server/common/server_context.py b/src/yaml_api_generator/server/common/server_context.py index f48e7f8b..0ec9db90 100644 --- a/src/yaml_api_generator/server/common/server_context.py +++ b/src/yaml_api_generator/server/common/server_context.py @@ -10,7 +10,7 @@ IRRequestBinding, WhereRhs, ) -from yaml_api_generator.ir.api import IRDto, IRPagination, IRRolesConfig +from yaml_api_generator.ir.api import CallerDemand, IRDto, IRPagination, IRRolesConfig from yaml_api_generator.ir.when import WhenExpr from yaml_api_generator.runtime.access import Roles @@ -59,7 +59,7 @@ def get_binding(self, provider: str) -> "IRProviderBinding | None": @dataclass(frozen=True) -class RuntimeAccessRule: +class RuntimeAccessRule(CallerDemand): principal: str required_role: str | None = None identity_symbols: dict[str, IRIdentityBinding] = field(default_factory=dict) @@ -84,6 +84,8 @@ class RuntimeRouteDescriptor: service_forwarding: dict[str, Any] = field(default_factory=dict) pagination: IRPagination | None = None body_dto: IRDto | None = None + record_param: str | None = None + scope: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/src/yaml_api_generator/server/common/steps/authorization.py b/src/yaml_api_generator/server/common/steps/authorization.py index f5f54b54..8d598108 100644 --- a/src/yaml_api_generator/server/common/steps/authorization.py +++ b/src/yaml_api_generator/server/common/steps/authorization.py @@ -22,6 +22,7 @@ Decision, Field, Predicate, + Unadmitted, decide, row_filter, ) @@ -108,7 +109,7 @@ def _rule_matches_principal( auth_config: RuntimeAuthConfig | None, ) -> bool: if principal is None: - return False + return rule.admits_anonymous if rule.principal and rule.principal.startswith("identity:"): expected_collection = rule.principal[len("identity:"):] @@ -264,10 +265,10 @@ async def _where_documents(rule: RuntimeAccessRule, ctx: RequestContext) -> Wher def ResolveWhereBindingsStep(access_rules: list[RuntimeAccessRule]) -> Step: async def step(ctx: RequestContext) -> RequestContext: - if ctx.principal is None: - return ctx - - ctx.where_documents = [await _where_documents(rule, ctx) for rule in access_rules] + ctx.where_documents = [ + await _where_documents(rule, ctx) if ctx.principal is not None or rule.admits_anonymous else {} + for rule in access_rules + ] return ctx return step @@ -294,6 +295,16 @@ def step(ctx: RequestContext) -> RequestContext: Admission = Callable[[RequestContext, list[Decision]], None] +def _admitting(admission: Admission, ctx: RequestContext, decisions: list[Decision]) -> None: + """Apply ``admission``, refusing an anonymous caller as one who may yet authenticate.""" + try: + admission(ctx, decisions) + except Unadmitted as exc: + if ctx.principal is None: + raise UnauthorizedError("Unauthorized") from exc + raise + + def admit_record(ctx: RequestContext, decisions: list[Decision]) -> None: """Routes without a current record: the first rule that holds decides.""" ctx.authorized_rule_index = decide(decisions) @@ -314,11 +325,13 @@ def AuthorizeRouteStep( auth_config: RuntimeAuthConfig | None = None, admission: Admission = admit_record, ) -> Step: + admits_anonymous = any(rule.admits_anonymous for rule in access_rules or ()) + def step(ctx: RequestContext) -> RequestContext: - if ctx.principal is None: + if ctx.principal is None and not admits_anonymous: raise UnauthorizedError("Unauthorized") if access_rules: - admission(ctx, _decisions(access_rules, RECORD, ctx, auth_config)) + _admitting(admission, ctx, _decisions(access_rules, RECORD, ctx, auth_config)) return ctx return step @@ -330,7 +343,7 @@ def PostFetchAuthorizeStep( ) -> Step: def step(ctx: RequestContext) -> RequestContext: record = ctx.response if isinstance(ctx.response, dict) else RECORD - admit_record(ctx, _decisions(access_rules, record, ctx, auth_config)) + _admitting(admit_record, ctx, _decisions(access_rules, record, ctx, auth_config)) return ctx return step diff --git a/src/yaml_api_generator/server/common/steps/repositories.py b/src/yaml_api_generator/server/common/steps/repositories.py index 81714da3..3bc3e83d 100644 --- a/src/yaml_api_generator/server/common/steps/repositories.py +++ b/src/yaml_api_generator/server/common/steps/repositories.py @@ -1,4 +1,3 @@ -import re from dataclasses import dataclass from functools import partial from typing import Any, cast @@ -7,13 +6,13 @@ from ..request_context import RequestContext from ..server_context import RuntimeIdentityDescriptor from yaml_api_generator.ir.api import IRPagination +from yaml_api_generator.runtime.access import conjoin, matching from ..pagination import Page, build_response, map_request from ..repositories.base import BaseRepository from ..utils.ids import generate_id as _generate_id from . import NotFoundError, Step -_PATH_TOKEN = re.compile(r"\{(\w+)\}") _REQUEST_BODY_MESSAGE = "Request body must be a JSON object" _NOT_FOUND_MESSAGE = "Resource not found" @@ -31,39 +30,28 @@ def _require_param(url_params: dict[str, str], token: str) -> str: @dataclass(frozen=True) class RecordLocator: - """Locates a single record from a route's path tokens. + """Locates the rows a route reaches from its path parameters. - The final path token identifies the record by its id field; each preceding - token constrains a same-named field, scoping the record to its parents. + The record parameter names the record by its id field; each scope parameter + constrains the same-named field. """ id_field: str - id_token: str = "id" - parents: tuple[tuple[str, str], ...] = () + record_param: str | None = "id" + scope: tuple[str, ...] = () @property def is_scoped(self) -> bool: - return bool(self.parents) + return bool(self.scope) + + def scope_criteria(self, url_params: dict[str, str]) -> dict[str, str]: + return {name: _require_param(url_params, name) for name in self.scope} def document_id(self, url_params: dict[str, str]) -> str: - return _require_param(url_params, self.id_token) + return _require_param(url_params, self.record_param) def criteria(self, url_params: dict[str, str]) -> dict[str, Any]: - constraints = { - field_name: _require_param(url_params, token) - for field_name, token in self.parents - } - constraints[self.id_field] = _require_param(url_params, self.id_token) - return constraints - - -def record_locator(path: str, id_field: str) -> RecordLocator: - *parents, id_token = _PATH_TOKEN.findall(path) or ["id"] - return RecordLocator( - id_field=id_field, - id_token=id_token, - parents=tuple((token, token) for token in parents), - ) + return {**self.scope_criteria(url_params), self.id_field: self.document_id(url_params)} async def _fetch_record( @@ -131,15 +119,19 @@ def RepositoryListStep( collection_name: str, id_field: str, pagination: IRPagination | None = None, + locator: RecordLocator | None = None, ) -> Step: + locator = locator or RecordLocator(id_field, None) + async def step(ctx: RequestContext) -> RequestContext: + rows = conjoin(ctx.row_filter, matching(locator.scope_criteria(ctx.url_params))) if pagination is None: - result = await _repository(ctx).list(collection_name, ctx.row_filter) + result = await _repository(ctx).list(collection_name, rows) ctx.response = _inject_canonical_id_list(result, id_field) return ctx query = ctx.query or {} primitives = map_request(pagination, query) - page = await _repository(ctx).paginate(collection_name, primitives, ctx.row_filter) + page = await _repository(ctx).paginate(collection_name, primitives, rows) canonical = Page( rows=_inject_canonical_id_list(page.rows, id_field), total=page.total, @@ -171,12 +163,16 @@ def RepositoryCreateStep( id_format: str, identity: RuntimeIdentityDescriptor | None = None, assignable: bool = False, + locator: RecordLocator | None = None, ) -> Step: - """A new identity record keeps the roles it names only when they are ``assignable``.""" + """A new identity record keeps the roles it names only when they are ``assignable``; + a new record holds the values of the path parameters scoping the route.""" grant = partial(identity.role_store.granted, assignable=assignable) if identity else dict + locator = locator or RecordLocator(id_field, None) async def step(ctx: RequestContext) -> RequestContext: - payload = _prepare_payload(grant(_payload(ctx)), identity) + scoped = {**_payload(ctx), **locator.scope_criteria(ctx.url_params)} + payload = _prepare_payload(grant(scoped), identity) if id_format != "natural": payload[id_field] = _generate_id(id_format) result = await _repository(ctx).create(collection_name, payload) diff --git a/tests/integration/test_mongo_access_http.py b/tests/integration/test_mongo_access_http.py index 677f67f8..2bf4eb70 100644 --- a/tests/integration/test_mongo_access_http.py +++ b/tests/integration/test_mongo_access_http.py @@ -1,19 +1,21 @@ -"""The access HTTP suite, served by the generated MongoDB backends: raw ASGI and DRF.""" +"""The access HTTP suite, served on MongoDB by the dev server and every generated backend.""" +import itertools from uuid import uuid4 import pytest -from tests.unit.support.live_apps import MONGO_SERVERS, mongo, serve, write_spec +from tests.unit.support.live_apps import MONGO_SERVERS, mongo from tests.unit.test_access_http import * # noqa: F403 -from tests.unit.test_access_http import _SPEC +from tests.unit.test_access_http import VERSIONS, case_id, serving pytestmark = pytest.mark.integration -@pytest.fixture(scope="module", params=MONGO_SERVERS, ids=lambda server: server.name) +@pytest.fixture(scope="module", params=list(itertools.product(MONGO_SERVERS, VERSIONS)), ids=case_id) def client(request, tmp_path_factory, mongo_uri): - server = request.param - spec = write_spec(tmp_path_factory.mktemp(server.name), _SPEC, mongo(mongo_uri, f"{server.name}_{uuid4().hex}")) - with serve(server, spec) as client: + server, version = request.param + name = case_id(request.param) + datastore = mongo(mongo_uri, f"{name.replace('.', '')}_{uuid4().hex}") + with serving(server, version, tmp_path_factory.mktemp(name), datastore) as client: yield client diff --git a/tests/integration/test_mongo_identity_http.py b/tests/integration/test_mongo_identity_http.py index 56c8a59e..5ab99697 100644 --- a/tests/integration/test_mongo_identity_http.py +++ b/tests/integration/test_mongo_identity_http.py @@ -1,4 +1,4 @@ -"""The identity HTTP suite, served by the generated MongoDB backends: raw ASGI and DRF.""" +"""The identity HTTP suite, served on MongoDB by the dev server and every generated backend.""" from uuid import uuid4 diff --git a/tests/unit/support/live_apps.py b/tests/unit/support/live_apps.py index 98e0bd38..9a9eb2ea 100644 --- a/tests/unit/support/live_apps.py +++ b/tests/unit/support/live_apps.py @@ -51,7 +51,7 @@ def process(self, port: int) -> list[str]: class DjangoApp(GeneratedApp): - """The generated DRF backend under gunicorn; its tests skip where Django REST framework is not installed.""" + """The generated DRF backend under uvicorn, run by the interpreter YAML_API_TEST_DRF_PYTHON names; skipped without one.""" environment = {"DJANGO_SECRET_KEY": "test", "JWT_SECRET": "test"} @@ -59,19 +59,14 @@ def __init__(self) -> None: super().__init__("drf") def process(self, port): - for module in ("rest_framework", "gunicorn"): - pytest.importorskip(module) - return [sys.executable, "-m", "gunicorn", "--bind", f"127.0.0.1:{port}", "project.wsgi:application"] + python = os.environ.get("YAML_API_TEST_DRF_PYTHON") + if python is None: + pytest.skip("YAML_API_TEST_DRF_PYTHON is not set") + return [python, "-m", "uvicorn", "project.asgi:application", "--port", str(port)] SERVERS = (DevServer(), GeneratedApp("fastapi")) -MONGO_SERVERS = ( - GeneratedApp("raw"), - pytest.param( - DjangoApp(), - marks=pytest.mark.xfail(reason="generated DRF views define async handlers on the synchronous APIView", strict=True), - ), -) +MONGO_SERVERS = (*SERVERS, GeneratedApp("raw"), DjangoApp()) def sqlite(directory: Path) -> str: diff --git a/tests/unit/test_access_http.py b/tests/unit/test_access_http.py index 55e32468..35e2cff7 100644 --- a/tests/unit/test_access_http.py +++ b/tests/unit/test_access_http.py @@ -1,11 +1,14 @@ -"""Access rules over real HTTP, in the dev server and the generated FastAPI app.""" +"""Access rules over real HTTP, in the dev server and the generated FastAPI app, for every spec version that declares them.""" + +import itertools +from contextlib import contextmanager import pytest from tests.unit.support.live_apps import SERVERS, serve, write_spec _SPEC = """ -version: "0.3" +version: "{version}" api: basePath: /api operations: @@ -24,12 +27,27 @@ title: { type: string } ownerId: { type: string } archivedAt: { type: string } + comment: + fields: + id: { type: id } + postId: { type: string } + text: { type: string } + draft: + fields: + id: { type: id } + title: { type: string } + ownerId: { type: string } repositories: users: { entity: user } posts: { entity: post } + comments: { entity: comment } + drafts: { entity: draft } services: users: { repository: users } posts: { repository: posts } + comments: { repository: comments } + drafts: { repository: drafts } + publish: { handler: handlers.publish } controllers: users: service: users @@ -81,23 +99,105 @@ access: - given: { principal: "identity:user", owner: users by ownerId } when: owner.id equals principal.sub + publish: + base_path: /publish + routes: + run: + service: publish + method: POST + path: /{id} + access: + - given: { principal: "identity:user", document: posts } + when: document.ownerId equals principal.sub + comments: + base_path: /posts + routes: + create: + service: comments + method: POST + path: /{postId}/comments + access: + - given: { principal: "identity:user", parent: posts by postId } + when: parent.ownerId equals principal.sub + list: + service: comments + method: GET + path: /{postId}/comments + access: + - given: { principal: "identity:user", parent: posts by postId } + when: parent.ownerId equals principal.sub + get: + service: comments + method: GET + path: /{postId}/comments/{id} + access: + - given: { principal: "identity:user", parent: posts by postId } + when: parent.ownerId equals principal.sub + delete: + service: comments + method: DELETE + path: /{postId}/comments/{id} + access: + - given: { principal: "identity:user", parent: posts by postId } + when: parent.ownerId equals principal.sub + drafts: + base_path: /drafts + routes: + create: + service: drafts + method: POST + path: "" + access: + - given: { body: request.body } + when: body.ownerId is not null + then: deny + reason: Drafts start without an owner. + - then: allow """ +# The dev server hands a handler its request context; a generated backend hands it the request +_HANDLERS = ''' +async def publish(target): + if hasattr(target, "response"): + target.response = {"published": True} + return target + return {"published": True} +''' -@pytest.fixture(scope="module", params=SERVERS, ids=lambda server: server.name) +VERSIONS = ("0.2", "0.3") +CASES = list(itertools.product(SERVERS, VERSIONS)) + + +def case_id(case): + server, version = case + return f"{server.name}-{version}" + + +@contextmanager +def serving(server, version, directory, datastore=None): + """``server`` serving the spec at ``version``, its handlers module beside it.""" + (directory / "handlers.py").write_text(_HANDLERS) + spec = write_spec(directory, _SPEC.replace("{version}", version), datastore) + with serve(server, spec, env={"PYTHONPATH": str(directory)}) as client: + yield client + + +@pytest.fixture(scope="module", params=CASES, ids=case_id) def client(request, tmp_path_factory): - with serve(request.param, write_spec(tmp_path_factory.mktemp(request.param.name), _SPEC)) as client: + server, version = request.param + with serving(server, version, tmp_path_factory.mktemp(case_id(request.param))) as client: yield client @pytest.fixture(scope="module") def accounts(client): - """Two users, each owning one post, as ``{email: (basic auth, user id, post)}``.""" + """Two users, each owning one post with one comment, as ``{email: (basic auth, user id, post, comment)}``.""" def account(email): user = client.post("/api/users/register", json={"email": email, "password": "pw"}).json() basic = (email, "pw") post = client.post("/api/posts", json={"title": email, "ownerId": user["id"]}, auth=basic).json() - return basic, user["id"], post + comment = client.post(f"/api/posts/{post['id']}/comments", json={"text": email}, auth=basic).json() + return basic, user["id"], post, comment return {email: account(email) for email in ("ann@x.io", "bob@x.io")} @@ -113,7 +213,7 @@ def _get(client, accounts, path, **fields): def test_list_returns_only_the_rows_the_rule_admits(client, accounts): - basic, user_id, _ = accounts["ann@x.io"] + basic, user_id, *_ = accounts["ann@x.io"] rows = client.get("/api/posts", auth=basic).json() @@ -121,13 +221,70 @@ def test_list_returns_only_the_rows_the_rule_admits(client, accounts): def test_document_binding_by_parameter_loads_the_named_record(client, accounts): - ann, ann_id, ann_post = accounts["ann@x.io"] - _, bob_id, bob_post = accounts["bob@x.io"] + ann, ann_id, ann_post, _ = accounts["ann@x.io"] + _, bob_id, bob_post, _ = accounts["bob@x.io"] assert client.get(f"/api/owned/{ann_id}/{ann_post['id']}", auth=ann).status_code == 200 assert client.get(f"/api/owned/{bob_id}/{bob_post['id']}", auth=ann).status_code == 403 +def test_path_parameters_naming_fields_scope_the_record(client, accounts): + ann, ann_id, _, _ = accounts["ann@x.io"] + _, _, bob_post, _ = accounts["bob@x.io"] + + assert client.get(f"/api/owned/{ann_id}/{bob_post['id']}", auth=ann).status_code == 404 + + +def test_nested_create_checks_the_parent_and_stores_its_id(client, accounts): + ann, _, ann_post, ann_comment = accounts["ann@x.io"] + _, _, bob_post, _ = accounts["bob@x.io"] + + assert ann_comment["postId"] == ann_post["id"] + assert client.post(f"/api/posts/{bob_post['id']}/comments", json={"text": "x"}, auth=ann).status_code == 403 + + +def test_nested_list_returns_only_the_parent_rows(client, accounts): + ann, _, ann_post, ann_comment = accounts["ann@x.io"] + + rows = client.get(f"/api/posts/{ann_post['id']}/comments", auth=ann).json() + + assert [row["id"] for row in rows] == [ann_comment["id"]] + + +def test_nested_detail_routes_only_reach_the_parent_rows(client, accounts): + ann, _, ann_post, _ = accounts["ann@x.io"] + bob, _, bob_post, bob_comment = accounts["bob@x.io"] + foreign = f"/api/posts/{ann_post['id']}/comments/{bob_comment['id']}" + + assert client.get(foreign, auth=ann).status_code == 404 + assert client.delete(foreign, auth=ann).status_code == 404 + assert client.get(f"/api/posts/{bob_post['id']}/comments/{bob_comment['id']}", auth=bob).status_code == 200 + + +def test_handler_route_binds_the_route_record(client, accounts): + ann, _, ann_post, _ = accounts["ann@x.io"] + _, _, bob_post, _ = accounts["bob@x.io"] + + published = client.post(f"/api/publish/{ann_post['id']}", auth=ann) + + assert (published.status_code, published.json()) == (200, {"published": True}) + assert client.post(f"/api/publish/{bob_post['id']}", auth=ann).status_code == 403 + + +def test_unconditional_allow_admits_anonymous_callers(client, accounts): + _, _, ann_post, _ = accounts["ann@x.io"] + + assert client.get(f"/api/posts/{ann_post['id']}").status_code == 200 + assert client.post("/api/drafts", json={"title": "t"}).status_code == 200 + + +def test_request_is_null_holds_for_absent_body_fields(client): + denied = client.post("/api/drafts", json={"title": "t", "ownerId": "u"}) + + assert client.post("/api/drafts", json={"title": "t"}).status_code == 200 + assert (denied.status_code, denied.json()["detail"]) == (403, "Drafts start without an owner.") + + @pytest.mark.parametrize( ("fields", "status"), [({"archivedAt": None}, 200), ({}, 200), (ARCHIVED, 403)], diff --git a/tests/unit/test_access_reason.py b/tests/unit/test_access_reason.py index 40b57aed..e1224e4f 100644 --- a/tests/unit/test_access_reason.py +++ b/tests/unit/test_access_reason.py @@ -180,7 +180,7 @@ def test_generated_fastapi_surfaces_prefetch_and_postfetch_reasons(tmp_path): from fastapi import FastAPI from fastapi.testclient import TestClient - from dependencies.auth import require_principal + from dependencies.auth import get_optional_principal from dependencies.services import get_posts_service from routes._handlers import router as handlers_router from routes.posts import router @@ -205,7 +205,7 @@ async def principal(): app = FastAPI() app.include_router(router) app.include_router(handlers_router) - app.dependency_overrides[require_principal] = principal + app.dependency_overrides[get_optional_principal] = principal app.dependency_overrides[get_posts_service] = Service client = TestClient(app) diff --git a/tests/unit/test_fastapi_files_codegen.py b/tests/unit/test_fastapi_files_codegen.py index cb36f4d4..340a30ae 100644 --- a/tests/unit/test_fastapi_files_codegen.py +++ b/tests/unit/test_fastapi_files_codegen.py @@ -341,7 +341,7 @@ def test_generated_app_serves_uploaded_files(backend, tmp_path): - given: { principal: "identity:user", document: documents } when: principal.sub not equals document.ownerId then: deny - - then: allow + - given: { principal: "identity:user" } """ _GUARDED_SCRIPT = """ diff --git a/tests/unit/test_free_slug_fetch.py b/tests/unit/test_free_slug_fetch.py index 6928c3b5..2e5f3ec2 100644 --- a/tests/unit/test_free_slug_fetch.py +++ b/tests/unit/test_free_slug_fetch.py @@ -12,6 +12,8 @@ import pytest from tests.unit.support.repositories import InMemoryRepository +from yaml_api_generator.ir.api import IRCollection, IRRoute +from yaml_api_generator.parsers.transformers.v02.record_location import id_param, last_param, locate_records from yaml_api_generator.server.common.repositories import RepositoryRegistry from yaml_api_generator.server.common.repositories.base import BaseRepository from yaml_api_generator.server.common.request_context import RequestContext @@ -29,7 +31,6 @@ RepositoryGetStep, RepositoryReplaceStep, RepositoryUpdateStep, - record_locator, ) from yaml_api_generator.server.fastapi.factory import register_fastapi_routes @@ -60,31 +61,48 @@ def _ctx( # --- token derivation ----------------------------------------------------- -def test_record_locator_v02_id_token_unchanged(): - locator = record_locator("/api/v1/items/{id}", "id") - assert locator == RecordLocator("id", "id", ()) - assert not locator.is_scoped - assert locator.criteria({"id": "i1"}) == {"id": "i1"} +def _located(path: str, record_param, operation: str = "get", fields=("orgId",)) -> IRRoute: + collection = IRCollection( + name="c", entity_name="C", type="data", idField="id", idFormat="uuidv7", + fields=dict.fromkeys(fields, {"type": "string"}), + ) + route = IRRoute(method="GET", path=path, operation=operation, collection="c") + [located] = locate_records([route], {"c": collection}, record_param) + return located + + +def test_v02_routes_name_their_record_by_id(): + route = _located("/api/v1/items/{id}", id_param) + assert (route.record_param, route.scope) == ("id", ()) + + +def test_v02_renamed_token_names_no_record(): + assert _located("/api/v1/categories/{categoryId}", id_param).record_param is None + + +def test_v03_final_token_names_the_record(): + route = _located("/api/v1/categories/{categoryId}", last_param) + assert (route.record_param, route.scope) == ("categoryId", ()) + + +def test_v03_preceding_collection_fields_scope_the_record(): + route = _located("/api/v1/orgs/{orgId}/members/{memberId}", last_param) + assert (route.record_param, route.scope) == ("memberId", ("orgId",)) -def test_record_locator_renamed_single_token_maps_to_id_field(): - locator = record_locator("/api/v1/categories/{categoryId}", "id") - assert locator == RecordLocator("id", "categoryId", ()) - assert not locator.is_scoped - assert locator.criteria({"categoryId": "c1"}) == {"id": "c1"} +def test_list_routes_are_scoped_without_a_record(): + route = _located("/api/v1/orgs/{orgId}/members", last_param, operation="list") + assert (route.record_param, route.scope) == (None, ("orgId",)) -def test_record_locator_multi_token_scopes_by_parents(): - locator = record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") - assert locator == RecordLocator("id", "memberId", (("orgId", "orgId"),)) +def test_record_locator_criteria_join_scope_and_id(): + locator = RecordLocator("id", "memberId", ("orgId",)) assert locator.is_scoped assert locator.criteria({"orgId": "o1", "memberId": "m1"}) == {"orgId": "o1", "id": "m1"} def test_record_locator_honours_custom_id_field(): - locator = record_locator("/api/v1/countries/{code}", "code") - assert locator == RecordLocator("code", "code", ()) - assert locator.criteria({"code": "PT"}) == {"code": "PT"} + assert RecordLocator("code", "code").criteria({"code": "PT"}) == {"code": "PT"} # --- single-record steps: renamed token ----------------------------------- @@ -95,7 +113,7 @@ async def test_get_step_renamed_token_fetches_record(): repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "Books"}]}) repo.register_id_field("assetCategories", "id") step = RepositoryGetStep( - "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", RecordLocator("id", "categoryId") ) result = await step(_ctx(repo, {"categoryId": "c1"}, collection="assetCategories")) assert result.response["title"] == "Books" @@ -106,7 +124,7 @@ async def test_delete_step_renamed_token_removes_record(): repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "Books"}]}) repo.register_id_field("assetCategories", "id") step = RepositoryDeleteStep( - "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", RecordLocator("id", "categoryId") ) result = await step( _ctx(repo, {"categoryId": "c1"}, collection="assetCategories", operation="delete") @@ -128,7 +146,7 @@ async def test_get_step_multi_token_fetches_by_all_tokens(): ) repo.register_id_field("members", "id") step = RepositoryGetStep( - "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", RecordLocator("id", "memberId", ("orgId",)) ) result = await step( _ctx(repo, {"orgId": "o1", "memberId": "m1"}, collection="members") @@ -141,7 +159,7 @@ async def test_get_step_multi_token_wrong_parent_is_404(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "Ada"}]}) repo.register_id_field("members", "id") step = RepositoryGetStep( - "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", RecordLocator("id", "memberId", ("orgId",)) ) with pytest.raises(NotFoundError): await step(_ctx(repo, {"orgId": "o2", "memberId": "m1"}, collection="members")) @@ -152,7 +170,7 @@ async def test_update_step_multi_token_scoped_mutates_matching_record(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryUpdateStep( - "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", RecordLocator("id", "memberId", ("orgId",)) ) result = await step( _ctx( @@ -171,7 +189,7 @@ async def test_replace_step_multi_token_wrong_parent_is_404(): repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "old"}]}) repo.register_id_field("members", "id") step = RepositoryReplaceStep( - "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", RecordLocator("id", "memberId", ("orgId",)) ) with pytest.raises(NotFoundError): await step( @@ -204,7 +222,7 @@ async def test_update_step_renamed_token_mutates_record(): repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "old"}]}) repo.register_id_field("assetCategories", "id") step = RepositoryUpdateStep( - "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", RecordLocator("id", "categoryId") ) result = await step( _ctx( @@ -243,7 +261,7 @@ async def test_replace_step_renamed_token_replaces_record(): ) repo.register_id_field("assetCategories", "id") step = RepositoryReplaceStep( - "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", RecordLocator("id", "categoryId") ) result = await step( _ctx( @@ -276,7 +294,7 @@ async def test_missing_renamed_token_names_the_token(): repo = InMemoryRepository({"assetCategories": []}) repo.register_id_field("assetCategories", "id") step = RepositoryGetStep( - "assetCategories", "id", record_locator("/api/v1/categories/{categoryId}", "id") + "assetCategories", "id", RecordLocator("id", "categoryId") ) with pytest.raises(ValueError, match="^Missing route parameter: categoryId$"): await step(_ctx(repo, {}, collection="assetCategories")) @@ -287,7 +305,7 @@ async def test_missing_parent_token_names_the_token(): repo = InMemoryRepository({"members": []}) repo.register_id_field("members", "id") step = RepositoryGetStep( - "members", "id", record_locator("/api/v1/orgs/{orgId}/members/{memberId}", "id") + "members", "id", RecordLocator("id", "memberId", ("orgId",)) ) with pytest.raises(ValueError, match="^Missing route parameter: orgId$"): await step(_ctx(repo, {"memberId": "m1"}, collection="members")) @@ -357,6 +375,7 @@ def test_fastapi_renamed_token_get_fetches(): operation="get", collection="assetCategories", is_public=True, + record_param="categoryId", ) repo = InMemoryRepository({"assetCategories": [{"id": "c1", "title": "Books"}]}) client = TestClient( @@ -377,6 +396,8 @@ def test_fastapi_multi_token_get_fetches_by_all_tokens(): operation="get", collection="members", is_public=True, + record_param="memberId", + scope=("orgId",), ) repo = InMemoryRepository({"members": [{"id": "m1", "orgId": "o1", "name": "Ada"}]}) client = TestClient( @@ -399,6 +420,7 @@ def test_fastapi_v02_id_token_get_regression(): operation="get", collection="items", is_public=True, + record_param="id", ) repo = InMemoryRepository({"items": [{"id": "i1", "title": "kept"}]}) client = TestClient( diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py index 20190387..8f614183 100644 --- a/tests/unit/test_pipeline.py +++ b/tests/unit/test_pipeline.py @@ -321,6 +321,7 @@ async def test_builder_update_with_document_rule_allows_matching_principal(): route = RuntimeRouteDescriptor( method="PATCH", path="/items/{id}", + record_param="id", operation="update", collection="items", is_public=False, @@ -362,6 +363,7 @@ async def test_builder_update_with_document_rule_denies_non_matching_principal() route = RuntimeRouteDescriptor( method="PATCH", path="/items/{id}", + record_param="id", operation="update", collection="items", is_public=False, diff --git a/tests/unit/test_request_access_generation.py b/tests/unit/test_request_access_generation.py index 319de2b3..8357d416 100644 --- a/tests/unit/test_request_access_generation.py +++ b/tests/unit/test_request_access_generation.py @@ -33,7 +33,7 @@ def test_create_route_uses_typed_body_and_request_locals(tmp_path): route = (tmp_path / "routes" / "memberships.py").read_text() assert "from models._dto import CreateMembership" in route assert "payload: CreateMembership" in route - assert '_request_body = payload.model_dump(mode="json")' in route + assert '_request_body = payload.model_dump(mode="json", exclude_unset=True)' in route assert "request: Request" not in route or "_request_query" in route diff --git a/tests/unit/test_where_binding.py b/tests/unit/test_where_binding.py index b9401fae..64f44abd 100644 --- a/tests/unit/test_where_binding.py +++ b/tests/unit/test_where_binding.py @@ -138,11 +138,11 @@ def test_post_fetch_rule_sees_its_where_document(): assert PostFetchAuthorizeStep([rule])(ctx) is ctx -def test_step_is_a_no_op_without_principal(): +def test_anonymous_callers_load_nothing_for_rules_demanding_an_identity(): repo = InMemoryRepository({"assetCategories": []}) ctx = _ctx(repo, {"categorySlug": "books"}, principal=None) assert asyncio.run(ResolveWhereBindingsStep([_owner_rule()])(ctx)) is ctx - assert ctx.where_documents is None + assert ctx.where_documents == [{}] _SPEC = """ From 1577a95647a18315b387eafdac77160ebc16e80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 00:12:07 +0100 Subject: [PATCH 30/89] Reject operation names that shadow generated repository or service members --- .../schemas/yaml-spec-v0.2.schema.json | 71 +++++++++++++++++++ .../schemas/yaml-spec-v0.3.schema.json | 11 +++ tests/unit/test_operation_names.py | 51 +++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 tests/unit/test_operation_names.py diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json index 9fad9227..f1fccaf4 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.2.schema.json @@ -781,6 +781,9 @@ "type": "object", "additionalProperties": { "$ref": "#/$defs/OperationConfig" + }, + "propertyNames": { + "$ref": "#/$defs/OperationName" } } } @@ -830,6 +833,9 @@ } } ] + }, + "propertyNames": { + "$ref": "#/$defs/OperationName" } } } @@ -875,6 +881,11 @@ "additionalProperties": { "$ref": "#/$defs/RouteOverride" } + }, + "operations": { + "propertyNames": { + "$ref": "#/$defs/OperationName" + } } } }, @@ -1191,6 +1202,66 @@ ] } } + }, + "OperationName": { + "description": "An operation name, which becomes a method of the generated repository and service classes", + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "not": { + "enum": [ + "False", + "None", + "True", + "and", + "as", + "assert", + "async", + "await", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "except", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "nonlocal", + "not", + "or", + "pass", + "raise", + "return", + "try", + "while", + "with", + "yield", + "COLLECTION", + "DEFAULT_ROLE", + "IDENTITY_FIELDS", + "ID_FIELD", + "ID_FORMAT", + "TABLE", + "delete_by", + "find_by", + "find_one", + "get_by", + "insert", + "paginate", + "replace_by", + "run_operation", + "update_by", + "upsert_by" + ] + } } } } diff --git a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json index 1ee6155e..445507bc 100644 --- a/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json +++ b/src/yaml_api_generator/parsers/schemas/yaml-spec-v0.3.schema.json @@ -705,6 +705,9 @@ "type": "object", "additionalProperties": { "$ref": "yaml-spec-v0.2.schema.json#/$defs/OperationConfig" + }, + "propertyNames": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/OperationName" } } } @@ -760,6 +763,9 @@ } } ] + }, + "propertyNames": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/OperationName" } } } @@ -805,6 +811,11 @@ "additionalProperties": { "$ref": "#/$defs/RouteOverride" } + }, + "operations": { + "propertyNames": { + "$ref": "yaml-spec-v0.2.schema.json#/$defs/OperationName" + } } } }, diff --git a/tests/unit/test_operation_names.py b/tests/unit/test_operation_names.py new file mode 100644 index 00000000..6d067d17 --- /dev/null +++ b/tests/unit/test_operation_names.py @@ -0,0 +1,51 @@ +"""Operation names that would shadow a generated repository or service member are rejected by the schema.""" + +import shutil +from pathlib import Path + +import pytest +import yaml + +from yaml_api_generator.parsers.yaml_parser import ConfigErrors, load_and_generate_ir + +_SPEC = Path(__file__).parents[1] / "integration" / "specs" / "services_mongo" / "spec.yaml" + + +@pytest.fixture(autouse=True) +def _mongo_settings(monkeypatch): + monkeypatch.setenv("MONGO_URI", "mongodb://localhost:27017") + monkeypatch.setenv("MONGO_DB", "services") + + +def _renamed(spec: dict, section: str, owner: str, old: str, new: str) -> dict: + operations = spec[section][owner]["operations"] + operations[new] = operations.pop(old) + return spec + + +def _load(spec: dict, tmp_path: Path): + path = tmp_path / "spec.yaml" + path.write_text(yaml.safe_dump(spec)) + shutil.copytree(_SPEC.parent / "hooks", tmp_path / "hooks") + return load_and_generate_ir(str(path)) + + +@pytest.mark.parametrize("name", ["insert", "paginate", "find_one", "run_operation", "get_by", "upsert_by", "class"]) +@pytest.mark.parametrize(("section", "owner", "old"), [ + ("repositories", "tasks", "retitle"), + ("services", "tasks", "echo"), + ("controllers", "tasks", "echo"), +]) +def test_operation_named_after_a_generated_member_is_rejected(tmp_path, section, owner, old, name): + spec = _renamed(yaml.safe_load(_SPEC.read_text()), section, owner, old, name) + with pytest.raises(ConfigErrors) as raised: + _load(spec, tmp_path) + assert any(error.code == "SCHEMA_VALIDATION_ERROR" for error in raised.value.errors) + + +def test_route_names_of_a_controller_without_service_stay_free(tmp_path): + spec = yaml.safe_load(_SPEC.read_text()) + totals = spec["controllers"]["totals"] + totals["routes"]["tasks-total"] = totals["routes"].pop("tasks") + totals["operations"]["tasks-total"] = totals["operations"].pop("tasks") + assert _load(spec, tmp_path).ir is not None From d19f78ba46ec35c6a37c378fe51a1bad00855204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A3o=20Teixeira?= Date: Sun, 25 Oct 2026 01:16:36 +0100 Subject: [PATCH 31/89] Send typed, writable-only bodies from every generated form through a shared fields module and record-form hook --- .../frontend/packs/builtin/generic/pack.yaml | 2 + .../components/bulk-edit-drawer.jinja | 59 +++------ .../components/form-inline-row.jinja | 87 ++------------ .../templates/components/form-modal.jinja | 96 ++------------- .../templates/components/form-preview.jinja | 113 ++---------------- .../templates/components/form-sectioned.jinja | 113 ++---------------- .../templates/components/form-stacked.jinja | 99 +-------------- .../components/list-spreadsheet.jinja | 34 +++--- .../templates/macros/field_input.jinja | 106 ++++++++++++++++ .../templates/scaffold/fields.ts.jinja | 80 +++++++++++++ .../templates/scaffold/useRecordForm.ts.jinja | 46 +++++++ src/yaml_api_generator/ir/ui.py | 40 +++++-- src/yaml_api_generator/parsers/ui_parser.py | 14 +-- tests/integration/test_mongo_payloads.py | 20 +++- tests/unit/support/form_payloads.py | 89 ++++++++++++++ tests/unit/support/form_payloads.ts | 11 ++ tests/unit/test_form_payloads.py | 51 ++++++++ 17 files changed, 512 insertions(+), 548 deletions(-) create mode 100644 src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/macros/field_input.jinja create mode 100644 src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/fields.ts.jinja create mode 100644 src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/scaffold/useRecordForm.ts.jinja create mode 100644 tests/unit/support/form_payloads.py create mode 100644 tests/unit/support/form_payloads.ts create mode 100644 tests/unit/test_form_payloads.py diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/pack.yaml b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/pack.yaml index d372909c..4ee1afd4 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/pack.yaml +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/pack.yaml @@ -322,6 +322,8 @@ scaffold: - api - config.ts - auth.tsx + - fields.ts + - useRecordForm.ts - index.css - tailwind.config.js - postcss.config.js diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/bulk-edit-drawer.jinja b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/bulk-edit-drawer.jinja index e4e480e8..80e97071 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/bulk-edit-drawer.jinja +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/bulk-edit-drawer.jinja @@ -1,7 +1,9 @@ +{% from 'macros/field_input.jinja' import field_control %} import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useAuth } from '../__generated__/App'; import { API_BASE_URL } from '../__generated__/config'; +import { ENTITY_FIELDS, FormValues, formValues, payload } from '../__generated__/fields'; interface {{ entity.name | pascal_case }} { {{ entity.pk_field }}: string; @@ -21,9 +23,7 @@ interface {{ entity.name | pascal_case }} { {% set other_fields = editable_fields | rejectattr('name', 'in', role_names) | list %} {% set display_fields = (role_fields + other_fields)[:4] %} -interface BulkPatch { - [field: string]: string; -} +const FIELDS = ENTITY_FIELDS[{{ entity.name | tojson }}]; export default function {{ view.name | pascal_case }}() { const [items, setItems] = useState<{{ entity.name | pascal_case }}[]>([]); @@ -32,7 +32,7 @@ export default function {{ view.name | pascal_case }}() { const [selectedIds, setSelectedIds] = useState>(new Set()); const [drawerOpen, setDrawerOpen] = useState(false); const [enabledFields, setEnabledFields] = useState>(new Set()); - const [patch, setPatch] = useState({}); + const [patch, setPatch] = useState(() => formValues(FIELDS)); const [applying, setApplying] = useState(false); const { token } = useAuth(); const navigate = useNavigate(); @@ -70,34 +70,28 @@ export default function {{ view.name | pascal_case }}() { return next; }); + const setPatchValue = (name: string, value: unknown) => setPatch(prev => ({ ...prev, [name]: value })); + const applyBulkEdit = async () => { - const activePatch = Object.fromEntries( - Object.entries(patch).filter(([k]) => enabledFields.has(k)) - ); - if (Object.keys(activePatch).length === 0) return; + const enabled = Object.fromEntries(Object.entries(FIELDS).filter(([name]) => enabledFields.has(name))); + const body = JSON.stringify(payload(enabled, patch)); + if (body === '{}') return; setApplying(true); const headers: Record = { 'Content-Type': 'application/json' }; if (token) headers['Authorization'] = `Bearer ${token}`; - await Promise.all([...selectedIds].map(id => - fetch(`${API_BASE_URL}/{{ entity.repository_name | kebab_case }}/${id}`, { - method: 'PATCH', - headers, - body: JSON.stringify(activePatch), - }) + const updated: {{ entity.name | pascal_case }}[] = await Promise.all([...selectedIds].map(id => + fetch(`${API_BASE_URL}/{{ entity.repository_name | kebab_case }}/${id}`, { method: 'PATCH', headers, body }) + .then(res => (res.ok ? res.json() : null)) )); + const updatedById = new Map(updated.filter(Boolean).map(item => [item.{{ entity.pk_field }}, item])); - // Optimistically update local state - setItems(prev => prev.map(item => - selectedIds.has(item.{{ entity.pk_field }}) - ? { ...item, ...activePatch } - : item - )); + setItems(prev => prev.map(item => updatedById.get(item.{{ entity.pk_field }}) ?? item)); setApplying(false); setDrawerOpen(false); setEnabledFields(new Set()); - setPatch({}); + setPatch(formValues(FIELDS)); setSelectedIds(new Set()); }; @@ -228,26 +222,9 @@ export default function {{ view.name | pascal_case }}() { {{ field.name | title }} {enabledFields.has('{{ field.name }}') && ( - {% if field.type == 'enum' and field.options %} - - {% else %} - setPatch(p => ({ ...p, '{{ field.name }}': e.target.value }))} - placeholder="New value..." - type="{% if field.type == 'number' %}number{% elif field.type == 'boolean' %}text{% else %}text{% endif %}" - className="w-full wf-input text-sm" - /> - {% endif %} +
+ {{ field_control(field, 'patch', 'setPatchValue', 'bulk-' ~ field.name) }} +
)} {% endfor %} diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-inline-row.jinja b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-inline-row.jinja index 8a8393e8..a71adbe2 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-inline-row.jinja +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-inline-row.jinja @@ -1,58 +1,9 @@ -{% macro inline_field(field, formData='editData', handler='handleFieldChange') %} -{% set is_enum = field.type == 'enum' %} -{% set is_boolean = field.type == 'boolean' %} -{% set input_type = 'number' if field.type == 'number' else ('datetime-local' if field.type == 'datetime' else 'text') %} -
- - {% if is_boolean %} - - {% elif is_enum and field.enum_values %} -
- -
- {% else %} -
- -
- {% endif %} -
-{% endmacro %} +{% from 'macros/field_input.jinja' import field_input %} import { useEffect, useState } from 'react'; import { Link, useNavigate } from 'react-router-dom'; import { useAuth } from '../__generated__/App'; import { API_BASE_URL } from '../__generated__/config'; +import { useRecordForm } from '../__generated__/useRecordForm'; interface {{ entity.name | pascal_case }} { {{ entity.pk_field }}: string; @@ -66,10 +17,10 @@ export default function {{ view.name | pascal_case }}() { const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [editingId, setEditingId] = useState(null); - const [editData, setEditData] = useState>({}); const [saving, setSaving] = useState(false); const { token } = useAuth(); const navigate = useNavigate(); + const { values, setValue, submit } = useRecordForm<{{ entity.name | pascal_case }}>({{ entity.name | tojson }}, '{{ entity.repository_name | kebab_case }}', editingId ?? undefined); useEffect(() => { const headers: Record = {}; @@ -90,19 +41,9 @@ export default function {{ view.name | pascal_case }}() { }); }, [token]); - const startEdit = (item: {{ entity.name | pascal_case }}) => { - setEditingId(item.{{ entity.pk_field }}); - setEditData({ ...item }); - }; - - const cancelEdit = () => { - setEditingId(null); - setEditData({}); - }; + const startEdit = (item: {{ entity.name | pascal_case }}) => setEditingId(item.{{ entity.pk_field }}); - const handleFieldChange = ( - e: React.ChangeEvent - ) => setEditData(prev => ({ ...prev, [e.target.name]: e.target.value })); + const cancelEdit = () => setEditingId(null); const handleSave = async (e: React.FormEvent) => { e.preventDefault(); @@ -110,21 +51,7 @@ export default function {{ view.name | pascal_case }}() { setSaving(true); try { - const headers: Record = { 'Content-Type': 'application/json' }; - if (token) headers['Authorization'] = `Bearer ${token}`; - - const res = await fetch(`${API_BASE_URL}/{{ entity.repository_name | kebab_case }}/${editingId}`, { - method: 'PUT', - headers, - body: JSON.stringify(editData), - }); - - if (!res.ok) { - const data = await res.json().catch(() => ({})); - throw new Error(data.detail || 'Failed to save'); - } - - const updated = await res.json(); + const updated = await submit(); setItems(prev => prev.map(i => i.{{ entity.pk_field }} === editingId ? updated : i)); cancelEdit(); } catch (err: unknown) { @@ -212,7 +139,7 @@ export default function {{ view.name | pascal_case }}() { > {% for field in edit_fields %} {% if loop.index0 < 4 %} - {{ inline_field(field) }} + {{ field_input(field, id_prefix='inline-') }} {% endif %} {% endfor %} diff --git a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-modal.jinja b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-modal.jinja index e68c583b..5561e168 100644 --- a/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-modal.jinja +++ b/src/yaml_api_generator/generators/frontend/packs/builtin/generic/templates/components/form-modal.jinja @@ -1,88 +1,22 @@ -{% macro field_input(field, formData='formData', handler='handleFieldChange') %} -{% set is_textarea = field.name in ['content', 'body', 'description', 'excerpt', 'bio'] %} -{% set is_enum = field.type == 'enum' %} -{% set is_boolean = field.type == 'boolean' %} -{% set input_type = 'number' if field.type == 'number' else ('datetime-local' if field.type == 'datetime' else 'text') %} -
- - {% if is_boolean %} - - {% elif is_enum and field.enum_values %} -
- -
- {% elif is_textarea %} -
-