bedrock.generators.generate-openapi-specification

  1"""
  2This script generates the OpenAPI specification for the Bedrock application.
  3"""
  4import re
  5import sys
  6from enum import Enum
  7
  8import yaml
  9from sqlalchemy.orm import RelationshipProperty
 10
 11from bedrock._helpers.classes import find_all_attributes, find_all_methods
 12from bedrock._helpers.string import camelCase_to_snake_case
 13from bedrock.config import get_version
 14from bedrock.config._modules import _get_mappings
 15from bedrock.config.models import get_models
 16from bedrock.generators.introspection.decorators import get_function_decorators
 17from bedrock.generators._helpers.spec_generation_helpers import dump_dict_to_file, dump_file
 18from bedrock.meta.model import convert_model_to_schema, python_type_to_openapi_type, get_looping_properties
 19
 20BASE_PATH = None
 21APP_NAME = None
 22VERSION = None
 23ENVIRONMENTS = None
 24IS_DOCS_SPEC = None
 25mappings = None
 26models = None
 27OPENAPI_PATH = None
 28USE_ENVELOPED_SCHEMA = True
 29
 30
 31def generate_files():
 32    """
 33    The do-all-the-things method.
 34
 35    See `get_endpoint_info` and `get_endpoint_schemas`.
 36    """
 37    # Paths
 38    dump_file(OPENAPI_PATH, "errors.yml", """"400":
 39  description: Bad request
 40  content:
 41    application/json:
 42      schema:
 43        $ref: "../schemas/error.yml#/Error"
 44"401":
 45  description: Unauthorized
 46  content:
 47    application/json:
 48      schema:
 49        $ref: "../schemas/error.yml#/Error"
 50"403":
 51  description: Forbidden
 52  content:
 53    application/json:
 54      schema:
 55        $ref: "../schemas/error.yml#/Error"
 56"404":
 57  description: Not Found
 58  content:
 59    application/json:
 60      schema:
 61        $ref: "../schemas/error.yml#/Error"
 62"409":
 63  description: Conflict
 64  content:
 65    application/json:
 66      schema:
 67        $ref: "../schemas/error.yml#/Error"
 68""", "paths")
 69    paths = {}
 70    for key, obj in mappings.items():
 71        endpoint_file_name, content_dict = get_endpoint_info(key, obj, USE_ENVELOPED_SCHEMA)
 72        dump_dict_to_file(OPENAPI_PATH, endpoint_file_name, content_dict, "paths")
 73        paths[key] = {"$ref": f"paths/{endpoint_file_name}"}
 74
 75    # Schemas
 76    dump_file(OPENAPI_PATH, "error.yml", """Error:
 77  type: object
 78  properties:
 79    _errors:
 80      type: array
 81      items:
 82        type: object
 83        properties:
 84          code:
 85            type: string
 86          message:
 87            type: string
 88          exception:
 89            type: string""", "schemas")
 90    schemas = {}
 91    for model in models:
 92        model_name = model.__name__
 93        model_schema = convert_model_to_schema(model)
 94        dump_dict_to_file(OPENAPI_PATH, f"{model_name}.yml", model_schema, "schemas")
 95    for obj in mappings.values():
 96        for schema_name, schema_content in get_endpoint_schemas(obj).items():
 97            schemas[schema_name] = {"$ref": f"schemas/{schema_name}.yml#/{schema_name}"}
 98            dump_dict_to_file(OPENAPI_PATH, f"{schema_name}.yml", schema_content, "schemas")
 99
100    servers = [{
101        "description": f'{environment["name"]} server',
102        "url": f'https://{environment["host"]}'
103    } for environment in ENVIRONMENTS]
104
105    # Main
106    main_content = {
107        "openapi": "3.0.3",
108        "info": {
109            "version": VERSION,
110            "title": f"{APP_NAME} API",
111            "description": f"API Documentation for {APP_NAME}"
112        },
113        "servers": servers,
114        "components": {
115            "securitySchemes": {
116                "bearerAuth": {
117                    "type": "http",
118                    "scheme": "bearer",
119                    "bearerFormat": "JWT"
120                },
121                "apiKeyAuth": {
122                    "type": "apiKey",
123                    "in": "header",
124                    "name": "x-api-key"
125                }
126            },
127            "schemas": schemas
128        },
129        "paths": paths
130    }
131
132    dump_dict_to_file(OPENAPI_PATH, "main.yml", main_content)
133
134
135def get_endpoint_info(endpoint_path, endpoint_object, enveloped_schema: bool = True) -> tuple[str, dict]:
136    """
137    Generate the OpenAPI definition for a given endpoint.
138    :param endpoint_path: The endpoint path (e.g. `/countries/{{countryUuid}}/regions`)
139    :param endpoint_object: The endpoint instance (e.g. `RegionsEndpoint()`)
140    :param enveloped_schema: Whether to include the envelope header and change the returned schema to be in a response envelope
141    :return: A tuple with the yaml filename and the OpenAPI definition for the endpoint methods
142    """
143    file_name = get_endpoint_file_name(endpoint_path)
144    cls = endpoint_object.__class__
145    ending_param = cls().param
146    methods = sorted(m for m in find_all_methods(cls, with_parents=False)
147                     if (endpoint_path.endswith(ending_param) and m.endswith("_single"))
148                     or not endpoint_path.endswith(ending_param) and m.endswith("_global"))
149    if not IS_DOCS_SPEC:
150        options_method = auto_define_options(methods)
151        all_methods = [*methods, options_method]
152    else:
153        all_methods = methods
154    methods_dict = {}
155    for m in all_methods:
156        methods_dict.update(generate_endpoint_http_method(m, endpoint_object, enveloped_schema))
157    return file_name, methods_dict
158
159
160def get_endpoint_schemas(endpoint) -> dict:
161    """
162    Gets both request and response schemas.
163    :param endpoint: The endpoint instance
164    """
165    return {
166        **get_endpoint_schema(endpoint, "body"),
167        **get_endpoint_schema(endpoint, "return")
168    }
169
170
171def get_endpoint_schema(endpoint, schema_type) -> dict:
172    """
173    Get the schema for a given endpoint.
174
175    Checks whether the endpoint has custom schema, a related model or nothing at all.
176    :param endpoint: The endpoint instance
177    :param schema_type: Either `"body"` or `"return"` (for request body or responses respectively)
178    """
179    name = getattr(endpoint, f"_get_{schema_type}_schema_name")()
180    if getattr(endpoint, f"has_custom_{schema_type}_schema")():
181        endpoint_schema = getattr(endpoint, f"get_{schema_type}_schema")()
182        if isinstance(endpoint_schema, str):
183            return {name: yaml.load(endpoint_schema, yaml.CLoader)}
184        return {name: endpoint_schema}
185    if endpoint.related_model is None:
186        return {}
187    return {name: convert_model_to_schema(endpoint.related_model, name)}
188
189
190def apigateway_integration_block(lambda_arn_variable, endpoint, http_method) -> dict:
191    """
192    Returns the `x-amazon-apigateway-integration` block for a given lambda ARN variable.
193    :param lambda_arn_variable: The Lambda ARN variable, e.g. `cities_endpoint_lambda_arn`. It'll be used as a placeholder for terraform to then replace.
194    :param endpoint: The endpoint instance
195    :param http_method: The HTTP method (e.g. `GET`, `POST`, `PUT`, `DELETE`, etc.)
196    """
197    if IS_DOCS_SPEC:
198        return {}
199    http_method = http_method.upper()
200    integration_type = endpoint._get_integration_type(http_method)
201
202    apigateway_integration = {"x-amazon-apigateway-integration":
203        {
204            "type": integration_type,
205            "uri": f"arn:aws:apigateway:${{region}}:lambda:path/2015-03-31/functions/${{{lambda_arn_variable}}}/invocations",
206            "passthroughBehavior": "when_no_match",
207            "httpMethod": "POST",  # For Lambda function invocations, the value must be POST.
208            "timeoutInMillis": 29000,
209        }
210    }
211
212    if integration_type != "aws_proxy":
213        apigateway_integration["x-amazon-apigateway-integration"]["requestTemplates"] = {
214            f"{endpoint._get_request_content_type(http_method)}": f"{endpoint._get_request_mapping_template(http_method)}",
215        }
216        apigateway_integration["x-amazon-apigateway-integration"]["responses"] = {
217            f"{endpoint._get_response_mapping_regex__(http_method)}": {
218                "statusCode": "201",
219                "responseTemplates": {
220                    f"{endpoint._get_response_content_type__(http_method)}": f"{endpoint._get_response_mapping_template(http_method)}",
221                }
222            }
223        }
224        apigateway_integration["x-amazon-apigateway-integration"]["passthroughBehavior"] = "when_no_templates"
225
226    return apigateway_integration
227
228
229
230def get_model_properties_as_query_params(model_cls):
231    """
232    Get the model properties as query parameters.
233    :param model_cls: Class of the model
234    """
235    if issubclass(model_cls, Enum):
236        return {}
237    properties = [
238        getattr(model_cls, prop).prop for prop in find_all_attributes(model_cls) if prop not in ['metadata', 'registry']
239    ]
240    query_params = {}
241    for p in properties:
242        if isinstance(p, RelationshipProperty):
243            if p.key.endswith("s"):
244                query_params[f"{p.key}[empty]"] = {"type": "boolean", "example": False}
245        else:
246            data_type = python_type_to_openapi_type(p.columns[0].type.python_type, p.key, True)
247            if data_type:
248                query_params[f"{p.key}[OPER]"] = data_type
249    return query_params
250
251
252def get_endpoint_file_name(endpoint):
253    """
254    Get the filename for a given endpoint.
255    :param endpoint: The endpoint name
256    """
257    return endpoint[1:].replace("/", ".") \
258        .replace("{", "_") \
259        .replace("}", "_") + ".yml"
260
261
262def auto_define_options(methods):
263    """
264    Auto-define the `OPTIONS` method for a given endpoint (for CORS).
265    """
266    method = [m for m in methods if m.endswith("_single") or m.endswith("_global")][0]
267    return re.sub(r"\w+_", "options_", method)
268
269
270def generate_http_method_security(cls, method) -> dict:
271    """
272    Generate the security definition for a given method, based on whether it includes a `@protected` decorator.
273    :param cls: Endpoint class
274    :param method: Method name (e.g. `get_single`, `get_global`, `post_single`, `post_global`, etc.)
275    """
276    decorators = get_function_decorators(vars(cls)[method]) if "options" not in method else []
277    protected_decorators = [decorator for decorator in decorators if decorator.name == "@protected"]
278    if len(protected_decorators) > 0:
279        security_modes = [{"bearerAuth": []}]  # if using @protected, then it always has bearer auth
280        has_api_key_auth = len([True for decorator in protected_decorators if
281                                "keys" in decorator.kwargs or len(decorator.args) >= 3]) > 0
282        if has_api_key_auth and IS_DOCS_SPEC:
283            security_modes.append({"apiKeyAuth": []})
284        return {"security": security_modes}
285    return {}
286
287
288def generate_http_method_parameters(method: str, endpoint, enveloped_schema: bool = True) -> dict:
289    """
290    Generate the `path` and `query` parameters for a given method.
291    :param method: Name of the method (e.g. `get_single`, `get_global`, `post_single`, `post_global`, etc.)
292    :param endpoint: The endpoint instance
293    :param enveloped_schema: Whether the header should include the envelope toggle
294    :return: A dictionary `{"parameters": [...]}`.
295    """
296    cls = endpoint.__class__
297    is_global = "_global" in method
298
299    path_parameter_keys = endpoint.path_params if is_global else [*endpoint.path_params, endpoint.param_key]
300    path_params = [{
301        "in": "path",
302        "name": param,
303        "required": True,
304        "schema": {"type": "string"}
305    } for param in path_parameter_keys]
306
307    query_parameters = {}
308    if IS_DOCS_SPEC:
309        decorators = get_function_decorators(vars(cls)[method]) if "options" not in method else []
310        with_query_param_decorators = [decorator for decorator in decorators if decorator.name == "@with_query_param"]
311        for decorator in with_query_param_decorators:
312            query_parameters[decorator.args[0]] = python_type_to_openapi_type(decorator.args[1], decorator.args[0],
313                                                                              True)
314        filter_decorators = [decorator for decorator in decorators if decorator.name == "@filter_on_columns"]
315        if filter_decorators and endpoint.related_model:
316            query_parameters = {
317                **query_parameters,
318                **get_model_properties_as_query_params(endpoint.related_model)
319            }
320    query_params = [make_query_param(param, param_type) for param, param_type in query_parameters.items()]
321    header_params = [{
322        "in": "header",
323        "name": 'bedrock-response-envelope',
324        "required": False,
325        "schema": {"type": "string", "enum": ["true", "false"]}
326    }] if enveloped_schema else []
327    return {
328        "parameters": [*path_params, *query_params, *header_params]
329    }
330
331
332def make_query_param(param, param_type):
333    schema_info = {"schema": param_type}
334    if "examples" in param_type:
335        _param_type = param_type.copy()
336        del _param_type["examples"]
337        schema_info = {
338            "examples": param_type["examples"],
339            "schema": _param_type
340        }
341    return {
342        "in": "query",
343        "name": param,
344        "required": False,
345        **schema_info,
346        "allowReserved": True,
347        "description": "OPER can be one of `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in`, `ni`, `like`, `is`, `isnt`, `any` and `empty`" if "[OPER]" in param else ""
348    }
349
350
351def make_return_body_schema(endpoint):
352    body_schema_path = f"../schemas/{endpoint._get_return_schema_name()}.yml#/{endpoint._get_return_schema_name()}"
353    body_schema = {"$ref": body_schema_path}
354
355    if not endpoint.__return_schema_name__ and endpoint.related_model:
356        looping_properties = get_looping_properties(endpoint.related_model, as_camel_case=True)
357        body_schema = {
358            "allOf": [
359                body_schema,
360                {
361                    "properties": looping_properties
362                }
363            ]
364        }
365    return body_schema
366
367
368def generate_http_method_ok_response(method, endpoint, is_envelope: bool = True) -> dict:
369    """
370    Generate the success responses for a given method (i.e. `201` for create-type actions and `200` for everything else).
371    :param method: Name of the method (e.g. `get_single`, `get_global`, `post_single`, `post_global`, etc.)
372    :param endpoint: The endpoint instance
373    :param is_envelope: Whether the generated schema should be enveloped in a response object
374    :return: A dictionary, e.g. `{"200": {...}}`
375    """
376    is_global = "_global" in method
377    body_schema = make_return_body_schema(endpoint)
378    if not endpoint.__class__.has_custom_return_schema():
379        body_schema = {"allOf": [
380            {
381                "type": "object",
382                "properties": {
383                    "uuid": {
384                        "type": "string",
385                        "example": "43f5e9fd-1ffd-4469-8e01-f6d1b0b78d34"
386                    }
387                }
388            },
389            body_schema
390        ]}
391    if "delete" in method:
392        body_schema = {
393            "type": "object",
394            "properties": {
395                "deleted": {"type": "boolean"},
396                "originalData": body_schema
397            }
398        }
399    if is_global and endpoint.__class__.global_is_array():
400        body_schema = {
401            "type": "array",
402            "items": body_schema
403        }
404    http_code = "201" if "post" in method else "200"
405    description = "Created" if "post" in method else "Ok"
406    if "delete" in method:
407        enveloped_schema = {**body_schema}
408        enveloped_schema["properties"]["data"] = {"type": "object", "nullable": True}
409    else:
410        enveloped_schema = {
411            "type": "object",
412            "properties": {
413                "data": body_schema,
414                "_metadata": {
415                    "type": "object",
416                    "properties": {
417                        "pagination": {
418                            "type": "object",
419                            "properties": {
420                                "sortColumn": {"type": "string", "example": "created_at"},
421                                "sortOrder": {"type": "string", "enum": ["asc", "desc"], "example": "desc"},
422                                "limit": {"type": "integer", "example": 50, "nullable": True},
423                                "offset": {"type": "integer", "example": 10, "nullable": True},
424                                "total": {"type": "integer", "example": 1642, "nullable": True}
425                            },
426                            "nullable": True
427                        }
428                    }
429                },
430                "_warnings": {
431                    "type": "array",
432                    "items": {
433                        "type": "object",
434                        "properties": {
435                            "code": {"type": "string"},
436                            "message": {"type": "string"},
437                            "exception": {"type": "string"}
438                        }
439                    }
440                },
441                "_errors": {
442                    "type": "array",
443                    "items": {
444                        "type": "object",
445                        "properties": {
446                            "code": {"type": "string"},
447                            "message": {"type": "string"},
448                            "exception": {"type": "string"}
449                        }
450                    }
451                }
452            }
453        }
454    return {
455        http_code: {
456            "content": {"application/json": {"schema": enveloped_schema if is_envelope else body_schema}},
457            "description": description
458        }
459    }
460
461
462def generate_http_method_request_body(http_method, endpoint) -> dict:
463    """
464    Generate the request body for a given method (i.e. the body schema).
465    :param http_method: Name of the method (e.g. `get`, `post`, etc.)
466    :param endpoint: The endpoint instance
467    """
468    if http_method not in ["post", "put"]:
469        return {}
470    body_schema_path = f"../schemas/{endpoint._get_body_schema_name()}.yml#/{endpoint._get_body_schema_name()}"
471    body_schema = {"$ref": body_schema_path}
472    request_content_type = endpoint._get_request_content_type(http_method)
473    return {
474        "requestBody": {
475            "required": False,
476            "content": {request_content_type: {"schema": body_schema}}
477        }
478    }
479
480
481def http_method_summary(method, endpoint):
482    """
483    Generate the summary for a given method (i.e. "Get City list" or "Create Country")
484    :param method:
485    :param endpoint:
486    :return:
487    """
488    http_method = method.replace("_global", "").replace("_single", "")
489    is_global = "_global" in method and endpoint.__class__.global_is_array()
490    prefix_map = {
491        "get": "Get",
492        "post": "Create",
493        "put": "Update",
494        "delete": "Delete",
495        "options": "CORS for"
496    }
497    suffix_map = {
498        "get": " list" if is_global else "",
499        "post": "",
500        "put": "",
501        "delete": "",
502        "options": ""
503    }
504    return f"{prefix_map[http_method.lower()]} {endpoint.__class__.__name__}{suffix_map[http_method.lower()]}"
505
506
507def generate_endpoint_http_method(method: str, endpoint, enveloped_schema: bool = True) -> dict:
508    """
509    Generate the OpenAPI definition for a given endpoint method.
510    :param method: Name of the method (e.g. `get_single`, `get_global`, `post_single`, `post_global`, etc.)
511    :param endpoint: The endpoint instance
512    :param enveloped_schema: Whether to include the envelope header and change the returned schema to be in a response envelope
513    """
514    cls = endpoint.__class__
515    http_method = method.replace("_global", "").replace("_single", "")
516    lambda_arn_variable = f"{camelCase_to_snake_case(cls.__name__)}_lambda_arn"
517    return {
518        http_method: {
519            "summary": http_method_summary(method, endpoint),
520            "tags": [cls.__name__],
521            **generate_http_method_security(cls, method),
522            **generate_http_method_parameters(method, endpoint, enveloped_schema),
523            **generate_http_method_request_body(http_method, endpoint),
524            "responses": {
525                **generate_http_method_ok_response(method, endpoint, enveloped_schema),
526                "400": {"$ref": "./errors.yml#/400"},
527                "401": {"$ref": "./errors.yml#/401"},
528                "403": {"$ref": "./errors.yml#/403"},
529                "404": {"$ref": "./errors.yml#/404"},
530                "409": {"$ref": "./errors.yml#/409"}
531            },
532            **apigateway_integration_block(lambda_arn_variable, endpoint, http_method)
533        }
534    }
535
536
537if __name__ == '__main__':
538    BASE_PATH = sys.argv[1]
539    APP_NAME = sys.argv[2] if len(sys.argv) > 2 else "bedrock"
540    VERSION = sys.argv[3] if len(sys.argv) > 3 else get_version()
541    default_environments = [f"{env}|{APP_NAME}-api.{env}.keyholding.com" for env in ["testing", "staging"]]
542    default_environments.append(f"production|{APP_NAME}-api.keyholding.com")
543    environments_with_hosts = sys.argv[4].split(",") if len(sys.argv) > 4 else default_environments
544    ENVIRONMENTS = [{"name": env_with_host.split("|")[0], "host": env_with_host.split("|")[1]} for env_with_host in
545                    environments_with_hosts]
546    docs_param = sys.argv[5] if len(sys.argv) > 5 else "api-gw-spec"
547    IS_DOCS_SPEC = docs_param.lower() == "docs-spec"
548    if len(sys.argv) > 6 and sys.argv[6].lower() == "no-envelope":
549        USE_ENVELOPED_SCHEMA = False
550
551    mappings = _get_mappings(f"{BASE_PATH}/app/endpoints")
552    models = get_models(f"{BASE_PATH}/app/model", "model")
553    OPENAPI_PATH = f"{BASE_PATH}/openapi/"
554    generate_files()