loader

This module is responsible for loading the documentation from Python objects.

It uses inspect for introspecting objects, iterating over their members, etc.

Loader

This class contains the object documentation loading mechanisms.

Any error that occurred during collection of the objects and their documentation is stored in the errors list.

Source code in src/pytkdocs/loader.py

class Loader:
    """This class contains the object documentation loading mechanisms.

    Any error that occurred during collection of the objects and their documentation is stored in the `errors` list.
    """

    def __init__(
        self,
        filters: Optional[list[str]] = None,
        docstring_style: str = "google",
        docstring_options: Optional[dict] = None,
        inherited_members: bool = False,  # noqa: FBT001, FBT002
        new_path_syntax: bool = False,  # noqa: FBT001, FBT002
    ) -> None:
        """Initialize the object.

        Arguments:
            filters: A list of regular expressions to fine-grain select members. It is applied recursively.
            docstring_style: The style to use when parsing docstrings.
            docstring_options: The options to pass to the docstrings parser.
            inherited_members: Whether to select inherited members for classes.
            new_path_syntax: Whether to use the "colon" syntax for the path.
        """
        if not filters:
            filters = []

        self.filters = [(filtr, re.compile(filtr.lstrip("!"))) for filtr in filters]
        self.docstring_parser = PARSERS[docstring_style](**(docstring_options or {}))
        self.errors: list[str] = []
        self.select_inherited_members = inherited_members
        self.new_path_syntax = new_path_syntax

    def get_object_documentation(self, dotted_path: str, members: Optional[Union[set[str], bool]] = None) -> Object:
        """Get the documentation for an object and its children.

        Arguments:
            dotted_path: The Python dotted path to the desired object.
            members: `True` to select members and filter them, `False` to select no members,
                or a list of names to explicitly select the members with these names.
                It is applied only on the root object.

        Returns:
            The documented object.
        """
        if members is True:
            members = set()

        root_object: Object
        leaf = get_object_tree(dotted_path, self.new_path_syntax)

        if leaf.is_module():
            root_object = self.get_module_documentation(leaf, members)
        elif leaf.is_class():
            root_object = self.get_class_documentation(leaf, members)
        elif leaf.is_staticmethod():
            root_object = self.get_staticmethod_documentation(leaf)
        elif leaf.is_classmethod():
            root_object = self.get_classmethod_documentation(leaf)
        elif leaf.is_method_descriptor() or leaf.is_method():
            root_object = self.get_regular_method_documentation(leaf)
        elif leaf.is_function():
            root_object = self.get_function_documentation(leaf)
        elif leaf.is_property():
            root_object = self.get_property_documentation(leaf)
        else:
            root_object = self.get_attribute_documentation(leaf)

        root_object.parse_all_docstrings(self.docstring_parser)

        return root_object

    def get_module_documentation(
        self,
        node: ObjectNode,
        select_members: Optional[Union[set[str], bool]] = None,
    ) -> Module:
        """Get the documentation for a module and its children.

        Arguments:
            node: The node representing the module and its parents.
            select_members: Explicit members to select.

        Returns:
            The documented module object.
        """
        module = node.obj
        path = node.dotted_path
        name = path.split(".")[-1]
        source: Optional[Source]

        try:
            source = Source(inspect.getsource(module), 1)
        except OSError:
            try:
                code = Path(node.file_path).read_text()
            except (OSError, UnicodeDecodeError):
                source = None
            else:
                source = Source(code, 1) if code else None

        root_object = Module(
            name=name,
            path=path,
            file_path=node.file_path,
            docstring=inspect.getdoc(module),
            source=source,
        )

        if select_members is False:
            return root_object

        select_members = select_members or set()

        attributes_data = get_module_attributes(module)
        root_object.parse_docstring(self.docstring_parser, attributes=attributes_data)

        for member_name, member in inspect.getmembers(module):
            if self.select(member_name, select_members):  # type: ignore[arg-type]
                child_node = ObjectNode(member, member_name, parent=node)
                if child_node.is_class() and node.root.obj is inspect.getmodule(child_node.obj):
                    root_object.add_child(self.get_class_documentation(child_node))
                elif child_node.is_function() and node.root.obj is inspect.getmodule(child_node.obj):
                    root_object.add_child(self.get_function_documentation(child_node))
                elif member_name in attributes_data:
                    root_object.add_child(self.get_attribute_documentation(child_node, attributes_data[member_name]))

        if hasattr(module, "__path__"):
            for _, modname, _ in pkgutil.iter_modules(module.__path__):
                if self.select(modname, select_members):  # type: ignore[arg-type]
                    leaf = get_object_tree(f"{path}.{modname}")
                    root_object.add_child(self.get_module_documentation(leaf))

        return root_object

    @staticmethod
    def _class_path(cls: type) -> str:  # noqa: PLW0211
        mod = cls.__module__
        qname = cls.__qualname__
        if mod == "builtins":
            return qname
        return f"{mod}.{qname}"

    def get_class_documentation(
        self,
        node: ObjectNode,
        select_members: Optional[Union[set[str], bool]] = None,
    ) -> Class:
        """Get the documentation for a class and its children.

        Arguments:
            node: The node representing the class and its parents.
            select_members: Explicit members to select.

        Returns:
            The documented class object.
        """
        class_ = node.obj
        docstring = inspect.cleandoc(class_.__doc__ or "")
        bases = [self._class_path(b) for b in class_.__bases__]

        source: Optional[Source]

        try:
            source = Source(*inspect.getsourcelines(node.obj))
        except (OSError, TypeError):
            source = None

        root_object = Class(
            name=node.name,
            path=node.dotted_path,
            file_path=node.file_path,
            docstring=docstring,
            bases=bases,
            source=source,
        )

        # Even if we don't select members, we want to correctly parse the docstring
        attributes_data: dict[str, dict[str, Any]] = {}
        for parent_class in reversed(class_.__mro__[:-1]):
            merge(attributes_data, get_class_attributes(parent_class))
        context: dict[str, Any] = {"attributes": attributes_data}
        if "__init__" in class_.__dict__:
            try:
                attributes_data.update(get_instance_attributes(class_.__init__))
                context["signature"] = inspect.signature(class_.__init__)
            except (TypeError, ValueError):
                pass
        root_object.parse_docstring(self.docstring_parser, **context)

        if select_members is False:
            return root_object

        select_members = select_members or set()

        # Build the list of members
        members = {}
        inherited = set()
        direct_members = class_.__dict__
        all_members = dict(inspect.getmembers(class_))
        for member_name, member in all_members.items():
            if member is class_:
                continue
            if not (member is type or member is object) and self.select(member_name, select_members):  # type: ignore[arg-type]
                if member_name not in direct_members:
                    if self.select_inherited_members:
                        members[member_name] = member
                        inherited.add(member_name)
                else:
                    members[member_name] = member

        # Iterate on the selected members
        child: Object
        for member_name, member in members.items():
            child_node = ObjectNode(member, member_name, parent=node)
            if child_node.is_class():
                child = self.get_class_documentation(child_node)
            elif child_node.is_classmethod():
                child = self.get_classmethod_documentation(child_node)
            elif child_node.is_staticmethod():
                child = self.get_staticmethod_documentation(child_node)
            elif child_node.is_method():
                child = self.get_regular_method_documentation(child_node)
            elif child_node.is_property():
                child = self.get_property_documentation(child_node)
            elif member_name in attributes_data:
                child = self.get_attribute_documentation(child_node, attributes_data[member_name])
            else:
                continue
            if member_name in inherited:
                child.properties.append("inherited")
            root_object.add_child(child)

        for attr_name, properties, add_method in (
            ("__fields__", ["pydantic-model"], self.get_pydantic_field_documentation),
            ("_declared_fields", ["marshmallow-model"], self.get_marshmallow_field_documentation),
            ("_meta.get_fields", ["django-model"], self.get_django_field_documentation),
            ("__dataclass_fields__", ["dataclass"], self.get_annotated_dataclass_field),
        ):
            if self.detect_field_model(attr_name, direct_members, all_members):
                root_object.properties.extend(properties)
                self.add_fields(
                    node,
                    root_object,
                    attr_name,
                    all_members,
                    select_members,
                    class_,
                    add_method,
                )
                break

        return root_object

    def detect_field_model(self, attr_name: str, direct_members: Sequence[str], all_members: dict) -> bool:
        """Detect if an attribute is present in members.

        Arguments:
            attr_name: The name of the attribute to detect, can contain dots.
            direct_members: The direct members of the class.
            all_members: All members of the class.

        Returns:
            Whether the attribute is present.
        """
        first_order_attr_name, remainder = split_attr_name(attr_name)
        if not (
            first_order_attr_name in direct_members
            or (self.select_inherited_members and first_order_attr_name in all_members)
        ):
            return False

        if remainder:
            with suppress(AttributeError):
                return bool(attrgetter(remainder)(all_members[first_order_attr_name]))
        return True

    def add_fields(
        self,
        node: ObjectNode,
        root_object: Object,
        attr_name: str,
        members: Mapping[str, Any],
        select_members: Optional[Union[set[str], bool]],
        base_class: type,
        add_method: Callable,
    ) -> None:
        """Add detected fields to the current object.

        Arguments:
            node: The current object node.
            root_object: The current object.
            attr_name: The fields attribute name.
            members: The members to pick the fields attribute in.
            select_members: The members to select.
            base_class: The class declaring the fields.
            add_method: The method to add the children object.
        """
        fields = get_fields(attr_name, members=members)

        for field_name, field in fields.items():
            select_field = self.select(field_name, select_members)  # type: ignore[arg-type]
            is_inherited = field_is_inherited(field_name, attr_name, base_class)

            if select_field and (self.select_inherited_members or not is_inherited):
                child_node = ObjectNode(obj=field, name=field_name, parent=node)
                root_object.add_child(add_method(child_node))

    def get_function_documentation(self, node: ObjectNode) -> Function:
        """Get the documentation for a function.

        Arguments:
            node: The node representing the function and its parents.

        Returns:
            The documented function object.
        """
        function = node.obj
        source: Optional[Source]
        signature: Optional[inspect.Signature]

        try:
            signature = inspect.signature(function)
        except TypeError:
            signature = None

        try:
            source = Source(*inspect.getsourcelines(function))
        except OSError:
            source = None

        properties: list[str] = []
        if node.is_coroutine_function():
            properties.append("async")

        return Function(
            name=node.name,
            path=node.dotted_path,
            file_path=node.file_path,
            docstring=inspect.getdoc(function),
            signature=signature,
            source=source,
            properties=properties,
        )

    def get_property_documentation(self, node: ObjectNode) -> Attribute:
        """Get the documentation for a property.

        Arguments:
            node: The node representing the property and its parents.

        Returns:
            The documented attribute object (properties are considered attributes for now).
        """
        prop = node.obj
        path = node.dotted_path
        properties = ["property"]
        if node.is_cached_property():
            # cached_property is always writable, see the docs
            properties.extend(["writable", "cached"])
            sig_source_func = prop.func
        else:
            properties.append("readonly" if prop.fset is None else "writable")
            sig_source_func = prop.fget

        source: Optional[Source]

        try:
            signature = inspect.signature(sig_source_func)
        except (TypeError, ValueError):
            attr_type = None
        else:
            attr_type = signature.return_annotation

        try:
            source = Source(*inspect.getsourcelines(sig_source_func))
        except (OSError, TypeError):
            source = None

        return Attribute(
            name=node.name,
            path=path,
            file_path=node.file_path,
            docstring=inspect.getdoc(prop),
            attr_type=attr_type,
            properties=properties,
            source=source,
        )

    @staticmethod
    def get_pydantic_field_documentation(node: ObjectNode) -> Attribute:
        """Get the documentation for a Pydantic Field.

        Arguments:
            node: The node representing the Field and its parents.

        Returns:
            The documented attribute object.
        """
        prop = node.obj
        path = node.dotted_path
        properties = ["pydantic-field"]
        if prop.required:
            properties.append("required")

        return Attribute(
            name=node.name,
            path=path