Add support for PEP-735 dependency groups. (#2584)

You can now specify one or more `--group <name>@<project_dir>` as
sources of requirements when building a PEX or creating a lock.

See: https://peps.python.org/pep-0735

Author: jsirois

CHANGES.md

@@ -1,5 +1,17 @@
 # Release Notes
 
+## 2.23.0
+
+This release adds support for drawing requirements from
+[PEP-735][PEP-735] dependency groups when creating PEXes or lock files.
+Groups are requested via `--group <name>@<project dir>` or just
+`--group <name>` if the project directory is the current working
+directory.
+
+* Add support for PEP-735 dependency groups. (#2584)
+
+[PEP-735]: https://peps.python.org/pep-0735/
+
 ## 2.22.0
 
 This release adds support for `--pip-version 24.3.1`.

pex/bin/pex.py

@@ -746,7 +746,7 @@ def configure_clp_sources(parser):
 
     project.register_options(
         parser,
-        help=(
+        project_help=(
             "Add the local project at the specified path to the generated .pex file along with "
             "its transitive dependencies."
         ),
@@ -1016,6 +1016,14 @@ def build_pex(
         else resolver_configuration.pip_configuration
     )
 
+    group_requirements = project.get_group_requirements(options)
+    if group_requirements:
+        requirements = OrderedSet(requirement_configuration.requirements)
+        requirements.update(str(req) for req in group_requirements)
+        requirement_configuration = attr.evolve(
+            requirement_configuration, requirements=requirements
+        )
+
     project_dependencies = OrderedSet()  # type: OrderedSet[Requirement]
     with TRACER.timed(
         "Adding distributions built from local projects and collecting their requirements: "

pex/build_system/pep_518.py

@@ -6,6 +6,7 @@
 import os.path
 import subprocess
 
+from pex import toml
 from pex.build_system import DEFAULT_BUILD_BACKEND
 from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode
 from pex.dist_metadata import Distribution
@@ -26,9 +27,8 @@
     from typing import Iterable, Mapping, Optional, Tuple, Union
 
     import attr  # vendor:skip
-    import toml  # vendor:skip
 else:
-    from pex.third_party import attr, toml
+    from pex.third_party import attr
 
 
 @attr.s(frozen=True)
@@ -43,8 +43,7 @@ def _read_build_system_table(
 ):
     # type: (...) -> Union[Optional[BuildSystemTable], Error]
     try:
-        with open(pyproject_toml) as fp:
-            data = toml.load(fp)
+        data = toml.load(pyproject_toml)
     except toml.TomlDecodeError as e:
         return Error(
             "Problem parsing toml in {pyproject_toml}: {err}".format(

pex/cli/commands/lock.py

@@ -416,7 +416,7 @@ def _add_resolve_options(cls, parser):
         requirement_options.register(options_group)
         project.register_options(
             options_group,
-            help=(
+            project_help=(
                 "Add the transitive dependencies of the local project at the specified path to "
                 "the lock but do not lock project itself."
             ),
@@ -846,25 +846,28 @@ def _gather_requirements(
     ):
         # type: (...) -> RequirementConfiguration
         requirement_configuration = requirement_options.configure(self.options)
+        group_requirements = project.get_group_requirements(self.options)
         projects = project.get_projects(self.options)
-        if not projects:
+        if not projects and not group_requirements:
             return requirement_configuration
 
         requirements = OrderedSet(requirement_configuration.requirements)
-        with TRACER.timed(
-            "Collecting requirements from {count} local {projects}".format(
-                count=len(projects), projects=pluralize(projects, "project")
-            )
-        ):
-            requirements.update(
-                str(req)
-                for req in projects.collect_requirements(
-                    resolver=ConfiguredResolver(pip_configuration),
-                    interpreter=targets.interpreter,
-                    pip_version=pip_configuration.version,
-                    max_jobs=pip_configuration.max_jobs,
+        requirements.update(str(req) for req in group_requirements)
+        if projects:
+            with TRACER.timed(
+                "Collecting requirements from {count} local {projects}".format(
+                    count=len(projects), projects=pluralize(projects, "project")
+                )
+            ):
+                requirements.update(
+                    str(req)
+                    for req in projects.collect_requirements(
+                        resolver=ConfiguredResolver(pip_configuration),
+                        interpreter=targets.interpreter,
+                        pip_version=pip_configuration.version,
+                        max_jobs=pip_configuration.max_jobs,
+                    )
                 )
-            )
         return attr.evolve(requirement_configuration, requirements=requirements)
 
     def _create(self):

pex/pep_723.py

@@ -7,6 +7,7 @@
 import re
 from collections import OrderedDict
 
+from pex import toml
 from pex.common import pluralize
 from pex.compatibility import string
 from pex.dist_metadata import Requirement, RequirementParseError
@@ -17,9 +18,8 @@
     from typing import Any, List, Mapping, Tuple
 
     import attr  # vendor:skip
-    import toml  # vendor:skip
 else:
-    from pex.third_party import attr, toml
+    from pex.third_party import attr
 
 
 _UNSPECIFIED_SOURCE = "<unspecified source>"

pex/resolve/project.py

@@ -3,17 +3,21 @@
 
 from __future__ import absolute_import
 
+import os.path
 from argparse import Namespace, _ActionsContainer
 
-from pex import requirements
+from pex import requirements, toml
 from pex.build_system import pep_517
 from pex.common import pluralize
+from pex.compatibility import string
 from pex.dependency_configuration import DependencyConfiguration
-from pex.dist_metadata import DistMetadata, Requirement
+from pex.dist_metadata import DistMetadata, Requirement, RequirementParseError
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.interpreter import PythonInterpreter
 from pex.jobs import Raise, SpawnedJob, execute_parallel
+from pex.orderedset import OrderedSet
 from pex.pep_427 import InstallableType
+from pex.pep_503 import ProjectName
 from pex.pip.version import PipVersionValue
 from pex.requirements import LocalProjectRequirement, ParseError
 from pex.resolve.configured_resolve import resolve
@@ -25,7 +29,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, Iterator, List, Optional, Set, Tuple
+    from typing import Any, Iterable, Iterator, List, Mapping, Optional, Set, Tuple, Union
 
     import attr  # vendor:skip
 else:
@@ -148,9 +152,147 @@ def __len__(self):
         return len(self.projects)
 
 
+@attr.s(frozen=True)
+class GroupName(ProjectName):
+    # N.B.: A dependency group name follows the same rules, including canonicalization, as project
+    # names.
+    pass
+
+
+@attr.s(frozen=True)
+class DependencyGroup(object):
+    @classmethod
+    def parse(cls, spec):
+        # type: (str) -> DependencyGroup
+
+        group, sep, project_dir = spec.partition("@")
+        abs_project_dir = os.path.realpath(project_dir)
+        if not os.path.isdir(abs_project_dir):
+            raise ValueError(
+                "The project directory specified by '{spec}' is not a directory".format(spec=spec)
+            )
+
+        pyproject_toml = os.path.join(abs_project_dir, "pyproject.toml")
+        if not os.path.isfile(pyproject_toml):
+            raise ValueError(
+                "The project directory specified by '{spec}' does not contain a pyproject.toml "
+                "file".format(spec=spec)
+            )
+
+        group_name = GroupName(group)
+        try:
+            dependency_groups = {
+                GroupName(name): group
+                for name, group in toml.load(pyproject_toml)["dependency-groups"].items()
+            }  # type: Mapping[GroupName, Any]
+        except (IOError, OSError, KeyError, ValueError, AttributeError) as e:
+            raise ValueError(
+                "Failed to read `[dependency-groups]` metadata from {pyproject_toml} when parsing "
+                "dependency group spec '{spec}': {err}".format(
+                    pyproject_toml=pyproject_toml, spec=spec, err=e
+                )
+            )
+        if group_name not in dependency_groups:
+            raise KeyError(
+                "The dependency group '{group}' specified by '{spec}' does not exist in "
+                "{pyproject_toml}".format(group=group, spec=spec, pyproject_toml=pyproject_toml)
+            )
+
+        return cls(project_dir=abs_project_dir, name=group_name, groups=dependency_groups)
+
+    project_dir = attr.ib()  # type: str
+    name = attr.ib()  # type: GroupName
+    _groups = attr.ib()  # type: Mapping[GroupName, Any]
+
+    def _parse_group_items(
+        self,
+        group,  # type: GroupName
+        required_by=None,  # type: Optional[GroupName]
+    ):
+        # type: (...) -> Iterator[Union[GroupName, Requirement]]
+
+        members = self._groups.get(group)
+        if not members:
+            if not required_by:
+                raise KeyError(
+                    "The dependency group '{group}' does not exist in the project at "
+                    "{project_dir}.".format(group=group, project_dir=self.project_dir)
+                )
+            else:
+                raise KeyError(
+                    "The dependency group '{group}' required by dependency group '{required_by}' "
+                    "does not exist in the project at {project_dir}.".format(
+                        group=group, required_by=required_by, project_dir=self.project_dir
+                    )
+                )
+
+        if not isinstance(members, list):
+            raise ValueError(
+                "Invalid dependency group '{group}' in the project at {project_dir}.\n"
+                "The value must be a list containing dependency specifiers or dependency group "
+                "includes.\n"
+                "See https://peps.python.org/pep-0735/#specification for the specification "
+                "of [dependency-groups] syntax."
+            )
+
+        for index, item in enumerate(members, start=1):
+            if isinstance(item, string):
+                try:
+                    yield Requirement.parse(item)
+                except RequirementParseError as e:
+                    raise ValueError(
+                        "Invalid [dependency-group] entry '{name}'.\n"
+                        "Item {index}: '{req}', is an invalid dependency specifier: {err}".format(
+                            name=group.raw, index=index, req=item, err=e
+                        )
+                    )
+            elif isinstance(item, dict):
+                try:
+                    yield GroupName(item["include-group"])
+                except KeyError:
+                    raise ValueError(
+                        "Invalid [dependency-group] entry '{name}'.\n"
+                        "Item {index} is a non 'include-group' table and only dependency "
+                        "specifiers and single entry 'include-group' tables are allowed in group "
+                        "dependency lists.\n"
+                        "See https://peps.python.org/pep-0735/#specification for the specification "
+                        "of [dependency-groups] syntax.\n"
+                        "Given: {item}".format(name=group.raw, index=index, item=item)
+                    )
+            else:
+                raise ValueError(
+                    "Invalid [dependency-group] entry '{name}'.\n"
+                    "Item {index} is not a dependency specifier or a dependency group include.\n"
+                    "See https://peps.python.org/pep-0735/#specification for the specification "
+                    "of [dependency-groups] syntax.\n"
+                    "Given: {item}".format(name=group.raw, index=index, item=item)
+                )
+
+    def iter_requirements(self):
+        # type: () -> Iterator[Requirement]
+
+        visited_groups = set()  # type: Set[GroupName]
+
+        def iter_group(
+            group,  # type: GroupName
+            required_by=None,  # type: Optional[GroupName]
+        ):
+            # type: (...) -> Iterator[Requirement]
+            if group not in visited_groups:
+                visited_groups.add(group)
+                for item in self._parse_group_items(group, required_by=required_by):
+                    if isinstance(item, Requirement):
+                        yield item
+                    else:
+                        for req in iter_group(item, required_by=group):
+                            yield req
+
+        return iter_group(self.name)
+
+
 def register_options(
     parser,  # type: _ActionsContainer
-    help,  # type: str
+    project_help,  # type: str
 ):
     # type: (...) -> None
 
@@ -161,7 +303,27 @@ def register_options(
         default=[],
         type=str,
         action="append",
-        help=help,
+        help=project_help,
+    )
+
+    parser.add_argument(
+        "--group",
+        "--dependency-group",
+        dest="dependency_groups",
+        metavar="GROUP[@DIR]",
+        default=[],
+        type=DependencyGroup.parse,
+        action="append",
+        help=(
+            "Pull requirements from the specified PEP-735 dependency group. Dependency groups are "
+            "specified by referencing the group name in a given project's pyproject.toml in the "
+            "form `<group name>@<project directory>`; e.g.: `test@local/project/directory`. If "
+            "either the `@<project directory>` suffix is not present or the suffix is just `@`, "
+            "the current working directory is assumed to be the project directory to read the "
+            "dependency group information from. Multiple dependency groups across any number of "
+            "projects can be specified. Read more about dependency groups at "
+            "https://peps.python.org/pep-0735/."
+        ),
     )
 
 
@@ -207,3 +369,13 @@ def get_projects(options):
         )
 
     return Projects(projects=tuple(projects))
+
+
+def get_group_requirements(options):
+    # type: (Namespace) -> Iterable[Requirement]
+
+    group_requirements = OrderedSet()  # type: OrderedSet[Requirement]
+    for dependency_group in options.dependency_groups:
+        for requirement in dependency_group.iter_requirements():
+            group_requirements.add(requirement)
+    return group_requirements