From 054440d4367eefa93832523c13761ea5be78dad3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 20 Feb 2024 21:07:22 +0000 Subject: [PATCH 01/16] Add poetry support Fixes #43 --- changelog/43.breaking.rst | 3 + pyproject.toml | 5 +- requirements/base.txt | 1 + src/ptscripts/__init__.py | 27 +++- src/ptscripts/models.py | 237 ++++++++++++++++++++++++++++++++++++ src/ptscripts/parser.py | 158 ++++++------------------ src/ptscripts/utils.py | 34 ++++++ src/ptscripts/virtualenv.py | 109 +++-------------- 8 files changed, 351 insertions(+), 223 deletions(-) create mode 100644 changelog/43.breaking.rst create mode 100644 src/ptscripts/models.py create mode 100644 src/ptscripts/utils.py diff --git a/changelog/43.breaking.rst b/changelog/43.breaking.rst new file mode 100644 index 0000000..03e957c --- /dev/null +++ b/changelog/43.breaking.rst @@ -0,0 +1,3 @@ +Added support for using poetry and it's requirements groups. + +**THIS IS A BREAKING CHANGE AND ALL OLD CODE MUST BE UPDATED** diff --git a/pyproject.toml b/pyproject.toml index 18ae0fc..928bfd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,8 +79,6 @@ ignore = [ "ANN102", # Missing type annotation for `cls` in classmethod ] ignore-init-module-imports = true -# Group violations by containing file. -format = "grouped" [tool.ruff.per-file-ignores] "src/**/*.py" = [ @@ -100,6 +98,9 @@ format = "grouped" "ANN002", # Missing type annotation for `*args` "ANN003", # Missing type annotation for `**kwargs` ] +"src/ptscripts/models.py" = [ + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in * +] "src/ptscripts/virtualenv.py" = [ "PTH110", # `os.path.exists()` should be replaced by `Path.exists()` "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator diff --git a/requirements/base.txt b/requirements/base.txt index ade3442..952828d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,5 @@ attrs +pydantic >= 2.0 rich requests >= 2.31.0 typing-extensions; python_version < "3.11" diff --git a/src/ptscripts/__init__.py b/src/ptscripts/__init__.py index 20948cc..6ea0e52 100644 --- a/src/ptscripts/__init__.py +++ b/src/ptscripts/__init__.py @@ -5,17 +5,17 @@ import sys from typing import TYPE_CHECKING +from pydantic import NonNegativeFloat + import ptscripts.logs +from ptscripts.models import DefaultConfig +from ptscripts.models import VirtualEnvConfig from ptscripts.parser import Context from ptscripts.parser import DefaultToolsPythonRequirements from ptscripts.parser import DefaultVirtualEnv from ptscripts.parser import RegisteredImports from ptscripts.parser import command_group -if TYPE_CHECKING: - from ptscripts.parser import DefaultRequirementsConfig - from ptscripts.virtualenv import VirtualEnvConfig - __all__ = ["command_group", "register_tools_module", "Context"] @@ -23,6 +23,12 @@ def register_tools_module(import_module: str, venv_config: VirtualEnvConfig | No """ Register a module to be imported when instantiating the tools parser. """ + if venv_config and not isinstance(venv_config, VirtualEnvConfig): + msg = ( + "The 'venv_config' keyword argument must be an instance " + f"of '{VirtualEnvConfig.__module__}.VirtualEnvConfig'" + ) + raise RuntimeError(msg) RegisteredImports.register_import(import_module, venv_config=venv_config) @@ -30,11 +36,20 @@ def set_default_virtualenv_config(venv_config: VirtualEnvConfig) -> None: """ Define the default tools virtualenv configuration. """ + if venv_config and not isinstance(venv_config, VirtualEnvConfig): + msg = ( + "The 'venv_config' keyword argument must be an instance " + f"of '{VirtualEnvConfig.__module__}.VirtualEnvConfig'" + ) + raise RuntimeError(msg) DefaultVirtualEnv.set_default_virtualenv_config(venv_config) -def set_default_requirements_config(reqs_config: DefaultRequirementsConfig) -> None: +def set_default_config(config: DefaultConfig) -> None: """ Define the default tools requirements configuration. """ - DefaultToolsPythonRequirements.set_default_requirements_config(reqs_config) + if config and not isinstance(config, DefaultConfig): + msg = f"The 'config' keyword argument must be an instance of '{DefaultConfig.__module__}.DefaultConfig'" + raise RuntimeError(msg) + DefaultToolsPythonRequirements.set_default_requirements_config(config) diff --git a/src/ptscripts/models.py b/src/ptscripts/models.py new file mode 100644 index 0000000..03b1d5c --- /dev/null +++ b/src/ptscripts/models.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import hashlib +import os +import shutil +import sys +import tempfile +from functools import cached_property +from typing import TYPE_CHECKING +from typing import Any + +from pydantic import BaseModel +from pydantic import Field +from pydantic import model_validator + +from ptscripts.utils import cast_to_pathlib_path +from ptscripts.utils import file_digest + +if TYPE_CHECKING: + import pathlib + + from ptscripts.parser import Context + + +class Pip(BaseModel): + """ + Pip dependencies support. + """ + + requirements: list[str] = Field(default_factory=list) + requirements_files: list[pathlib.Path] = Field(default_factory=list) + install_args: list[str] = Field(default_factory=list) + + def get_config_hash(self) -> bytes: + """ + Return a hash digest of the configuration. + """ + config_hash = hashlib.sha256() + for argument in self.install_args: + config_hash.update(argument.encode()) + for requirement in sorted(self.requirements): + config_hash.update(requirement.encode()) + for fpath in sorted(self.requirements_files): + config_hash.update(file_digest(cast_to_pathlib_path(fpath))) + return config_hash.digest() + + def install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install requirements. + """ + if python_executable is None: + python_executable = sys.executable + args = [] + if self.requirements_files: + for fpath in self.requirements_files: + args.extend(["-r", str(fpath)]) + if self.requirements: + args.extend(self.requirements) + ctx.info("Installing base tools requirements ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + *self.install_args, + *args, + ) + + +class Poetry(BaseModel): + """ + Poetry dependencies support. + """ + + no_root: bool = Field(default=True) + groups: list[str] = Field(default_factory=list) + export_args: list[str] = Field(default_factory=list) + install_args: list[str] = Field(default_factory=list) + + def get_config_hash(self) -> bytes: + """ + Return a hash digest of the configuration. + """ + config_hash = hashlib.sha256() + config_hash.update(str(self.no_root).encode()) + for argument in self.export_args: + config_hash.update(argument.encode()) + for argument in self.install_args: + config_hash.update(argument.encode()) + for group in self.groups: + config_hash.update(group.encode()) + + # Late import to avoid circular import errors + from ptscripts.__main__ import CWD + + config_hash.update(file_digest(CWD / "poetry.lock")) + return config_hash.digest() + + def install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + if python_executable is None: + python_executable = sys.executable + with tempfile.NamedTemporaryFile(prefix="reqs-", suffix=".txt") as tfile: + args: list[str] = [] + if self.no_root is True: + param_name = "only" + else: + param_name = "with" + args.extend(f"--{param_name}={group}" for group in self.groups) + args.append(f"--output={tfile.name}") + poetry = shutil.which("poetry") + if poetry is None: + ctx.error("Did not find the 'poetry' binary in path") + ctx.exit(1) + ctx.info("Exporting requirements from poetry ...") + ctx.run(poetry, "export", *self.export_args, *args) + ctx.info("Installing requirements ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + *self.install_args, + "-r", + tfile.name, + ) + + +class DefaultConfig(BaseModel): + """ + Default tools configuration model. + """ + + pip: Pip = Field(default=None) + poetry: Poetry = Field(default=None) + + @model_validator(mode="before") + @classmethod + def _pip_or_poetry_not_both(cls, data: Any) -> Any: + if isinstance(data, dict) and "poetry" in data and "pip" in data: + msg = "Only configure 'pip' or 'poetry', not both." + raise ValueError(msg) + return data + + @cached_property + def config_hash(self) -> str: + """ + Returns a sha256 hash of the requirements. + """ + config_hash = hashlib.sha256() + # The first part of the hash should be the path to the tools executable + config_hash.update(sys.argv[0].encode()) + # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set + hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") + config_hash.update(hash_seed.encode()) + if self.pip: + config_hash.update(self.pip.get_config_hash()) + if self.poetry: + config_hash.update(self.poetry.get_config_hash()) + + return config_hash.hexdigest() + + def install(self, ctx: Context) -> None: + """ + Install default requirements. + """ + from ptscripts.__main__ import TOOLS_VENVS_PATH + + config_hash_file = TOOLS_VENVS_PATH / ".default-config.hash" + if config_hash_file.exists() and config_hash_file.read_text() == self.config_hash: + # Requirements are up to date + ctx.debug( + f"Base tools requirements haven't changed. Hash file: '{config_hash_file}'; " + f"Hash: '{self.config_hash}'" + ) + return + + if self.pip: + self.pip.install(ctx) + if self.poetry: + self.poetry.install(ctx) + config_hash_file.parent.mkdir(parents=True, exist_ok=True) + config_hash_file.write_text(self.config_hash) + ctx.debug(f"Wrote '{config_hash_file}' with contents: '{self.config_hash}'") + + +class VirtualEnvConfig(BaseModel): + """ + Virtualenv Configuration Typing. + """ + + name: str = Field(default=None) + env: dict[str, str] = Field(default=None) + system_site_packages: bool = Field(default=False) + pip_requirement: str = Field(default="pip>=22.3.1,<23.0") + setuptools_requirement: str = Field(default="setuptools>=65.6.3,<66") + poetry_requirement: str = Field(default=">=1.7") + add_as_extra_site_packages: bool = Field(default=False) + pip: Pip = Field(default=None) + poetry: Poetry = Field(default=None) + + @model_validator(mode="before") + @classmethod + def _pip_or_poetry_not_both(cls, data: Any) -> Any: + if isinstance(data, dict) and "poetry" in data and "pip" in data: + msg = "Only configure 'pip' or 'poetry', not both." + raise ValueError(msg) + return data + + def get_config_hash(self) -> str: + """ + Return a hash digest of the configuration. + """ + config_hash = hashlib.sha256() + # The first part of the hash should be the path to the tools executable + config_hash.update(sys.argv[0].encode()) + # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set + hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") + config_hash.update(hash_seed.encode()) + if self.pip: + config_hash.update(self.pip.get_config_hash()) + if self.poetry: + config_hash.update(self.poetry.get_config_hash()) + + return config_hash.hexdigest() + + def install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install requirements. + """ + if self.pip: + return self.pip.install(ctx, python_executable=python_executable) + if self.poetry: + return self.poetry.install(ctx, python_executable=python_executable) + return None diff --git a/src/ptscripts/parser.py b/src/ptscripts/parser.py index f10225f..df4aee8 100644 --- a/src/ptscripts/parser.py +++ b/src/ptscripts/parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import hashlib import importlib import inspect import logging @@ -16,7 +15,6 @@ from contextlib import AbstractContextManager from contextlib import contextmanager from contextlib import nullcontext -from functools import cached_property from functools import partial from subprocess import CompletedProcess from types import FunctionType @@ -28,7 +26,6 @@ from typing import TypeVar from typing import cast -import attr import requests import rich from rich.console import Console @@ -36,9 +33,8 @@ from ptscripts import logs from ptscripts import process +from ptscripts.models import VirtualEnvConfig from ptscripts.virtualenv import VirtualEnv -from ptscripts.virtualenv import VirtualEnvConfig -from ptscripts.virtualenv import _cast_to_pathlib_path if sys.version_info < (3, 10): from typing_extensions import Concatenate @@ -69,6 +65,8 @@ from argparse import _SubParsersAction from collections.abc import Iterator + from ptscripts.models import DefaultConfig + Param = ParamSpec("Param") RetType = TypeVar("RetType") @@ -103,92 +101,6 @@ class FullArgumentOptions(ArgumentOptions): type: type[Any] -@attr.s(frozen=True) -class DefaultRequirementsConfig: - """ - Default tools requirements configuration typing. - """ - - requirements: list[str] = attr.ib(factory=list) - requirements_files: list[pathlib.Path] = attr.ib(factory=list) - pip_args: list[str] = attr.ib(factory=list) - - @cached_property - def requirements_hash(self) -> str: - """ - Returns a sha256 hash of the requirements. - """ - requirements_hash = hashlib.sha256() - # The first part of the hash should be the path to the tools executable - requirements_hash.update(sys.argv[0].encode()) - # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set - hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") - requirements_hash.update(hash_seed.encode()) - # Third, any custom pip cli argument defined - if self.pip_args: - for argument in self.pip_args: - requirements_hash.update(argument.encode()) - # Forth, each passed requirement - if self.requirements: - for requirement in sorted(self.requirements): - requirements_hash.update(requirement.encode()) - # And, lastly, any requirements files passed in - if self.requirements_files: - for fpath in sorted(self.requirements_files): - with _cast_to_pathlib_path(fpath).open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = hashlib.sha256() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - requirements_hash.update(digest.digest()) - return requirements_hash.hexdigest() - - def install(self, ctx: Context) -> None: - """ - Install default requirements. - """ - from ptscripts.__main__ import TOOLS_VENVS_PATH - - requirements_hash_file = TOOLS_VENVS_PATH / ".default-requirements.hash" - if ( - requirements_hash_file.exists() - and requirements_hash_file.read_text() == self.requirements_hash - ): - # Requirements are up to date - ctx.debug( - f"Base tools requirements haven't changed. Hash file: '{requirements_hash_file}'; " - f"Hash: '{self.requirements_hash}'" - ) - return - requirements = [] - if self.requirements_files: - for fpath in self.requirements_files: - requirements.extend(["-r", str(fpath)]) - if self.requirements: - requirements.extend(self.requirements) - if requirements: - ctx.info("Installing base tools requirements ...") - ctx.run( - sys.executable, - "-m", - "pip", - "install", - *self.pip_args, - *requirements, - ) - requirements_hash_file.parent.mkdir(parents=True, exist_ok=True) - requirements_hash_file.write_text(self.requirements_hash) - ctx.debug(f"Wrote '{requirements_hash_file}' with contents: '{self.requirements_hash}'") - - class Context: """ Context class passed to every command group function as the first argument. @@ -348,18 +260,15 @@ def chdir(self, path: pathlib.Path) -> Iterator[pathlib.Path]: os.chdir(cwd) @contextmanager - def virtualenv( - self, - name: str, - requirements: list[str] | None = None, - requirements_files: list[pathlib.Path] | None = None, - ) -> Iterator[VirtualEnv]: + def virtualenv(self, name: str, config: VirtualEnvConfig | None = None) -> Iterator[VirtualEnv]: """ Create and use a virtual environment. """ - with VirtualEnv( - ctx=self, name=name, requirements=requirements, requirements_files=requirements_files - ) as venv: + if config is None: + config = VirtualEnvConfig(name=name) + elif config.name is None: + config.name = name + with VirtualEnv(ctx=self, config=config) as venv: yield venv @property @@ -405,7 +314,7 @@ class DefaultToolsPythonRequirements: """ _instance: DefaultToolsPythonRequirements | None = None - reqs_config: DefaultRequirementsConfig | None + config: DefaultConfig | None def __new__(cls) -> DefaultToolsPythonRequirements: """ @@ -413,19 +322,19 @@ def __new__(cls) -> DefaultToolsPythonRequirements: """ if cls._instance is None: instance = super().__new__(cls) - instance.reqs_config = None + instance.config = None cls._instance = instance return cls._instance @classmethod - def set_default_requirements_config(cls, reqs_config: DefaultRequirementsConfig) -> None: + def set_default_requirements_config(cls, config: DefaultConfig) -> None: """ Set the default tools requirements configuration. """ instance = cls._instance if instance is None: instance = cls() - instance.reqs_config = reqs_config + instance.config = config class RegisteredImports: @@ -565,26 +474,26 @@ def __new__(cls) -> Parser: return cls._instance def _process_registered_tool_modules(self) -> None: - default_reqs_config = DefaultToolsPythonRequirements().reqs_config - if default_reqs_config: - default_reqs_config.install(self.context) + default_config = DefaultToolsPythonRequirements().config + if default_config: + default_config.install(self.context) default_venv: VirtualEnv | AbstractContextManager[None] default_venv_config = DefaultVirtualEnv().venv_config if default_venv_config: - if "name" not in default_venv_config: - default_venv_config["name"] = "default" - default_venv_config["add_as_extra_site_packages"] = True - default_venv = VirtualEnv(ctx=self.context, **default_venv_config) + if not default_venv_config.name: + default_venv_config.name = "default" + default_venv_config.add_as_extra_site_packages = True + default_venv = VirtualEnv(ctx=self.context, config=default_venv_config) else: default_venv = nullcontext() with default_venv: for module_name, venv_config in RegisteredImports(): venv: VirtualEnv | AbstractContextManager[None] if venv_config: - if "name" not in venv_config: - venv_config["name"] = module_name - venv = VirtualEnv(ctx=self.context, **venv_config) + if not venv_config.name: + venv_config.name = module_name + venv = VirtualEnv(ctx=self.context, config=venv_config) else: venv = nullcontext() with venv: @@ -688,7 +597,7 @@ def __init__( help: str, description: str | None = None, parent: Parser | CommandGroup | list[str] | tuple[str] | str | None = None, - venv_config: VirtualEnvConfig | None = None, + venv_config: VirtualEnvConfig | dict[str, Any] | None = None, ) -> None: self.name = name if description is None: @@ -705,9 +614,14 @@ def __init__( GroupReference.add_command((*parent, name), self) parent = GroupReference()[tuple(parent)] - if venv_config and "name" not in venv_config: - venv_config["name"] = self.name - self.venv_config = venv_config or {} + if venv_config is None: + venv_config = VirtualEnvConfig(name=self.name) + elif isinstance(venv_config, dict): + venv_config = VirtualEnvConfig(**venv_config) + if venv_config.name is None: + venv_config.name = self.name + self.venv_config: VirtualEnvConfig = venv_config + if TYPE_CHECKING: assert parent self.parser = parent.subparsers.add_parser( # type: ignore[union-attr, has-type] @@ -886,11 +800,11 @@ def __call__( bound = signature.bind_partial(*args, **kwargs) venv: VirtualEnv | None = None if venv_config: - if "name" not in venv_config: - venv_config["name"] = getattr(options, f"{self.name}_command") - venv = VirtualEnv(ctx=self.context, **venv_config) + if venv_config.name is None: + venv_config.name = getattr(options, f"{self.name}_command") + venv = VirtualEnv(ctx=self.context, config=venv_config) elif self.venv_config: - venv = VirtualEnv(ctx=self.context, **self.venv_config) + venv = VirtualEnv(ctx=self.context, config=self.venv_config) if venv: with venv: previous_venv = self.context.venv diff --git a/src/ptscripts/utils.py b/src/ptscripts/utils.py new file mode 100644 index 0000000..16d4ac7 --- /dev/null +++ b/src/ptscripts/utils.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import hashlib +import pathlib +from typing import cast + + +def cast_to_pathlib_path(value: str | pathlib.Path) -> pathlib.Path: + """ + Cast passed in string to an instance of `pathlib.Path`. + """ + if isinstance(value, pathlib.Path): + return value + return pathlib.Path(str(value)) + + +def file_digest(path: pathlib.Path) -> bytes: + """ + Return a SHA256 digest of a file. + """ + with path.open("rb") as rfh: + try: + digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] + except AttributeError: + # Python < 3.11 + buf = bytearray(2**18) # Reusable buffer to reduce allocations. + view = memoryview(buf) + digest = hashlib.sha256() + while True: + size = rfh.readinto(buf) + if size == 0: + break # EOF + digest.update(view[:size]) + return cast(bytes, digest.digest()) diff --git a/src/ptscripts/virtualenv.py b/src/ptscripts/virtualenv.py index b768691..4b840b4 100644 --- a/src/ptscripts/virtualenv.py +++ b/src/ptscripts/virtualenv.py @@ -1,10 +1,8 @@ from __future__ import annotations -import hashlib import json import logging import os -import pathlib import shutil import subprocess import sys @@ -12,74 +10,31 @@ from subprocess import CompletedProcess from typing import TYPE_CHECKING -if sys.version_info < (3, 11): - from typing_extensions import NotRequired - from typing_extensions import TypedDict -else: - from typing import NotRequired - from typing import TypedDict - import attr if TYPE_CHECKING: + import pathlib + + from ptscripts.models import VirtualEnvConfig from ptscripts.parser import Context log = logging.getLogger(__name__) -class VirtualEnvConfig(TypedDict): - """ - Virtualenv Configuration Typing. - """ - - name: NotRequired[str] - requirements: NotRequired[list[str]] - requirements_files: NotRequired[list[pathlib.Path]] - env: NotRequired[dict[str, str]] - system_site_packages: NotRequired[bool] - pip_requirement: NotRequired[str] - setuptools_requirement: NotRequired[str] - add_as_extra_site_packages: NotRequired[bool] - pip_args: NotRequired[list[str]] - - -def _cast_to_pathlib_path(value: str | pathlib.Path) -> pathlib.Path: - if isinstance(value, pathlib.Path): - return value - return pathlib.Path(str(value)) - - @attr.s(frozen=True, slots=True) class VirtualEnv: """ Helper class to create and user virtual environments. """ - name: str = attr.ib() ctx: Context = attr.ib() - requirements: list[str] | None = attr.ib(repr=False, default=None) - requirements_files: list[pathlib.Path] | None = attr.ib(repr=False, default=None) - env: dict[str, str] | None = attr.ib(default=None) - system_site_packages: bool = attr.ib(default=False) - pip_requirement: str = attr.ib(repr=False) - setuptools_requirement: str = attr.ib(repr=False) - add_as_extra_site_packages: bool = attr.ib(default=False) - pip_args: list[str] = attr.ib(factory=list, repr=False) + config: VirtualEnvConfig = attr.ib() environ: dict[str, str] = attr.ib(init=False, repr=False) venv_dir: pathlib.Path = attr.ib(init=False) venv_python: pathlib.Path = attr.ib(init=False, repr=False) venv_bin_dir: pathlib.Path = attr.ib(init=False, repr=False) requirements_hash: str = attr.ib(init=False, repr=False) - @pip_requirement.default - def _default_pip_requiremnt(self) -> str: - return "pip>=22.3.1,<23.0" - - @setuptools_requirement.default - def _default_setuptools_requirement(self) -> str: - # https://github.com/pypa/setuptools/commit/137ab9d684075f772c322f455b0dd1f992ddcd8f - return "setuptools>=65.6.3,<66" - @venv_dir.default def _default_venv_dir(self) -> pathlib.Path: # Late import to avoid circular import errors @@ -87,13 +42,13 @@ def _default_venv_dir(self) -> pathlib.Path: venvs_path = TOOLS_VENVS_PATH venvs_path.mkdir(parents=True, exist_ok=True) - return venvs_path / self.name + return venvs_path / self.config.name @environ.default def _default_environ(self) -> dict[str, str]: environ = os.environ.copy() - if self.env: - environ.update(self.env) + if self.config.env: + environ.update(self.config.env) return environ @venv_python.default @@ -108,31 +63,7 @@ def _default_venv_bin_dir(self) -> pathlib.Path: @requirements_hash.default def __default_requirements_hash(self) -> str: - requirements_hash = hashlib.sha256(self.name.encode()) - hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") - requirements_hash.update(hash_seed.encode()) - if self.pip_args: - requirements_hash.update(str(sorted(self.pip_args)).encode()) - if self.requirements: - for requirement in sorted(self.requirements): - requirements_hash.update(requirement.encode()) - if self.requirements_files: - for fpath in sorted(self.requirements_files): - with _cast_to_pathlib_path(fpath).open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = hashlib.sha256() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - requirements_hash.update(digest.digest()) - return requirements_hash.hexdigest() + return self.config.get_config_hash() def _install_requirements(self) -> None: requirements_hash_file = self.venv_dir / ".requirements.hash" @@ -141,17 +72,9 @@ def _install_requirements(self) -> None: and requirements_hash_file.read_text() == self.requirements_hash ): # Requirements are up to date - self.ctx.debug(f"Requirements for virtualenv({self.name}) haven't changed.") + self.ctx.debug(f"Requirements for virtualenv({self.config.name}) haven't changed.") return - requirements = [] - if self.requirements_files: - for fpath in sorted(self.requirements_files): - requirements.extend(["-r", str(fpath)]) - if self.requirements: - requirements.extend(sorted(self.requirements)) - if requirements: - self.ctx.info(f"Install requirements for virtualenv({self.name}) ...") - self.install(*self.pip_args, *requirements) + self.config.install(self.ctx, python_executable=str(self.venv_python)) self.venv_dir.joinpath(".requirements.hash").write_text(self.requirements_hash) def _create_virtualenv(self) -> None: @@ -189,24 +112,24 @@ def _create_virtualenv(self) -> None: "-m", "venv", ] - if self.system_site_packages: + if self.config.system_site_packages: cmd.append("--system-site-packages") cmd.append(str(self.venv_dir)) try: relative_venv_path = self.venv_dir.relative_to(CWD) except ValueError: relative_venv_path = self.venv_dir - self.ctx.info(f"Creating virtualenv({self.name}) in {relative_venv_path}") + self.ctx.info(f"Creating virtualenv({self.config.name}) in {relative_venv_path}") self.run(*cmd, cwd=str(self.venv_dir.parent)) self.install( "-U", "wheel", - self.pip_requirement, - self.setuptools_requirement, + self.config.pip_requirement, + self.config.setuptools_requirement, ) def _add_as_extra_site_packages(self) -> None: - if self.add_as_extra_site_packages is False: + if self.config.add_as_extra_site_packages is False: return ret = self.run_code( "import json,site; print(json.dumps(site.getsitepackages()))", @@ -223,7 +146,7 @@ def _add_as_extra_site_packages(self) -> None: sys.path.append(path) def _remove_extra_site_packages(self) -> None: - if self.add_as_extra_site_packages is False: + if self.config.add_as_extra_site_packages is False: return ret = self.run_code( "import json,site; print(json.dumps(site.getsitepackages()))", From 9e4ba2b0ec2781f5e806b4eefa5813787a477662 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 21 Feb 2024 05:32:59 +0000 Subject: [PATCH 02/16] Simplify how venv configs are passed in. Fix typing. --- src/ptscripts/__init__.py | 36 ++++++++++++++++++------------------ src/ptscripts/parser.py | 36 +++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/ptscripts/__init__.py b/src/ptscripts/__init__.py index 6ea0e52..9d35a69 100644 --- a/src/ptscripts/__init__.py +++ b/src/ptscripts/__init__.py @@ -4,6 +4,7 @@ import pathlib import sys from typing import TYPE_CHECKING +from typing import Any from pydantic import NonNegativeFloat @@ -19,37 +20,36 @@ __all__ = ["command_group", "register_tools_module", "Context"] -def register_tools_module(import_module: str, venv_config: VirtualEnvConfig | None = None) -> None: +def register_tools_module( + import_module: str, venv_config: VirtualEnvConfig | dict[str, Any] | None = None +) -> None: """ Register a module to be imported when instantiating the tools parser. """ - if venv_config and not isinstance(venv_config, VirtualEnvConfig): - msg = ( - "The 'venv_config' keyword argument must be an instance " - f"of '{VirtualEnvConfig.__module__}.VirtualEnvConfig'" - ) - raise RuntimeError(msg) + if venv_config and isinstance(venv_config, dict): + venv_config = VirtualEnvConfig(**venv_config) + if TYPE_CHECKING: + assert isinstance(venv_config, VirtualEnvConfig) RegisteredImports.register_import(import_module, venv_config=venv_config) -def set_default_virtualenv_config(venv_config: VirtualEnvConfig) -> None: +def set_default_virtualenv_config(venv_config: VirtualEnvConfig | dict[str, Any]) -> None: """ Define the default tools virtualenv configuration. """ - if venv_config and not isinstance(venv_config, VirtualEnvConfig): - msg = ( - "The 'venv_config' keyword argument must be an instance " - f"of '{VirtualEnvConfig.__module__}.VirtualEnvConfig'" - ) - raise RuntimeError(msg) + if venv_config and isinstance(venv_config, dict): + venv_config = VirtualEnvConfig(**venv_config) + if TYPE_CHECKING: + assert isinstance(venv_config, VirtualEnvConfig) DefaultVirtualEnv.set_default_virtualenv_config(venv_config) -def set_default_config(config: DefaultConfig) -> None: +def set_default_config(config: DefaultConfig | dict[str, Any]) -> None: """ Define the default tools requirements configuration. """ - if config and not isinstance(config, DefaultConfig): - msg = f"The 'config' keyword argument must be an instance of '{DefaultConfig.__module__}.DefaultConfig'" - raise RuntimeError(msg) + if config and isinstance(config, dict): + config = DefaultConfig(**config) + if TYPE_CHECKING: + assert isinstance(config, DefaultConfig) DefaultToolsPythonRequirements.set_default_requirements_config(config) diff --git a/src/ptscripts/parser.py b/src/ptscripts/parser.py index df4aee8..a895e51 100644 --- a/src/ptscripts/parser.py +++ b/src/ptscripts/parser.py @@ -260,13 +260,20 @@ def chdir(self, path: pathlib.Path) -> Iterator[pathlib.Path]: os.chdir(cwd) @contextmanager - def virtualenv(self, name: str, config: VirtualEnvConfig | None = None) -> Iterator[VirtualEnv]: + def virtualenv( + self, name: str, config: VirtualEnvConfig | dict[str, Any] | None = None + ) -> Iterator[VirtualEnv]: """ Create and use a virtual environment. """ if config is None: config = VirtualEnvConfig(name=name) - elif config.name is None: + elif isinstance(config, dict): + config = VirtualEnvConfig(**config) + if TYPE_CHECKING: + assert isinstance(config, VirtualEnvConfig) + + if config.name is None: config.name = name with VirtualEnv(ctx=self, config=config) as venv: yield venv @@ -597,7 +604,7 @@ def __init__( help: str, description: str | None = None, parent: Parser | CommandGroup | list[str] | tuple[str] | str | None = None, - venv_config: VirtualEnvConfig | dict[str, Any] | None = None, + venv_config: VirtualEnvConfig | None = None, ) -> None: self.name = name if description is None: @@ -614,13 +621,10 @@ def __init__( GroupReference.add_command((*parent, name), self) parent = GroupReference()[tuple(parent)] - if venv_config is None: - venv_config = VirtualEnvConfig(name=self.name) - elif isinstance(venv_config, dict): - venv_config = VirtualEnvConfig(**venv_config) - if venv_config.name is None: + if venv_config is not None and venv_config.name is None: venv_config.name = self.name - self.venv_config: VirtualEnvConfig = venv_config + + self.venv_config: VirtualEnvConfig | None = venv_config if TYPE_CHECKING: assert parent @@ -644,7 +648,7 @@ def command( # noqa: ANN201,C901,PLR0912,PLR0915 help: str | None = None, description: str | None = None, arguments: dict[str, ArgumentOptions] | None = None, - venv_config: VirtualEnvConfig | None = None, + venv_config: VirtualEnvConfig | dict[str, Any] | None = None, ): """ Register a sub-command in the command group. @@ -766,6 +770,12 @@ def command( # noqa: ANN201,C901,PLR0912,PLR0915 flags = [f"--{parameter.name.replace('_', '-')}"] log.debug("Adding Command %r. Flags: %s; KwArgs: %s", name, flags, kwargs) command.add_argument(*flags, **kwargs) # type: ignore[arg-type] + + if venv_config and isinstance(venv_config, dict): + venv_config = VirtualEnvConfig(**venv_config) + if TYPE_CHECKING: + assert isinstance(venv_config, VirtualEnvConfig) + command.set_defaults(func=partial(self, func, venv_config=venv_config)) return func @@ -821,10 +831,14 @@ def command_group( name: str, help: str, description: str | None = None, - venv_config: VirtualEnvConfig | None = None, + venv_config: VirtualEnvConfig | dict[str, Any] | None = None, parent: Parser | CommandGroup | list[str] | tuple[str] | str | None = None, ) -> CommandGroup: """ Create a new command group. """ + if venv_config and isinstance(venv_config, dict): + venv_config = VirtualEnvConfig(**venv_config) + if TYPE_CHECKING: + assert isinstance(venv_config, VirtualEnvConfig) return CommandGroup(name, help, description=description, venv_config=venv_config, parent=parent) From 02ec1c775897dd806ee58bddd9bc5d09eeb9db6c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 21 Feb 2024 05:34:55 +0000 Subject: [PATCH 03/16] Generate Changelog for version 0.20.0rc2 --- CHANGELOG.rst | 11 +++++++++++ changelog/43.breaking.rst | 3 --- 2 files changed, 11 insertions(+), 3 deletions(-) delete mode 100644 changelog/43.breaking.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 73897b2..c2a0b86 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,17 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.0rc2 (2024-02-21) +====================== + +Breaking Changes +---------------- + +- Added support for using poetry and it's requirements groups. + + **THIS IS A BREAKING CHANGE AND ALL OLD CODE MUST BE UPDATED** (`#43 `_) + + 0.18.6 (2023-11-26) =================== diff --git a/changelog/43.breaking.rst b/changelog/43.breaking.rst deleted file mode 100644 index 03e957c..0000000 --- a/changelog/43.breaking.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added support for using poetry and it's requirements groups. - -**THIS IS A BREAKING CHANGE AND ALL OLD CODE MUST BE UPDATED** From 428e756ed68c50f949ec6e71bcd95f0e28740c47 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 21 Feb 2024 05:44:16 +0000 Subject: [PATCH 04/16] Prevent having to rebuild the pydantic modules due to typing forward referencing --- pyproject.toml | 1 + src/ptscripts/models.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 928bfd8..21a0f45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ ignore-init-module-imports = true ] "src/ptscripts/models.py" = [ "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in * + "TCH003", # Move standard library import `*` into a type-checking block" ] "src/ptscripts/virtualenv.py" = [ "PTH110", # `os.path.exists()` should be replaced by `Path.exists()` diff --git a/src/ptscripts/models.py b/src/ptscripts/models.py index 03b1d5c..0bd6b98 100644 --- a/src/ptscripts/models.py +++ b/src/ptscripts/models.py @@ -2,6 +2,7 @@ import hashlib import os +import pathlib import shutil import sys import tempfile @@ -17,8 +18,6 @@ from ptscripts.utils import file_digest if TYPE_CHECKING: - import pathlib - from ptscripts.parser import Context From e62f5b2de10a0b666ee1e9a9e939c7b4ea24f0f3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 21 Feb 2024 05:45:22 +0000 Subject: [PATCH 05/16] Generate Changelog for version 0.20.0rc3 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c2a0b86..0367976 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.0rc3 (2024-02-21) +====================== + +No significant changes. + + 0.20.0rc2 (2024-02-21) ====================== From 2adb2445efb62b7d1987077e89a022479e54f150 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 22 Feb 2024 12:54:38 +0000 Subject: [PATCH 06/16] Require pydantic models to configure virtualenvs --- src/ptscripts/__init__.py | 32 +++--------- src/ptscripts/models.py | 103 +++++++++++++++++++++----------------- src/ptscripts/parser.py | 25 ++------- 3 files changed, 69 insertions(+), 91 deletions(-) diff --git a/src/ptscripts/__init__.py b/src/ptscripts/__init__.py index 9d35a69..33b3193 100644 --- a/src/ptscripts/__init__.py +++ b/src/ptscripts/__init__.py @@ -1,55 +1,37 @@ from __future__ import annotations -import os -import pathlib -import sys from typing import TYPE_CHECKING -from typing import Any - -from pydantic import NonNegativeFloat import ptscripts.logs -from ptscripts.models import DefaultConfig -from ptscripts.models import VirtualEnvConfig from ptscripts.parser import Context from ptscripts.parser import DefaultToolsPythonRequirements from ptscripts.parser import DefaultVirtualEnv from ptscripts.parser import RegisteredImports from ptscripts.parser import command_group +if TYPE_CHECKING: + from ptscripts.models import DefaultConfig + from ptscripts.models import VirtualEnvConfig + __all__ = ["command_group", "register_tools_module", "Context"] -def register_tools_module( - import_module: str, venv_config: VirtualEnvConfig | dict[str, Any] | None = None -) -> None: +def register_tools_module(import_module: str, venv_config: VirtualEnvConfig | None = None) -> None: """ Register a module to be imported when instantiating the tools parser. """ - if venv_config and isinstance(venv_config, dict): - venv_config = VirtualEnvConfig(**venv_config) - if TYPE_CHECKING: - assert isinstance(venv_config, VirtualEnvConfig) RegisteredImports.register_import(import_module, venv_config=venv_config) -def set_default_virtualenv_config(venv_config: VirtualEnvConfig | dict[str, Any]) -> None: +def set_default_virtualenv_config(venv_config: VirtualEnvConfig) -> None: """ Define the default tools virtualenv configuration. """ - if venv_config and isinstance(venv_config, dict): - venv_config = VirtualEnvConfig(**venv_config) - if TYPE_CHECKING: - assert isinstance(venv_config, VirtualEnvConfig) DefaultVirtualEnv.set_default_virtualenv_config(venv_config) -def set_default_config(config: DefaultConfig | dict[str, Any]) -> None: +def set_default_config(config: DefaultConfig) -> None: """ Define the default tools requirements configuration. """ - if config and isinstance(config, dict): - config = DefaultConfig(**config) - if TYPE_CHECKING: - assert isinstance(config, DefaultConfig) DefaultToolsPythonRequirements.set_default_requirements_config(config) diff --git a/src/ptscripts/models.py b/src/ptscripts/models.py index 0bd6b98..c93c94e 100644 --- a/src/ptscripts/models.py +++ b/src/ptscripts/models.py @@ -8,11 +8,9 @@ import tempfile from functools import cached_property from typing import TYPE_CHECKING -from typing import Any from pydantic import BaseModel from pydantic import Field -from pydantic import model_validator from ptscripts.utils import cast_to_pathlib_path from ptscripts.utils import file_digest @@ -21,7 +19,7 @@ from ptscripts.parser import Context -class Pip(BaseModel): +class _PipMixin(BaseModel): """ Pip dependencies support. """ @@ -30,7 +28,7 @@ class Pip(BaseModel): requirements_files: list[pathlib.Path] = Field(default_factory=list) install_args: list[str] = Field(default_factory=list) - def get_config_hash(self) -> bytes: + def _get_config_hash(self) -> bytes: """ Return a hash digest of the configuration. """ @@ -43,7 +41,7 @@ def get_config_hash(self) -> bytes: config_hash.update(file_digest(cast_to_pathlib_path(fpath))) return config_hash.digest() - def install(self, ctx: Context, python_executable: str | None = None) -> None: + def _install(self, ctx: Context, python_executable: str | None = None) -> None: """ Install requirements. """ @@ -66,7 +64,7 @@ def install(self, ctx: Context, python_executable: str | None = None) -> None: ) -class Poetry(BaseModel): +class _PoetryMixin(BaseModel): """ Poetry dependencies support. """ @@ -76,9 +74,9 @@ class Poetry(BaseModel): export_args: list[str] = Field(default_factory=list) install_args: list[str] = Field(default_factory=list) - def get_config_hash(self) -> bytes: + def _get_config_hash(self) -> bytes: """ - Return a hash digest of the configuration. + Return a hash of the configuration. """ config_hash = hashlib.sha256() config_hash.update(str(self.no_root).encode()) @@ -95,7 +93,7 @@ def get_config_hash(self) -> bytes: config_hash.update(file_digest(CWD / "poetry.lock")) return config_hash.digest() - def install(self, ctx: Context, python_executable: str | None = None) -> None: + def _install(self, ctx: Context, python_executable: str | None = None) -> None: """ Install default requirements. """ @@ -132,16 +130,17 @@ class DefaultConfig(BaseModel): Default tools configuration model. """ - pip: Pip = Field(default=None) - poetry: Poetry = Field(default=None) + def _get_config_hash(self) -> bytes: + """ + Return a hash of the configuration. + """ + raise NotImplementedError - @model_validator(mode="before") - @classmethod - def _pip_or_poetry_not_both(cls, data: Any) -> Any: - if isinstance(data, dict) and "poetry" in data and "pip" in data: - msg = "Only configure 'pip' or 'poetry', not both." - raise ValueError(msg) - return data + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + raise NotImplementedError @cached_property def config_hash(self) -> str: @@ -154,11 +153,7 @@ def config_hash(self) -> str: # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") config_hash.update(hash_seed.encode()) - if self.pip: - config_hash.update(self.pip.get_config_hash()) - if self.poetry: - config_hash.update(self.poetry.get_config_hash()) - + config_hash.update(self._get_config_hash()) return config_hash.hexdigest() def install(self, ctx: Context) -> None: @@ -176,15 +171,25 @@ def install(self, ctx: Context) -> None: ) return - if self.pip: - self.pip.install(ctx) - if self.poetry: - self.poetry.install(ctx) + self._install(ctx) + config_hash_file.parent.mkdir(parents=True, exist_ok=True) config_hash_file.write_text(self.config_hash) ctx.debug(f"Wrote '{config_hash_file}' with contents: '{self.config_hash}'") +class DefaultPipConfig(_PipMixin, DefaultConfig): + """ + Default tools pip configuration model. + """ + + +class DefaultPoetryConfig(_PoetryMixin, DefaultConfig): + """ + Default tools poetry configuration model. + """ + + class VirtualEnvConfig(BaseModel): """ Virtualenv Configuration Typing. @@ -197,16 +202,18 @@ class VirtualEnvConfig(BaseModel): setuptools_requirement: str = Field(default="setuptools>=65.6.3,<66") poetry_requirement: str = Field(default=">=1.7") add_as_extra_site_packages: bool = Field(default=False) - pip: Pip = Field(default=None) - poetry: Poetry = Field(default=None) - @model_validator(mode="before") - @classmethod - def _pip_or_poetry_not_both(cls, data: Any) -> Any: - if isinstance(data, dict) and "poetry" in data and "pip" in data: - msg = "Only configure 'pip' or 'poetry', not both." - raise ValueError(msg) - return data + def _get_config_hash(self) -> bytes: + """ + Return a hash of the configuration. + """ + raise NotImplementedError + + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + raise NotImplementedError def get_config_hash(self) -> str: """ @@ -218,19 +225,23 @@ def get_config_hash(self) -> str: # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") config_hash.update(hash_seed.encode()) - if self.pip: - config_hash.update(self.pip.get_config_hash()) - if self.poetry: - config_hash.update(self.poetry.get_config_hash()) - + config_hash.update(self._get_config_hash()) return config_hash.hexdigest() def install(self, ctx: Context, python_executable: str | None = None) -> None: """ Install requirements. """ - if self.pip: - return self.pip.install(ctx, python_executable=python_executable) - if self.poetry: - return self.poetry.install(ctx, python_executable=python_executable) - return None + return self._install(ctx, python_executable=python_executable) + + +class VirtualEnvPipConfig(_PipMixin, VirtualEnvConfig): + """ + Virtualenv pip configuration. + """ + + +class VirtualEnvPoetryConfig(_PoetryMixin, VirtualEnvConfig): + """ + Virtualenv poetry configuration. + """ diff --git a/src/ptscripts/parser.py b/src/ptscripts/parser.py index a895e51..80768f8 100644 --- a/src/ptscripts/parser.py +++ b/src/ptscripts/parser.py @@ -34,6 +34,7 @@ from ptscripts import logs from ptscripts import process from ptscripts.models import VirtualEnvConfig +from ptscripts.models import VirtualEnvPipConfig from ptscripts.virtualenv import VirtualEnv if sys.version_info < (3, 10): @@ -260,19 +261,12 @@ def chdir(self, path: pathlib.Path) -> Iterator[pathlib.Path]: os.chdir(cwd) @contextmanager - def virtualenv( - self, name: str, config: VirtualEnvConfig | dict[str, Any] | None = None - ) -> Iterator[VirtualEnv]: + def virtualenv(self, name: str, config: VirtualEnvConfig | None = None) -> Iterator[VirtualEnv]: """ Create and use a virtual environment. """ if config is None: - config = VirtualEnvConfig(name=name) - elif isinstance(config, dict): - config = VirtualEnvConfig(**config) - if TYPE_CHECKING: - assert isinstance(config, VirtualEnvConfig) - + config = VirtualEnvPipConfig(name=name) if config.name is None: config.name = name with VirtualEnv(ctx=self, config=config) as venv: @@ -648,7 +642,7 @@ def command( # noqa: ANN201,C901,PLR0912,PLR0915 help: str | None = None, description: str | None = None, arguments: dict[str, ArgumentOptions] | None = None, - venv_config: VirtualEnvConfig | dict[str, Any] | None = None, + venv_config: VirtualEnvConfig | None = None, ): """ Register a sub-command in the command group. @@ -771,11 +765,6 @@ def command( # noqa: ANN201,C901,PLR0912,PLR0915 log.debug("Adding Command %r. Flags: %s; KwArgs: %s", name, flags, kwargs) command.add_argument(*flags, **kwargs) # type: ignore[arg-type] - if venv_config and isinstance(venv_config, dict): - venv_config = VirtualEnvConfig(**venv_config) - if TYPE_CHECKING: - assert isinstance(venv_config, VirtualEnvConfig) - command.set_defaults(func=partial(self, func, venv_config=venv_config)) return func @@ -831,14 +820,10 @@ def command_group( name: str, help: str, description: str | None = None, - venv_config: VirtualEnvConfig | dict[str, Any] | None = None, + venv_config: VirtualEnvConfig | None = None, parent: Parser | CommandGroup | list[str] | tuple[str] | str | None = None, ) -> CommandGroup: """ Create a new command group. """ - if venv_config and isinstance(venv_config, dict): - venv_config = VirtualEnvConfig(**venv_config) - if TYPE_CHECKING: - assert isinstance(venv_config, VirtualEnvConfig) return CommandGroup(name, help, description=description, venv_config=venv_config, parent=parent) From 5c0a4692561255128d2d8b680ceee4f41ebc95f9 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 22 Feb 2024 12:55:41 +0000 Subject: [PATCH 07/16] Generate Changelog for version 0.20.0rc4 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0367976..f8af944 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.0rc4 (2024-02-22) +====================== + +No significant changes. + + 0.20.0rc3 (2024-02-21) ====================== From 1274cf2ecc913a5b6e7b11cef3062fe7e6cb097c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 22 Feb 2024 16:05:54 +0000 Subject: [PATCH 08/16] Generate Changelog for version 0.20.0 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f8af944..00363d4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.0 (2024-02-22) +=================== + +No significant changes. + + 0.20.0rc4 (2024-02-22) ====================== From 017c9711ff08542038afb65baf72dae84d32f6a5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:40:12 +0000 Subject: [PATCH 09/16] Add poetry as an extra when installing --- .pre-commit-hooks.yaml | 11 +++++++++++ requirements/poetry.txt | 2 ++ setup.cfg | 1 + 3 files changed, 14 insertions(+) create mode 100644 requirements/poetry.txt diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 80c0ea9..185a44a 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -6,3 +6,14 @@ args: [] require_serial: true additional_dependencies: [] + +- id: tools-poetry + name: tools + entry: tools + language: python + types: [file] + args: [] + require_serial: true + additional_dependencies: + - poetry >= "8.0", + - poetry-plugin-export diff --git a/requirements/poetry.txt b/requirements/poetry.txt new file mode 100644 index 0000000..fe2c318 --- /dev/null +++ b/requirements/poetry.txt @@ -0,0 +1,2 @@ +poetry >= 8.0 +poetry-plugin-export diff --git a/setup.cfg b/setup.cfg index 58e9480..a5fd07f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,7 @@ extras_require = lint = requirements/lint.txt tests = requirements/tests.txt changelog = requirements/changelog.txt + poetry = requirements/poetry.txt [options.entry_points] console_scripts = From 98b9936106bb51b19cfa13d5d23f9f5b1fe4c814 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:41:06 +0000 Subject: [PATCH 10/16] Generate Changelog for version 0.20.1 --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 00363d4..57f0045 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,13 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.1 (2024-02-28) +=================== + +Added poetry as an extra, ie, `python -m pip install python-tools-scripts[poetry]`. +Do note however that, if `poetry` is in path, the extra does not need to be provided. + + 0.20.0 (2024-02-22) =================== From fa1c91834921b609c8060223a7a34b548afec1ed Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:46:50 +0000 Subject: [PATCH 11/16] Fix additional requirements --- .pre-commit-hooks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 185a44a..9d2dd76 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -15,5 +15,5 @@ args: [] require_serial: true additional_dependencies: - - poetry >= "8.0", + - poetry>=8.0 - poetry-plugin-export From ea2ad2e77fba488ad8388d23a4d87da9dcbd6922 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:48:14 +0000 Subject: [PATCH 12/16] Generate Changelog for version 0.20.2 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 57f0045..2c8deeb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.2 (2024-02-28) +=================== + +No significant changes. + + 0.20.1 (2024-02-28) =================== From 8d1c5ca115bc367e7fa5ea5ae23a4104bb36980c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:49:57 +0000 Subject: [PATCH 13/16] Poetry version fix --- .pre-commit-hooks.yaml | 2 +- requirements/poetry.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 9d2dd76..b4d1d65 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -15,5 +15,5 @@ args: [] require_serial: true additional_dependencies: - - poetry>=8.0 + - poetry>=1.8.0 - poetry-plugin-export diff --git a/requirements/poetry.txt b/requirements/poetry.txt index fe2c318..152d1ef 100644 --- a/requirements/poetry.txt +++ b/requirements/poetry.txt @@ -1,2 +1,2 @@ -poetry >= 8.0 +poetry >= 1.8.0 poetry-plugin-export From ecc1a5b3ac68b6ead993fd02e85bd85778a7c906 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 28 Feb 2024 16:50:23 +0000 Subject: [PATCH 14/16] Generate Changelog for version 0.20.3 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c8deeb..ac1f249 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.3 (2024-02-28) +=================== + +No significant changes. + + 0.20.2 (2024-02-28) =================== From 3b5defad6b3f25cc61fb4b4df6b3e4f04d66e34f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 29 Feb 2024 09:34:45 +0000 Subject: [PATCH 15/16] More poetry support fixes --- .pre-commit-hooks.yaml | 5 +- CHANGELOG.rst | 8 +++ requirements/base.txt | 1 + src/ptscripts/models.py | 114 +++++++++++++++++++++++++----------- src/ptscripts/virtualenv.py | 44 ++++++++++---- 5 files changed, 122 insertions(+), 50 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b4d1d65..660d207 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,7 +4,7 @@ language: python types: [file] args: [] - require_serial: true + require_serial: false additional_dependencies: [] - id: tools-poetry @@ -13,7 +13,6 @@ language: python types: [file] args: [] - require_serial: true + require_serial: false additional_dependencies: - poetry>=1.8.0 - - poetry-plugin-export diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac1f249..4ff09e9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,14 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.4 (2024-02-29) +=================== + +Fix the poetry support. +Added a lockfile while setting up the virtual environments. This is specially important when +using the tools as a pre-commit hook since on a clean system, we just one on of the invocations +to setup the virtual environment, not all concurrent invocations. + 0.20.3 (2024-02-28) =================== diff --git a/requirements/base.txt b/requirements/base.txt index 952828d..4d36304 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,4 +2,5 @@ attrs pydantic >= 2.0 rich requests >= 2.31.0 +filelock typing-extensions; python_version < "3.11" diff --git a/src/ptscripts/models.py b/src/ptscripts/models.py index c93c94e..b761cce 100644 --- a/src/ptscripts/models.py +++ b/src/ptscripts/models.py @@ -3,9 +3,7 @@ import hashlib import os import pathlib -import shutil import sys -import tempfile from functools import cached_property from typing import TYPE_CHECKING @@ -27,12 +25,16 @@ class _PipMixin(BaseModel): requirements: list[str] = Field(default_factory=list) requirements_files: list[pathlib.Path] = Field(default_factory=list) install_args: list[str] = Field(default_factory=list) + pip_requirement: str = Field(default="pip>=22.3.1,<23.0") + setuptools_requirement: str = Field(default="setuptools>=65.6.3,<66") def _get_config_hash(self) -> bytes: """ Return a hash digest of the configuration. """ config_hash = hashlib.sha256() + config_hash.update(self.pip_requirement.encode()) + config_hash.update(self.setuptools_requirement.encode()) for argument in self.install_args: config_hash.update(argument.encode()) for requirement in sorted(self.requirements): @@ -63,6 +65,22 @@ def _install(self, ctx: Context, python_executable: str | None = None) -> None: *args, ) + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + if python_executable is None: + python_executable = sys.executable + ctx.info("Setting up virtual environment ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + self.pip_requirement, + self.setuptools_requirement, + ) + class _PoetryMixin(BaseModel): """ @@ -71,17 +89,16 @@ class _PoetryMixin(BaseModel): no_root: bool = Field(default=True) groups: list[str] = Field(default_factory=list) - export_args: list[str] = Field(default_factory=list) install_args: list[str] = Field(default_factory=list) + poetry_requirement: str = Field(default="poetry>=1.8.0") def _get_config_hash(self) -> bytes: """ Return a hash of the configuration. """ config_hash = hashlib.sha256() + config_hash.update(self.poetry_requirement.encode()) config_hash.update(str(self.no_root).encode()) - for argument in self.export_args: - config_hash.update(argument.encode()) for argument in self.install_args: config_hash.update(argument.encode()) for group in self.groups: @@ -99,33 +116,39 @@ def _install(self, ctx: Context, python_executable: str | None = None) -> None: """ if python_executable is None: python_executable = sys.executable - with tempfile.NamedTemporaryFile(prefix="reqs-", suffix=".txt") as tfile: - args: list[str] = [] - if self.no_root is True: - param_name = "only" - else: - param_name = "with" - args.extend(f"--{param_name}={group}" for group in self.groups) - args.append(f"--output={tfile.name}") - poetry = shutil.which("poetry") - if poetry is None: - ctx.error("Did not find the 'poetry' binary in path") - ctx.exit(1) - ctx.info("Exporting requirements from poetry ...") - ctx.run(poetry, "export", *self.export_args, *args) - ctx.info("Installing requirements ...") - ctx.run( - python_executable, - "-m", - "pip", - "install", - *self.install_args, - "-r", - tfile.name, - ) + args: list[str] = [] + if self.no_root is True: + args.append("--no-root") + args.extend(f"--with={group}" for group in self.groups) + ctx.info("Installing requirements ...") + ctx.run(python_executable, "-m", "poetry", "install", *self.install_args, *args) + + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + if python_executable is None: + python_executable = sys.executable + ctx.info("Setting up virtual environment ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + *self.install_args, + self.poetry_requirement, + ) + + +class _LockTimeoutMixin(BaseModel): + """ + Mixin class to provide the lock timeout field. + """ + lock_timeout_seconds: int = Field(default=5 * 60) -class DefaultConfig(BaseModel): + +class DefaultConfig(_LockTimeoutMixin, BaseModel): """ Default tools configuration model. """ @@ -142,6 +165,12 @@ def _install(self, ctx: Context, python_executable: str | None = None) -> None: """ raise NotImplementedError + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + raise NotImplementedError + @cached_property def config_hash(self) -> str: """ @@ -177,6 +206,12 @@ def install(self, ctx: Context) -> None: config_hash_file.write_text(self.config_hash) ctx.debug(f"Wrote '{config_hash_file}' with contents: '{self.config_hash}'") + def setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + self._setup_venv(ctx, python_executable=python_executable) + class DefaultPipConfig(_PipMixin, DefaultConfig): """ @@ -190,7 +225,7 @@ class DefaultPoetryConfig(_PoetryMixin, DefaultConfig): """ -class VirtualEnvConfig(BaseModel): +class VirtualEnvConfig(_LockTimeoutMixin, BaseModel): """ Virtualenv Configuration Typing. """ @@ -198,9 +233,6 @@ class VirtualEnvConfig(BaseModel): name: str = Field(default=None) env: dict[str, str] = Field(default=None) system_site_packages: bool = Field(default=False) - pip_requirement: str = Field(default="pip>=22.3.1,<23.0") - setuptools_requirement: str = Field(default="setuptools>=65.6.3,<66") - poetry_requirement: str = Field(default=">=1.7") add_as_extra_site_packages: bool = Field(default=False) def _get_config_hash(self) -> bytes: @@ -215,6 +247,12 @@ def _install(self, ctx: Context, python_executable: str | None = None) -> None: """ raise NotImplementedError + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + raise NotImplementedError + def get_config_hash(self) -> str: """ Return a hash digest of the configuration. @@ -232,7 +270,13 @@ def install(self, ctx: Context, python_executable: str | None = None) -> None: """ Install requirements. """ - return self._install(ctx, python_executable=python_executable) + self._install(ctx, python_executable=python_executable) + + def setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + self._setup_venv(ctx, python_executable=python_executable) class VirtualEnvPipConfig(_PipMixin, VirtualEnvConfig): diff --git a/src/ptscripts/virtualenv.py b/src/ptscripts/virtualenv.py index 4b840b4..fe1c6bd 100644 --- a/src/ptscripts/virtualenv.py +++ b/src/ptscripts/virtualenv.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING import attr +from filelock import FileLock if TYPE_CHECKING: import pathlib @@ -34,6 +35,7 @@ class VirtualEnv: venv_python: pathlib.Path = attr.ib(init=False, repr=False) venv_bin_dir: pathlib.Path = attr.ib(init=False, repr=False) requirements_hash: str = attr.ib(init=False, repr=False) + lockfile: FileLock = attr.ib(init=False, repr=False) @venv_dir.default def _default_venv_dir(self) -> pathlib.Path: @@ -65,6 +67,15 @@ def _default_venv_bin_dir(self) -> pathlib.Path: def __default_requirements_hash(self) -> str: return self.config.get_config_hash() + @lockfile.default + def __lockfile(self) -> FileLock: + # Late import to avoid circular import errors + from ptscripts.__main__ import CWD + + return FileLock( + CWD / f".venv-{self.config.name}.lock", timeout=self.config.lock_timeout_seconds + ) + def _install_requirements(self) -> None: requirements_hash_file = self.venv_dir / ".requirements.hash" if ( @@ -121,12 +132,7 @@ def _create_virtualenv(self) -> None: relative_venv_path = self.venv_dir self.ctx.info(f"Creating virtualenv({self.config.name}) in {relative_venv_path}") self.run(*cmd, cwd=str(self.venv_dir.parent)) - self.install( - "-U", - "wheel", - self.config.pip_requirement, - self.config.setuptools_requirement, - ) + self.setup() def _add_as_extra_site_packages(self) -> None: if self.config.add_as_extra_site_packages is False: @@ -162,10 +168,7 @@ def _remove_extra_site_packages(self) -> None: if path in sys.path: sys.path.remove(path) - def __enter__(self) -> VirtualEnv: - """ - Creates the virtual environment when entering context. - """ + def _enter(self) -> VirtualEnv: try: self._create_virtualenv() except subprocess.CalledProcessError: @@ -185,11 +188,28 @@ def __enter__(self) -> VirtualEnv: self._add_as_extra_site_packages() return self - def __exit__(self, *args) -> None: + def _exit(self) -> None: + self._remove_extra_site_packages() + + def __enter__(self) -> VirtualEnv: + """ + Creates the virtual environment when entering context. + """ + with self.lockfile: + return self._enter() + + def __exit__(self, *_) -> None: """ Exit the virtual environment context. """ - self._remove_extra_site_packages() + with self.lockfile: + self._exit() + + def setup(self) -> None: + """ + Setup the virtual environment. + """ + self.config.setup_venv(self.ctx, python_executable=str(self.venv_python)) def install(self, *args: str, **kwargs) -> CompletedProcess[bytes]: """ From dee6091ba46f3cf427541b46e9643136fb172495 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 29 Feb 2024 10:13:07 +0000 Subject: [PATCH 16/16] Change the path for lock files --- CHANGELOG.rst | 5 +++++ src/ptscripts/virtualenv.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4ff09e9..34d58e3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.5 (2024-02-29) +=================== + +Change the path for lock files + 0.20.4 (2024-02-29) =================== diff --git a/src/ptscripts/virtualenv.py b/src/ptscripts/virtualenv.py index fe1c6bd..a8befed 100644 --- a/src/ptscripts/virtualenv.py +++ b/src/ptscripts/virtualenv.py @@ -70,10 +70,11 @@ def __default_requirements_hash(self) -> str: @lockfile.default def __lockfile(self) -> FileLock: # Late import to avoid circular import errors - from ptscripts.__main__ import CWD + from ptscripts.__main__ import TOOLS_VENVS_PATH return FileLock( - CWD / f".venv-{self.config.name}.lock", timeout=self.config.lock_timeout_seconds + TOOLS_VENVS_PATH / "locks" / f"{self.config.name}.lock", + timeout=self.config.lock_timeout_seconds, ) def _install_requirements(self) -> None: