diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 80c0ea9..660d207 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,5 +4,15 @@ language: python types: [file] args: [] - require_serial: true + require_serial: false additional_dependencies: [] + +- id: tools-poetry + name: tools + entry: tools + language: python + types: [file] + args: [] + require_serial: false + additional_dependencies: + - poetry>=1.8.0 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 73897b2..34d58e3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,67 @@ Backward incompatible (breaking) changes will only be introduced in major versio .. towncrier release notes start +0.20.5 (2024-02-29) +=================== + +Change the path for lock files + +0.20.4 (2024-02-29) +=================== + +Fix the poetry support. +Added a lockfile while setting up the virtual environments. This is specially important when +using the tools as a pre-commit hook since on a clean system, we just one on of the invocations +to setup the virtual environment, not all concurrent invocations. + +0.20.3 (2024-02-28) +=================== + +No significant changes. + + +0.20.2 (2024-02-28) +=================== + +No significant changes. + + +0.20.1 (2024-02-28) +=================== + +Added poetry as an extra, ie, `python -m pip install python-tools-scripts[poetry]`. +Do note however that, if `poetry` is in path, the extra does not need to be provided. + + +0.20.0 (2024-02-22) +=================== + +No significant changes. + + +0.20.0rc4 (2024-02-22) +====================== + +No significant changes. + + +0.20.0rc3 (2024-02-21) +====================== + +No significant changes. + + +0.20.0rc2 (2024-02-21) +====================== + +Breaking Changes +---------------- + +- Added support for using poetry and it's requirements groups. + + **THIS IS A BREAKING CHANGE AND ALL OLD CODE MUST BE UPDATED** (`#43 `_) + + 0.18.6 (2023-11-26) =================== diff --git a/pyproject.toml b/pyproject.toml index 18ae0fc..21a0f45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,8 +79,6 @@ ignore = [ "ANN102", # Missing type annotation for `cls` in classmethod ] ignore-init-module-imports = true -# Group violations by containing file. -format = "grouped" [tool.ruff.per-file-ignores] "src/**/*.py" = [ @@ -100,6 +98,10 @@ format = "grouped" "ANN002", # Missing type annotation for `*args` "ANN003", # Missing type annotation for `**kwargs` ] +"src/ptscripts/models.py" = [ + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in * + "TCH003", # Move standard library import `*` into a type-checking block" +] "src/ptscripts/virtualenv.py" = [ "PTH110", # `os.path.exists()` should be replaced by `Path.exists()` "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator diff --git a/requirements/base.txt b/requirements/base.txt index ade3442..4d36304 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,6 @@ attrs +pydantic >= 2.0 rich requests >= 2.31.0 +filelock typing-extensions; python_version < "3.11" diff --git a/requirements/poetry.txt b/requirements/poetry.txt new file mode 100644 index 0000000..152d1ef --- /dev/null +++ b/requirements/poetry.txt @@ -0,0 +1,2 @@ +poetry >= 1.8.0 +poetry-plugin-export diff --git a/setup.cfg b/setup.cfg index 58e9480..a5fd07f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,7 @@ extras_require = lint = requirements/lint.txt tests = requirements/tests.txt changelog = requirements/changelog.txt + poetry = requirements/poetry.txt [options.entry_points] console_scripts = diff --git a/src/ptscripts/__init__.py b/src/ptscripts/__init__.py index 20948cc..33b3193 100644 --- a/src/ptscripts/__init__.py +++ b/src/ptscripts/__init__.py @@ -1,8 +1,5 @@ from __future__ import annotations -import os -import pathlib -import sys from typing import TYPE_CHECKING import ptscripts.logs @@ -13,8 +10,8 @@ from ptscripts.parser import command_group if TYPE_CHECKING: - from ptscripts.parser import DefaultRequirementsConfig - from ptscripts.virtualenv import VirtualEnvConfig + from ptscripts.models import DefaultConfig + from ptscripts.models import VirtualEnvConfig __all__ = ["command_group", "register_tools_module", "Context"] @@ -33,8 +30,8 @@ def set_default_virtualenv_config(venv_config: VirtualEnvConfig) -> None: DefaultVirtualEnv.set_default_virtualenv_config(venv_config) -def set_default_requirements_config(reqs_config: DefaultRequirementsConfig) -> None: +def set_default_config(config: DefaultConfig) -> None: """ Define the default tools requirements configuration. """ - DefaultToolsPythonRequirements.set_default_requirements_config(reqs_config) + DefaultToolsPythonRequirements.set_default_requirements_config(config) diff --git a/src/ptscripts/models.py b/src/ptscripts/models.py new file mode 100644 index 0000000..b761cce --- /dev/null +++ b/src/ptscripts/models.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import hashlib +import os +import pathlib +import sys +from functools import cached_property +from typing import TYPE_CHECKING + +from pydantic import BaseModel +from pydantic import Field + +from ptscripts.utils import cast_to_pathlib_path +from ptscripts.utils import file_digest + +if TYPE_CHECKING: + from ptscripts.parser import Context + + +class _PipMixin(BaseModel): + """ + Pip dependencies support. + """ + + requirements: list[str] = Field(default_factory=list) + requirements_files: list[pathlib.Path] = Field(default_factory=list) + install_args: list[str] = Field(default_factory=list) + pip_requirement: str = Field(default="pip>=22.3.1,<23.0") + setuptools_requirement: str = Field(default="setuptools>=65.6.3,<66") + + def _get_config_hash(self) -> bytes: + """ + Return a hash digest of the configuration. + """ + config_hash = hashlib.sha256() + config_hash.update(self.pip_requirement.encode()) + config_hash.update(self.setuptools_requirement.encode()) + for argument in self.install_args: + config_hash.update(argument.encode()) + for requirement in sorted(self.requirements): + config_hash.update(requirement.encode()) + for fpath in sorted(self.requirements_files): + config_hash.update(file_digest(cast_to_pathlib_path(fpath))) + return config_hash.digest() + + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install requirements. + """ + if python_executable is None: + python_executable = sys.executable + args = [] + if self.requirements_files: + for fpath in self.requirements_files: + args.extend(["-r", str(fpath)]) + if self.requirements: + args.extend(self.requirements) + ctx.info("Installing base tools requirements ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + *self.install_args, + *args, + ) + + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + if python_executable is None: + python_executable = sys.executable + ctx.info("Setting up virtual environment ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + self.pip_requirement, + self.setuptools_requirement, + ) + + +class _PoetryMixin(BaseModel): + """ + Poetry dependencies support. + """ + + no_root: bool = Field(default=True) + groups: list[str] = Field(default_factory=list) + install_args: list[str] = Field(default_factory=list) + poetry_requirement: str = Field(default="poetry>=1.8.0") + + def _get_config_hash(self) -> bytes: + """ + Return a hash of the configuration. + """ + config_hash = hashlib.sha256() + config_hash.update(self.poetry_requirement.encode()) + config_hash.update(str(self.no_root).encode()) + for argument in self.install_args: + config_hash.update(argument.encode()) + for group in self.groups: + config_hash.update(group.encode()) + + # Late import to avoid circular import errors + from ptscripts.__main__ import CWD + + config_hash.update(file_digest(CWD / "poetry.lock")) + return config_hash.digest() + + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + if python_executable is None: + python_executable = sys.executable + args: list[str] = [] + if self.no_root is True: + args.append("--no-root") + args.extend(f"--with={group}" for group in self.groups) + ctx.info("Installing requirements ...") + ctx.run(python_executable, "-m", "poetry", "install", *self.install_args, *args) + + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + if python_executable is None: + python_executable = sys.executable + ctx.info("Setting up virtual environment ...") + ctx.run( + python_executable, + "-m", + "pip", + "install", + *self.install_args, + self.poetry_requirement, + ) + + +class _LockTimeoutMixin(BaseModel): + """ + Mixin class to provide the lock timeout field. + """ + + lock_timeout_seconds: int = Field(default=5 * 60) + + +class DefaultConfig(_LockTimeoutMixin, BaseModel): + """ + Default tools configuration model. + """ + + def _get_config_hash(self) -> bytes: + """ + Return a hash of the configuration. + """ + raise NotImplementedError + + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + raise NotImplementedError + + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + raise NotImplementedError + + @cached_property + def config_hash(self) -> str: + """ + Returns a sha256 hash of the requirements. + """ + config_hash = hashlib.sha256() + # The first part of the hash should be the path to the tools executable + config_hash.update(sys.argv[0].encode()) + # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set + hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") + config_hash.update(hash_seed.encode()) + config_hash.update(self._get_config_hash()) + return config_hash.hexdigest() + + def install(self, ctx: Context) -> None: + """ + Install default requirements. + """ + from ptscripts.__main__ import TOOLS_VENVS_PATH + + config_hash_file = TOOLS_VENVS_PATH / ".default-config.hash" + if config_hash_file.exists() and config_hash_file.read_text() == self.config_hash: + # Requirements are up to date + ctx.debug( + f"Base tools requirements haven't changed. Hash file: '{config_hash_file}'; " + f"Hash: '{self.config_hash}'" + ) + return + + self._install(ctx) + + config_hash_file.parent.mkdir(parents=True, exist_ok=True) + config_hash_file.write_text(self.config_hash) + ctx.debug(f"Wrote '{config_hash_file}' with contents: '{self.config_hash}'") + + def setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + self._setup_venv(ctx, python_executable=python_executable) + + +class DefaultPipConfig(_PipMixin, DefaultConfig): + """ + Default tools pip configuration model. + """ + + +class DefaultPoetryConfig(_PoetryMixin, DefaultConfig): + """ + Default tools poetry configuration model. + """ + + +class VirtualEnvConfig(_LockTimeoutMixin, BaseModel): + """ + Virtualenv Configuration Typing. + """ + + name: str = Field(default=None) + env: dict[str, str] = Field(default=None) + system_site_packages: bool = Field(default=False) + add_as_extra_site_packages: bool = Field(default=False) + + def _get_config_hash(self) -> bytes: + """ + Return a hash of the configuration. + """ + raise NotImplementedError + + def _install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install default requirements. + """ + raise NotImplementedError + + def _setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + raise NotImplementedError + + def get_config_hash(self) -> str: + """ + Return a hash digest of the configuration. + """ + config_hash = hashlib.sha256() + # The first part of the hash should be the path to the tools executable + config_hash.update(sys.argv[0].encode()) + # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set + hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") + config_hash.update(hash_seed.encode()) + config_hash.update(self._get_config_hash()) + return config_hash.hexdigest() + + def install(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Install requirements. + """ + self._install(ctx, python_executable=python_executable) + + def setup_venv(self, ctx: Context, python_executable: str | None = None) -> None: + """ + Setup the virtual environment. + """ + self._setup_venv(ctx, python_executable=python_executable) + + +class VirtualEnvPipConfig(_PipMixin, VirtualEnvConfig): + """ + Virtualenv pip configuration. + """ + + +class VirtualEnvPoetryConfig(_PoetryMixin, VirtualEnvConfig): + """ + Virtualenv poetry configuration. + """ diff --git a/src/ptscripts/parser.py b/src/ptscripts/parser.py index f10225f..80768f8 100644 --- a/src/ptscripts/parser.py +++ b/src/ptscripts/parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import hashlib import importlib import inspect import logging @@ -16,7 +15,6 @@ from contextlib import AbstractContextManager from contextlib import contextmanager from contextlib import nullcontext -from functools import cached_property from functools import partial from subprocess import CompletedProcess from types import FunctionType @@ -28,7 +26,6 @@ from typing import TypeVar from typing import cast -import attr import requests import rich from rich.console import Console @@ -36,9 +33,9 @@ from ptscripts import logs from ptscripts import process +from ptscripts.models import VirtualEnvConfig +from ptscripts.models import VirtualEnvPipConfig from ptscripts.virtualenv import VirtualEnv -from ptscripts.virtualenv import VirtualEnvConfig -from ptscripts.virtualenv import _cast_to_pathlib_path if sys.version_info < (3, 10): from typing_extensions import Concatenate @@ -69,6 +66,8 @@ from argparse import _SubParsersAction from collections.abc import Iterator + from ptscripts.models import DefaultConfig + Param = ParamSpec("Param") RetType = TypeVar("RetType") @@ -103,92 +102,6 @@ class FullArgumentOptions(ArgumentOptions): type: type[Any] -@attr.s(frozen=True) -class DefaultRequirementsConfig: - """ - Default tools requirements configuration typing. - """ - - requirements: list[str] = attr.ib(factory=list) - requirements_files: list[pathlib.Path] = attr.ib(factory=list) - pip_args: list[str] = attr.ib(factory=list) - - @cached_property - def requirements_hash(self) -> str: - """ - Returns a sha256 hash of the requirements. - """ - requirements_hash = hashlib.sha256() - # The first part of the hash should be the path to the tools executable - requirements_hash.update(sys.argv[0].encode()) - # The second, TOOLS_VIRTUALENV_CACHE_SEED env variable, if set - hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") - requirements_hash.update(hash_seed.encode()) - # Third, any custom pip cli argument defined - if self.pip_args: - for argument in self.pip_args: - requirements_hash.update(argument.encode()) - # Forth, each passed requirement - if self.requirements: - for requirement in sorted(self.requirements): - requirements_hash.update(requirement.encode()) - # And, lastly, any requirements files passed in - if self.requirements_files: - for fpath in sorted(self.requirements_files): - with _cast_to_pathlib_path(fpath).open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = hashlib.sha256() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - requirements_hash.update(digest.digest()) - return requirements_hash.hexdigest() - - def install(self, ctx: Context) -> None: - """ - Install default requirements. - """ - from ptscripts.__main__ import TOOLS_VENVS_PATH - - requirements_hash_file = TOOLS_VENVS_PATH / ".default-requirements.hash" - if ( - requirements_hash_file.exists() - and requirements_hash_file.read_text() == self.requirements_hash - ): - # Requirements are up to date - ctx.debug( - f"Base tools requirements haven't changed. Hash file: '{requirements_hash_file}'; " - f"Hash: '{self.requirements_hash}'" - ) - return - requirements = [] - if self.requirements_files: - for fpath in self.requirements_files: - requirements.extend(["-r", str(fpath)]) - if self.requirements: - requirements.extend(self.requirements) - if requirements: - ctx.info("Installing base tools requirements ...") - ctx.run( - sys.executable, - "-m", - "pip", - "install", - *self.pip_args, - *requirements, - ) - requirements_hash_file.parent.mkdir(parents=True, exist_ok=True) - requirements_hash_file.write_text(self.requirements_hash) - ctx.debug(f"Wrote '{requirements_hash_file}' with contents: '{self.requirements_hash}'") - - class Context: """ Context class passed to every command group function as the first argument. @@ -348,18 +261,15 @@ def chdir(self, path: pathlib.Path) -> Iterator[pathlib.Path]: os.chdir(cwd) @contextmanager - def virtualenv( - self, - name: str, - requirements: list[str] | None = None, - requirements_files: list[pathlib.Path] | None = None, - ) -> Iterator[VirtualEnv]: + def virtualenv(self, name: str, config: VirtualEnvConfig | None = None) -> Iterator[VirtualEnv]: """ Create and use a virtual environment. """ - with VirtualEnv( - ctx=self, name=name, requirements=requirements, requirements_files=requirements_files - ) as venv: + if config is None: + config = VirtualEnvPipConfig(name=name) + if config.name is None: + config.name = name + with VirtualEnv(ctx=self, config=config) as venv: yield venv @property @@ -405,7 +315,7 @@ class DefaultToolsPythonRequirements: """ _instance: DefaultToolsPythonRequirements | None = None - reqs_config: DefaultRequirementsConfig | None + config: DefaultConfig | None def __new__(cls) -> DefaultToolsPythonRequirements: """ @@ -413,19 +323,19 @@ def __new__(cls) -> DefaultToolsPythonRequirements: """ if cls._instance is None: instance = super().__new__(cls) - instance.reqs_config = None + instance.config = None cls._instance = instance return cls._instance @classmethod - def set_default_requirements_config(cls, reqs_config: DefaultRequirementsConfig) -> None: + def set_default_requirements_config(cls, config: DefaultConfig) -> None: """ Set the default tools requirements configuration. """ instance = cls._instance if instance is None: instance = cls() - instance.reqs_config = reqs_config + instance.config = config class RegisteredImports: @@ -565,26 +475,26 @@ def __new__(cls) -> Parser: return cls._instance def _process_registered_tool_modules(self) -> None: - default_reqs_config = DefaultToolsPythonRequirements().reqs_config - if default_reqs_config: - default_reqs_config.install(self.context) + default_config = DefaultToolsPythonRequirements().config + if default_config: + default_config.install(self.context) default_venv: VirtualEnv | AbstractContextManager[None] default_venv_config = DefaultVirtualEnv().venv_config if default_venv_config: - if "name" not in default_venv_config: - default_venv_config["name"] = "default" - default_venv_config["add_as_extra_site_packages"] = True - default_venv = VirtualEnv(ctx=self.context, **default_venv_config) + if not default_venv_config.name: + default_venv_config.name = "default" + default_venv_config.add_as_extra_site_packages = True + default_venv = VirtualEnv(ctx=self.context, config=default_venv_config) else: default_venv = nullcontext() with default_venv: for module_name, venv_config in RegisteredImports(): venv: VirtualEnv | AbstractContextManager[None] if venv_config: - if "name" not in venv_config: - venv_config["name"] = module_name - venv = VirtualEnv(ctx=self.context, **venv_config) + if not venv_config.name: + venv_config.name = module_name + venv = VirtualEnv(ctx=self.context, config=venv_config) else: venv = nullcontext() with venv: @@ -705,9 +615,11 @@ def __init__( GroupReference.add_command((*parent, name), self) parent = GroupReference()[tuple(parent)] - if venv_config and "name" not in venv_config: - venv_config["name"] = self.name - self.venv_config = venv_config or {} + if venv_config is not None and venv_config.name is None: + venv_config.name = self.name + + self.venv_config: VirtualEnvConfig | None = venv_config + if TYPE_CHECKING: assert parent self.parser = parent.subparsers.add_parser( # type: ignore[union-attr, has-type] @@ -852,6 +764,7 @@ def command( # noqa: ANN201,C901,PLR0912,PLR0915 flags = [f"--{parameter.name.replace('_', '-')}"] log.debug("Adding Command %r. Flags: %s; KwArgs: %s", name, flags, kwargs) command.add_argument(*flags, **kwargs) # type: ignore[arg-type] + command.set_defaults(func=partial(self, func, venv_config=venv_config)) return func @@ -886,11 +799,11 @@ def __call__( bound = signature.bind_partial(*args, **kwargs) venv: VirtualEnv | None = None if venv_config: - if "name" not in venv_config: - venv_config["name"] = getattr(options, f"{self.name}_command") - venv = VirtualEnv(ctx=self.context, **venv_config) + if venv_config.name is None: + venv_config.name = getattr(options, f"{self.name}_command") + venv = VirtualEnv(ctx=self.context, config=venv_config) elif self.venv_config: - venv = VirtualEnv(ctx=self.context, **self.venv_config) + venv = VirtualEnv(ctx=self.context, config=self.venv_config) if venv: with venv: previous_venv = self.context.venv diff --git a/src/ptscripts/utils.py b/src/ptscripts/utils.py new file mode 100644 index 0000000..16d4ac7 --- /dev/null +++ b/src/ptscripts/utils.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import hashlib +import pathlib +from typing import cast + + +def cast_to_pathlib_path(value: str | pathlib.Path) -> pathlib.Path: + """ + Cast passed in string to an instance of `pathlib.Path`. + """ + if isinstance(value, pathlib.Path): + return value + return pathlib.Path(str(value)) + + +def file_digest(path: pathlib.Path) -> bytes: + """ + Return a SHA256 digest of a file. + """ + with path.open("rb") as rfh: + try: + digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] + except AttributeError: + # Python < 3.11 + buf = bytearray(2**18) # Reusable buffer to reduce allocations. + view = memoryview(buf) + digest = hashlib.sha256() + while True: + size = rfh.readinto(buf) + if size == 0: + break # EOF + digest.update(view[:size]) + return cast(bytes, digest.digest()) diff --git a/src/ptscripts/virtualenv.py b/src/ptscripts/virtualenv.py index b768691..a8befed 100644 --- a/src/ptscripts/virtualenv.py +++ b/src/ptscripts/virtualenv.py @@ -1,10 +1,8 @@ from __future__ import annotations -import hashlib import json import logging import os -import pathlib import shutil import subprocess import sys @@ -12,73 +10,32 @@ from subprocess import CompletedProcess from typing import TYPE_CHECKING -if sys.version_info < (3, 11): - from typing_extensions import NotRequired - from typing_extensions import TypedDict -else: - from typing import NotRequired - from typing import TypedDict - import attr +from filelock import FileLock if TYPE_CHECKING: + import pathlib + + from ptscripts.models import VirtualEnvConfig from ptscripts.parser import Context log = logging.getLogger(__name__) -class VirtualEnvConfig(TypedDict): - """ - Virtualenv Configuration Typing. - """ - - name: NotRequired[str] - requirements: NotRequired[list[str]] - requirements_files: NotRequired[list[pathlib.Path]] - env: NotRequired[dict[str, str]] - system_site_packages: NotRequired[bool] - pip_requirement: NotRequired[str] - setuptools_requirement: NotRequired[str] - add_as_extra_site_packages: NotRequired[bool] - pip_args: NotRequired[list[str]] - - -def _cast_to_pathlib_path(value: str | pathlib.Path) -> pathlib.Path: - if isinstance(value, pathlib.Path): - return value - return pathlib.Path(str(value)) - - @attr.s(frozen=True, slots=True) class VirtualEnv: """ Helper class to create and user virtual environments. """ - name: str = attr.ib() ctx: Context = attr.ib() - requirements: list[str] | None = attr.ib(repr=False, default=None) - requirements_files: list[pathlib.Path] | None = attr.ib(repr=False, default=None) - env: dict[str, str] | None = attr.ib(default=None) - system_site_packages: bool = attr.ib(default=False) - pip_requirement: str = attr.ib(repr=False) - setuptools_requirement: str = attr.ib(repr=False) - add_as_extra_site_packages: bool = attr.ib(default=False) - pip_args: list[str] = attr.ib(factory=list, repr=False) + config: VirtualEnvConfig = attr.ib() environ: dict[str, str] = attr.ib(init=False, repr=False) venv_dir: pathlib.Path = attr.ib(init=False) venv_python: pathlib.Path = attr.ib(init=False, repr=False) venv_bin_dir: pathlib.Path = attr.ib(init=False, repr=False) requirements_hash: str = attr.ib(init=False, repr=False) - - @pip_requirement.default - def _default_pip_requiremnt(self) -> str: - return "pip>=22.3.1,<23.0" - - @setuptools_requirement.default - def _default_setuptools_requirement(self) -> str: - # https://github.com/pypa/setuptools/commit/137ab9d684075f772c322f455b0dd1f992ddcd8f - return "setuptools>=65.6.3,<66" + lockfile: FileLock = attr.ib(init=False, repr=False) @venv_dir.default def _default_venv_dir(self) -> pathlib.Path: @@ -87,13 +44,13 @@ def _default_venv_dir(self) -> pathlib.Path: venvs_path = TOOLS_VENVS_PATH venvs_path.mkdir(parents=True, exist_ok=True) - return venvs_path / self.name + return venvs_path / self.config.name @environ.default def _default_environ(self) -> dict[str, str]: environ = os.environ.copy() - if self.env: - environ.update(self.env) + if self.config.env: + environ.update(self.config.env) return environ @venv_python.default @@ -108,31 +65,17 @@ def _default_venv_bin_dir(self) -> pathlib.Path: @requirements_hash.default def __default_requirements_hash(self) -> str: - requirements_hash = hashlib.sha256(self.name.encode()) - hash_seed = os.environ.get("TOOLS_VIRTUALENV_CACHE_SEED", "") - requirements_hash.update(hash_seed.encode()) - if self.pip_args: - requirements_hash.update(str(sorted(self.pip_args)).encode()) - if self.requirements: - for requirement in sorted(self.requirements): - requirements_hash.update(requirement.encode()) - if self.requirements_files: - for fpath in sorted(self.requirements_files): - with _cast_to_pathlib_path(fpath).open("rb") as rfh: - try: - digest = hashlib.file_digest(rfh, "sha256") # type: ignore[attr-defined] - except AttributeError: - # Python < 3.11 - buf = bytearray(2**18) # Reusable buffer to reduce allocations. - view = memoryview(buf) - digest = hashlib.sha256() - while True: - size = rfh.readinto(buf) - if size == 0: - break # EOF - digest.update(view[:size]) - requirements_hash.update(digest.digest()) - return requirements_hash.hexdigest() + return self.config.get_config_hash() + + @lockfile.default + def __lockfile(self) -> FileLock: + # Late import to avoid circular import errors + from ptscripts.__main__ import TOOLS_VENVS_PATH + + return FileLock( + TOOLS_VENVS_PATH / "locks" / f"{self.config.name}.lock", + timeout=self.config.lock_timeout_seconds, + ) def _install_requirements(self) -> None: requirements_hash_file = self.venv_dir / ".requirements.hash" @@ -141,17 +84,9 @@ def _install_requirements(self) -> None: and requirements_hash_file.read_text() == self.requirements_hash ): # Requirements are up to date - self.ctx.debug(f"Requirements for virtualenv({self.name}) haven't changed.") + self.ctx.debug(f"Requirements for virtualenv({self.config.name}) haven't changed.") return - requirements = [] - if self.requirements_files: - for fpath in sorted(self.requirements_files): - requirements.extend(["-r", str(fpath)]) - if self.requirements: - requirements.extend(sorted(self.requirements)) - if requirements: - self.ctx.info(f"Install requirements for virtualenv({self.name}) ...") - self.install(*self.pip_args, *requirements) + self.config.install(self.ctx, python_executable=str(self.venv_python)) self.venv_dir.joinpath(".requirements.hash").write_text(self.requirements_hash) def _create_virtualenv(self) -> None: @@ -189,24 +124,19 @@ def _create_virtualenv(self) -> None: "-m", "venv", ] - if self.system_site_packages: + if self.config.system_site_packages: cmd.append("--system-site-packages") cmd.append(str(self.venv_dir)) try: relative_venv_path = self.venv_dir.relative_to(CWD) except ValueError: relative_venv_path = self.venv_dir - self.ctx.info(f"Creating virtualenv({self.name}) in {relative_venv_path}") + self.ctx.info(f"Creating virtualenv({self.config.name}) in {relative_venv_path}") self.run(*cmd, cwd=str(self.venv_dir.parent)) - self.install( - "-U", - "wheel", - self.pip_requirement, - self.setuptools_requirement, - ) + self.setup() def _add_as_extra_site_packages(self) -> None: - if self.add_as_extra_site_packages is False: + if self.config.add_as_extra_site_packages is False: return ret = self.run_code( "import json,site; print(json.dumps(site.getsitepackages()))", @@ -223,7 +153,7 @@ def _add_as_extra_site_packages(self) -> None: sys.path.append(path) def _remove_extra_site_packages(self) -> None: - if self.add_as_extra_site_packages is False: + if self.config.add_as_extra_site_packages is False: return ret = self.run_code( "import json,site; print(json.dumps(site.getsitepackages()))", @@ -239,10 +169,7 @@ def _remove_extra_site_packages(self) -> None: if path in sys.path: sys.path.remove(path) - def __enter__(self) -> VirtualEnv: - """ - Creates the virtual environment when entering context. - """ + def _enter(self) -> VirtualEnv: try: self._create_virtualenv() except subprocess.CalledProcessError: @@ -262,11 +189,28 @@ def __enter__(self) -> VirtualEnv: self._add_as_extra_site_packages() return self - def __exit__(self, *args) -> None: + def _exit(self) -> None: + self._remove_extra_site_packages() + + def __enter__(self) -> VirtualEnv: + """ + Creates the virtual environment when entering context. + """ + with self.lockfile: + return self._enter() + + def __exit__(self, *_) -> None: """ Exit the virtual environment context. """ - self._remove_extra_site_packages() + with self.lockfile: + self._exit() + + def setup(self) -> None: + """ + Setup the virtual environment. + """ + self.config.setup_venv(self.ctx, python_executable=str(self.venv_python)) def install(self, *args: str, **kwargs) -> CompletedProcess[bytes]: """