diff --git a/Cargo.lock b/Cargo.lock index bed33466eb6..e2a2f05a733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1721,9 +1721,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libffi" diff --git a/Cargo.toml b/Cargo.toml index 4813ab159cc..a1c0ee6a370 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,7 +171,7 @@ insta = "1.46" itertools = "0.14.0" is-macro = "0.3.7" junction = "1.4.2" -libc = "0.2.180" +libc = "0.2.182" libffi = "5" log = "0.4.29" nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 7076e42b9a5..acc54e5ec91 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -61,14 +61,6 @@ pub mod module { use strum::IntoEnumIterator; use strum_macros::{EnumIter, EnumString}; - #[cfg(any(target_os = "android", target_os = "linux"))] - #[pyattr] - use libc::{SCHED_DEADLINE, SCHED_NORMAL}; - - #[cfg(target_os = "freebsd")] - #[pyattr] - use libc::{MFD_HUGE_MASK, SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; - #[cfg(target_os = "linux")] #[pyattr] use libc::PIDFD_NONBLOCK; @@ -80,42 +72,72 @@ pub mod module { PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, PRIO_DARWIN_THREAD, }; + #[cfg(target_os = "freebsd")] + #[pyattr] + use libc::{SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; + #[cfg(any(target_os = "android", target_os = "linux"))] #[pyattr] use libc::{ CLONE_FILES, CLONE_FS, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS, CLONE_NEWPID, CLONE_NEWUSER, CLONE_NEWUTS, CLONE_SIGHAND, CLONE_SYSVSEM, CLONE_THREAD, - CLONE_VM, EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, O_NOATIME, O_TMPFILE, P_PIDFD, - SCHED_BATCH, SCHED_IDLE, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE, + CLONE_VM, MFD_HUGE_SHIFT, O_NOATIME, O_TMPFILE, P_PIDFD, SCHED_BATCH, SCHED_DEADLINE, + SCHED_IDLE, SCHED_NORMAL, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE, SPLICE_F_NONBLOCK, }; - #[cfg(any(target_os = "android", unix))] - #[pyattr] - use libc::{ - F_OK, O_CLOEXEC, O_DIRECTORY, O_NOFOLLOW, O_NONBLOCK, PRIO_PGRP, PRIO_PROCESS, PRIO_USER, - R_OK, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW, W_OK, WCONTINUED, WNOHANG, WUNTRACED, - X_OK, - }; - #[cfg(any(target_os = "macos", target_os = "redox"))] #[pyattr] use libc::O_SYMLINK; + #[cfg(any(target_os = "android", target_os = "redox", unix))] + #[pyattr] + use libc::{O_NOFOLLOW, PRIO_PGRP, PRIO_PROCESS, PRIO_USER}; + + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "netbsd"))] + #[pyattr] + use libc::{XATTR_CREATE, XATTR_REPLACE}; + + #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))] + #[pyattr] + use libc::O_RSYNC; + #[cfg(any(target_os = "android", target_os = "freebsd", target_os = "linux"))] #[pyattr] use libc::{ - MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGETLB, POSIX_FADV_DONTNEED, POSIX_FADV_NOREUSE, - POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL, POSIX_FADV_WILLNEED, + MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGE_MASK, MFD_HUGETLB, POSIX_FADV_DONTNEED, + POSIX_FADV_NOREUSE, POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL, + POSIX_FADV_WILLNEED, }; - #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))] + #[cfg(any(target_os = "android", target_os = "linux", target_os = "redox", unix))] #[pyattr] - use libc::{TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME, TFD_TIMER_CANCEL_ON_SET}; + use libc::{RTLD_LAZY, RTLD_NOW, WNOHANG}; - #[cfg(any(target_os = "linux", target_os = "macos", target_os = "netbsd"))] + #[cfg(any(target_os = "android", target_os = "macos", target_os = "redox", unix))] #[pyattr] - use libc::{XATTR_CREATE, XATTR_REPLACE}; + use libc::RTLD_GLOBAL; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "redox" + ))] + #[pyattr] + use libc::O_PATH; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use libc::{ + EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME, + TFD_TIMER_CANCEL_ON_SET, + }; #[cfg(any( target_os = "android", @@ -128,21 +150,43 @@ pub mod module { #[cfg(any( target_os = "android", - target_os = "freebsd", target_os = "linux", - target_os = "redox" + target_os = "macos", + target_os = "redox", + unix ))] #[pyattr] - use libc::O_PATH; + use libc::{F_OK, R_OK, W_OK, X_OK}; #[cfg(any( target_os = "android", - target_os = "linux", + target_os = "freebsd", target_os = "netbsd", - target_os = "openbsd" + target_os = "redox", + unix ))] #[pyattr] - use libc::O_RSYNC; + use libc::O_NONBLOCK; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] + #[pyattr] + use libc::O_DSYNC; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] + #[pyattr] + use libc::SCHED_OTHER; #[cfg(any( target_os = "android", @@ -165,48 +209,62 @@ pub mod module { use libc::O_DIRECT; #[cfg(any( - target_os = "android", target_os = "dragonfly", target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "redox" + ))] + #[pyattr] + use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK}; + + #[cfg(any( + target_os = "android", target_os = "linux", target_os = "macos", - target_os = "netbsd" + target_os = "netbsd", + target_os = "redox", + unix ))] #[pyattr] - use libc::RTLD_NOLOAD; + use libc::RTLD_LOCAL; #[cfg(any( target_os = "android", target_os = "freebsd", target_os = "linux", - target_os = "macos", target_os = "netbsd", - target_os = "openbsd" + target_os = "redox", + unix ))] #[pyattr] - use libc::O_DSYNC; + use libc::WUNTRACED; #[cfg(any( + target_os = "android", target_os = "dragonfly", target_os = "freebsd", target_os = "linux", target_os = "macos", - target_os = "netbsd", - target_os = "openbsd" + target_os = "netbsd" ))] #[pyattr] - use libc::SCHED_OTHER; + use libc::{ + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, O_SYNC, P_ALL, + P_PGID, P_PID, RTLD_NOLOAD, SCHED_FIFO, SCHED_RR, + }; #[cfg(any( + target_os = "android", target_os = "dragonfly", target_os = "freebsd", target_os = "macos", target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" + target_os = "redox", + unix ))] #[pyattr] - use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK}; + use libc::O_DIRECTORY; #[cfg(any( target_os = "android", @@ -215,12 +273,11 @@ pub mod module { target_os = "linux", target_os = "macos", target_os = "netbsd", - target_os = "openbsd" + target_os = "redox" ))] #[pyattr] use libc::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, F_LOCK, - F_TEST, F_TLOCK, F_ULOCK, O_SYNC, P_ALL, P_PGID, P_PID, SCHED_FIFO, SCHED_RR, + F_LOCK, F_TEST, F_TLOCK, F_ULOCK, O_ASYNC, O_NDELAY, O_NOCTTY, WEXITED, WNOWAIT, WSTOPPED, }; #[cfg(any( @@ -230,11 +287,11 @@ pub mod module { target_os = "linux", target_os = "macos", target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" + target_os = "redox", + unix ))] #[pyattr] - use libc::{O_ASYNC, O_NDELAY, O_NOCTTY, WEXITED, WNOWAIT, WSTOPPED}; + use libc::{O_CLOEXEC, WCONTINUED}; #[pyattr] const EX_OK: i8 = exitcode::OK as i8; diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py index be375aebe8f..82f3eb96d83 100644 --- a/scripts/libc_posix.py +++ b/scripts/libc_posix.py @@ -1,19 +1,24 @@ #!/usr/bin/env python import collections +import dataclasses +import pathlib import re +import subprocess import urllib.request -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from collections.abc import Iterator +import tomllib -CONSTS_PAT = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b") -OS_CONSTS_PAT = re.compile( - r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)" -) # TODO: Exclude matches if they have `(` after (those are functions) +CPYTHON_VERSION = "3.14" +CONSTS_PATTERN = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b") -LIBC_VERSION = "0.2.180" +CARGO_TOML_FILE = pathlib.Path(__file__).parents[1] / "Cargo.toml" +CARGO_TOML = tomllib.loads(CARGO_TOML_FILE.read_text()) +LIBC_DATA = CARGO_TOML["workspace"]["dependencies"]["libc"] + + +LIBC_VERSION = LIBC_DATA["version"] if isinstance(LIBC_DATA, dict) else LIBC_DATA +BASE_URL = f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}" EXCLUDE = frozenset( { @@ -42,89 +47,201 @@ } ) -EXTRAS = { - frozenset({"macos"}): {"COPYFILE_DATA"}, -} -RENAMES = {"COPYFILE_DATA": "_COPYFILE_DATA"} - - -def build_url(fname: str) -> str: - return f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}/libc-test/semver/{fname}.txt" - - -TARGET_OS = { - "android": "android", - "dragonfly": "dragonfly", - "freebsd": "freebsd", - "linux": "linux", - "macos": "apple", - "netbsd": "netbsd", - "openbsd": "openbsd", - "redox": "redox", - # solaris? - "unix": "unix", -} - - -def get_consts(url: str, pattern: re.Pattern = CONSTS_PAT) -> frozenset[str]: - with urllib.request.urlopen(url) as f: - resp = f.read().decode() - - return frozenset(pattern.findall(resp)) - EXCLUDE +def rustfmt(code: str) -> str: + return subprocess.check_output(["rustfmt", "--emit=stdout"], input=code, text=True) + + +@dataclasses.dataclass(eq=True, frozen=True, slots=True) +class Cfg: + inner: str + + def __str__(self) -> str: + return self.inner + + def __lt__(self, other) -> bool: + si, oi = map(str, (self.inner, other.inner)) + + # Smaller length cfgs are smaller, regardless of value. + return (len(si), si) < (len(oi), oi) + + +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) +class Target: + cfgs: set[Cfg] + sources: set[str] = dataclasses.field(default_factory=set) + extras: set[str] = dataclasses.field(default_factory=set) + + +TARGETS = ( + Target( + cfgs={Cfg('target_os = "android"')}, + sources={ + f"{BASE_URL}/src/unix/linux_like/android/mod.rs", + f"{BASE_URL}/libc-test/semver/android.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "dragonfly"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/freebsdlike/dragonfly/mod.rs", + f"{BASE_URL}/libc-test/semver/dragonfly.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "freebsd"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/freebsdlike/freebsd/mod.rs", + f"{BASE_URL}/libc-test/semver/freebsd.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "linux"')}, + sources={ + f"{BASE_URL}/src/unix/linux_like/mod.rs", + f"{BASE_URL}/src/unix/linux_like/linux_l4re_shared.rs", + f"{BASE_URL}/libc-test/semver/linux.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "macos"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/apple/mod.rs", + f"{BASE_URL}/libc-test/semver/apple.txt", + }, + extras={"COPYFILE_DATA as _COPYFILE_DATA"}, + ), + Target( + cfgs={Cfg('target_os = "netbsd"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/netbsdlike/netbsd/mod.rs", + f"{BASE_URL}/libc-test/semver/netbsd.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "redox"')}, + sources={ + f"{BASE_URL}/src/unix/redox/mod.rs", + f"{BASE_URL}/libc-test/semver/redox.txt", + }, + ), + Target(cfgs={Cfg("unix")}, sources={f"{BASE_URL}/libc-test/semver/unix.txt"}), +) -def format_groups(groups: dict) -> "Iterator[tuple[str, str]]": - # sort by length, then alphabet. so we will have a consistent output - for targets, consts in sorted( - groups.items(), key=lambda t: (len(t[0]), sorted(t[0])) - ): - cond = ", ".join( - f'target_os = "{target_os}"' if target_os != "unix" else target_os - for target_os in sorted(targets) - ) - if len(targets) > 1: - cond = f"any({cond})" - cfg = f"#[cfg({cond})]" - imports = ", ".join( - const if const not in RENAMES else f"{const} as {RENAMES[const]}" - for const in sorted(consts) - ) - use = f"use libc::{{{imports}}};" - yield cfg, use +def extract_consts( + contents: str, + *, + pattern: re.Pattern = CONSTS_PATTERN, + exclude: frozenset[str] = EXCLUDE, +) -> frozenset[str]: + """ + Extract all words that are comprised from only uppercase letters + underscores. + + Parameters + ---------- + contents : str + Contents to extract the constants from. + pattern : re.Pattern, Optional + RE compiled pattern for extracting the consts. + exclude : frozenset[str], Optional + Items to exclude from the returned value. + + Returns + ------- + frozenset[str] + All constant names. + """ + result = frozenset(pattern.findall(contents)) + return result - exclude + + +def consts_from_url( + url: str, *, pattern: re.Pattern = CONSTS_PATTERN, exclude: frozenset[str] = EXCLUDE +) -> frozenset[str]: + """ + Extract all consts from the contents found at the given URL. + + Parameters + ---------- + url : str + URL to fetch the contents from. + pattern : re.Pattern, Optional + RE compiled pattern for extracting the consts. + exclude : frozenset[str], Optional + Items to exclude from the returned value. + + Returns + ------- + frozenset[str] + All constant names at the URL. + """ + try: + with urllib.request.urlopen(url) as f: + contents = f.read().decode() + except urllib.error.HTTPError as err: + err.add_note(url) + raise + + return extract_consts(contents, pattern=pattern, exclude=exclude) def main(): - wanted_consts = get_consts( - "https://docs.python.org/3.14/library/os.html", # Should we read from https://github.com/python/cpython/blob/bcee1c322115c581da27600f2ae55e5439c027eb/Modules/posixmodule.c#L17023 instead? - pattern=OS_CONSTS_PAT, + # Step 1: Get all OS contants that we do want from upstream + wanted_consts = consts_from_url( + f"https://docs.python.org/{CPYTHON_VERSION}/library/os.html", + # TODO: Exclude matches if they have `(` after (those are functions) + pattern=re.compile(r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)"), ) - available = { - target_os: get_consts(build_url(fname)) - for target_os, fname in TARGET_OS.items() - } - group_consts = collections.defaultdict(set) - for const in wanted_consts: - target_oses = frozenset( - target_os for target_os, consts in available.items() if const in consts + # Step 2: build dict of what consts are available per cfg. `cfg -> {consts}` + available = collections.defaultdict(set) + for target in TARGETS: + consts = set() + for source in target.sources: + consts |= consts_from_url(source) + + for cfg in target.cfgs: + available[cfg] |= consts + + # Step 3: Keep only the "wanted" consts. Build a groupped mapping of `{cfgs} -> {consts}' + groups = collections.defaultdict(set) + available_items = available.items() + for wanted_const in wanted_consts: + cfgs = frozenset( + cfg for cfg, consts in available_items if wanted_const in consts ) - if not target_oses: + if not cfgs: + # We have no cfgs for a wanted const :/ continue - group_consts[target_oses].add(const) - group_consts = {grp: v | EXTRAS.get(grp, set()) for grp, v in group_consts.items()} + groups[cfgs].add(wanted_const) + + # Step 4: Build output + output = "" + for cfgs, consts in sorted(groups.items(), key=lambda t: (len(t[0]), sorted(t[0]))): + target = next((target for target in TARGETS if target.cfgs == cfgs), None) + if target: + # If we found an exact target. Add its "extras" as-is + consts |= target.extras - code = "\n\n".join( - f""" -{cfg} + cfgs_inner = ",".join(sorted(map(str, cfgs))) + + if len(cfgs) >= 2: + cfgs_rust = f"#[cfg(any({cfgs_inner}))]" + else: + cfgs_rust = f"#[cfg({cfgs_inner})]" + + imports = ",".join(consts) + entry = f""" +{cfgs_rust} #[pyattr] -{use} +use libc::{{{imports}}}; """.strip() - for cfg, use in format_groups(group_consts) - ) - print(code) + output += f"{entry}\n\n" + + print(rustfmt(output)) if __name__ == "__main__":