generated from goodbyekansas/opensource-template
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🔨 ⚡ 🦀 🔒 Make rust sandbox safe with crates
This adds a set of rust crates, under base.languages.rust.crates, and tooling to extend, override or replace that set. This set is built from fixed output derivations so rust will no longer access crates.io during build. It also simplifies the logic by treating external and internal crates the same. It also removes a limitation where users could not create a custom .cargo/config.toml file. Removed the test for internal dependencies, because the use of setupHooks the vendoring happens build time and the eval time test can't verify anything.
- Loading branch information
1 parent
e5957b1
commit eeb6074
Showing
14 changed files
with
373 additions
and
132 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| #! /usr/bin/env bash | ||
|
|
||
| crates="$NIX_BUILD_TOP"/.nedryglot/.dependency-crates | ||
| name="$crates/$(echo "@out@" | sed -E 's|/nix/store/\w{32}-||')" | ||
| mkdir -p "$crates" | ||
| if [ ! -e "@out@"/Cargo.toml ]; then | ||
| for crate in "@out@"/*; do | ||
| if [ ! -e "$crates/$(basename "$crate")" ] && [ -f "$crate"/Cargo.toml ]; then | ||
| ln -s "$crate" "$crates/$(basename "$crate")" | ||
| fi | ||
| done | ||
| elif [ ! -e "$name" ]; then | ||
| ln -s "@out@" "$name" | ||
| fi | ||
|
|
||
| echo "[source.crates-io] | ||
| directory=\"$NIX_BUILD_TOP/.nedryglot/.dependency-crates\"" >"$NIX_BUILD_TOP"/.nedryglot/config.toml | ||
| export CARGO_HOME="$NIX_BUILD_TOP"/.nedryglot |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| from pathlib import Path | ||
| import argparse | ||
| import io | ||
| import json | ||
| import sys | ||
| import tarfile | ||
| import tempfile | ||
| import typing | ||
| import urllib.request | ||
|
|
||
|
|
||
| FETCH_CRATE_EXPR = ' {name} = fetchCrate {{ name="{name}"; version="{version}"; sha256="{sha256}"; deps=[{deps}];}};' | ||
|
|
||
|
|
||
| def comparable_version(semver: str) -> str: | ||
| return ".".join(map(lambda i: i.zfill(4), semver.split("."))) | ||
|
|
||
|
|
||
| def to_nix_expression(content: io.BufferedReader, version: typing.Optional[str], include_optional_deps: bool) -> (str, typing.List[str]): | ||
| """Generate a nix expression from a file inside a tar archive. | ||
| Args: | ||
| content: A file inside a tar archive. | ||
| version: Use a specific version instead of latest. | ||
| include_optional_deps: If optional dependencies should be returned in the deps list | ||
| Returns: | ||
| (str, list): The nix expression to fetch the crate and a list of dependencies. | ||
| """ | ||
| versions = map(json.loads, content.readlines()) | ||
| for version_info in sorted(versions, key=lambda v: comparable_version(v.get("vers")), reverse=True): | ||
| if version is not None and version_info["vers"] != version: | ||
| continue | ||
| if not version_info["yanked"]: | ||
| deps = set( | ||
| map( | ||
| lambda dep: dep.get("package", dep["name"]), | ||
| filter( | ||
| lambda dep: True if dep.get("optional", False) and include_optional_deps else not dep.get("optional", False), | ||
| filter( | ||
| lambda dep: dep.get("kind") in ["normal", "build"] and dep.get("name") != version_info["name"], | ||
| version_info["deps"] | ||
| ) | ||
| ) | ||
| ) | ||
| ) | ||
| return ( | ||
| FETCH_CRATE_EXPR.format( | ||
| name=version_info["name"], | ||
| version=version_info["vers"], | ||
| sha256=version_info["cksum"], | ||
| deps=" ".join(deps), | ||
| ), | ||
| list(deps), | ||
| ) | ||
| return ("", []) | ||
|
|
||
| def is_crate(tar_member: tarfile.TarInfo) -> bool: | ||
| """Filter tar member to only include files describing crates. | ||
| Exclude everything in the .github folder, everything with an extension and | ||
| directories. | ||
| Args: | ||
| tar_member: The object representing a file in the tar archive. | ||
| Returns: | ||
| bool: whether a file is a crate description. | ||
| """ | ||
| return ( | ||
| tar_member.isreg() | ||
| and tar_member.name.split("/")[1] != ".github" | ||
| and "." not in tar_member.name.split("/")[-1] | ||
| ) | ||
|
|
||
| def main(github_ref: str, crates: typing.List[str], output: typing.Optional[Path], include_deps: bool, optional_deps: bool, silent: bool) -> None: | ||
| """Generate a nix expression to fetch crates from crates.io index. | ||
| Args: | ||
| github_ref: The ref to use on creates.io-index. | ||
| creates: The list of crate names to look up. | ||
| output: An optional file to output to instead of stdout. | ||
| include_deps: If nix expressions should be generated for dependencies. | ||
| optional_deps: If optional dependencies should be considered. | ||
| silent: If progress and messages should be omitted. | ||
| """ | ||
|
|
||
| visited_crates = [] | ||
| crates_to_visit = crates | ||
| result = [] | ||
|
|
||
| if not silent: | ||
| print(f"Downloading crates index at {github_ref}...", file=sys.stderr) | ||
| with urllib.request.urlopen(f"https://github.com/rust-lang/crates.io-index/tarball/{github_ref}") as tarball: | ||
| with tempfile.NamedTemporaryFile("w+b") as temp_tar: | ||
| temp_tar.write(tarball.read()) | ||
| with tarfile.open(temp_tar.name) as tar: | ||
| for crate in crates_to_visit: | ||
| if not silent: | ||
| print(f"Looking for {crate}...", file=sys.stderr) | ||
| visited_crates.append(crate) | ||
| for tar_member in filter(is_crate, tar.getmembers()): | ||
| if tar_member.name.split("/")[-1] == crate.split(":")[0]: | ||
| version = crate.split(":")[1] if ":" in crate else None | ||
| expr, deps = to_nix_expression(tar.extractfile(tar_member.name), version, optional_deps) | ||
| if expr: | ||
| result.append(expr) | ||
| if include_deps and deps: | ||
| crates_to_visit.extend([c for c in deps if c not in visited_crates and c not in crates_to_visit]) | ||
| break | ||
|
|
||
| expression = "fetchCrate: rec{\n" + "\n\n".join(result) + "\n}" | ||
|
|
||
| if output: | ||
| with open(output, "w") as out: | ||
| out.write(expression) | ||
| else: | ||
| print(expression) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser(description="Generate a nix expression to fetch rust crates when building with Nedryglot's rust language tooling.") | ||
| parser.add_argument("--ref", type=str, help="Git ref of https://github.com/rust-lang/crates.io-index to use for crate lookup, if omitted master is used.", default="master") | ||
| parser.add_argument("--output", type=Path, help="Write output to a file instead of stdout.") | ||
| parser.add_argument("--no-deps", action="store_true", help="Turn off dependency traversal and only download the listed crates.") | ||
| parser.add_argument("--optional-deps", action="store_true", help="Include optional dependencies.") | ||
| parser.add_argument("--silent", action="store_true", help="Do not print progress and info to stderr.") | ||
| parser.add_argument("crates", nargs="*", help="A space separated list of crates to generate expressions for. Use name:version to fetch a specific version.", default=[]) | ||
| args = parser.parse_args() | ||
| main(args.ref, args.crates, args.output, not args.no_deps, args.optional_deps, args.silent) | ||
| sys.exit(0) | ||
|
|
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.