Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Upgrade RTS Dependencies #4677

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/niv-updater-rare.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
uses: knl/niv-updater-action@v13
with:
# might be too noisy
whitelist: 'ic-ref,musl-wasi,ic-wasm'
whitelist: 'ic-ref,ic-wasm'
labels: |
automerge-squash
keep_updating: true
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/niv-updater-trial.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
- name: niv-updater-action
uses: knl/niv-updater-action@v13
with:
whitelist: 'nixpkgs,musl-wasi,ic'
whitelist: 'nixpkgs,ic'
labels: |
autoclose
keep_updating: true
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/niv-updater.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
uses: knl/niv-updater-action@v13
with:
# might be too noisy
blacklist: 'nixpkgs,ic-ref,musl-wasi,ic'
blacklist: 'nixpkgs,ic-ref,ic'
labels: |
automerge-squash
keep_updating: true
Expand Down
33 changes: 8 additions & 25 deletions default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -35,35 +35,31 @@ in

let
rtsBuildInputs = with nixpkgs; [
# pulls in clang (wrapped) and clang-13 (unwrapped)
llvmPackages_13.clang
# pulls in wasm-ld
llvmPackages_13.lld
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this package now merged into bintools? We do use wasm-ld-18 below...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, wasm-ld is part of bintools.

llvmPackages_13.bintools
llvmPackages_18.clang
llvmPackages_18.bintools
rustc-nightly
cargo-nightly
wasmtime
rust-bindgen
python3
emscripten
] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
libiconv
];

llvmEnv = ''
# When compiling to wasm, we want to have more control over the flags,
# so we do not use the nix-provided wrapper in clang
export WASM_CLANG="clang-13"
export WASM_CLANG="clang-18"
export WASM_LD=wasm-ld
# because we use the unwrapped clang, we have to pass in some flags/paths
# that otherwise the wrapped clang would take care for us
export WASM_CLANG_LIB="${nixpkgs.llvmPackages_13.clang-unwrapped.lib}"
export WASM_CLANG_LIB="${nixpkgs.llvmPackages_18.clang-unwrapped.lib}"
# When compiling natively, we want to use `clang` (which is a nixpkgs
# provided wrapper that sets various include paths etc).
# But for some reason it does not handle building for Wasm well, so
# there we use plain clang-13. There is no stdlib there anyways.
export CLANG="${nixpkgs.clang_13}/bin/clang"
# there we use plain clang-18. There is no stdlib there anyways.
export CLANG="${nixpkgs.clang_18}/bin/clang"
'';
in

Expand Down Expand Up @@ -153,15 +149,6 @@ let ocaml_exe = name: bin: rts:
$out/bin/* --help >/dev/null
'';
};

musl-wasi-sysroot = stdenv.mkDerivation {
name = "musl-wasi-sysroot";
src = nixpkgs.sources.musl-wasi;
phases = [ "unpackPhase" "installPhase" ];
installPhase = ''
make SYSROOT="$out" include_dirs
'';
};
in

rec {
Expand All @@ -178,7 +165,7 @@ rec {
vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps";

# SHA256 of Rust std deps
rustStdDepsHash = "sha256-A3WPIx+weu4wIYV7cweGkRxYGAPt7srxBAtMEyPOkhI=";
rustStdDepsHash = "sha256-U4BTr1CzFuOMdyLuhw5ry3/u8bkRiPmnMr4pLo3IdOQ=";

# Vendor directory for Rust std deps
rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation {
Expand Down Expand Up @@ -255,8 +242,6 @@ rec {
${llvmEnv}
export TOMMATHSRC=${nixpkgs.sources.libtommath}
export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
export MUSL_WASI_SYSROOT=${musl-wasi-sysroot}
'';

doCheck = true;
Expand Down Expand Up @@ -343,7 +328,7 @@ rec {

# extra deps for test/ld
ldTestDeps =
with nixpkgs; [ llvmPackages_13.bintools llvmPackages_13.clang ];
with nixpkgs; [ llvmPackages_18.lld llvmPackages_18.clang ];

testDerivation = args:
stdenv.mkDerivation (testDerivationArgs // args);
Expand Down Expand Up @@ -865,8 +850,6 @@ EOF
'';
ESM=nixpkgs.sources.esm;
TOMMATHSRC = nixpkgs.sources.libtommath;
MUSLSRC = "${nixpkgs.sources.musl-wasi}/libc-top-half/musl";
MUSL_WASI_SYSROOT = musl-wasi-sysroot;
LOCALE_ARCHIVE = nixpkgs.lib.optionalString stdenv.isLinux "${nixpkgs.glibcLocales}/lib/locale/locale-archive";
MOTOKO_BASE = base-src;
CANDID_TESTS = "${nixpkgs.sources.candid}/test";
Expand Down
6 changes: 2 additions & 4 deletions nix/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ let
};
};

# No testing of atdgen, as it pulls in python stuff, tricky on musl
# No testing of atdgen, as it pulls in python stuff
atdgen = super.ocamlPackages.atdgen.overrideAttrs { doCheck = false; };
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be doCheck = true now?

Copy link
Contributor

@ggreif ggreif Nov 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or rather leave it uncustomized?

};
}
Expand All @@ -91,13 +91,11 @@ let

# Rust nightly
(self: super: let
rust-channel = self.moz_overlay.rustChannelOf { date = "2023-04-21"; channel = "nightly"; };
rust-channel = self.moz_overlay.rustChannelOf { date = "2024-07-28"; channel = "nightly"; };
in rec {
rustc-nightly = rust-channel.rust.override {
targets = [
"wasm32-unknown-emscripten"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, why were these here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We used emscripten to get PIC-relocatable shared Wasm modules for the RTS. This is now redundant by the customized RTS backends.

"wasm32-wasi"
"i686-unknown-linux-gnu"
];
extensions = ["rust-src"];
};
Expand Down
8 changes: 4 additions & 4 deletions nix/drun.nix
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,17 @@ EOF

buildInputs = with pkgs; [
openssl
llvm_13
llvmPackages_13.libclang
llvm_18
llvmPackages_18.libclang
lmdb
libunwind
libiconv
] ++ pkgs.lib.optional pkgs.stdenv.isDarwin
pkgs.darwin.apple_sdk.frameworks.Security;

# needed for bindgen
LIBCLANG_PATH = "${pkgs.llvmPackages_13.libclang.lib}/lib";
CLANG_PATH = "${pkgs.llvmPackages_13.clang}/bin/clang";
LIBCLANG_PATH = "${pkgs.llvmPackages_18.libclang.lib}/lib";
CLANG_PATH = "${pkgs.llvmPackages_18.clang}/bin/clang";

# needed for ic-protobuf
PROTOC="${pkgs.protobuf}/bin/protoc";
Expand Down
32 changes: 9 additions & 23 deletions nix/sources.json
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,15 @@
"version": "v1.2.0"
},
"motoko-base": {
"branch": "next-moc",
"branch": "pull/589/head",
"description": "The Motoko base library",
"homepage": "",
"owner": "dfinity",
"repo": "motoko-base",
"rev": "21a80ab250501a5de3365114abb242559573fecc",
"sha256": "0zm60azafdfczdiqkqs1fwgxf3crb2bk0vs7y09rvnwkqgqkxxw8",
"rev": "1bece926bfc417db5661c66d8617f6b09ab06b95",
"sha256": "02i8s4ybjmbbf72jclpvqcsxh4dypgsxqsagiga64jqyprmqzvlr",
"type": "tarball",
"url": "https://github.com/dfinity/motoko-base/archive/21a80ab250501a5de3365114abb242559573fecc.tar.gz",
"url": "https://github.com/dfinity/motoko-base/archive/1bece926bfc417db5661c66d8617f6b09ab06b95.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
Comment on lines -73 to 82
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is to be changed back to next-moc once dfinity/motoko-base#589 is merged.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's not forget this

},
"motoko-matchers": {
Expand All @@ -94,30 +94,16 @@
"url": "https://github.com/kritzcreek/motoko-matchers/archive/cb838c192df3328ff9ae172e2dc7338cf55e74bf.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"musl-wasi": {
"branch": "main",
"builtin": false,
"description": "WASI libc implementation for WebAssembly",
"homepage": "https://wasi.dev",
"owner": "WebAssembly",
"repo": "wasi-libc",
"rev": "c5264e2bbe532994d06b039005f2af91bedcc1a6",
"sha256": "1skw2jqzaidr2zbzbjw32b36xvqniwf4if8cr4kbpp8vjvh7j2lr",
"type": "tarball",
"url": "https://github.com/WebAssembly/wasi-libc/archive/c5264e2bbe532994d06b039005f2af91bedcc1a6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs": {
"branch": "release-24.05",
"builtin": true,
"description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels",
"homepage": null,
"description": "Nix Packages collection & NixOS",
"homepage": "",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "68f19df4809ea68e782cddc5f3e9827cb5db65cb",
"sha256": "1caizbyi4sc5b47il8kbr2a8aripyb6wqw56dpgdhcmfhng17wb0",
"rev": "404f983f8c1da3ed403c95df9bd18d7c3b23399d",
"sha256": "025alb35qf1iwmz45615ahp400q54582kqzc0238a51ai2w88pxc",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/68f19df4809ea68e782cddc5f3e9827cb5db65cb.tar.gz",
"url": "https://github.com/NixOS/nixpkgs/archive/404f983f8c1da3ed403c95df9bd18d7c3b23399d.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-mozilla": {
Expand Down
Loading
Loading