acheechee
commented
3 years ago EDIT: a 10 minute save with this patch (down to 45mins from 56mins) to emerge 'rust' on Gentoo.
I've decided that it's unlikely this will be implemented soon enough so I went back to re-make my hacky patch(that avoids compression and keeps the files as .tar) to work for 1.52.0 rust version, and this is it:
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
@@ -6,7 +6,7 @@ use tar::{Builder, Header};
use walkdir::WalkDir;
use crate::{
- compression::{CombinedEncoder, CompressionFormats},
+ //compression::{CombinedEncoder, CompressionFormats},
util::*,
};
@@ -22,8 +22,8 @@ actor! {
/// The folder in which the input is to be found.
work_dir: String = "./workdir",
- /// The formats used to compress the tarball.
- compression_formats: CompressionFormats = CompressionFormats::default(),
+ ///// The formats used to compress the tarball.
+ //compression_formats: CompressionFormats = CompressionFormats::default(),
}
}
@@ -31,12 +31,13 @@ impl Tarballer {
/// Generates the actual tarballs
pub fn run(self) -> Result<()> {
let tarball_name = self.output.clone() + ".tar";
- let encoder = CombinedEncoder::new(
- self.compression_formats
- .iter()
- .map(|f| f.encode(&tarball_name))
- .collect::<Result<Vec<_>>>()?,
- );
+ let encoder = ensure_new_file(tarball_name)?;
+ //let encoder = CombinedEncoder::new(
+ // self.compression_formats
+ // .iter()
+ // .map(|f| f.encode(&tarball_name))
+ // .collect::<Result<Vec<_>>>()?,
+ //);
// Sort files by their suffix, to group files with the same name from
// different locations (likely identical) and files with the same
#@@ -51,7 +52,7 @@ impl Tarballer {
# let mut builder = Builder::new(buf);
#
# let pool = rayon::ThreadPoolBuilder::new()
#- .num_threads(2)
#+ .num_threads(12)
# .build()
# .unwrap();
# pool.install(move || {
@@ -71,8 +72,8 @@ impl Tarballer {
.context("failed to finish writing .tar stream")?
.into_inner()
.ok()
- .unwrap()
- .finish()?;
+ .unwrap();
+ //.finish()?;
Ok(())
})
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
@@ -1,7 +1,7 @@
use super::Scripter;
use super::Tarballer;
use crate::{
- compression::{CompressionFormat, CompressionFormats},
+ //compression::{CompressionFormat, CompressionFormats},
util::*,
};
use anyhow::{bail, Context, Result};
@@ -39,8 +39,8 @@ actor! {
/// The location to put the final image and tarball.
output_dir: String = "./dist",
- /// The formats used to compress the tarball
- compression_formats: CompressionFormats = CompressionFormats::default(),
+ // /// The formats used to compress the tarball
+ //compression_formats: CompressionFormats = CompressionFormats::default(),
}
}
@@ -64,11 +64,11 @@ impl Combiner {
.filter(|s| !s.is_empty())
{
// Extract the input tarballs
- let compression =
- CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
- anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
- })?;
- Archive::new(compression.decode(input_tarball)?)
+ //let compression =
+ // CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
+ // anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
+ /* })?;*/ let input_tarball=input_tarball.trim_end_matches(".gz").trim_end_matches(".xz");
+ Archive::new(open_file(&input_tarball)?)//compression.decode(input_tarball)?)
.unpack(&self.work_dir)
.with_context(|| {
format!(
#@@ -68,6 +68,7 @@ impl Combiner {
# // CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
# // anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
# // })?;
#+ let input_tarball=input_tarball.trim_end_matches(".gz").trim_end_matches(".xz");
# Archive::new(open_file(&input_tarball)?)//compression.decode(input_tarball)?)
# .unpack(&self.work_dir)
# .with_context(|| {
@@ -78,7 +78,8 @@ impl Combiner {
})?;
let pkg_name =
- input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
+ input_tarball.trim_end_matches(".tar");
+ //input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
let pkg_name = Path::new(pkg_name).file_name().unwrap();
let pkg_dir = Path::new(&self.work_dir).join(&pkg_name);
@@ -142,8 +143,8 @@ impl Combiner {
tarballer
.work_dir(self.work_dir)
.input(self.package_name)
- .output(path_to_str(&output)?.into())
- .compression_formats(self.compression_formats.clone());
+ .output(path_to_str(&output)?.into());
+ //.compression_formats(self.compression_formats.clone());
tarballer.run()?;
Ok(())
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
@@ -42,8 +42,8 @@ actor! {
/// The location to put the final image and tarball
output_dir: String = "./dist",
- /// The formats used to compress the tarball
- compression_formats: CompressionFormats = CompressionFormats::default(),
+ // /// The formats used to compress the tarball
+ // compression_formats: CompressionFormats = CompressionFormats::default(),
}
}
@@ -99,8 +99,8 @@ impl Generator {
tarballer
.work_dir(self.work_dir)
.input(self.package_name)
- .output(path_to_str(&output)?.into())
- .compression_formats(self.compression_formats.clone());
+ .output(path_to_str(&output)?.into());
+ //.compression_formats(self.compression_formats.clone());
tarballer.run()?;
Ok(())
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
@@ -41,7 +41,7 @@ fn combine(matches: &ArgMatches<'_>) ->
"non-installed-overlay" => non_installed_overlay,
"work-dir" => work_dir,
"output-dir" => output_dir,
- "compression-formats" => compression_formats,
+ //"compression-formats" => compression_formats,
});
combiner.run().context("failed to combine installers")?;
@@ -61,7 +61,7 @@ fn generate(matches: &ArgMatches<'_>) ->
"image-dir" => image_dir,
"work-dir" => work_dir,
"output-dir" => output_dir,
- "compression-formats" => compression_formats,
+ //"compression-formats" => compression_formats,
});
generator.run().context("failed to generate installer")?;
@@ -88,7 +88,7 @@ fn tarball(matches: &ArgMatches<'_>) ->
"input" => input,
"output" => output,
"work-dir" => work_dir,
- "compression-formats" => compression_formats,
+ //"compression-formats" => compression_formats,
});
tarballer.run().context("failed to generate tarballs")?;
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
@@ -73,6 +73,16 @@ pub fn create_new_file<P: AsRef<Path>>(p
Ok(file)
}
+/// Ensure a new file or truncates existing one
+pub fn ensure_new_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
+ let file = fs::OpenOptions::new()
+ .write(true)
+ .create(true).truncate(true)
+ .open(&path)
+ .with_context(|| format!("failed to truncate existing or create new file '{}'", path.as_ref().display()))?;
+ Ok(file)
+}
+
/// Wraps `fs::File::open()` with a nicer error message.
pub fn open_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
let file = fs::File::open(&path)I'm using this on Gentoo if it matters... and with this ebuild:
```ebuild
# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
EAPI=7
PYTHON_COMPAT=( python3_{7..9} )
inherit bash-completion-r1 check-reqs estack flag-o-matic llvm multiprocessing \
multilib-build python-any-r1 rust-toolchain toolchain-funcs verify-sig
if [[ ${PV} = *beta* ]]; then
betaver=${PV//*beta}
BETA_SNAPSHOT="${betaver:0:4}-${betaver:4:2}-${betaver:6:2}"
MY_P="rustc-beta"
SLOT="beta/${PV}"
SRC="${BETA_SNAPSHOT}/rustc-beta-src.tar.xz -> rustc-${PV}-src.tar.xz"
else
ABI_VER="$(ver_cut 1-2)"
SLOT="stable/${ABI_VER}"
MY_P="rustc-${PV}"
SRC="${MY_P}-src.tar.xz"
KEYWORDS="~amd64 ~arm ~arm64 ~ppc64 ~x86"
fi
RUST_STAGE0_VERSION="1.$(($(ver_cut 2) - 1)).0"
DESCRIPTION="Systems programming language from Mozilla"
HOMEPAGE="https://www.rust-lang.org/"
SRC_URI="
https://static.rust-lang.org/dist/${SRC}
verify-sig? ( https://static.rust-lang.org/dist/${SRC}.asc )
!system-bootstrap? ( $(rust_all_arch_uris rust-${RUST_STAGE0_VERSION}) )
"
# keep in sync with llvm ebuild of the same version as bundled one.
ALL_LLVM_TARGETS=( AArch64 AMDGPU ARM AVR BPF Hexagon Lanai Mips MSP430
NVPTX PowerPC RISCV Sparc SystemZ WebAssembly X86 XCore )
ALL_LLVM_TARGETS=( "${ALL_LLVM_TARGETS[@]/#/llvm_targets_}" )
LLVM_TARGET_USEDEPS=${ALL_LLVM_TARGETS[@]/%/(-)?}
LICENSE="|| ( MIT Apache-2.0 ) BSD-1 BSD-2 BSD-4 UoI-NCSA"
IUSE="clippy cpu_flags_x86_sse2 debug doc miri nightly parallel-compiler rls rustfmt system-bootstrap system-llvm test wasm ${ALL_LLVM_TARGETS[*]}"
# Please keep the LLVM dependency block separate. Since LLVM is slotted,
# we need to *really* make sure we're not pulling more than one slot
# simultaneously.
# How to use it:
# List all the working slots in LLVM_VALID_SLOTS, newest first.
LLVM_VALID_SLOTS=( 12 )
LLVM_MAX_SLOT="${LLVM_VALID_SLOTS[0]}"
# splitting usedeps needed to avoid CI/pkgcheck's UncheckableDep limitation
# (-) usedep needed because we may build with older llvm without that target
LLVM_DEPEND="|| ( "
for _s in ${LLVM_VALID_SLOTS[@]}; do
LLVM_DEPEND+=" ( "
for _x in ${ALL_LLVM_TARGETS[@]}; do
LLVM_DEPEND+="
${_x}? ( sys-devel/llvm:${_s}[${_x}(-)] )"
done
LLVM_DEPEND+=" )"
done
unset _s _x
LLVM_DEPEND+=" )
/dev/null) )
rustc_version=${rustc_version[0]#rust-bin-}
rustc_version=${rustc_version#rust-}
[[ -z "${rustc_version}" ]] && die "Failed to determine rust version, check 'eselect rust' output"
if ver_test "${rustc_version}" -lt "${rustc_wanted}" ; then
eerror "Rust >=${rustc_wanted} is required"
eerror "please run 'eselect rust' and set correct rust version"
die "selected rust version is too old"
elif ver_test "${rustc_version}" -ge "${rustc_toonew}" ; then
eerror "Rust <${rustc_toonew} is required"
eerror "please run 'eselect rust' and set correct rust version"
die "selected rust version is too new"
else
einfo "Using rust ${rustc_version} to build"
fi
}
pre_build_checks() {
local M=8192
# multiply requirements by 1.5 if we are doing x86-multilib
if use amd64; then
M=$(( $(usex abi_x86_32 15 10) * ${M} / 10 ))
fi
M=$(( $(usex clippy 128 0) + ${M} ))
M=$(( $(usex miri 128 0) + ${M} ))
M=$(( $(usex rls 512 0) + ${M} ))
M=$(( $(usex rustfmt 256 0) + ${M} ))
# add 2G if we compile llvm and 256M per llvm_target
if ! use system-llvm; then
M=$(( 2048 + ${M} ))
local ltarget
for ltarget in ${ALL_LLVM_TARGETS[@]}; do
M=$(( $(usex ${ltarget} 256 0) + ${M} ))
done
fi
M=$(( $(usex wasm 256 0) + ${M} ))
M=$(( $(usex debug 2 1) * ${M} ))
eshopts_push -s extglob
if is-flagq '-g?(gdb)?([1-9])'; then
M=$(( 15 * ${M} / 10 ))
fi
eshopts_pop
M=$(( $(usex system-bootstrap 0 1024) + ${M} ))
M=$(( $(usex doc 256 0) + ${M} ))
M=$(( 61000 + ${M} )) #takes 61G so to be sure add wtw else they think it would take, for crazy safety.
#current ebuild (29nov2020) took 60mins to compile (mitigations=2) and 60G in /var/tmp/portage (oddly 610M left even tho it was a 64G ext4 zram
CHECKREQS_DISK_BUILD=${M}M check-reqs_pkg_${EBUILD_PHASE}
}
llvm_check_deps() {
has_version -r "sys-devel/llvm:${LLVM_SLOT}[${LLVM_TARGET_USEDEPS// /,}]"
}
pkg_pretend() {
pre_build_checks
}
pkg_setup() {
pre_build_checks
python-any-r1_pkg_setup
export LIBGIT2_NO_PKG_CONFIG=1 #749381
use system-bootstrap && boostrap_rust_version_check
if use system-llvm; then
llvm_pkg_setup
local llvm_config="$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
export LLVM_LINK_SHARED=1
export RUSTFLAGS="${RUSTFLAGS} -Lnative=$("${llvm_config}" --libdir)"
fi
}
src_prepare() {
if ! use system-bootstrap; then
local rust_stage0_root="${WORKDIR}"/rust-stage0
local rust_stage0="rust-${RUST_STAGE0_VERSION}-$(rust_abi)"
"${WORKDIR}/${rust_stage0}"/install.sh --disable-ldconfig \
--without=rust-docs --destdir="${rust_stage0_root}" --prefix=/ || die
fi
default
}
src_configure() {
local rust_target="" rust_targets="" arch_cflags
# Collect rust target names to compile standard libs for all ABIs.
for v in $(multilib_get_enabled_abi_pairs); do
rust_targets="${rust_targets},\"$(rust_abi $(get_abi_CHOST ${v##*.}))\""
done
if use wasm; then
rust_targets="${rust_targets},\"wasm32-unknown-unknown\""
if use system-llvm; then
# un-hardcode rust-lld linker for this target
# https://bugs.gentoo.org/715348
sed -i '/linker:/ s/rust-lld/wasm-ld/' compiler/rustc_target/src/spec/wasm32_base.rs || die
fi
fi
rust_targets="${rust_targets#,}"
local tools="\"cargo\","
if use clippy; then
tools="\"clippy\",$tools"
fi
if use miri; then
tools="\"miri\",$tools"
fi
if use rls; then
tools="\"rls\",\"analysis\",\"src\",$tools"
fi
if use rustfmt; then
tools="\"rustfmt\",$tools"
fi
local rust_stage0_root
if use system-bootstrap; then
rust_stage0_root="$(rustc --print sysroot)"
else
rust_stage0_root="${WORKDIR}"/rust-stage0
fi
rust_target="$(rust_abi)"
cat <<- _EOF_ > "${S}"/config.toml
[llvm]
download-ci-llvm = false
optimize = $(toml_usex !debug)
release-debuginfo = $(toml_usex debug)
assertions = $(toml_usex debug)
ccache = "/usr/bin/ccache"
ninja = true
targets = "${LLVM_TARGETS// /;}"
experimental-targets = ""
link-shared = $(toml_usex system-llvm)
[build]
build = "${rust_target}"
host = ["${rust_target}"]
target = [${rust_targets}]
cargo = "${rust_stage0_root}/bin/cargo"
rustc = "${rust_stage0_root}/bin/rustc"
rustfmt = "${rust_stage0_root}/bin/rustfmt"
docs = $(toml_usex doc)
compiler-docs = false
submodules = false
python = "${EPYTHON}"
locked-deps = true
vendor = true
extended = true
tools = [${tools}]
# Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
verbose = 2
sanitizers = false
profiler = false
cargo-native-static = false
low-priority = true
print-step-timings = true
[install]
prefix = "${EPREFIX}/usr/lib/${PN}/${PV}"
sysconfdir = "etc"
docdir = "share/doc/rust"
bindir = "bin"
libdir = "lib"
mandir = "share/man"
[rust]
# https://github.com/rust-lang/rust/issues/54872
codegen-units-std = 1
optimize = true
debug = $(toml_usex debug)
debug-assertions = $(toml_usex debug)
debug-assertions-std = $(toml_usex debug)
debuginfo-level = 2
debuginfo-level-rustc = 2
debuginfo-level-std = 2
debuginfo-level-tools = 2
#debuginfo-level = $(usex debug 2 0)
#debuginfo-level-rustc = $(usex debug 2 0)
#debuginfo-level-std = $(usex debug 2 0)
#debuginfo-level-tools = $(usex debug 2 0)
debuginfo-level-tests = 0
backtrace = true
incremental = false
default-linker = "$(tc-getCC)"
parallel-compiler = $(toml_usex parallel-compiler)
channel = "$(usex nightly nightly stable)"
description = "gentoo"
rpath = false
verbose-tests = true
optimize-tests = $(toml_usex !debug)
codegen-tests = true
dist-src = false
remap-debuginfo = true
lld = $(usex system-llvm false $(toml_usex wasm))
# only deny warnings if doc+wasm are NOT requested, documenting stage0 wasm std fails without it
# https://github.com/rust-lang/rust/issues/74976
# https://github.com/rust-lang/rust/issues/76526
deny-warnings = $(usex wasm $(usex doc false true) true)
backtrace-on-ice = true
jemalloc = false
[dist]
src-tarball = false
compression-formats = ["gz"]
#TODO: try compression-formats 'cat'(not valid!) or just make sure it doesn't use any! However list must not be empty! as per https://github.com/rust-lang/rust/blob/abf3ec5b3353be973b18269fcdda76a59743f235/config.toml.example#L696 see: https://github.com/rust-lang/rust-installer/issues/110
#missing-tools = false
#^ this won't work (either with =false or =true), it fails to compile miri even though USE=-miri due to /var/db/repos/gentoo/profiles/base/package.use.mask:30 so it has to be =true
_EOF_
for v in $(multilib_get_enabled_abi_pairs); do
rust_target=$(rust_abi $(get_abi_CHOST ${v##*.}))
arch_cflags="$(get_abi_CFLAGS ${v##*.})"
cat <<- _EOF_ >> "${S}"/config.env
CFLAGS_${rust_target}=${arch_cflags}
_EOF_
cat <<- _EOF_ >> "${S}"/config.toml
[target.${rust_target}]
cc = "$(tc-getBUILD_CC)"
cxx = "$(tc-getBUILD_CXX)"
linker = "$(tc-getCC)"
ar = "$(tc-getAR)"
_EOF_
# librustc_target/spec/linux_musl_base.rs sets base.crt_static_default = true;
if use elibc_musl; then
cat <<- _EOF_ >> "${S}"/config.toml
crt-static = false
_EOF_
fi
if use system-llvm; then
cat <<- _EOF_ >> "${S}"/config.toml
llvm-config = "$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
_EOF_
fi
done
if use wasm; then
cat <<- _EOF_ >> "${S}"/config.toml
[target.wasm32-unknown-unknown]
linker = "$(usex system-llvm lld rust-lld)"
_EOF_
fi
if [[ -n ${I_KNOW_WHAT_I_AM_DOING_CROSS} ]]; then # whitespace intentionally shifted below
# experimental cross support
# discussion: https://bugs.gentoo.org/679878
# TODO: c*flags, clang, system-llvm, cargo.eclass target support
# it would be much better if we could split out stdlib
# complilation to separate ebuild and abuse CATEGORY to
# just install to /usr/lib/rustlib/
# extra targets defined as a bash array
# spec format: ::
# best place would be /etc/portage/env/dev-lang/rust
# Example:
# RUST_CROSS_TARGETS=(
# "AArch64:aarch64-unknown-linux-gnu:aarch64-unknown-linux-gnu"
# )
# no extra hand holding is done, no target transformations, all
# values are passed as-is with just basic checks, so it's up to user to supply correct values
# valid rust targets can be obtained with
# rustc --print target-list
# matching cross toolchain has to be installed
# matching LLVM_TARGET has to be enabled for both rust and llvm (if using system one)
# only gcc toolchains installed with crossdev are checked for now.
# BUG: we can't pass host flags to cross compiler, so just filter for now
# BUG: this should be more fine-grained.
filter-flags '-mcpu=*' '-march=*' '-mtune=*'
local cross_target_spec
for cross_target_spec in "${RUST_CROSS_TARGETS[@]}";do
# extracts first element form ::
local cross_llvm_target="${cross_target_spec%%:*}"
# extracts toolchain triples, :
local cross_triples="${cross_target_spec#*:}"
# extracts first element after before : separator
local cross_rust_target="${cross_triples%%:*}"
# extracts last element after : separator
local cross_toolchain="${cross_triples##*:}"
use llvm_targets_${cross_llvm_target} || die "need llvm_targets_${cross_llvm_target} target enabled"
command -v ${cross_toolchain}-gcc > /dev/null 2>&1 || die "need ${cross_toolchain} cross toolchain"
cat <<- _EOF_ >> "${S}"/config.toml
[target.${cross_rust_target}]
cc = "${cross_toolchain}-gcc"
cxx = "${cross_toolchain}-g++"
linker = "${cross_toolchain}-gcc"
ar = "${cross_toolchain}-ar"
_EOF_
if use system-llvm; then
cat <<- _EOF_ >> "${S}"/config.toml
llvm-config = "$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
_EOF_
fi
# append cross target to "normal" target list
# example 'target = ["powerpc64le-unknown-linux-gnu"]'
# becomes 'target = ["powerpc64le-unknown-linux-gnu","aarch64-unknown-linux-gnu"]'
rust_targets="${rust_targets},\"${cross_rust_target}\""
sed -i "/^target = \[/ s#\[.*\]#\[${rust_targets}\]#" config.toml || die
ewarn
ewarn "Enabled ${cross_rust_target} rust target"
ewarn "Using ${cross_toolchain} cross toolchain"
ewarn
if ! has_version -b 'sys-devel/binutils[multitarget]' ; then
ewarn "'sys-devel/binutils[multitarget]' is not installed"
ewarn "'strip' will be unable to strip cross libraries"
ewarn "cross targets will be installed with full debug information"
ewarn "enable 'multitarget' USE flag for binutils to be able to strip object files"
ewarn
ewarn "Alternatively llvm-strip can be used, it supports stripping any target"
ewarn "define STRIP=\"llvm-strip\" to use it (experimental)"
ewarn
fi
done
fi # I_KNOW_WHAT_I_AM_DOING_CROSS
einfo "Rust configured with the following flags:"
echo
echo "RUSTFLAGS=\"${RUSTFLAGS:-}\""
echo "RUSTFLAGS_BOOTSTRAP=\"${RUSTFLAGS_BOOTSTRAP:-}\""
echo "RUSTFLAGS_NOT_BOOTSTRAP=\"${RUSTFLAGS_NOT_BOOTSTRAP:-}\""
cat "${S}"/config.env || die
echo
einfo "config.toml contents:"
cat "${S}"/config.toml || die
echo
}
src_compile() {
# we need \n IFS to have config.env with spaces loaded properly. #734018
(
IFS=$'\n'
env $(cat "${S}"/config.env) RUST_BACKTRACE=1\
"${EPYTHON}" ./x.py dist -vv --config="${S}"/config.toml -j$(makeopts_jobs) || die
)
}
src_test() {
# https://rustc-dev-guide.rust-lang.org/tests/intro.html
# those are basic and codegen tests.
local tests=(
codegen
codegen-units
compile-fail
incremental
mir-opt
pretty
run-make
)
# fails if llvm is not built with ALL targets.
# and known to fail with system llvm sometimes.
use system-llvm || tests+=( assembly )
# fragile/expensive/less important tests
# or tests that require extra builds
# TODO: instead of skipping, just make some nonfatal.
if [[ ${ERUST_RUN_EXTRA_TESTS:-no} != no ]]; then
tests+=(
rustdoc
rustdoc-js
rustdoc-js-std
rustdoc-ui
run-make-fulldeps
ui
ui-fulldeps
)
fi
local i failed=()
einfo "rust_src_test: enabled tests ${tests[@]/#/src/test/}"
for i in "${tests[@]}"; do
local t="src/test/${i}"
einfo "rust_src_test: running ${t}"
if ! (
IFS=$'\n'
env $(cat "${S}"/config.env) RUST_BACKTRACE=1 \
"${EPYTHON}" ./x.py test -vv --config="${S}"/config.toml \
-j$(makeopts_jobs) --no-doc --no-fail-fast "${t}"
)
then
failed+=( "${t}" )
eerror "rust_src_test: ${t} failed"
fi
done
if [[ ${#failed[@]} -ne 0 ]]; then
eerror "rust_src_test: failure summary: ${failed[@]}"
die "aborting due to test failures"
fi
}
src_install() {
(
IFS=$'\n'
env $(cat "${S}"/config.env) DESTDIR="${D}" \
"${EPYTHON}" ./x.py install -vv --config="${S}"/config.toml -j$(makeopts_jobs) || die
)
# bug #689562, #689160
rm -v "${ED}/usr/lib/${PN}/${PV}/etc/bash_completion.d/cargo" || die
rmdir -v "${ED}/usr/lib/${PN}/${PV}"/etc{/bash_completion.d,} || die
newbashcomp src/tools/cargo/src/etc/cargo.bashcomp.sh cargo
local symlinks=(
cargo
rustc
rustdoc
rust-gdb
rust-gdbgui
rust-lldb
)
use clippy && symlinks+=( clippy-driver cargo-clippy )
use miri && symlinks+=( miri cargo-miri )
use rls && symlinks+=( rls )
use rustfmt && symlinks+=( rustfmt cargo-fmt )
einfo "installing eselect-rust symlinks and paths: ${symlinks[@]}"
local i
for i in "${symlinks[@]}"; do
# we need realpath on /usr/bin/* symlink return version-appended binary path.
# so /usr/bin/rustc should point to /usr/lib/rust//bin/rustc-
# need to fix eselect-rust to remove this hack.
local ver_i="${i}-${PV}"
if [[ -f "${ED}/usr/lib/${PN}/${PV}/bin/${i}" ]]; then
einfo "Installing ${i} symlink"
ln -v "${ED}/usr/lib/${PN}/${PV}/bin/${i}" "${ED}/usr/lib/${PN}/${PV}/bin/${ver_i}" || die
else
ewarn "${i} symlink requested, but source file not found"
ewarn "please report this"
fi
dosym "../lib/${PN}/${PV}/bin/${ver_i}" "/usr/bin/${ver_i}"
done
# symlinks to switch components to active rust in eselect
dosym "${PV}/lib" "/usr/lib/${PN}/lib-${PV}"
dosym "${PV}/libexec" "/usr/lib/${PN}/libexec-${PV}"
dosym "${PV}/share/man" "/usr/lib/${PN}/man-${PV}"
dosym "rust/${PV}/lib/rustlib" "/usr/lib/rustlib-${PV}"
dosym "../../lib/${PN}/${PV}/share/doc/rust" "/usr/share/doc/${P}"
newenvd - "50${P}" <<-_EOF_
LDPATH="${EPREFIX}/usr/lib/rust/lib"
MANPATH="${EPREFIX}/usr/lib/rust/man"
$(use amd64 && usex elibc_musl 'CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=-crt-static"' '')
$(use arm64 && usex elibc_musl 'CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=-crt-static"' '')
_EOF_
rm -rf "${ED}/usr/lib/${PN}/${PV}"/*.old || die
rm -rf "${ED}/usr/lib/${PN}/${PV}/doc"/*.old || die
# note: eselect-rust adds EROOT to all paths below
cat <<-_EOF_ > "${T}/provider-${P}"
/usr/bin/cargo
/usr/bin/rustdoc
/usr/bin/rust-gdb
/usr/bin/rust-gdbgui
/usr/bin/rust-lldb
/usr/lib/rustlib
/usr/lib/rust/lib
/usr/lib/rust/libexec
/usr/lib/rust/man
/usr/share/doc/rust
_EOF_
if use clippy; then
echo /usr/bin/clippy-driver >> "${T}/provider-${P}"
echo /usr/bin/cargo-clippy >> "${T}/provider-${P}"
fi
if use miri; then
echo /usr/bin/miri >> "${T}/provider-${P}"
echo /usr/bin/cargo-miri >> "${T}/provider-${P}"
fi
if use rls; then
echo /usr/bin/rls >> "${T}/provider-${P}"
fi
if use rustfmt; then
echo /usr/bin/rustfmt >> "${T}/provider-${P}"
echo /usr/bin/cargo-fmt >> "${T}/provider-${P}"
fi
insinto /etc/env.d/rust
doins "${T}/provider-${P}"
}
pkg_postinst() {
eselect rust update
if has_version sys-devel/gdb || has_version dev-util/lldb; then
elog "Rust installs a helper script for calling GDB and LLDB,"
elog "for your convenience it is installed under /usr/bin/rust-{gdb,lldb}-${PV}."
fi
if has_version app-editors/emacs; then
elog "install app-emacs/rust-mode to get emacs support for rust."
fi
if has_version app-editors/gvim || has_version app-editors/vim; then
elog "install app-vim/rust-vim to get vim support for rust."
fi
}
pkg_postrm() {
eselect rust cleanup
}
```
Without patch it used to emerge rust in 56 mins (ran it twice and got the same time), but now, with this patch, it does it in ... about 43 mins (second run was: 45mins because I was also watching a movie meanwhile
real 44m56.608s
user 353m59.402s
sys 5m25.139s)
In the 'compile' phase(./x.py dist)(this takes like 33 mins to complete), some `.tar` (instead of `.tar.gz`) files get generated in dir: `/var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/build/dist`
```
total 10628612
drwxr-xr-x 6 portage portage 4096 May 11 23:48 ..
-rw-r--r-- 1 portage portage 1450638336 May 12 00:38 rustc-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 178041344 May 12 00:38 rust-std-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 4334052864 May 12 00:38 rustc-dev-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 38921216 May 12 00:38 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 40742400 May 12 00:38 rust-src-nightly.tar
-rw-r--r-- 1 portage portage 202902016 May 12 00:38 cargo-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 196488704 May 12 00:38 rustfmt-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 538494464 May 12 00:38 rls-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 436925952 May 12 00:39 rust-analyzer-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 134438912 May 12 00:39 clippy-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 77839360 May 12 00:39 miri-nightly-x86_64-unknown-linux-gnu.tar
drwxr-xr-x 2 portage portage 4096 May 12 00:39 .
-rw-r--r-- 1 portage portage 3254147072 May 12 00:39 rust-nightly-x86_64-unknown-linux-gnu.tar
```
In the 'install' phase(./x.py install)(this takes like 10 mins to complete), some of the above files get regenerated(those from timestamp 0:41am inclusive):
```
real 9m53.212s
user 65m38.853s
sys 1m54.637s
```
```
total 10628596
drwxr-xr-x 6 portage portage 4096 May 11 23:48 ..
-rw-r--r-- 1 portage portage 4334052864 May 12 00:38 rustc-dev-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 436925952 May 12 00:39 rust-analyzer-nightly-x86_64-unknown-linux-gnu.tar
drwxr-xr-x 2 portage portage 4096 May 12 00:39 .
-rw-r--r-- 1 portage portage 3254147072 May 12 00:39 rust-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 178041344 May 12 00:41 rust-std-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 202902016 May 12 00:43 cargo-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 538494464 May 12 00:47 rls-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 196488704 May 12 00:48 rustfmt-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 134438912 May 12 00:48 clippy-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 77839360 May 12 00:49 miri-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 38921216 May 12 00:49 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 40742400 May 12 00:49 rust-src-nightly.tar
-rw-r--r-- 1 portage portage 1450638336 May 12 00:50 rustc-nightly-x86_64-unknown-linux-gnu.tar
```
Actual `config.toml` used while building (based on the above ebuild):
```toml
[llvm]
download-ci-llvm = false
optimize = true
release-debuginfo = false
assertions = false
ccache = "/usr/bin/ccache"
ninja = true
targets = "X86"
experimental-targets = ""
link-shared = true
[build]
build = "x86_64-unknown-linux-gnu"
host = ["x86_64-unknown-linux-gnu"]
target = ["x86_64-unknown-linux-gnu"]
cargo = "/usr/lib/rust/1.52.0/bin/cargo"
rustc = "/usr/lib/rust/1.52.0/bin/rustc"
rustfmt = "/usr/lib/rust/1.52.0/bin/rustfmt"
docs = false
compiler-docs = false
submodules = false
python = "python3.9"
locked-deps = true
vendor = true
extended = true
tools = ["rustfmt","rls","analysis","src","miri","clippy","cargo",]
# Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
verbose = 2
sanitizers = false
profiler = false
cargo-native-static = false
low-priority = true
print-step-timings = true
[install]
prefix = "/usr/lib/rust/1.52.0"
sysconfdir = "etc"
docdir = "share/doc/rust"
bindir = "bin"
libdir = "lib"
mandir = "share/man"
[rust]
# https://github.com/rust-lang/rust/issues/54872
codegen-units-std = 1
optimize = true
debug = false
debug-assertions = false
debug-assertions-std = false
debuginfo-level = 2
debuginfo-level-rustc = 2
debuginfo-level-std = 2
debuginfo-level-tools = 2
#debuginfo-level = 0
#debuginfo-level-rustc = 0
#debuginfo-level-std = 0
#debuginfo-level-tools = 0
debuginfo-level-tests = 0
backtrace = true
incremental = false
default-linker = "x86_64-pc-linux-gnu-gcc"
parallel-compiler = true
channel = "nightly"
description = "gentoo"
rpath = false
verbose-tests = true
optimize-tests = true
codegen-tests = true
dist-src = false
remap-debuginfo = true
lld = false
# only deny warnings if doc+wasm are NOT requested, documenting stage0 wasm std fails without it
# https://github.com/rust-lang/rust/issues/74976
# https://github.com/rust-lang/rust/issues/76526
deny-warnings = true
backtrace-on-ice = true
jemalloc = false
[dist]
src-tarball = false
compression-formats = ["gz"]
#nvm patch not made! //keep 'gz' here so that 0600_fabricate_neutering_try2_just_tar_not_gz_or_xz_for_rust_1_52_0.patch applies neatly
#TODO: try compression-formats 'cat'(not valid!) or just make sure it doesn't use any! However list must not be empty! as per https://github.com/rust-lang/rust/blob/abf3ec5b3353be973b18269fcdda76a59743f235/config.toml.example#L696 see: https://github.com/rust-lang/rust-installer/issues/110
#missing-tools = false
#^ this won't work (either with =false or =true), it fails to compile miri even though USE=-miri due to /var/db/repos/gentoo/profiles/base/package.use.mask:30 so it has to be =true
[target.x86_64-unknown-linux-gnu]
cc = "x86_64-pc-linux-gnu-gcc"
cxx = "x86_64-pc-linux-gnu-g++"
linker = "x86_64-pc-linux-gnu-gcc"
ar = "x86_64-pc-linux-gnu-ar"
llvm-config = "/usr/lib/llvm/12/bin/llvm-config"
```
aswild
correabuscar
onur-ozkan
Unless I'm misunderstanding something, currently one must use
gzorxzas compression formats. Because of this, during the building of rust itself, thefabricateprocess uses 100% CPU (aka 1 core only) and thus wastes a lot of time trying to compress to one or both of the two formats (specified in config.toml)On systems where the filesystem is btrfs and using zstd compression already, some may wish to use no compression at all, thus just
.tar(instead of.tar.gzor.tar.xzwould be preferred). Maybe such people don't care about any extra used space(presumably filesystem compression if poorer than .gz or .xz ?), or they do care about how much time rust compilation takes (it would take less time this way with avoiding doing any compression especially since it's using only one CPU core).Either way, I'd like to suggest supporting no compression at all - which would probably mean simply just
.tarIn the past I used to hack a patch to do this (
.tarinstead of.tar.gz) and it worked just fine, but since v1.48.0 rust's changed that code a bit in 1.52.0 and I'm unable to understand how to change the patch to apply it, so far.