Auto merge of #42433 - marco-c:profiling, r=alexcrichton

Build instruction profiler runtime as part of compiler-rt

r? @alexcrichton

This is #38608 with some fixes.

Still missing:
- [x] testing with profiler enabled on some builders (on which ones? Should I add the option to some of the already existing configurations, or create a new configuration?);
- [x] enabling distribution (on which builders?);
- [x] documentation.

Author: bors

.travis.yml

@@ -54,7 +54,7 @@ matrix:
     # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
     - env: >
         RUST_CHECK_TARGET=check
-        RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers"
+        RUST_CONFIGURE_ARGS="--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler"
         SRC=.
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
         SCCACHE_ERROR_LOG=/tmp/sccache.log
@@ -87,7 +87,7 @@ matrix:
     # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7.
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended"
+        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended --enable-profiler"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -101,7 +101,7 @@ matrix:
         - *osx_install_sccache
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers"
+        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers --enable-profiler"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1

appveyor.yml

@@ -7,7 +7,7 @@ environment:
   matrix:
   # 32/64 bit MSVC tests
   - MSYS_BITS: 64
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
     SCRIPT: python x.py test
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc
@@ -48,12 +48,14 @@ environment:
   - RUST_CONFIGURE_ARGS: >
       --build=x86_64-pc-windows-msvc
       --enable-extended
+      --enable-profiler
     SCRIPT: python x.py dist
     DEPLOY: 1
   - RUST_CONFIGURE_ARGS: >
       --build=i686-pc-windows-msvc
       --target=i586-pc-windows-msvc
       --enable-extended
+      --enable-profiler
     SCRIPT: python x.py dist
     DEPLOY: 1
   - MSYS_BITS: 32

configure

@@ -452,6 +452,7 @@ opt vendor 0 "enable usage of vendored Rust crates"
 opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)"
 opt dist-src 1 "when building tarballs enables building a source tarball"
 opt cargo-openssl-static 0 "static openssl in cargo"
+opt profiler 0 "build the profiler runtime"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

src/Cargo.lock

@@ -299,6 +299,10 @@ pub fn compiletest(build: &Build,
         cmd.env("SANITIZER_SUPPORT", "1");
     }
 
+    if build.config.profiler {
+        cmd.env("PROFILER_SUPPORT", "1");
+    }
+
     cmd.arg("--adb-path").arg("adb");
     cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);
     if target.contains("android") {

src/bootstrap/check.rs

@@ -50,6 +50,7 @@ pub struct Config {
     pub full_bootstrap: bool,
     pub extended: bool,
     pub sanitizers: bool,
+    pub profiler: bool,
 
     // llvm codegen options
     pub llvm_assertions: bool,
@@ -162,6 +163,7 @@ struct Build {
     extended: Option<bool>,
     verbose: Option<usize>,
     sanitizers: Option<bool>,
+    profiler: Option<bool>,
     openssl_static: Option<bool>,
 }
 
@@ -318,6 +320,7 @@ impl Config {
         set(&mut config.extended, build.extended);
         set(&mut config.verbose, build.verbose);
         set(&mut config.sanitizers, build.sanitizers);
+        set(&mut config.profiler, build.profiler);
         set(&mut config.openssl_static, build.openssl_static);
 
         if let Some(ref install) = toml.install {
@@ -471,6 +474,7 @@ impl Config {
                 ("FULL_BOOTSTRAP", self.full_bootstrap),
                 ("EXTENDED", self.extended),
                 ("SANITIZERS", self.sanitizers),
+                ("PROFILER", self.profiler),
                 ("DIST_SRC", self.rust_dist_src),
                 ("CARGO_OPENSSL_STATIC", self.openssl_static),
             }

src/bootstrap/config.rs

@@ -147,6 +147,9 @@
 # Build the sanitizer runtimes
 #sanitizers = false
 
+# Build the profiler runtime
+#profiler = false
+
 # Indicates whether the OpenSSL linked into Cargo will be statically linked or
 # not. If static linkage is specified then the build system will download a
 # known-good version of OpenSSL, compile it, and link it to Cargo.

src/bootstrap/config.toml.example

@@ -570,6 +570,7 @@ pub fn rust_src(build: &Build) {
         "src/libgetopts",
         "src/compiler-rt",
         "src/jemalloc",
+        "src/libprofiler_builtins",
     ];
     let std_src_dirs_exclude = [
         "src/compiler-rt/test",

src/bootstrap/dist.rs

@@ -594,6 +594,9 @@ impl Build {
         if self.config.backtrace {
             features.push_str(" backtrace");
         }
+        if self.config.profiler {
+            features.push_str(" profiler");
+        }
         return features
     }
 

src/bootstrap/lib.rs

@@ -90,7 +90,8 @@ ENV HOSTS=i686-unknown-linux-gnu
 ENV RUST_CONFIGURE_ARGS \
       --host=$HOSTS \
       --enable-extended \
-      --enable-sanitizers
+      --enable-sanitizers \
+      --enable-profiler
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 
 # This is the only builder which will create source tarballs

src/ci/docker/dist-i686-linux/Dockerfile

@@ -90,7 +90,8 @@ ENV HOSTS=x86_64-unknown-linux-gnu
 ENV RUST_CONFIGURE_ARGS \
       --host=$HOSTS \
       --enable-extended \
-      --enable-sanitizers
+      --enable-sanitizers \
+      --enable-profiler
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 
 # This is the only builder which will create source tarballs

src/ci/docker/dist-x86_64-linux/Dockerfile

@@ -22,5 +22,5 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers
+ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-sanitizers --enable-profiler
 ENV SCRIPT python2.7 ../x.py test

src/ci/docker/x86_64-gnu/Dockerfile

@@ -2,6 +2,7 @@
 
 - [Compiler flags](compiler-flags.md)
     - [linker_flavor](compiler-flags/linker-flavor.md)
+    - [profile](compiler-flags/profile.md)
     - [remap_path_prefix](compiler-flags/remap-path-prefix.md)
 - [Language features](language-features.md)
     - [abi_msp430_interrupt](language-features/abi-msp430-interrupt.md)
@@ -70,6 +71,7 @@
     - [plugin_registrar](language-features/plugin-registrar.md)
     - [prelude_import](language-features/prelude-import.md)
     - [proc_macro](language-features/proc-macro.md)
+    - [profiler_runtime](language-features/profiler-runtime.md)
     - [quote](language-features/quote.md)
     - [repr_align](language-features/repr-align.md)
     - [repr_simd](language-features/repr-simd.md)
@@ -178,6 +180,7 @@
     - [placement_new_protocol](library-features/placement-new-protocol.md)
     - [print_internals](library-features/print-internals.md)
     - [proc_macro_internals](library-features/proc-macro-internals.md)
+    - [profiler_runtime_lib](library-features/sanitizer-runtime-lib.md)
     - [question_mark_carrier](library-features/question-mark-carrier.md)
     - [rand](library-features/rand.md)
     - [range_contains](library-features/range-contains.md)

src/doc/unstable-book/src/SUMMARY.md

@@ -0,0 +1,21 @@
+# `profile`
+
+The tracking issue for this feature is: [#42524](https://github.com/rust-lang/rust/issues/42524).
+
+------------------------
+
+This feature allows the generation of code coverage reports.
+
+Set the `-Zprofile` compiler flag in order to enable gcov profiling.
+
+For example:
+```Bash
+cargo new testgcov --bin
+cd testgcov
+export RUSTFLAGS="-Zprofile"
+cargo build
+cargo run
+```
+
+Once you've built and run your program, files with the `gcno` (after build) and `gcda` (after execution) extensions will be created.
+You can parse them with [llvm-cov gcov](http://llvm.org/docs/CommandGuide/llvm-cov.html#llvm-cov-gcov) or [grcov](https://github.com/marco-c/grcov).

src/doc/unstable-book/src/compiler-flags/profile.md

@@ -0,0 +1,5 @@
+# `profiler_runtime`
+
+The tracking issue for this feature is: [#42524](https://github.com/rust-lang/rust/issues/42524).
+
+------------------------

src/doc/unstable-book/src/language-features/profiler-runtime.md

@@ -0,0 +1,5 @@
+# `profiler_runtime_lib`
+
+This feature is internal to the Rust compiler and is not intended for general use.
+
+------------------------

src/doc/unstable-book/src/library-features/profiler-runtime-lib.md

@@ -0,0 +1,18 @@
+[package]
+authors = ["The Rust Project Developers"]
+build = "build.rs"
+name = "profiler_builtins"
+version = "0.0.0"
+
+[lib]
+name = "profiler_builtins"
+path = "lib.rs"
+test = false
+bench = false
+doc = false
+
+[dependencies]
+core = { path = "../libcore" }
+
+[build-dependencies]
+gcc = "0.3.50"

src/libprofiler_builtins/Cargo.toml

@@ -0,0 +1,60 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Compiles the profiler part of the `compiler-rt` library.
+//!
+//! See the build.rs for libcompiler_builtins crate for details.
+
+extern crate gcc;
+
+use std::env;
+use std::path::Path;
+
+fn main() {
+    let target = env::var("TARGET").expect("TARGET was not set");
+    let cfg = &mut gcc::Config::new();
+
+    let mut profile_sources = vec!["GCDAProfiling.c",
+                                   "InstrProfiling.c",
+                                   "InstrProfilingBuffer.c",
+                                   "InstrProfilingFile.c",
+                                   "InstrProfilingMerge.c",
+                                   "InstrProfilingMergeFile.c",
+                                   "InstrProfilingPlatformDarwin.c",
+                                   "InstrProfilingPlatformLinux.c",
+                                   "InstrProfilingPlatformOther.c",
+                                   "InstrProfilingRuntime.cc",
+                                   "InstrProfilingUtil.c",
+                                   "InstrProfilingValue.c",
+                                   "InstrProfilingWriter.c"];
+
+    if target.contains("msvc") {
+        // Don't pull in extra libraries on MSVC
+        cfg.flag("/Zl");
+        profile_sources.push("WindowsMMap.c");
+        cfg.define("strdup", Some("_strdup"));
+        cfg.define("open", Some("_open"));
+        cfg.define("fdopen", Some("_fdopen"));
+    } else {
+        // Turn off various features of gcc and such, mostly copying
+        // compiler-rt's build system already
+        cfg.flag("-fno-builtin");
+        cfg.flag("-fvisibility=hidden");
+        cfg.flag("-fomit-frame-pointer");
+        cfg.flag("-ffreestanding");
+        cfg.define("VISIBILITY_HIDDEN", None);
+    }
+
+    for src in profile_sources {
+        cfg.file(Path::new("../compiler-rt/lib/profile").join(src));
+    }
+
+    cfg.compile("libprofiler-rt.a");
+}

src/libprofiler_builtins/build.rs

@@ -0,0 +1,20 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![no_std]
+#![cfg_attr(not(stage0), feature(profiler_runtime))]
+#![cfg_attr(not(stage0), profiler_runtime)]
+#![unstable(feature = "profiler_runtime_lib",
+            reason = "internal implementation detail of rustc right now",
+            issue = "0")]
+#![crate_name = "profiler_builtins"]
+#![crate_type = "rlib"]
+#![allow(unused_features)]
+#![feature(staged_api)]

src/libprofiler_builtins/lib.rs

@@ -258,6 +258,7 @@ pub trait CrateStore {
     fn is_panic_runtime(&self, cnum: CrateNum) -> bool;
     fn is_compiler_builtins(&self, cnum: CrateNum) -> bool;
     fn is_sanitizer_runtime(&self, cnum: CrateNum) -> bool;
+    fn is_profiler_runtime(&self, cnum: CrateNum) -> bool;
     fn panic_strategy(&self, cnum: CrateNum) -> PanicStrategy;
     fn extern_crate(&self, cnum: CrateNum) -> Option<ExternCrate>;
     /// The name of the crate as it is referred to in source code of the current
@@ -381,6 +382,7 @@ impl CrateStore for DummyCrateStore {
     fn is_allocator(&self, cnum: CrateNum) -> bool { bug!("is_allocator") }
     fn is_panic_runtime(&self, cnum: CrateNum) -> bool { bug!("is_panic_runtime") }
     fn is_compiler_builtins(&self, cnum: CrateNum) -> bool { bug!("is_compiler_builtins") }
+    fn is_profiler_runtime(&self, cnum: CrateNum) -> bool { bug!("is_profiler_runtime") }
     fn is_sanitizer_runtime(&self, cnum: CrateNum) -> bool { bug!("is_sanitizer_runtime") }
     fn panic_strategy(&self, cnum: CrateNum) -> PanicStrategy {
         bug!("panic_strategy")

src/librustc/middle/cstore.rs

@@ -1033,6 +1033,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "a single extra argument to prepend the linker invocation (can be used several times)"),
     pre_link_args: Option<Vec<String>> = (None, parse_opt_list, [UNTRACKED],
         "extra arguments to prepend to the linker invocation (space separated)"),
+    profile: bool = (false, parse_bool, [TRACKED],
+                     "insert profiling code"),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc/session/config.rs

@@ -204,7 +204,8 @@ pub fn compile_input(sess: &Session,
                 println!("Pre-trans");
                 tcx.print_debug_stats();
             }
-            let trans = phase_4_translate_to_llvm(tcx, analysis, &incremental_hashes_map);
+            let trans = phase_4_translate_to_llvm(tcx, analysis, &incremental_hashes_map,
+                                                  &outputs);
 
             if log_enabled!(::log::LogLevel::Info) {
                 println!("Post-trans");
@@ -1042,7 +1043,8 @@ pub fn phase_3_run_analysis_passes<'tcx, F, R>(sess: &'tcx Session,
 /// be discarded.
 pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                            analysis: ty::CrateAnalysis,
-                                           incremental_hashes_map: &IncrementalHashesMap)
+                                           incremental_hashes_map: &IncrementalHashesMap,
+                                           output_filenames: &OutputFilenames)
                                            -> trans::CrateTranslation {
     let time_passes = tcx.sess.time_passes();
 
@@ -1053,7 +1055,7 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
     let translation =
         time(time_passes,
              "translation",
-             move || trans::trans_crate(tcx, analysis, &incremental_hashes_map));
+             move || trans::trans_crate(tcx, analysis, &incremental_hashes_map, output_filenames));
 
     time(time_passes,
          "assert dep graph",