Rollup of 13 pull requests

Cargo.lock

@@ -4528,12 +4528,14 @@ name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
  "rustc_abi",
+ "rustc_ast",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_hir",
  "rustc_index",
  "rustc_macros",
  "rustc_middle",
+ "rustc_serialize",
  "rustc_session",
  "rustc_span",
  "rustc_symbol_mangling",
@@ -4599,7 +4601,6 @@ dependencies = [
  "rustc_crate_store",
  "rustc_data_structures",
  "rustc_errors",
- "rustc_expand",
  "rustc_feature",
  "rustc_hir",
  "rustc_index",

compiler/rustc_borrowck/src/lib.rs

@@ -612,21 +612,26 @@ fn get_flow_results<'a, 'tcx>(
 ) -> Results<'tcx, Borrowck<'a, 'tcx>> {
     // We compute these three analyses individually, but them combine them into
     // a single results so that `mbcx` can visit them all together.
-    let borrows = Borrows::new(tcx, body, regioncx, borrow_set).iterate_to_fixpoint(
-        tcx,
-        body,
-        Some("borrowck"),
-    );
-    let uninits = MaybeUninitializedPlaces::new(tcx, body, move_data).iterate_to_fixpoint(
-        tcx,
-        body,
-        Some("borrowck"),
-    );
-    let ever_inits = EverInitializedPlaces::new(body, move_data).iterate_to_fixpoint(
-        tcx,
-        body,
-        Some("borrowck"),
-    );
+    let borrows = {
+        let _timer = tcx.prof.generic_activity("borrowck_dataflow_borrows");
+        Borrows::new(tcx, body, regioncx, borrow_set).iterate_to_fixpoint(
+            tcx,
+            body,
+            Some("borrowck"),
+        )
+    };
+    let uninits = {
+        let _timer = tcx.prof.generic_activity("borrowck_dataflow_maybe_uninits");
+        MaybeUninitializedPlaces::new(tcx, body, move_data).iterate_to_fixpoint(
+            tcx,
+            body,
+            Some("borrowck"),
+        )
+    };
+    let ever_inits = {
+        let _timer = tcx.prof.generic_activity("borrowck_dataflow_ever_inits");
+        EverInitializedPlaces::new(body, move_data).iterate_to_fixpoint(tcx, body, Some("borrowck"))
+    };
 
     let analysis = Borrowck {
         borrows: borrows.analysis,

compiler/rustc_borrowck/src/nll.rs

@@ -154,6 +154,7 @@ pub(crate) fn compute_regions<'tcx>(
     // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints
     // and use them to compute loan liveness.
     if let Some(polonius_context) = polonius_context.as_mut() {
+        let _timer = infcx.tcx.prof.generic_activity("borrowck_polonius_loan_liveness");
         polonius_context.compute_loan_liveness(&mut regioncx, body, borrow_set)
     }
 

compiler/rustc_borrowck/src/type_check/liveness/mod.rs

@@ -32,6 +32,7 @@ pub(super) fn generate<'tcx>(
     move_data: &MoveData<'tcx>,
 ) {
     debug!("liveness::generate");
+    let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness");
 
     let mut free_regions = regions_that_outlive_free_regions(
         typeck.infcx.num_region_vars(),

compiler/rustc_borrowck/src/type_check/liveness/trace.rs

@@ -45,6 +45,8 @@ pub(super) fn trace<'tcx>(
     relevant_live_locals: Vec<Local>,
     boring_locals: Vec<Local>,
 ) {
+    let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness_trace");
+
     let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, typeck.body);
     let cx = LivenessContext {
         typeck,
@@ -485,6 +487,7 @@ impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> {
             // a much, much smaller domain: in our benchmarks, when it's not zero (the most likely
             // case), there are a few dozens compared to e.g. thousands or tens of thousands of
             // locals and move paths.
+            let _timer = tcx.prof.generic_activity("borrowck_dataflow_maybe_inits");
             let flow_inits = MaybeInitializedPlaces::new(tcx, body, self.move_data)
                 .iterate_to_fixpoint(tcx, body, Some("borrowck"))
                 .into_results_cursor(body);

compiler/rustc_builtin_macros/src/offload.rs

@@ -1,6 +1,6 @@
+use rustc_ast::ast;
 use rustc_ast::token::{Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
-use rustc_ast::{AttrItem, ast};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_session::config::Offload;
 use rustc_span::{DUMMY_SP, Ident, Span, sym};
@@ -9,7 +9,7 @@ use thin_vec::thin_vec;
 use crate::diagnostics;
 
 fn compile_for_device(ecx: &mut ExtCtxt<'_>) -> bool {
-    ecx.sess.opts.unstable_opts.offload.contains(&Offload::Device)
+    ecx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Device(_)))
 }
 
 fn outer_normal_attr(normal: &Box<ast::NormalAttr>, id: ast::AttrId, span: Span) -> ast::Attribute {
@@ -45,7 +45,6 @@ fn extract_fn(
 /// This expands to the host-side function:
 ///
 /// ```
-/// #[unsafe(no_mangle)]
 /// #[inline(never)]
 /// fn foo(_: &[f32], _: &[f32], _: *mut f32) {
 ///     ::core::panicking::panic("not implemented")
@@ -56,7 +55,6 @@ fn extract_fn(
 ///
 /// ```
 /// #[rustc_offload_kernel]
-/// #[unsafe(no_mangle)]
 /// unsafe extern "gpu-kernel" fn foo(a: &[f32], b: &[f32], c: *mut f32) {
 ///     *c = a[0] + b[0];
 /// }
@@ -110,24 +108,9 @@ pub(crate) fn expand_kernel(
         span,
     );
 
-    // unsafe(no_mangle) attr
-    let unsafe_item = AttrItem {
-        unsafety: ast::Safety::Unsafe(span),
-        path: ast::Path::from_ident(Ident::new(sym::no_mangle, span)),
-        args: ast::AttrArgs::Empty,
-        span,
-    };
-
-    let no_mangle_attr = Box::new(ast::NormalAttr { item: unsafe_item, tokens: None });
-    let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id();
-    let unsafe_no_mangle = outer_normal_attr(&no_mangle_attr, new_id, span);
-
     let device_item = {
-        let mut item = ecx.item(
-            span,
-            thin_vec![rustc_offload_kernel, unsafe_no_mangle],
-            ast::ItemKind::Fn(device_fn),
-        );
+        let mut item =
+            ecx.item(span, thin_vec![rustc_offload_kernel.clone()], ast::ItemKind::Fn(device_fn));
         item.vis = vis.clone();
         Annotatable::Item(item)
     };
@@ -187,12 +170,12 @@ pub(crate) fn expand_kernel(
     let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id();
     let inline_never = outer_normal_attr(&inline_never_attr, new_id, span);
 
-    let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id();
-    let unsafe_no_mangle = outer_normal_attr(&no_mangle_attr, new_id, span);
-
     let host_item = {
-        let mut item =
-            ecx.item(span, thin_vec![unsafe_no_mangle, inline_never], ast::ItemKind::Fn(host_fn));
+        let mut item = ecx.item(
+            span,
+            thin_vec![rustc_offload_kernel, inline_never],
+            ast::ItemKind::Fn(host_fn),
+        );
         item.vis = vis.clone();
         Annotatable::Item(item)
     };

compiler/rustc_builtin_macros/src/source_util.rs

@@ -9,7 +9,7 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::{join_path_idents, token};
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{
-    DummyResult, ExpandResult, ExtCtxt, MacEager, MacResult, MacroExpanderResult, resolve_path,
+    DummyResult, ExpandResult, ExtCtxt, MacEager, MacResult, MacroExpanderResult,
 };
 use rustc_expand::module::DirOwnership;
 use rustc_parse::lexer::StripTokens;
@@ -117,7 +117,7 @@ pub(crate) fn expand_include<'cx>(
         Err(guar) => return ExpandResult::Ready(DummyResult::any(sp, guar)),
     };
     // The file will be added to the code map by the parser
-    let path = match resolve_path(&cx.sess, path.as_str(), sp) {
+    let path = match cx.sess.resolve_path(path.as_str(), sp) {
         Ok(path) => path,
         Err(err) => {
             let guar = err.emit();
@@ -267,7 +267,7 @@ fn load_binary_file(
     macro_span: Span,
     path_span: Span,
 ) -> Result<(Arc<[u8]>, Span), Box<dyn MacResult>> {
-    let resolved_path = match resolve_path(&cx.sess, original_path, macro_span) {
+    let resolved_path = match cx.sess.resolve_path(original_path, macro_span) {
         Ok(path) => path,
         Err(err) => {
             let guar = err.emit();

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -742,7 +742,9 @@ pub(crate) unsafe fn llvm_optimize(
         llvm::set_value_name(new_fn, &name);
     }
 
-    if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) {
+    if cgcx.target_is_like_gpu
+        && config.offload.iter().any(|o| matches!(o, config::Offload::Device(_)))
+    {
         let cx =
             SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size);
         for func in cx.get_functions() {
@@ -813,7 +815,9 @@ pub(crate) unsafe fn llvm_optimize(
         )
     };
 
-    if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) {
+    if cgcx.target_is_like_gpu
+        && config.offload.iter().any(|o| matches!(o, config::Offload::Device(_)))
+    {
         let device_path = cgcx.output_filenames.path(OutputType::Object);
         let device_dir = device_path.parent().unwrap();
         let device_out = device_dir.join("device.bin");

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -26,7 +26,9 @@ use rustc_session::config::CrateType;
 use rustc_session::diagnostics::feature_err;
 use rustc_session::lint::builtin::DEPRECATED_LLVM_INTRINSIC;
 use rustc_span::{ErrorGuaranteed, Span, Symbol, sym};
-use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate};
+use rustc_symbol_mangling::{
+    mangle_internal_symbol, mangle_offload_export, symbol_name_for_instance_in_crate,
+};
 use rustc_target::callconv::PassMode;
 use rustc_target::spec::Arch;
 use tracing::debug;
@@ -1850,9 +1852,9 @@ fn codegen_offload<'ll, 'tcx>(
         _ => panic!("unparsable"),
     };
     let args = get_args_from_tuple(bx, args[4], fn_target);
-    let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE);
+    let target_symbol = mangle_offload_export(tcx, fn_target);
 
-    let sig = tcx.fn_sig(fn_target.def_id()).skip_binder();
+    let sig = tcx.fn_sig(fn_target.def_id()).instantiate(tcx, fn_target.args).skip_norm_wip();
     let sig = tcx.instantiate_bound_regions_with_erased(sig);
     let inputs = sig.inputs();
 

compiler/rustc_codegen_llvm/src/lib.rs

@@ -375,7 +375,7 @@ impl CodegenBackend for LlvmCodegenBackend {
     fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Box<dyn Any> {
         use rustc_session::config::Offload;
 
-        if tcx.sess.opts.unstable_opts.offload.contains(&Offload::Device)
+        if tcx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Device(_)))
             || tcx.sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Host(_)))
         {
             match llvm::RustOffloadWrapper::get_or_init(&tcx.sess.opts.sysroot) {

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -3,6 +3,7 @@ use std::collections::hash_map::Entry::*;
 use rustc_abi::{CanonAbi, X86Call};
 use rustc_ast::expand::allocator::{AllocatorKind, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name};
 use rustc_crate_store::CrateDepKind;
+use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::unord::UnordMap;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
@@ -19,7 +20,7 @@ use rustc_middle::ty::{
 use rustc_middle::util::Providers;
 use rustc_session::config::CrateType;
 use rustc_span::Span;
-use rustc_symbol_mangling::mangle_internal_symbol;
+use rustc_symbol_mangling::{is_offload_kernel, mangle_internal_symbol};
 use rustc_target::spec::{Arch, Os, TlsModel};
 use tracing::debug;
 
@@ -244,6 +245,51 @@ pub fn exported_non_generic_symbols_helper<'tcx>(
         ));
     }
 
+    let is_device_offload = tcx
+        .sess
+        .opts
+        .unstable_opts
+        .offload
+        .iter()
+        .any(|o| matches!(o, rustc_session::config::Offload::Device(_)));
+    if is_device_offload {
+        let crate_items = tcx.hir_crate_items(());
+        let mut seen: rustc_data_structures::fx::FxHashSet<DefId> = symbols
+            .iter()
+            .filter_map(|(s, _)| match s {
+                ExportedSymbol::NonGeneric(d) => Some(*d),
+                _ => None,
+            })
+            .collect();
+
+        let mut try_emit_offload_kernel = |def_id: DefId, seen: &mut FxHashSet<DefId>| {
+            if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) {
+                return;
+            }
+            if !tcx.generics_of(def_id).requires_monomorphization(tcx)
+                && is_offload_kernel(tcx.codegen_fn_attrs(def_id))
+                && seen.insert(def_id)
+            {
+                symbols.push((
+                    ExportedSymbol::NonGeneric(def_id),
+                    SymbolExportInfo {
+                        level: SymbolExportLevel::C,
+                        kind: SymbolExportKind::Text,
+                        used: false,
+                        rustc_std_internal_symbol: false,
+                    },
+                ));
+            }
+        };
+
+        for id in crate_items.free_items() {
+            try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen);
+        }
+        for id in crate_items.impl_items() {
+            try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen);
+        }
+    }
+
     // Sort so we get a stable incr. comp. hash.
     symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx));
 
@@ -260,7 +306,16 @@ fn exported_generic_symbols_provider_local<'tcx>(
 
     let mut symbols: Vec<_> = vec![];
 
-    if tcx.local_crate_exports_generics() {
+    let export_generics = tcx.local_crate_exports_generics();
+    let is_device_offload = tcx
+        .sess
+        .opts
+        .unstable_opts
+        .offload
+        .iter()
+        .any(|o| matches!(o, rustc_session::config::Offload::Device(_)));
+
+    if export_generics || is_device_offload {
         use rustc_hir::attrs::Linkage;
         use rustc_middle::mono::{MonoItem, Visibility};
         use rustc_middle::ty::InstanceKind;
@@ -306,6 +361,14 @@ fn exported_generic_symbols_provider_local<'tcx>(
                     })
             };
 
+        let is_offload_instance = |mono_item: &MonoItem<'tcx>| {
+            if let MonoItem::Fn(instance) = mono_item {
+                is_offload_kernel(tcx.codegen_fn_attrs(instance.def_id()))
+            } else {
+                false
+            }
+        };
+
         // The symbols created in this loop are sorted below it
         #[allow(rustc::potential_query_instability)]
         for (mono_item, data) in cgus.iter().flat_map(|cgu| cgu.items().iter()) {
@@ -321,7 +384,9 @@ fn exported_generic_symbols_provider_local<'tcx>(
                 continue;
             }
 
-            if !tcx.sess.opts.share_generics() {
+            let item_is_offload = is_offload_instance(mono_item);
+
+            if !item_is_offload && !tcx.sess.opts.share_generics() {
                 if tcx.codegen_fn_attrs(mono_item.def_id()).inline
                     == rustc_hir::attrs::InlineAttr::Never
                 {
@@ -338,15 +403,22 @@ fn exported_generic_symbols_provider_local<'tcx>(
                 MonoItem::Fn(Instance { def: InstanceKind::Item(def), args }) => {
                     let has_generics = args.non_erasable_generics().next().is_some();
 
-                    let should_export =
-                        has_generics && is_instantiable_downstream(Some(def), &args);
+                    let should_export = if item_is_offload {
+                        has_generics
+                    } else {
+                        has_generics && is_instantiable_downstream(Some(def), &args)
+                    };
 
                     if should_export {
                         let symbol = ExportedSymbol::Generic(def, args);
                         symbols.push((
                             symbol,
                             SymbolExportInfo {
-                                level: SymbolExportLevel::Rust,
+                                level: if item_is_offload {
+                                    SymbolExportLevel::C
+                                } else {
+                                    SymbolExportLevel::Rust
+                                },
                                 kind: SymbolExportKind::Text,
                                 used: false,
                                 rustc_std_internal_symbol: false,