Merge pull request #918 from bjorn3/incr_cache

Implement incremental caching of object files

Readme.md

@@ -18,6 +18,7 @@ $ ./test.sh --release
 ### Cargo
 
 ```bash
+$ export CG_CLIF_INCR_CACHE=1 # Enable caching of object files in the incremental cache
 $ CHANNEL="release" $cg_clif_dir/cargo.sh run
 ```
 
@@ -26,6 +27,7 @@ If you compiled cg_clif in debug mode you should use `CHANNEL="debug"` instead o
 ### Rustc
 
 ```bash
+$ export CG_CLIF_INCR_CACHE=1 # Enable caching of object files in the incremental cache
 $ rustc -Cpanic=abort -Zcodegen-backend=$cg_clif_dir/target/release/librustc_codegen_cranelift.so --sysroot $cg_clif_dir/build_sysroot/sysroot my_crate.rs
 ```
 

src/driver.rs

@@ -2,9 +2,11 @@ use std::any::Any;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
+use rustc::dep_graph::{WorkProduct, WorkProductFileKind, WorkProductId};
 use rustc::middle::cstore::EncodedMetadata;
-use rustc::mir::mono::{Linkage as RLinkage, Visibility};
+use rustc::mir::mono::{CodegenUnit, Linkage as RLinkage, Visibility};
 use rustc::session::config::{DebugInfo, OutputType};
+use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_codegen_ssa::back::linker::LinkerInfo;
 use rustc_codegen_ssa::CrateInfo;
 
@@ -172,20 +174,32 @@ fn run_aot(
     tcx: TyCtxt<'_>,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
-) -> Box<CodegenResults> {
-    let new_module = |name: String| {
+) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    let mut work_products = FxHashMap::default();
+
+    fn new_module(tcx: TyCtxt<'_>, name: String) -> Module<crate::backend::Backend> {
         let module = crate::backend::make_module(tcx.sess, name);
         assert_eq!(pointer_ty(tcx), module.target_config().pointer_type());
         module
     };
 
+    struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
+
+    use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+
+    impl<HCX> HashStable<HCX> for ModuleCodegenResult {
+        fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) {
+            // do nothing
+        }
+    }
+
     fn emit_module<B: Backend>(
         tcx: TyCtxt<'_>,
         name: String,
         kind: ModuleKind,
         mut module: Module<B>,
         debug: Option<DebugContext>,
-    ) -> CompiledModule
+    ) -> ModuleCodegenResult
         where B::Product: Emit + WriteDebugInfo,
     {
             module.finalize_definitions();
@@ -200,54 +214,146 @@ fn run_aot(
                 .temp_path(OutputType::Object, Some(&name));
             let obj = product.emit();
             std::fs::write(&tmp_file, obj).unwrap();
-            CompiledModule {
-                name,
-                kind,
-                object: Some(tmp_file),
-                bytecode: None,
-                bytecode_compressed: None,
-            }
+
+            let work_product = if std::env::var("CG_CLIF_INCR_CACHE").is_ok() {
+                rustc_incremental::copy_cgu_workproducts_to_incr_comp_cache_dir(
+                    tcx.sess,
+                    &name,
+                    &[(WorkProductFileKind::Object, tmp_file.clone())],
+                )
+            } else {
+                None
+            };
+
+            ModuleCodegenResult(
+                CompiledModule {
+                    name,
+                    kind,
+                    object: Some(tmp_file),
+                    bytecode: None,
+                    bytecode_compressed: None,
+                },
+                work_product,
+            )
         };
 
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
 
-    let modules = time(tcx.sess, "codegen mono items", move || {
+    if tcx.dep_graph.is_fully_enabled() {
+        for cgu in &*cgus {
+            tcx.codegen_unit(cgu.name());
+        }
+    }
+
+    let modules = time(tcx.sess, "codegen mono items", || {
         cgus.iter().map(|cgu| {
-            let mono_items = cgu.items_in_deterministic_order(tcx);
+            let cgu_reuse = determine_cgu_reuse(tcx, cgu);
+            tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
+
+            match cgu_reuse {
+                CguReuse::No => {}
+                CguReuse::PreLto => {
+                    let incr_comp_session_dir = tcx.sess.incr_comp_session_dir();
+                    let mut object = None;
+                    let work_product = cgu.work_product(tcx);
+                    for (kind, saved_file) in &work_product.saved_files {
+                        let obj_out = match kind {
+                            WorkProductFileKind::Object => {
+                                let path = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&cgu.name().as_str()));
+                                object = Some(path.clone());
+                                path
+                            }
+                            WorkProductFileKind::Bytecode | WorkProductFileKind::BytecodeCompressed => {
+                                panic!("cg_clif doesn't use bytecode");
+                            }
+                        };
+                        let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file);
+                        if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
+                            tcx.sess.err(&format!(
+                                "unable to copy {} to {}: {}",
+                                source_file.display(),
+                                obj_out.display(),
+                                err
+                            ));
+                        }
+                    }
+
+                    work_products.insert(cgu.work_product_id(), work_product);
 
-            let mut module = new_module(cgu.name().as_str().to_string());
+                    return CompiledModule {
+                        name: cgu.name().to_string(),
+                        kind: ModuleKind::Regular,
+                        object,
+                        bytecode: None,
+                        bytecode_compressed: None,
+                    };
+                }
+                CguReuse::PostLto => unreachable!(),
+            }
+
+            let dep_node = cgu.codegen_dep_node(tcx);
+            let (ModuleCodegenResult(module, work_product), _) =
+                tcx.dep_graph.with_task(dep_node, tcx, cgu.name(), module_codegen, rustc::dep_graph::hash_result);
 
-            let mut debug = if tcx.sess.opts.debuginfo != DebugInfo::None {
-                let debug = DebugContext::new(
+            fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodegenResult {
+                let cgu = tcx.codegen_unit(cgu_name);
+                let mono_items = cgu.items_in_deterministic_order(tcx);
+
+                let mut module = new_module(tcx, cgu_name.as_str().to_string());
+
+                let mut debug = if tcx.sess.opts.debuginfo != DebugInfo::None {
+                    let debug = DebugContext::new(
+                        tcx,
+                        module.target_config().pointer_type().bytes() as u8,
+                    );
+                    Some(debug)
+                } else {
+                    None
+                };
+
+                codegen_mono_items(tcx, &mut module, debug.as_mut(), mono_items);
+                crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module);
+
+                emit_module(
                     tcx,
-                    module.target_config().pointer_type().bytes() as u8,
-                );
-                Some(debug)
-            } else {
-                None
-            };
+                    cgu.name().as_str().to_string(),
+                    ModuleKind::Regular,
+                    module,
+                    debug,
+                )
+            }
 
-            codegen_mono_items(tcx, &mut module, debug.as_mut(), mono_items);
-            crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module);
+            if let Some((id, product)) = work_product {
+                work_products.insert(id, product);
+            }
 
-            emit_module(
-                tcx,
-                cgu.name().as_str().to_string(),
-                ModuleKind::Regular,
-                module,
-                debug,
-            )
+            module
         }).collect::<Vec<_>>()
     });
 
     tcx.sess.abort_if_errors();
 
-    let mut allocator_module = new_module("allocator_shim".to_string());
+    let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
     let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);
 
+    let allocator_module = if created_alloc_shim {
+        let ModuleCodegenResult(module, work_product) = emit_module(
+            tcx,
+            "allocator_shim".to_string(),
+            ModuleKind::Allocator,
+            allocator_module,
+            None,
+        );
+        if let Some((id, product)) = work_product {
+            work_products.insert(id, product);
+        }
+        Some(module)
+    } else {
+        None
+    };
+
     rustc_incremental::assert_dep_graph(tcx);
     rustc_incremental::save_dep_graph(tcx);
-    rustc_incremental::finalize_session_directory(tcx.sess, tcx.crate_hash(LOCAL_CRATE));
 
     let metadata_module = if need_metadata_module {
         let _timer = tcx.prof.generic_activity("codegen crate metadata");
@@ -284,27 +390,17 @@ fn run_aot(
         None
     };
 
-    Box::new(CodegenResults {
+    Box::new((CodegenResults {
         crate_name: tcx.crate_name(LOCAL_CRATE),
         modules,
-        allocator_module: if created_alloc_shim {
-            Some(emit_module(
-                tcx,
-                "allocator_shim".to_string(),
-                ModuleKind::Allocator,
-                allocator_module,
-                None,
-            ))
-        } else {
-            None
-        },
+        allocator_module,
         metadata_module,
         crate_hash: tcx.crate_hash(LOCAL_CRATE),
         metadata,
         windows_subsystem: None, // Windows is not yet supported
         linker_info: LinkerInfo::new(tcx),
         crate_info: CrateInfo::new(tcx),
-    })
+    }, work_products))
 }
 
 fn codegen_mono_items<'tcx>(
@@ -402,3 +498,36 @@ fn time<R>(sess: &Session, name: &'static str, f: impl FnOnce() -> R) -> R {
     println!("[{}] end time: {:?}", name, after - before);
     res
 }
+
+// Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953
+fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {
+    if !tcx.dep_graph.is_fully_enabled() {
+        return CguReuse::No;
+    }
+
+    let work_product_id = &cgu.work_product_id();
+    if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
+        // We don't have anything cached for this CGU. This can happen
+        // if the CGU did not exist in the previous session.
+        return CguReuse::No;
+    }
+
+    // Try to mark the CGU as green. If it we can do so, it means that nothing
+    // affecting the LLVM module has changed and we can re-use a cached version.
+    // If we compile with any kind of LTO, this means we can re-use the bitcode
+    // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only
+    // know that later). If we are not doing LTO, there is only one optimized
+    // version of each module, so we re-use that.
+    let dep_node = cgu.codegen_dep_node(tcx);
+    assert!(
+        !tcx.dep_graph.dep_node_exists(&dep_node),
+        "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",
+        cgu.name()
+    );
+
+    if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {
+        CguReuse::PreLto
+    } else {
+        CguReuse::No
+    }
+}

src/lib.rs

@@ -21,7 +21,7 @@ extern crate rustc_ast;
 
 use std::any::Any;
 
-use rustc::dep_graph::DepGraph;
+use rustc::dep_graph::{DepGraph, WorkProduct, WorkProductId};
 use rustc::middle::cstore::{EncodedMetadata, MetadataLoader};
 use rustc::session::config::OutputFilenames;
 use rustc::ty::query::Providers;
@@ -216,10 +216,16 @@ impl CodegenBackend for CraneliftCodegenBackend {
     fn join_codegen(
         &self,
         ongoing_codegen: Box<dyn Any>,
-        _sess: &Session,
-        _dep_graph: &DepGraph,
+        sess: &Session,
+        dep_graph: &DepGraph,
     ) -> Result<Box<dyn Any>, ErrorReported> {
-        Ok(ongoing_codegen)
+        let (codegen_results, work_products) = *ongoing_codegen.downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>().unwrap();
+
+        sess.time("serialize_work_products", move || {
+            rustc_incremental::save_work_product_index(sess, &dep_graph, work_products)
+        });
+
+        Ok(Box::new(codegen_results))
     }
 
     fn link(
@@ -247,6 +253,8 @@ impl CodegenBackend for CraneliftCodegenBackend {
             );
         });
 
+        rustc_incremental::finalize_session_directory(sess, codegen_results.crate_hash);
+
         Ok(())
     }
 }