Track object file dependencies for TH accurately (#20604)

`hscCompileCoreExprHook` is changed to return a list of `Module`s required
by a splice. These modules are accumulated in the TcGblEnv (tcg_th_needed_mods).
Dependencies on the object files of these modules are recording in the
interface.

The data structures in `LoaderState` are replaced with more efficient versions
to keep track of all the information required. The
MultiLayerModulesTH_Make allocations increase slightly but runtime is
faster.

Fixes #20604

-------------------------
Metric Increase:
    MultiLayerModulesTH_Make
-------------------------

3 files changed, 23 insertions, 10 deletions

diff --git a/compiler/GHC/Tc/Gen/Splice.hs b/compiler/GHC/Tc/Gen/Splice.hs
index 7264f2232a..f2efb93f2d 100644
--- a/compiler/GHC/Tc/Gen/Splice.hs
+++ b/compiler/GHC/Tc/Gen/Splice.hs
@@ -1011,13 +1011,12 @@ runMeta' show_code ppr_hs run_and_convert expr
 
         -- Compile and link it; might fail if linking fails
         ; src_span <- getSrcSpanM
-        ; mnwib <- getMnwib
         ; traceTc "About to run (desugared)" (ppr ds_expr)
         ; either_hval <- tryM $ liftIO $
-                         GHC.Driver.Main.hscCompileCoreExpr hsc_env (src_span, Just mnwib) ds_expr
+                         GHC.Driver.Main.hscCompileCoreExpr hsc_env src_span ds_expr
         ; case either_hval of {
             Left exn   -> fail_with_exn "compile and link" exn ;
-            Right hval -> do
+            Right (hval, needed_mods, needed_pkgs) -> do
 
         {       -- Coerce it to Q t, and run it
 
@@ -1031,6 +1030,7 @@ runMeta' show_code ppr_hs run_and_convert expr
                 --
                 -- See Note [Exceptions in TH]
           let expr_span = getLocA expr
+        ; recordThNeededRuntimeDeps needed_mods needed_pkgs
         ; either_tval <- tryAllM $
                          setSrcSpan expr_span $ -- Set the span so that qLocation can
                                                 -- see where this splice is
diff --git a/compiler/GHC/Tc/Types.hs b/compiler/GHC/Tc/Types.hs
index e1f0400e44..b49bc718cd 100644
--- a/compiler/GHC/Tc/Types.hs
+++ b/compiler/GHC/Tc/Types.hs
@@ -175,6 +175,7 @@ import GHCi.RemoteTypes
 
 import qualified Language.Haskell.TH as TH
 import GHC.Driver.Env.KnotVars
+import GHC.Linker.Types
 
 -- | A 'NameShape' is a substitution on 'Name's that can be used
 -- to refine the identities of a hole while we are renaming interfaces
@@ -521,6 +522,10 @@ data TcGblEnv
           --
           -- Splices disable recompilation avoidance (see #481)
 
+        tcg_th_needed_deps :: TcRef ([Linkable], PkgsLoaded),
+          -- ^ The set of runtime dependencies required by this module
+          -- See Note [Object File Dependencies]
+
         tcg_dfun_n  :: TcRef OccSet,
           -- ^ Allows us to choose unique DFun names.
 
diff --git a/compiler/GHC/Tc/Utils/Monad.hs b/compiler/GHC/Tc/Utils/Monad.hs
index ca2915e8fa..72670e6b06 100644
--- a/compiler/GHC/Tc/Utils/Monad.hs
+++ b/compiler/GHC/Tc/Utils/Monad.hs
@@ -58,7 +58,7 @@ module GHC.Tc.Utils.Monad(
   getRdrEnvs, getImports,
   getFixityEnv, extendFixityEnv, getRecFieldEnv,
   getDeclaredDefaultTys,
-  addDependentFiles, getMnwib,
+  addDependentFiles,
 
   -- * Error management
   getSrcSpanM, setSrcSpan, setSrcSpanA, addLocM, addLocMA, inGeneratedCode,
@@ -116,7 +116,7 @@ module GHC.Tc.Utils.Monad(
   emitNamedTypeHole, IsExtraConstraint(..), emitAnonTypeHole,
 
   -- * Template Haskell context
-  recordThUse, recordThSpliceUse,
+  recordThUse, recordThSpliceUse, recordThNeededRuntimeDeps,
   keepAlive, getStage, getStageAndBindLevel, setStage,
   addModFinalizersWithLclEnv,
 
@@ -222,6 +222,8 @@ import {-# SOURCE #-} GHC.Tc.Utils.Env    ( tcInitTidyEnv )
 
 import qualified Data.Map as Map
 import GHC.Driver.Env.KnotVars
+import GHC.Linker.Types
+import GHC.Types.Unique.DFM
 
 {-
 ************************************************************************
@@ -263,6 +265,7 @@ initTc hsc_env hsc_src keep_rn_syntax mod loc do_this
         th_state_var         <- newIORef Map.empty ;
         th_remote_state_var  <- newIORef Nothing ;
         th_docs_var          <- newIORef Map.empty ;
+        th_needed_deps_var   <- newIORef ([], emptyUDFM) ;
         next_wrapper_num     <- newIORef emptyModuleEnv ;
         let {
              -- bangs to avoid leaking the env (#19356)
@@ -311,6 +314,7 @@ initTc hsc_env hsc_src keep_rn_syntax mod loc do_this
                 tcg_ann_env        = emptyAnnEnv,
                 tcg_th_used        = th_var,
                 tcg_th_splice_used = th_splice_var,
+                tcg_th_needed_deps = th_needed_deps_var,
                 tcg_exports        = [],
                 tcg_imports        = emptyImportAvails,
                 tcg_used_gres     = used_gre_var,
@@ -963,11 +967,6 @@ getSrcSpanM :: TcRn SrcSpan
         -- Avoid clash with Name.getSrcLoc
 getSrcSpanM = do { env <- getLclEnv; return (RealSrcSpan (tcl_loc env) Strict.Nothing) }
 
-getMnwib :: TcRn ModuleNameWithIsBoot
-getMnwib = do
-  gbl_env <- getGblEnv
-  return $ GWIB (moduleName $ tcg_mod gbl_env) (hscSourceToIsBoot (tcg_src gbl_env))
-
 -- See Note [Error contexts in generated code]
 inGeneratedCode :: TcRn Bool
 inGeneratedCode = tcl_in_gen_code <$> getLclEnv
@@ -2010,6 +2009,15 @@ recordThUse = do { env <- getGblEnv; writeTcRef (tcg_th_used env) True }
 recordThSpliceUse :: TcM ()
 recordThSpliceUse = do { env <- getGblEnv; writeTcRef (tcg_th_splice_used env) True }
 
+recordThNeededRuntimeDeps :: [Linkable] -> PkgsLoaded -> TcM ()
+recordThNeededRuntimeDeps new_links new_pkgs
+  = do { env <- getGblEnv
+       ; updTcRef (tcg_th_needed_deps env) $ \(needed_links, needed_pkgs) ->
+           let links = new_links ++ needed_links
+               !pkgs = plusUDFM needed_pkgs new_pkgs
+               in (links, pkgs)
+       }
+
 keepAlive :: Name -> TcRn ()     -- Record the name in the keep-alive set
 keepAlive name
   = do { env <- getGblEnv