Driver Rework Patch

This patch comprises of four different but closely related ideas. The
net result is fixing a large number of open issues with the driver
whilst making it simpler to understand.

1. Use the hash of the source file to determine whether the source file
has changed or not. This makes the recompilation checking more robust to
modern build systems which are liable to copy files around changing
their modification times.

2. Remove the concept of a "stable module", a stable module was one
where the object file was older than the source file, and all transitive
dependencies were also stable. Now we don't rely on the modification
time of the source file, the notion of stability is moot.

3. Fix TH/plugin recompilation after the removal of stable modules. The
TH recompilation check used to rely on stable modules. Now there is a
uniform and simple way, we directly track the linkables which were
loaded into the interpreter whilst compiling a module. This is an
over-approximation but more robust wrt package dependencies changing.

4. Fix recompilation checking for dynamic object files. Now we actually
check if the dynamic object file exists when compiling with -dynamic-too

Fixes #19774 #19771 #19758 #17434 #11556 #9121 #8211 #16495 #7277 #16093

5 files changed, 73 insertions, 21 deletions

diff --git a/compiler/GHC/Unit/Module/Deps.hs b/compiler/GHC/Unit/Module/Deps.hs
index 2de3fe710d..2cafbbdcd4 100644
--- a/compiler/GHC/Unit/Module/Deps.hs
+++ b/compiler/GHC/Unit/Module/Deps.hs
@@ -32,9 +32,6 @@ data Dependencies = Deps
       -- ^ All packages directly imported by this module
       -- I.e. packages to which this module's direct imports belong.
       --
-   , dep_plgins :: [ModuleName]
-      -- ^ All the plugins used while compiling this module.
-
 
     -- Transitive information below here
    , dep_sig_mods :: ![ModuleName]
@@ -82,7 +79,6 @@ instance Binary Dependencies where
                       put_ bh (dep_boot_mods deps)
                       put_ bh (dep_orphs deps)
                       put_ bh (dep_finsts deps)
-                      put_ bh (dep_plgins deps)
 
     get bh = do dms <- get bh
                 dps <- get bh
@@ -91,17 +87,16 @@ instance Binary Dependencies where
                 sms <- get bh
                 os <- get bh
                 fis <- get bh
-                pl <- get bh
                 return (Deps { dep_direct_mods = dms
                              , dep_direct_pkgs = dps
                              , dep_sig_mods = hsigms
                              , dep_boot_mods = sms
                              , dep_trusted_pkgs = tps
                              , dep_orphs = os,
-                               dep_finsts = fis, dep_plgins = pl })
+                               dep_finsts = fis })
 
 noDependencies :: Dependencies
-noDependencies = Deps [] [] [] [] [] [] [] []
+noDependencies = Deps [] [] [] [] [] [] []
 
 -- | Records modules for which changes may force recompilation of this module
 -- See wiki: https://gitlab.haskell.org/ghc/ghc/wikis/commentary/compiler/recompilation-avoidance
@@ -141,14 +136,31 @@ data Usage
         usg_file_path  :: FilePath,
         -- ^ External file dependency. From a CPP #include or TH
         -- addDependentFile. Should be absolute.
-        usg_file_hash  :: Fingerprint
+        usg_file_hash  :: Fingerprint,
         -- ^ 'Fingerprint' of the file contents.
 
+        usg_file_label :: Maybe String
+        -- ^ An optional string which is used in recompilation messages if
+        -- file in question has changed.
+
         -- Note: We don't consider things like modification timestamps
         -- here, because there's no reason to recompile if the actual
         -- contents don't change.  This previously lead to odd
         -- recompilation behaviors; see #8114
   }
+  | UsageHomeModuleInterface {
+        usg_mod_name :: ModuleName
+        -- ^ Name of the module
+        , usg_iface_hash :: Fingerprint
+        -- ^ The *interface* hash of the module, not the ABI hash.
+        -- This changes when anything about the interface (and hence the
+        -- module) has changed.
+
+        -- UsageHomeModuleInterface is *only* used for recompilation
+        -- checking when using TemplateHaskell in the interpreter (where
+        -- some modules are loaded as BCOs).
+
+  }
   -- | A requirement which was merged into this one.
   | UsageMergedRequirement {
         usg_mod :: Module,
@@ -187,12 +199,18 @@ instance Binary Usage where
         putByte bh 2
         put_ bh (usg_file_path usg)
         put_ bh (usg_file_hash usg)
+        put_ bh (usg_file_label usg)
 
     put_ bh usg@UsageMergedRequirement{} = do
         putByte bh 3
         put_ bh (usg_mod      usg)
         put_ bh (usg_mod_hash usg)
 
+    put_ bh usg@UsageHomeModuleInterface{} = do
+        putByte bh 4
+        put_ bh (usg_mod_name usg)
+        put_ bh (usg_iface_hash usg)
+
     get bh = do
         h <- getByte bh
         case h of
@@ -212,11 +230,16 @@ instance Binary Usage where
           2 -> do
             fp   <- get bh
             hash <- get bh
-            return UsageFile { usg_file_path = fp, usg_file_hash = hash }
+            label <- get bh
+            return UsageFile { usg_file_path = fp, usg_file_hash = hash, usg_file_label = label }
           3 -> do
             mod <- get bh
             hash <- get bh
             return UsageMergedRequirement { usg_mod = mod, usg_mod_hash = hash }
+          4 -> do
+            mod <- get bh
+            hash <- get bh
+            return UsageHomeModuleInterface { usg_mod_name = mod, usg_iface_hash = hash }
           i -> error ("Binary.get(Usage): " ++ show i)
 
 
diff --git a/compiler/GHC/Unit/Module/ModGuts.hs b/compiler/GHC/Unit/Module/ModGuts.hs
index 640c258273..b40c980744 100644
--- a/compiler/GHC/Unit/Module/ModGuts.hs
+++ b/compiler/GHC/Unit/Module/ModGuts.hs
@@ -1,5 +1,6 @@
 module GHC.Unit.Module.ModGuts
    ( ModGuts (..)
+   , mg_mnwib
    , CgGuts (..)
    )
 where
@@ -31,7 +32,7 @@ import GHC.Types.ForeignStubs
 import GHC.Types.HpcInfo
 import GHC.Types.Name.Reader
 import GHC.Types.SafeHaskell
-import GHC.Types.SourceFile ( HscSource(..) )
+import GHC.Types.SourceFile ( HscSource(..), hscSourceToIsBoot )
 import GHC.Types.SrcLoc
 
 
@@ -96,6 +97,9 @@ data ModGuts
         mg_arg_docs      :: !ArgDocMap       -- ^ Docs on arguments.
     }
 
+mg_mnwib :: ModGuts -> ModuleNameWithIsBoot
+mg_mnwib mg = GWIB (moduleName (mg_module mg)) (hscSourceToIsBoot (mg_hsc_src mg))
+
 -- The ModGuts takes on several slightly different forms:
 --
 -- After simplification, the following fields change slightly:
diff --git a/compiler/GHC/Unit/Module/ModIface.hs b/compiler/GHC/Unit/Module/ModIface.hs
index 18101e309b..695e1ff6c2 100644
--- a/compiler/GHC/Unit/Module/ModIface.hs
+++ b/compiler/GHC/Unit/Module/ModIface.hs
@@ -242,13 +242,17 @@ data ModIface_ (phase :: ModIfacePhase)
                 -- ^ Either `()` or `ModIfaceBackend` for
                 -- a fully instantiated interface.
 
-        mi_ext_fields :: ExtensibleFields
+        mi_ext_fields :: ExtensibleFields,
                 -- ^ Additional optional fields, where the Map key represents
                 -- the field name, resulting in a (size, serialized data) pair.
                 -- Because the data is intended to be serialized through the
                 -- internal `Binary` class (increasing compatibility with types
                 -- using `Name` and `FastString`, such as HIE), this format is
                 -- chosen over `ByteString`s.
+                --
+
+        mi_src_hash :: !Fingerprint
+                -- ^ Hash of the .hs source, used for recompilation checking.
      }
 
 -- | Old-style accessor for whether or not the ModIface came from an hs-boot
@@ -305,6 +309,9 @@ instance Binary ModIface where
                  mi_module    = mod,
                  mi_sig_of    = sig_of,
                  mi_hsc_src   = hsc_src,
+                 mi_src_hash = _src_hash, -- Don't `put_` this in the instance
+                                          -- because we are going to write it
+                                          -- out separately in the actual file
                  mi_deps      = deps,
                  mi_usages    = usages,
                  mi_exports   = exports,
@@ -406,6 +413,8 @@ instance Binary ModIface where
                  mi_module      = mod,
                  mi_sig_of      = sig_of,
                  mi_hsc_src     = hsc_src,
+                 mi_src_hash = fingerprint0, -- placeholder because this is dealt
+                                             -- with specially when the file is read
                  mi_deps        = deps,
                  mi_usages      = usages,
                  mi_exports     = exports,
@@ -452,6 +461,7 @@ emptyPartialModIface mod
   = ModIface { mi_module      = mod,
                mi_sig_of      = Nothing,
                mi_hsc_src     = HsSrcFile,
+               mi_src_hash    = fingerprint0,
                mi_deps        = noDependencies,
                mi_usages      = [],
                mi_exports     = [],
@@ -512,11 +522,11 @@ emptyIfaceHashCache _occ = Nothing
 -- avoid major space leaks.
 instance (NFData (IfaceBackendExts (phase :: ModIfacePhase)), NFData (IfaceDeclExts (phase :: ModIfacePhase))) => NFData (ModIface_ phase) where
   rnf (ModIface f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12
-                f13 f14 f15 f16 f17 f18 f19 f20 f21 f22 f23 f24) =
+                f13 f14 f15 f16 f17 f18 f19 f20 f21 f22 f23 f24 f25) =
     rnf f1 `seq` rnf f2 `seq` f3 `seq` f4 `seq` f5 `seq` f6 `seq` rnf f7 `seq` f8 `seq`
     f9 `seq` rnf f10 `seq` rnf f11 `seq` f12 `seq` rnf f13 `seq` rnf f14 `seq` rnf f15 `seq`
     rnf f16 `seq` f17 `seq` rnf f18 `seq` rnf f19 `seq` f20 `seq` f21 `seq` f22 `seq` rnf f23
-    `seq` rnf f24
+    `seq` rnf f24 `seq` f25 `seq` ()
 
 -- | Records whether a module has orphans. An \"orphan\" is one of:
 --
diff --git a/compiler/GHC/Unit/Module/ModSummary.hs b/compiler/GHC/Unit/Module/ModSummary.hs
index 324cca33a3..339afd4564 100644
--- a/compiler/GHC/Unit/Module/ModSummary.hs
+++ b/compiler/GHC/Unit/Module/ModSummary.hs
@@ -10,6 +10,7 @@ module GHC.Unit.Module.ModSummary
    , ms_installed_mod
    , ms_mod_name
    , ms_imps
+   , ms_mnwib
    , ms_home_allimps
    , ms_home_srcimps
    , ms_home_imps
@@ -39,6 +40,7 @@ import GHC.Data.Maybe
 import GHC.Data.FastString
 import GHC.Data.StringBuffer ( StringBuffer )
 
+import GHC.Utils.Fingerprint
 import GHC.Utils.Outputable
 
 import Data.Time
@@ -73,14 +75,15 @@ data ModSummary
           -- ^ The module source either plain Haskell, hs-boot, or hsig
         ms_location     :: ModLocation,
           -- ^ Location of the various files belonging to the module
-        ms_hs_date      :: UTCTime,
-          -- ^ Timestamp of source file
+        ms_hs_hash      :: Fingerprint,
+          -- ^ Content hash of source file
         ms_obj_date     :: Maybe UTCTime,
           -- ^ Timestamp of object, if we have one
+        ms_dyn_obj_date     :: !(Maybe UTCTime),
+          -- ^ Timestamp of dynamic object, if we have one
         ms_iface_date   :: Maybe UTCTime,
-          -- ^ Timestamp of hi file, if we *only* are typechecking (it is
-          -- 'Nothing' otherwise.
-          -- See Note [Recompilation checking in -fno-code mode] and #9243
+          -- ^ Timestamp of hi file, if we have one
+          -- See Note [When source is considered modified] and #9243
         ms_hie_date   :: Maybe UTCTime,
           -- ^ Timestamp of hie file, if we have one
         ms_srcimps      :: [(Maybe FastString, Located ModuleName)],
@@ -145,7 +148,7 @@ ms_home_imps = home_imps . ms_imps
 -- and let @compile@ read from that file on the way back up.
 
 -- The ModLocation is stable over successive up-sweeps in GHCi, wheres
--- the ms_hs_date and imports can, of course, change
+-- the ms_hs_hash and imports can, of course, change
 
 msHsFilePath, msHiFilePath, msObjFilePath :: ModSummary -> FilePath
 msHsFilePath  ms = expectJust "msHsFilePath" (ml_hs_file  (ms_location ms))
@@ -159,10 +162,13 @@ msDynObjFilePath ms dflags = dynamicOutputFile dflags (msObjFilePath ms)
 isBootSummary :: ModSummary -> IsBootInterface
 isBootSummary ms = if ms_hsc_src ms == HsBootFile then IsBoot else NotBoot
 
+ms_mnwib :: ModSummary -> ModuleNameWithIsBoot
+ms_mnwib ms = GWIB (ms_mod_name ms) (isBootSummary ms)
+
 instance Outputable ModSummary where
    ppr ms
       = sep [text "ModSummary {",
-             nest 3 (sep [text "ms_hs_date = " <> text (show (ms_hs_date ms)),
+             nest 3 (sep [text "ms_hs_hash = " <> text (show (ms_hs_hash ms)),
                           text "ms_mod =" <+> ppr (ms_mod ms)
                                 <> text (hscSourceString (ms_hsc_src ms)) <> comma,
                           text "ms_textual_imps =" <+> ppr (ms_textual_imps ms),
@@ -170,6 +176,8 @@ instance Outputable ModSummary where
              char '}'
             ]
 
+-- | Find the first target in the provided list which matches the specified
+-- 'ModSummary'.
 findTarget :: ModSummary -> [Target] -> Maybe Target
 findTarget ms ts =
   case filter (matches ms) ts of
diff --git a/compiler/GHC/Unit/Module/Status.hs b/compiler/GHC/Unit/Module/Status.hs
index e4273de94b..6f926e3fb2 100644
--- a/compiler/GHC/Unit/Module/Status.hs
+++ b/compiler/GHC/Unit/Module/Status.hs
@@ -10,11 +10,13 @@ import GHC.Unit.Module.ModGuts
 import GHC.Unit.Module.ModIface
 
 import GHC.Utils.Fingerprint
+import GHC.Linker.Types
+import GHC.Utils.Outputable
 
 -- | Status of a module in incremental compilation
 data HscRecompStatus
     -- | Nothing to do because code already exists.
-    = HscUpToDate ModIface
+    = HscUpToDate ModIface (Maybe Linkable)
     -- | Recompilation of module, or update of interface is required. Optionally
     -- pass the old interface hash to avoid updating the existing interface when
     -- it has not changed.
@@ -38,3 +40,8 @@ data HscBackendAction
           -- avoid updating the existing interface when the interface isn't
           -- changed.
         }
+
+
+instance Outputable HscBackendAction where
+  ppr (HscUpdate mi) = text "Update:" <+> (ppr (mi_module mi))
+  ppr (HscRecomp _ ml _mi _mf) = text "Recomp:" <+> ppr ml