Add --enable-library-bytecode option for building bytecode libraries

The new `Cabal` and `cabal-install` option tells GHC to build a
bytecode library.

* When GHC is invoked, bytecode objects are also produced.
* A bytecode library is created during linking.
* The bytecode library is installed when the package is installed.
* The `bytecode-library-dirs` field can be populated to instruct GHC
  where to find the bytecode library for a package. (This can be useful
  if the bytecode libraries are distributed separately from the other
  libraries).

The bytecode object files are produced alongside an existing build way.
They are prioritised in this order (`-dyn-too`, `-dynamic`, `-static`).

Bytecode libraries can't be produced standalone at the moment. You have
to produce them alongside an existing build way. This is due to
limitations in GHC.

At the moment, there is not support implemented for creating profiled
bytecode libraries.

Implements haskell/cabal-proposals#2

Fixes #11188

Author: mpickering

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo.hs

@@ -73,6 +73,7 @@ data InstalledPackageInfo = InstalledPackageInfo
   , libraryDirs :: [FilePath]
   , libraryDirsStatic :: [FilePath]
   , libraryDynDirs :: [FilePath]
+  , libraryBytecodeDirs :: [FilePath]
   -- ^ overrides 'libraryDirs'
   , dataDir :: FilePath
   , hsLibraries :: [String]
@@ -160,6 +161,7 @@ emptyInstalledPackageInfo =
     , hsLibraries = []
     , extraLibraries = []
     , extraLibrariesStatic = []
+    , libraryBytecodeDirs = []
     , extraGHCiLibraries = []
     , includeDirs = []
     , includes = []

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo/FieldGrammar.hs

@@ -108,6 +108,7 @@ ipiFieldGrammar =
     <@> monoidalFieldAla "library-dirs" (alaList' FSep FilePathNT) L.libraryDirs
     <@> monoidalFieldAla "library-dirs-static" (alaList' FSep FilePathNT) L.libraryDirsStatic
     <@> monoidalFieldAla "dynamic-library-dirs" (alaList' FSep FilePathNT) L.libraryDynDirs
+    <@> monoidalFieldAla "bytecode-library-dirs" (alaList' FSep FilePathNT) L.libraryBytecodeDirs
     <@> optionalFieldDefAla "data-dir" FilePathNT L.dataDir ""
     <@> monoidalFieldAla "hs-libraries" (alaList' FSep Token) L.hsLibraries
     <@> monoidalFieldAla "extra-libraries" (alaList' FSep Token) L.extraLibraries

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo/Lens.hs

@@ -123,6 +123,10 @@ libraryDynDirs :: Lens' InstalledPackageInfo [FilePath]
 libraryDynDirs f s = fmap (\x -> s{T.libraryDynDirs = x}) (f (T.libraryDynDirs s))
 {-# INLINE libraryDynDirs #-}
 
+libraryBytecodeDirs :: Lens' InstalledPackageInfo [FilePath]
+libraryBytecodeDirs f s = fmap (\x -> s{T.libraryBytecodeDirs = x}) (f (T.libraryBytecodeDirs s))
+{-# INLINE libraryBytecodeDirs #-}
+
 dataDir :: Lens' InstalledPackageInfo FilePath
 dataDir f s = fmap (\x -> s{T.dataDir = x}) (f (T.dataDir s))
 {-# INLINE dataDir #-}

Cabal/src/Distribution/Simple/BuildPaths.hs

@@ -42,6 +42,7 @@ module Distribution.Simple.BuildPaths
   , mkSharedLibName
   , mkProfSharedLibName
   , mkStaticLibName
+  , mkBytecodeLibName
   , mkGenericSharedBundledLibName
   , exeExtension
   , objExtension
@@ -409,6 +410,9 @@ mkStaticLibName platform (CompilerId compilerFlavor compilerVersion) lib =
   where
     comp = prettyShow compilerFlavor ++ prettyShow compilerVersion
 
+mkBytecodeLibName :: CompilerId -> UnitId -> String
+mkBytecodeLibName _comp lib = getHSLibraryName lib <.> ".bytecodelib"
+
 -- | Create a library name for a bundled shared library from a given name.
 -- This matches the naming convention for shared libraries as implemented in
 -- GHC's packageHsLibs function in the Packages module.

Cabal/src/Distribution/Simple/BuildWay.hs

@@ -1,14 +1,21 @@
 {-# LANGUAGE LambdaCase #-}
 
-module Distribution.Simple.BuildWay where
+module Distribution.Simple.BuildWay
+  ( BuildWay (..)
+  , buildWayObjectExtension
+  , buildWayInterfaceExtension
+  ) where
 
 data BuildWay = StaticWay | DynWay | ProfWay | ProfDynWay
   deriving (Eq, Ord, Show, Read, Enum)
 
--- | Returns the object/interface extension prefix for the given build way (e.g. "dyn_" for 'DynWay')
-buildWayPrefix :: BuildWay -> String
-buildWayPrefix = \case
-  StaticWay -> ""
-  ProfWay -> "p_"
-  DynWay -> "dyn_"
-  ProfDynWay -> "p_dyn_"
+-- | Returns the object extension for the given build way (e.g. "dyn_o" for 'DynWay' on ELF)
+buildWayObjectExtension :: String -> BuildWay -> String
+buildWayObjectExtension ext = \case
+  StaticWay -> ext
+  ProfWay -> "p_" ++ ext
+  DynWay -> "dyn_" ++ ext
+  ProfDynWay -> "p_dyn_" ++ ext
+
+buildWayInterfaceExtension :: BuildWay -> String
+buildWayInterfaceExtension = buildWayObjectExtension "hi"

Cabal/src/Distribution/Simple/Compiler.hs

@@ -79,6 +79,7 @@ module Distribution.Simple.Compiler
   , profilingVanillaSupported
   , profilingVanillaSupportedOrUnknown
   , dynamicSupported
+  , bytecodeArtifactsSupported
   , backpackSupported
   , arResponseFilesSupported
   , arDashLSupported
@@ -575,6 +576,14 @@ profilingDynamicSupportedOrUnknown comp =
 dynamicSupported :: Compiler -> Maybe Bool
 dynamicSupported comp = waySupported "dyn" comp
 
+-- | Does the compiler support producing bytecode objects and bytecode libraries?
+bytecodeArtifactsSupported :: Compiler -> Bool
+bytecodeArtifactsSupported comp
+  | GHC <- compilerFlavor comp
+  , compilerVersion comp >= mkVersion [9, 15, 0] =
+      True
+  | otherwise = False
+
 -- | Does this compiler support a package database entry with:
 -- "visibility"?
 libraryVisibilitySupported :: Compiler -> Bool

Cabal/src/Distribution/Simple/Configure.hs

@@ -753,6 +753,9 @@ computeLocalBuildConfig cfg comp programDb = do
         -- into a huge .a archive) via GHCs -staticlib flag.
         fromFlagOrDefault False $ configStaticLib cfg
 
+      withBytecodeLib_ =
+        fromFlagOrDefault False $ configBytecodeLib cfg
+
       withDynExe_ = fromFlag $ configDynExe cfg
 
       withFullyStaticExe_ = fromFlag $ configFullyStaticExe cfg
@@ -780,12 +783,22 @@ computeLocalBuildConfig cfg comp programDb = do
   strip_lib <- strip_libexe "library" configStripLibs
   strip_exe <- strip_libexe "executable" configStripExes
 
+  checkedWithBytecodeLib <-
+    if bytecodeArtifactsSupported comp
+      then return withBytecodeLib_
+      else do
+        when withBytecodeLib_ $
+          warn verbosity $
+            "This compiler does not support bytecode libraries; ignoring --enable-library-bytecode"
+        return False
+
   let buildOptions =
         setCoverage . setProfiling $
           LBC.BuildOptions
             { withVanillaLib = fromFlag $ configVanillaLib cfg
             , withSharedLib = withSharedLib_
             , withStaticLib = withStaticLib_
+            , withBytecodeLib = checkedWithBytecodeLib
             , withDynExe = withDynExe_
             , withFullyStaticExe = withFullyStaticExe_
             , withProfLib = False

Cabal/src/Distribution/Simple/Errors.hs

@@ -170,6 +170,7 @@ data CabalException
   | MissingCoveredInstalledLibrary UnitId
   | SetupHooksException SetupHooksException
   | MultiReplDoesNotSupportComplexReexportedModules PackageName ComponentName
+  | StandaloneBytecodeNotSupportedYet
   deriving (Show)
 
 exceptionCode :: CabalException -> Int
@@ -304,6 +305,7 @@ exceptionCode e = case e of
   SetupHooksException err ->
     setupHooksExceptionCode err
   MultiReplDoesNotSupportComplexReexportedModules{} -> 9355
+  StandaloneBytecodeNotSupportedYet -> 9356
 
 versionRequirement :: VersionRange -> String
 versionRequirement range
@@ -804,3 +806,8 @@ exceptionMessage e = case e of
       ++ prettyShow pname
       ++ " a module renaming was found.\n"
       ++ "Multi-repl does not work with complicated reexported-modules until GHC-9.12."
+  StandaloneBytecodeNotSupportedYet ->
+    unlines $
+      [ "The ecosystem doesn't support building packages with just bytecode libraries yet."
+      , "If you have run into this error, please comment on issue #<TODO>"
+      ]

Cabal/src/Distribution/Simple/GHC.hs

@@ -986,6 +986,8 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
 
   -- copy the built library files over:
   when (has_code && hasLib) $ do
+    whenBytecodeLib $ installOrdinary builtDir targetDir bytecodeLibName
+
     forM_ libWays $ \w -> case w of
       StaticWay -> do
         sequence_
@@ -1095,6 +1097,7 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
     platform = hostPlatform lbi
     uid = componentUnitId clbi
     profileLibName = mkProfLibName uid
+    bytecodeLibName = mkBytecodeLibName compiler_id uid
     ghciLibName = Internal.mkGHCiLibName uid
     ghciProfLibName = Internal.mkGHCiProfLibName uid
 
@@ -1111,6 +1114,7 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
       _ -> False
     has_code = not (componentIsIndefinite clbi)
     whenGHCi = when (hasLib && withGHCiLib lbi && has_code)
+    whenBytecodeLib = when (hasLib && withBytecodeLib lbi && has_code)
 
 -- -----------------------------------------------------------------------------
 -- Registering

Cabal/src/Distribution/Simple/GHC/Build/Link.hs

@@ -279,6 +279,9 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
     staticLibFilePath =
       buildTargetDir
         </> makeRelativePathEx (mkStaticLibName (hostPlatform lbi) compiler_id uid)
+    bytecodeLibFilePath =
+      buildTargetDir
+        </> makeRelativePathEx (mkBytecodeLibName compiler_id uid)
     ghciLibFilePath = buildTargetDir </> makeRelativePathEx (Internal.mkGHCiLibName uid)
     ghciProfLibFilePath = buildTargetDir </> makeRelativePathEx (Internal.mkGHCiProfLibName uid)
     libInstallPath =
@@ -295,23 +298,29 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
       libInstallPath
         </> mkProfSharedLibName (hostPlatform lbi) compiler_id uid
 
-    getObjFiles :: BuildWay -> IO [SymbolicPath Pkg File]
-    getObjFiles way =
+    getObjWayFiles :: BuildWay -> IO [SymbolicPath Pkg File]
+    getObjWayFiles w = getObjFiles (buildWayObjectExtension objExtension w) (buildWayObjectExtension objExtension w)
+
+    getObjBytecodeWayFiles :: BuildWay -> IO [SymbolicPath Pkg File]
+    getObjBytecodeWayFiles companion_way = getObjFiles "gbc" (buildWayObjectExtension objExtension companion_way)
+
+    getObjFiles :: String -> String -> IO [SymbolicPath Pkg File]
+    getObjFiles hs_ext obj_ext =
       mconcat
         [ Internal.getHaskellObjects
             implInfo
             lib
             lbi
             clbi
             buildTargetDir
-            (buildWayPrefix way ++ objExtension)
+            hs_ext
             True
-        , pure $ map (srcObjPath way) extraSources
+        , pure $ map (srcObjPath obj_ext) extraSources
         , catMaybes
             <$> sequenceA
               [ findFileCwdWithExtension
                 mbWorkDir
-                [Suffix $ buildWayPrefix way ++ objExtension]
+                [Suffix $ obj_ext]
                 [buildTargetDir]
                 xPath
               | ghcVersion < mkVersion [7, 2] -- ghc-7.2+ does not make _stub.o files
@@ -323,16 +332,16 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
 
     -- Get the @.o@ path from a source path (e.g. @.hs@),
     -- in the library target build directory.
-    srcObjPath :: BuildWay -> SymbolicPath Pkg File -> SymbolicPath Pkg File
-    srcObjPath way srcPath =
+    srcObjPath :: String -> SymbolicPath Pkg File -> SymbolicPath Pkg File
+    srcObjPath obj_ext srcPath =
       case symbolicPathRelative_maybe objPath of
         -- Absolute path: should already be in the target build directory
         -- (e.g. a preprocessed file)
         -- TODO: assert this?
         Nothing -> objPath
         Just objRelPath -> coerceSymbolicPath buildTargetDir </> objRelPath
       where
-        objPath = srcPath `replaceExtensionSymbolicPath` (buildWayPrefix way ++ objExtension)
+        objPath = srcPath `replaceExtensionSymbolicPath` obj_ext
 
     -- I'm fairly certain that, just like the executable, we can keep just the
     -- module input list, and point to the right sources dir (as is already
@@ -343,6 +352,7 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
     -- we could more easily merge the two.
     --
     -- Right now, instead, we pass the path to each object file.
+    ghcBaseLinkArgs :: GhcOptions
     ghcBaseLinkArgs =
       mempty
         { -- TODO: This basically duplicates componentGhcOptions.
@@ -435,11 +445,17 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
         , -- TODO: Shouldn't this use cleanedExtraLibDirsStatic instead?
           ghcOptLinkLibPath = toNubListR $ cleanedExtraLibDirs
         }
+    ghcBytecodeLinkArgs objectFiles =
+      (ghcSharedLinkArgs objectFiles)
+        { ghcOptBytecodeLib = toFlag True
+        , ghcOptInputFiles = toNubListR $ map coerceSymbolicPath objectFiles
+        , ghcOptOutputFile = toFlag bytecodeLibFilePath
+        }
 
-  staticObjectFiles <- getObjFiles StaticWay
-  profObjectFiles <- getObjFiles ProfWay
-  dynamicObjectFiles <- getObjFiles DynWay
-  profDynamicObjectFiles <- getObjFiles ProfDynWay
+  staticObjectFiles <- getObjWayFiles StaticWay
+  profObjectFiles <- getObjWayFiles ProfWay
+  dynamicObjectFiles <- getObjWayFiles DynWay
+  profDynamicObjectFiles <- getObjWayFiles ProfDynWay
 
   let
     linkWay = \case
@@ -457,6 +473,10 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
         runGhcProg $ ghcProfSharedLinkArgs profDynamicObjectFiles
       DynWay -> do
         runGhcProg $ ghcSharedLinkArgs dynamicObjectFiles
+        -- The .gbc files were built with DynWay if both are enabled.
+        when (withBytecodeLib lbi) $ do
+          bytecodeObjectFiles <- getObjBytecodeWayFiles DynWay
+          runGhcProg $ ghcBytecodeLinkArgs bytecodeObjectFiles
       StaticWay -> do
         when (withVanillaLib lbi) $ do
           Ar.createArLibArchive verbosity lbi vanillaLibFilePath staticObjectFiles
@@ -470,6 +490,10 @@ linkLibrary buildTargetDir cleanedExtraLibDirs pkg_descr verbosity runGhcProg li
               staticObjectFiles
         when (withStaticLib lbi) $ do
           runGhcProg $ ghcStaticLinkArgs staticObjectFiles
+        -- The .gbc files were built with DynWay if both are enabled.
+        when (withBytecodeLib lbi && (DynWay `notElem` wantedWays)) $ do
+          bytecodeObjectFiles <- getObjBytecodeWayFiles StaticWay
+          runGhcProg $ ghcBytecodeLinkArgs bytecodeObjectFiles
 
   -- ROMES: Why exactly branch on staticObjectFiles, rather than any other build
   -- kind that we might have wanted instead?