Fooling around with a new setup for Text.Pandoc.Data.initialize-data-files

It is Now an exported module, exporting initializeDataFiles which must
be run in PandocMonad if you are going to be looking for any data files.

The point of this is to decouple pandoc's data from "pandoc-core,"
including the infrastructure for PandocMonad.

12 files changed, 60 insertions, 16 deletions

diff --git a/pandoc.cabal b/pandoc.cabal
index 7c74a26f0..dc83f856e 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -452,7 +452,6 @@ library
   if flag(embed_data_files)
      cpp-options:   -DEMBED_DATA_FILES
      build-depends: file-embed >= 0.0 && < 0.1
-     other-modules: Text.Pandoc.Data
   if os(windows)
     cpp-options:      -D_WINDOWS
   ghc-options:     -Wall -fno-warn-unused-do-bind
@@ -476,6 +475,7 @@ library
 
   exposed-modules: Text.Pandoc,
                    Text.Pandoc.App,
+                   Text.Pandoc.Data,
                    Text.Pandoc.Options,
                    Text.Pandoc.Extensions,
                    Text.Pandoc.Shared,
diff --git a/src/Text/Pandoc.hs b/src/Text/Pandoc.hs
index 549aeddfb..d541965e8 100644
--- a/src/Text/Pandoc.hs
+++ b/src/Text/Pandoc.hs
@@ -47,6 +47,8 @@ module Text.Pandoc
                , module Text.Pandoc.Logging
                -- * Typeclass
                , module Text.Pandoc.Class
+               -- * Data Files
+               , module Text.Pandoc.Data
                -- * Error handling
                , module Text.Pandoc.Error
                -- * Readers: converting /to/ Pandoc format
@@ -60,6 +62,7 @@ module Text.Pandoc
              ) where
 
 import Text.Pandoc.Class
+import Text.Pandoc.Data
 import Text.Pandoc.Definition
 import Text.Pandoc.Error
 import Text.Pandoc.Generic
diff --git a/src/Text/Pandoc/App.hs b/src/Text/Pandoc/App.hs
index aa75436a4..cb8e844ff 100644
--- a/src/Text/Pandoc/App.hs
+++ b/src/Text/Pandoc/App.hs
@@ -64,6 +64,7 @@ import qualified Text.Pandoc.UTF8 as UTF8
 import System.Posix.IO (stdOutput)
 import System.Posix.Terminal (queryTerminal)
 #endif
+import Text.Pandoc.Data (initializeDataFiles)
 
 convertWithOpts :: Opt -> IO ()
 convertWithOpts opts = do
@@ -98,6 +99,7 @@ convertWithOpts opts = do
   let runIO' :: PandocIO a -> IO a
       runIO' f = do
         (res, reports) <- runIOorExplode $ do
+                             initializeDataFiles
                              setTrace (optTrace opts)
                              setVerbosity verbosity
                              x <- f
diff --git a/src/Text/Pandoc/App/CommandLineOptions.hs b/src/Text/Pandoc/App/CommandLineOptions.hs
index c3b05b70f..c8ff55516 100644
--- a/src/Text/Pandoc/App/CommandLineOptions.hs
+++ b/src/Text/Pandoc/App/CommandLineOptions.hs
@@ -52,7 +52,7 @@ import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs, f
 import Text.Printf
 
 #ifdef EMBED_DATA_FILES
-import Text.Pandoc.Data (dataFiles)
+import Text.Pandoc.Data (initializeDataFiles)
 #else
 import Paths_pandoc (getDataDir)
 import System.Directory (getDirectoryContents)
@@ -760,7 +760,8 @@ options =
                  (NoArg
                   (\_ -> do
                      datafiles <- getDataFileNames
-                     tpl <- runIOorExplode $
+                     tpl <- runIOorExplode $ do
+                              initializeDataFiles
                               UTF8.toString <$>
                                 readDefaultDataFile "bash_completion.tpl"
                      let optnames (Option shorts longs _ _) =
@@ -841,6 +842,7 @@ options =
                                         Just f  -> UTF8.writeFile f
                                         Nothing -> UTF8.hPutStr stdout
                      templ <- runIO $ do
+                                initializeDataFiles
                                 setUserDataDir Nothing
                                 getDefaultTemplate (T.pack arg)
                      case templ of
@@ -916,7 +918,9 @@ options =
 getDataFileNames :: IO [FilePath]
 getDataFileNames = do
 #ifdef EMBED_DATA_FILES
-  let allDataFiles = map fst dataFiles
+  allDataFiles <- runIOorExplode $ do
+    initializeDataFiles
+    map fst . stDataFiles <$> getCommonState
 #else
   allDataFiles <- filter (\x -> x /= "." && x /= "..") <$>
                       (getDataDir >>= getDirectoryContents)
diff --git a/src/Text/Pandoc/Class/CommonState.hs b/src/Text/Pandoc/Class/CommonState.hs
index 7e1735c2b..b535e0c1d 100644
--- a/src/Text/Pandoc/Class/CommonState.hs
+++ b/src/Text/Pandoc/Class/CommonState.hs
@@ -19,6 +19,7 @@ where
 
 import Data.Default (Default (def))
 import Data.Text (Text)
+import qualified Data.ByteString as B
 import Text.Pandoc.BCP47 (Lang)
 import Text.Pandoc.MediaBag (MediaBag)
 import Text.Pandoc.Logging (LogMessage, Verbosity (WARNING))
@@ -50,6 +51,8 @@ data CommonState = CommonState
   , stResourcePath :: [FilePath]
     -- ^ Path to search for resources like
     -- included images
+  , stDataFiles    :: [(FilePath, B.ByteString)]
+    -- ^ Data files baked into the binary if compiled with embed_data_files
   , stVerbosity    :: Verbosity
     -- ^ Verbosity level
   , stTrace        :: Bool
@@ -75,6 +78,7 @@ defaultCommonState = CommonState
   , stInputFiles = []
   , stOutputFile = Nothing
   , stResourcePath = ["."]
+  , stDataFiles = []
   , stVerbosity = WARNING
   , stTrace = False
   }
diff --git a/src/Text/Pandoc/Class/PandocMonad.hs b/src/Text/Pandoc/Class/PandocMonad.hs
index 991aeed41..ec704935c 100644
--- a/src/Text/Pandoc/Class/PandocMonad.hs
+++ b/src/Text/Pandoc/Class/PandocMonad.hs
@@ -86,9 +86,6 @@ import qualified Debug.Trace
 import qualified System.FilePath.Posix as Posix
 import qualified Text.Pandoc.MediaBag as MB
 import qualified Text.Pandoc.UTF8 as UTF8
-#ifdef EMBED_DATA_FILES
-import Text.Pandoc.Data (dataFiles)
-#endif
 
 -- | The PandocMonad typeclass contains all the potentially
 -- IO-related functions used in pandoc's readers and writers.
@@ -573,8 +570,9 @@ readDefaultDataFile "reference.pptx" =
   (B.concat . BL.toChunks . fromArchive) <$> getDefaultReferencePptx
 readDefaultDataFile "reference.odt" =
   (B.concat . BL.toChunks . fromArchive) <$> getDefaultReferenceODT
-readDefaultDataFile fname =
+readDefaultDataFile fname = do
 #ifdef EMBED_DATA_FILES
+  dataFiles <- stDataFiles <$> getCommonState
   case lookup (makeCanonical fname) dataFiles of
     Nothing       -> throwError $ PandocCouldNotFindDataFileError $ T.pack fname
     Just contents -> return contents
diff --git a/src/Text/Pandoc/Data.hs b/src/Text/Pandoc/Data.hs
index 38682b9f9..048d67956 100644
--- a/src/Text/Pandoc/Data.hs
+++ b/src/Text/Pandoc/Data.hs
@@ -1,4 +1,7 @@
+{-# LANGUAGE CPP #-}
+#ifdef EMBED_DATA_FILES
 {-# LANGUAGE TemplateHaskell #-}
+#endif
 {- |
 Module      : Text.Pandoc.Data
 Copyright   : Copyright (C) 2013-2020 John MacFarlane
@@ -10,13 +13,31 @@ Portability : portable
 
 Provide contents data files as Haskell values.
 -}
-module Text.Pandoc.Data (dataFiles) where
+module Text.Pandoc.Data (initializeDataFiles) where
 
+import Text.Pandoc.Class (modifyCommonState, CommonState(..), PandocMonad)
+#ifdef EMBED_DATA_FILES
 import qualified Data.ByteString as B
 import Data.FileEmbed
 import System.FilePath (splitDirectories)
 import qualified System.FilePath.Posix as Posix
+#endif
 
+-- | Inject data from data files into CommonState, so that
+-- we don't need to read data files from file system. This
+-- only has an effect if pandoc was compiled with
+-- the @embed_data_files@ flag.  Generally you'll want to
+-- put this at the beginning of any PandocMonad action
+-- that will require reading pandoc's data files (e.g. templates).
+initializeDataFiles :: PandocMonad m => m ()
+initializeDataFiles =
+#ifdef EMBED_DATA_FILES
+  modifyCommonState $ \st ->st{ stDataFiles = dataFiles }
+#else
+  return ()
+#endif
+
+#ifdef EMBED_DATA_FILES
 -- We ensure that the data files are stored using Posix
 -- path separators (/), even on Windows.
 dataFiles :: [(FilePath, B.ByteString)]
@@ -30,3 +51,4 @@ dataFiles' = ("MANUAL.txt", $(embedFile "MANUAL.txt")) :
              ("docx/_rels/.rels", $(embedFile "data/docx/_rels/.rels")) :
              ("pptx/_rels/.rels", $(embedFile "data/pptx/_rels/.rels")) :
              $(embedDir "data")
+#endif
diff --git a/src/Text/Pandoc/Lua/Module/Utils.hs b/src/Text/Pandoc/Lua/Module/Utils.hs
index 11a0bda84..f8f1707bd 100644
--- a/src/Text/Pandoc/Lua/Module/Utils.hs
+++ b/src/Text/Pandoc/Lua/Module/Utils.hs
@@ -30,6 +30,7 @@ import qualified Foreign.Lua as Lua
 import qualified Text.Pandoc.Builder as B
 import qualified Text.Pandoc.Filter.JSON as JSONFilter
 import qualified Text.Pandoc.Shared as Shared
+import Text.Pandoc.Data (initializeDataFiles)
 
 -- | Push the "pandoc.utils" module to the lua stack.
 pushModule :: Maybe FilePath -> Lua NumResults
@@ -79,6 +80,7 @@ runJSONFilter mbDatadir doc filterFile optArgs = do
               Lua.getglobal "FORMAT"
               (:[]) <$> Lua.popValue
   filterRes <- Lua.liftIO . runIO $ do
+    initializeDataFiles
     setUserDataDir mbDatadir
     JSONFilter.apply def args filterFile doc
   case filterRes of
diff --git a/src/Text/Pandoc/Lua/Packages.hs b/src/Text/Pandoc/Lua/Packages.hs
index ad338f4bd..a0c50b631 100644
--- a/src/Text/Pandoc/Lua/Packages.hs
+++ b/src/Text/Pandoc/Lua/Packages.hs
@@ -20,7 +20,7 @@ import Data.ByteString (ByteString)
 import Foreign.Lua (Lua, NumResults, liftIO)
 import Text.Pandoc.Class.PandocIO (runIO)
 import Text.Pandoc.Class.PandocMonad (readDataFile, setUserDataDir)
-
+import Text.Pandoc.Data (initializeDataFiles)
 import qualified Foreign.Lua as Lua
 import Text.Pandoc.Lua.Module.Pandoc as Pandoc
 import Text.Pandoc.Lua.Module.MediaBag as MediaBag
@@ -83,7 +83,10 @@ loadStringAsPackage pkgName script = do
 -- | Get the ByteString representation of the pandoc module.
 dataDirScript :: Maybe FilePath -> FilePath -> IO (Maybe ByteString)
 dataDirScript datadir moduleFile = do
-  res <- runIO $ setUserDataDir datadir >> readDataFile moduleFile
+  res <- runIO $ do
+    initializeDataFiles
+    setUserDataDir datadir
+    readDataFile moduleFile
   return $ case res of
     Left _ -> Nothing
     Right s -> Just s
diff --git a/src/Text/Pandoc/Lua/Util.hs b/src/Text/Pandoc/Lua/Util.hs
index d79fbb085..00325e61e 100644
--- a/src/Text/Pandoc/Lua/Util.hs
+++ b/src/Text/Pandoc/Lua/Util.hs
@@ -31,6 +31,7 @@ import Foreign.Lua ( Lua, NumArgs, NumResults, Peekable, Pushable, StackIndex
                    , Status, ToHaskellFunction )
 import Text.Pandoc.Class.PandocIO (runIOorExplode)
 import Text.Pandoc.Class.PandocMonad (readDataFile, setUserDataDir)
+import Text.Pandoc.Data (initializeDataFiles)
 import qualified Foreign.Lua as Lua
 import qualified Text.Pandoc.UTF8 as UTF8
 import Data.Text (Text)
@@ -90,8 +91,10 @@ pushViaConstructor pandocFn = pushViaCall ("pandoc." ++ pandocFn)
 -- | Load a file from pandoc's data directory.
 loadScriptFromDataDir :: Maybe FilePath -> FilePath -> Lua ()
 loadScriptFromDataDir datadir scriptFile = do
-  script <- Lua.liftIO . runIOorExplode $
-            setUserDataDir datadir >> readDataFile scriptFile
+  script <- Lua.liftIO . runIOorExplode $ do
+              initializeDataFiles
+              setUserDataDir datadir
+              readDataFile scriptFile
   status <- Lua.dostring script
   when (status /= Lua.OK) $
     throwTopMessageAsError' (("Couldn't load '" ++ scriptFile ++ "'.\n") ++)
diff --git a/test/Tests/Command.hs b/test/Tests/Command.hs
index e0c5a8408..ac5050163 100644
--- a/test/Tests/Command.hs
+++ b/test/Tests/Command.hs
@@ -100,8 +100,8 @@ runCommandTest pandocpath num code =
 extractCommandTest :: FilePath -> FilePath -> TestTree
 extractCommandTest pandocpath fp = unsafePerformIO $ do
   contents <- UTF8.toText <$> BS.readFile ("command" </> fp)
-  Pandoc _ blocks <- runIOorExplode (readMarkdown
-                        def{ readerExtensions = pandocExtensions } contents)
+  Pandoc _ blocks <- runIOorExplode $
+    readMarkdown def{ readerExtensions = pandocExtensions } contents
   let codeblocks = map extractCode $ filter isCodeBlock blocks
   let cases = zipWith (runCommandTest pandocpath) [1..] codeblocks
   return $ testGroup fp cases
diff --git a/test/Tests/Writers/OOXML.hs b/test/Tests/Writers/OOXML.hs
index f2762ddfe..509f5d6a8 100644
--- a/test/Tests/Writers/OOXML.hs
+++ b/test/Tests/Writers/OOXML.hs
@@ -56,7 +56,10 @@ testArchive :: (WriterOptions -> Pandoc -> PandocIO BL.ByteString)
             -> IO Archive
 testArchive writerFn opts fp = do
   txt <- T.readFile fp
-  bs <- runIOorExplode $ readNative def txt >>= writerFn opts
+  bs <- runIOorExplode $ do
+    initializeDataFiles
+    setUserDataDir (Just "../data")
+    readNative def txt >>= writerFn opts
   return $ toArchive bs
 
 compareFileList :: FilePath -> Archive -> Archive -> Maybe String