Switch Highlighting to Text, add Legacy

14 files changed, 87 insertions, 37 deletions

diff --git a/pandoc.cabal b/pandoc.cabal
index dae742940..cb33e2328 100644
--- a/pandoc.cabal
+++ b/pandoc.cabal
@@ -552,6 +552,7 @@ library
                    -- TODO text: temporarily exposed
                    , Text.Pandoc.Legacy.Extensions
                    , Text.Pandoc.Legacy.Shared
+                   , Text.Pandoc.Legacy.Highlighting
                    --
   other-modules:   Text.Pandoc.App.CommandLineOptions,
                    Text.Pandoc.App.FormatHeuristics,
diff --git a/src/Text/Pandoc/App/CommandLineOptions.hs b/src/Text/Pandoc/App/CommandLineOptions.hs
index 24f4cf3d1..6ed60ae66 100644
--- a/src/Text/Pandoc/App/CommandLineOptions.hs
+++ b/src/Text/Pandoc/App/CommandLineOptions.hs
@@ -45,7 +45,7 @@ import System.IO (stdout)
 import Text.Pandoc.Legacy.Builder (setMeta) -- TODO text: remove Legacy
 import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..))
 import Text.Pandoc.Filter (Filter (..))
-import Text.Pandoc.Highlighting (highlightingStyles)
+import Text.Pandoc.Legacy.Highlighting (highlightingStyles)
 import Text.Pandoc.Legacy.Shared (ordNub, safeRead, defaultUserDataDirs)
 import Text.Printf
 
diff --git a/src/Text/Pandoc/Highlighting.hs b/src/Text/Pandoc/Highlighting.hs
index b68934230..3e02355f7 100644
--- a/src/Text/Pandoc/Highlighting.hs
+++ b/src/Text/Pandoc/Highlighting.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE NoImplicitPrelude #-}
+{-# LANGUAGE OverloadedStrings #-}
 {- |
    Module      : Text.Pandoc.Highlighting
    Copyright   : Copyright (C) 2008-2019 John MacFarlane
@@ -35,15 +36,14 @@ module Text.Pandoc.Highlighting ( highlightingStyles
                                 ) where
 import Prelude
 import Control.Monad
-import Data.Char (toLower)
 import qualified Data.Map as M
 import Data.Maybe (fromMaybe)
 import qualified Data.Text as T
 import Skylighting
-import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Legacy.Shared (safeRead) -- TODO text: remove Legacy
+import Text.Pandoc.Definition
+import Text.Pandoc.Shared (safeRead)
 
-highlightingStyles :: [(String, Style)]
+highlightingStyles :: [(T.Text, Style)]
 highlightingStyles =
   [("pygments", pygments),
    ("tango", tango),
@@ -54,18 +54,18 @@ highlightingStyles =
    ("breezedark", breezeDark),
    ("haddock", haddock)]
 
-languages :: [String]
-languages = [T.unpack (T.toLower (sName s)) | s <- M.elems defaultSyntaxMap]
+languages :: [T.Text]
+languages = [T.toLower (sName s) | s <- M.elems defaultSyntaxMap]
 
-languagesByExtension :: String -> [String]
+languagesByExtension :: T.Text -> [T.Text]
 languagesByExtension ext =
-  [T.unpack (T.toLower (sName s)) | s <- syntaxesByExtension defaultSyntaxMap ext]
+  [T.toLower (sName s) | s <- syntaxesByExtension defaultSyntaxMap (T.unpack ext)]
 
 highlight :: SyntaxMap
           -> (FormatOptions -> [SourceLine] -> a) -- ^ Formatter
           -> Attr   -- ^ Attributes of the CodeBlock
-          -> String -- ^ Raw contents of the CodeBlock
-          -> Either String a
+          -> T.Text -- ^ Raw contents of the CodeBlock
+          -> Either T.Text a
 highlight syntaxmap formatter (ident, classes, keyvals) rawCode =
   let firstNum = fromMaybe 1 (safeRead (fromMaybe "1" $ lookup "startFrom" keyvals))
       fmtOpts = defaultFormatOpts{
@@ -74,38 +74,36 @@ highlight syntaxmap formatter (ident, classes, keyvals) rawCode =
                         ["line-anchors", "lineAnchors"]) classes,
                   numberLines = any (`elem`
                         ["number","numberLines", "number-lines"]) classes,
-                  lineIdPrefix = if null ident
+                  lineIdPrefix = if T.null ident
                                     then mempty
-                                    else T.pack (ident ++ "-") }
+                                    else ident <> "-" }
       tokenizeOpts = TokenizerConfig{ syntaxMap = syntaxmap
                                     , traceOutput = False }
-      classes' = map T.pack classes
-      rawCode' = T.pack rawCode
-  in  case msum (map (`lookupSyntax` syntaxmap) classes') of
+  in  case msum (map (`lookupSyntax` syntaxmap) classes) of
             Nothing
               | numberLines fmtOpts -> Right
                               $ formatter fmtOpts{ codeClasses = [],
-                                                   containerClasses = classes' }
+                                                   containerClasses = classes }
                               $ map (\ln -> [(NormalTok, ln)])
-                              $ T.lines rawCode'
+                              $ T.lines rawCode
               | otherwise  -> Left ""
-            Just syntax  ->
+            Just syntax  -> either (Left . T.pack) Right $
               formatter fmtOpts{ codeClasses =
                                    [T.toLower (sShortname syntax)],
-                                  containerClasses = classes' } <$>
-                tokenize tokenizeOpts syntax rawCode'
+                                  containerClasses = classes } <$>
+                tokenize tokenizeOpts syntax rawCode
 
 -- Functions for correlating latex listings package's language names
 -- with skylighting language names:
 
-langToListingsMap :: M.Map String String
+langToListingsMap :: M.Map T.Text T.Text
 langToListingsMap = M.fromList langsList
 
-listingsToLangMap :: M.Map String String
+listingsToLangMap :: M.Map T.Text T.Text
 listingsToLangMap = M.fromList $ map switch langsList
   where switch (a,b) = (b,a)
 
-langsList :: [(String, String)]
+langsList :: [(T.Text, T.Text)]
 langsList =
   [("abap","ABAP"),
   ("acm","ACM"),
@@ -212,9 +210,9 @@ langsList =
   ("xslt","XSLT")]
 
 -- | Determine listings language name from skylighting language name.
-toListingsLanguage :: String -> Maybe String
-toListingsLanguage lang = M.lookup (map toLower lang) langToListingsMap
+toListingsLanguage :: T.Text -> Maybe T.Text
+toListingsLanguage lang = M.lookup (T.toLower lang) langToListingsMap
 
 -- | Determine skylighting language name from listings language name.
-fromListingsLanguage :: String -> Maybe String
+fromListingsLanguage :: T.Text -> Maybe T.Text
 fromListingsLanguage lang = M.lookup lang listingsToLangMap
diff --git a/src/Text/Pandoc/Legacy/Highlighting.hs b/src/Text/Pandoc/Legacy/Highlighting.hs
new file mode 100644
index 000000000..203c3a96f
--- /dev/null
+++ b/src/Text/Pandoc/Legacy/Highlighting.hs
@@ -0,0 +1,51 @@
+module Text.Pandoc.Legacy.Highlighting ( highlightingStyles
+                                , languages
+                                , languagesByExtension
+                                , highlight
+                                , TP.formatLaTeXInline
+                                , TP.formatLaTeXBlock
+                                , TP.styleToLaTeX
+                                , TP.formatHtmlInline
+                                , TP.formatHtmlBlock
+                                , TP.styleToCss
+                                , TP.pygments
+                                , TP.espresso
+                                , TP.zenburn
+                                , TP.tango
+                                , TP.kate
+                                , TP.monochrome
+                                , TP.breezeDark
+                                , TP.haddock
+                                , TP.Style
+                                , fromListingsLanguage
+                                , toListingsLanguage
+                                ) where
+
+import qualified Text.Pandoc.Highlighting as TP
+import qualified Text.Pandoc.Legacy.Definition as TP
+import qualified Skylighting as S
+import qualified Data.Text as T
+
+highlightingStyles :: [(String, TP.Style)]
+highlightingStyles = map (\(x, y) -> (T.unpack x, y)) TP.highlightingStyles
+
+languages :: [String]
+languages = map T.unpack TP.languages
+
+languagesByExtension :: String -> [String]
+languagesByExtension = map T.unpack . TP.languagesByExtension . T.pack
+
+highlight :: S.SyntaxMap
+          -> (S.FormatOptions -> [S.SourceLine] -> a)
+          -> TP.Attr
+          -> String
+          -> Either String a
+highlight x y z = either (Left . T.unpack) Right . TP.highlight x y (go z) . T.pack
+  where
+    go (a, b, c) = (T.pack a, map T.pack b, map (\(r, s) -> (T.pack r, T.pack s)) c)
+
+toListingsLanguage :: String -> Maybe String
+toListingsLanguage = fmap T.unpack . TP.toListingsLanguage . T.pack
+
+fromListingsLanguage :: String -> Maybe String
+fromListingsLanguage = fmap T.unpack . TP.fromListingsLanguage . T.pack
diff --git a/src/Text/Pandoc/Options.hs b/src/Text/Pandoc/Options.hs
index be814eeda..80ee56056 100644
--- a/src/Text/Pandoc/Options.hs
+++ b/src/Text/Pandoc/Options.hs
@@ -45,7 +45,7 @@ import Data.Typeable (Typeable)
 import GHC.Generics (Generic)
 import Skylighting (SyntaxMap, defaultSyntaxMap)
 import Text.Pandoc.Legacy.Extensions -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (Style, pygments)
+import Text.Pandoc.Legacy.Highlighting (Style, pygments)
 import Text.Pandoc.Legacy.Shared (camelCaseToHyphenated)
 import Text.DocTemplates (Template)
 import Data.Aeson.TH (deriveJSON, defaultOptions, Options(..),
diff --git a/src/Text/Pandoc/Readers/LaTeX.hs b/src/Text/Pandoc/Readers/LaTeX.hs
index b43b50889..a110c5998 100644
--- a/src/Text/Pandoc/Readers/LaTeX.hs
+++ b/src/Text/Pandoc/Readers/LaTeX.hs
@@ -45,7 +45,7 @@ import Text.Pandoc.Class (PandocMonad, PandocPure, getResourcePath, lookupEnv,
                           readFileFromDirs, report, setResourcePath,
                           setTranslations, translateTerm, trace, fileExists)
 import Text.Pandoc.Error (PandocError ( PandocParseError, PandocParsecError))
-import Text.Pandoc.Highlighting (fromListingsLanguage, languagesByExtension)
+import Text.Pandoc.Legacy.Highlighting (fromListingsLanguage, languagesByExtension)
 import Text.Pandoc.ImageSize (numUnit, showFl)
 import Text.Pandoc.Logging
 import Text.Pandoc.Options
diff --git a/src/Text/Pandoc/Writers/Docbook.hs b/src/Text/Pandoc/Writers/Docbook.hs
index f96aa3c58..63309260d 100644
--- a/src/Text/Pandoc/Writers/Docbook.hs
+++ b/src/Text/Pandoc/Writers/Docbook.hs
@@ -24,7 +24,7 @@ import qualified Data.Text as T
 import qualified Text.Pandoc.Builder as B
 import Text.Pandoc.Class (PandocMonad, report)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (languages, languagesByExtension)
+import Text.Pandoc.Legacy.Highlighting (languages, languagesByExtension)
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
 import Text.Pandoc.Options
diff --git a/src/Text/Pandoc/Writers/Docx.hs b/src/Text/Pandoc/Writers/Docx.hs
index f6eb91e44..650fc595d 100644
--- a/src/Text/Pandoc/Writers/Docx.hs
+++ b/src/Text/Pandoc/Writers/Docx.hs
@@ -43,7 +43,7 @@ import Data.Time
 import Text.Pandoc.UTF8 (fromStringLazy)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
 import Text.Pandoc.Generic
-import Text.Pandoc.Highlighting (highlight)
+import Text.Pandoc.Legacy.Highlighting (highlight)
 import Text.Pandoc.Error
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
diff --git a/src/Text/Pandoc/Writers/HTML.hs b/src/Text/Pandoc/Writers/HTML.hs
index d7c2eecf1..0d7bc0ebb 100644
--- a/src/Text/Pandoc/Writers/HTML.hs
+++ b/src/Text/Pandoc/Writers/HTML.hs
@@ -53,7 +53,7 @@ import Text.Blaze.Internal (preEscapedString, preEscapedText)
 import Text.Blaze.Html hiding (contents)
 import Text.DocTemplates (Context (..))
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (formatHtmlBlock, formatHtmlInline, highlight,
+import Text.Pandoc.Legacy.Highlighting (formatHtmlBlock, formatHtmlInline, highlight,
                                  styleToCss)
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Options
diff --git a/src/Text/Pandoc/Writers/JATS.hs b/src/Text/Pandoc/Writers/JATS.hs
index 738f360a8..7a5890876 100644
--- a/src/Text/Pandoc/Writers/JATS.hs
+++ b/src/Text/Pandoc/Writers/JATS.hs
@@ -28,7 +28,7 @@ import qualified Data.Text as T
 import Data.Text (Text)
 import Text.Pandoc.Class (PandocMonad, report)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (languages, languagesByExtension)
+import Text.Pandoc.Legacy.Highlighting (languages, languagesByExtension)
 import Text.Pandoc.Logging
 import Text.Pandoc.MIME (getMimeType)
 import Text.Pandoc.Walk (walk)
diff --git a/src/Text/Pandoc/Writers/LaTeX.hs b/src/Text/Pandoc/Writers/LaTeX.hs
index 7ed22a04f..4f0757fff 100644
--- a/src/Text/Pandoc/Writers/LaTeX.hs
+++ b/src/Text/Pandoc/Writers/LaTeX.hs
@@ -34,7 +34,7 @@ import Text.DocTemplates (FromContext(lookupContext))
 import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)
 import Text.Pandoc.Class (PandocMonad, report, toLang)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,
+import Text.Pandoc.Legacy.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,
                                  styleToLaTeX, toListingsLanguage)
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
diff --git a/src/Text/Pandoc/Writers/Ms.hs b/src/Text/Pandoc/Writers/Ms.hs
index 81d1555f1..64ec81f73 100644
--- a/src/Text/Pandoc/Writers/Ms.hs
+++ b/src/Text/Pandoc/Writers/Ms.hs
@@ -33,7 +33,7 @@ import System.FilePath (takeExtension)
 import Text.Pandoc.Asciify (toAsciiChar)
 import Text.Pandoc.Class (PandocMonad, report)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting
+import Text.Pandoc.Legacy.Highlighting
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
 import Text.Pandoc.Options
diff --git a/src/Text/Pandoc/Writers/Powerpoint/Presentation.hs b/src/Text/Pandoc/Writers/Powerpoint/Presentation.hs
index 2eff567ed..0e333db32 100644
--- a/src/Text/Pandoc/Writers/Powerpoint/Presentation.hs
+++ b/src/Text/Pandoc/Writers/Powerpoint/Presentation.hs
@@ -59,7 +59,7 @@ import Text.Pandoc.Writers.Shared (lookupMetaInlines, lookupMetaBlocks
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Maybe (maybeToList, fromMaybe)
-import Text.Pandoc.Highlighting
+import Text.Pandoc.Legacy.Highlighting
 import qualified Data.Text as T
 import Control.Applicative ((<|>))
 import Skylighting
diff --git a/src/Text/Pandoc/Writers/TEI.hs b/src/Text/Pandoc/Writers/TEI.hs
index 99b75bc2d..53092f5fe 100644
--- a/src/Text/Pandoc/Writers/TEI.hs
+++ b/src/Text/Pandoc/Writers/TEI.hs
@@ -19,7 +19,7 @@ import Data.List (isPrefixOf, stripPrefix)
 import Data.Text (Text)
 import Text.Pandoc.Class (PandocMonad, report)
 import Text.Pandoc.Legacy.Definition -- TODO text: remove Legacy
-import Text.Pandoc.Highlighting (languages, languagesByExtension)
+import Text.Pandoc.Legacy.Highlighting (languages, languagesByExtension)
 import Text.Pandoc.ImageSize
 import Text.Pandoc.Logging
 import Text.Pandoc.Options