Expand existing variables (#42)

* Allow parsing of interpolation in variables.

* Extract ParsedVariable type to new file.

* Interpolate references to previous variables / environment.

* Fix nested many parser error.

* Derive Show instances for ParsedVariable types.

* Update tests to use ParsedVariable.

* Add tests for interpolation and variable referencing.

* Split long spec lines.

* Add (<$>) imports for old versions of GHC.

* Split long import line in ParseSpec.

* Only import <$> in ParseSpec if necessary.

* Add ParsedVariable to cabal file.

* Change variable types prefix V -> Var.

* Only import (<$>) for test if necessary.

Author: habibalamin

dotenv.cabal

@@ -70,6 +70,7 @@ executable dotenv
 library
   exposed-modules:      Configuration.Dotenv
                       , Configuration.Dotenv.Parse
+                      , Configuration.Dotenv.ParsedVariable
                       , Configuration.Dotenv.Text
 
   build-depends:         base >=4.6 && <5.0
@@ -97,6 +98,7 @@ test-suite dotenv-test
                        , Configuration.Dotenv
                        , Configuration.Dotenv.Text
                        , Configuration.Dotenv.Parse
+                       , Configuration.Dotenv.ParsedVariable
 
   build-depends:       base >=4.6 && <5.0
                      , base-compat >= 0.4

spec/Configuration/Dotenv/ParseSpec.hs

@@ -3,6 +3,9 @@
 module Configuration.Dotenv.ParseSpec (main, spec) where
 
 import Configuration.Dotenv.Parse (configParser)
+import Configuration.Dotenv.ParsedVariable (ParsedVariable(..),
+                                            VarValue(..),
+                                            VarFragment(..))
 import Test.Hspec (it, describe, Spec, hspec)
 import Test.Hspec.Megaparsec (shouldParse, shouldFailOn)
 import Text.Megaparsec (ParseError, Dec, parse)
@@ -13,76 +16,130 @@ main = hspec spec
 spec :: Spec
 spec = describe "parse" $ do
   it "parses unquoted values" $
-    parseConfig "FOO=bar" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO=bar"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"])]
 
   it "parses values with spaces around equal signs" $ do
-    parseConfig "FOO =bar" `shouldParse` [("FOO", "bar")]
-    parseConfig "FOO= bar" `shouldParse` [("FOO", "bar")]
-    parseConfig "FOO =\t bar" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO =bar"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"])]
+    parseConfig "FOO= bar"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"])]
+    parseConfig "FOO =\t bar"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"])]
 
   it "parses double-quoted values" $
-    parseConfig "FOO=\"bar\"" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO=\"bar\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "bar"])]
 
   it "parses single-quoted values" $
-    parseConfig "FOO='bar'" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO='bar'"
+      `shouldParse` [ParsedVariable "FOO" (SingleQuoted [VarLiteral "bar"])]
 
   it "parses escaped double quotes" $
     parseConfig "FOO=\"escaped\\\"bar\""
-      `shouldParse` [("FOO", "escaped\"bar")]
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "escaped\"bar"])]
 
   it "supports CRLF line breaks" $
     parseConfig "FOO=bar\r\nbaz=fbb"
-      `shouldParse` [("FOO", "bar"), ("baz", "fbb")]
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"]),
+                     ParsedVariable "baz" (Unquoted [VarLiteral "fbb"])]
 
   it "parses empty values" $
-    parseConfig "FOO=" `shouldParse` [("FOO", "")]
+    parseConfig "FOO="
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [])]
+
+  it "parses unquoted interpolated values" $ do
+    parseConfig "FOO=$HOME"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarInterpolation "HOME"])]
+    parseConfig "FOO=abc_$HOME"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "abc_",
+                                                     VarInterpolation "HOME"])
+                    ]
+    parseConfig "FOO=${HOME}"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarInterpolation "HOME"])]
+    parseConfig "FOO=abc_${HOME}"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "abc_",
+                                                     VarInterpolation "HOME"])
+                    ]
+
+  it "parses double-quoted interpolated values" $ do
+    parseConfig "FOO=\"$HOME\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarInterpolation "HOME"])]
+    parseConfig "FOO=\"abc_$HOME\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "abc_",
+                                                         VarInterpolation "HOME"])
+                    ]
+    parseConfig "FOO=\"${HOME}\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarInterpolation "HOME"])]
+    parseConfig "FOO=\"abc_${HOME}\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "abc_",
+                                                         VarInterpolation "HOME"])
+                    ]
+
+  it "parses single-quoted interpolated values as literals" $ do
+    parseConfig "FOO='$HOME'"
+      `shouldParse` [ParsedVariable "FOO" (SingleQuoted [VarLiteral "$HOME"])]
+    parseConfig "FOO='abc_$HOME'"
+      `shouldParse` [ParsedVariable "FOO" (SingleQuoted [VarLiteral "abc_$HOME"])]
+    parseConfig "FOO='${HOME}'"
+      `shouldParse` [ParsedVariable "FOO" (SingleQuoted [VarLiteral "${HOME}"])]
+    parseConfig "FOO='abc_${HOME}'"
+      `shouldParse` [ParsedVariable "FOO" (SingleQuoted [VarLiteral "abc_${HOME}"])]
 
   it "does not parse if line format is incorrect" $ do
     parseConfig `shouldFailOn` "lol$wut"
     parseConfig `shouldFailOn` "KEY=\nVALUE"
     parseConfig `shouldFailOn` "KEY\n=VALUE"
 
   it "expands newlines in quoted strings" $
-    parseConfig "FOO=\"bar\nbaz\"" `shouldParse` [("FOO", "bar\nbaz")]
+    parseConfig "FOO=\"bar\nbaz\""
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "bar\nbaz"])]
 
   it "does not parse variables with hyphens in the name" $
     parseConfig `shouldFailOn` "FOO-BAR=foobar"
 
   it "parses variables with \"_\" in the name" $
-    parseConfig "FOO_BAR=foobar" `shouldParse` [("FOO_BAR", "foobar")]
+    parseConfig "FOO_BAR=foobar"
+      `shouldParse` [ParsedVariable "FOO_BAR" (Unquoted [VarLiteral "foobar"])]
 
   it "parses variables with digits after the first character" $
-    parseConfig "FOO_BAR_12=foobar" `shouldParse` [("FOO_BAR_12", "foobar")]
+    parseConfig "FOO_BAR_12=foobar"
+      `shouldParse` [ParsedVariable "FOO_BAR_12" (Unquoted [VarLiteral "foobar"])]
 
   it "allows vertical spaces after a quoted variable" $
-    parseConfig "foo='bar' " `shouldParse` [("foo", "bar")]
+    parseConfig "foo='bar' "
+      `shouldParse` [ParsedVariable "foo" (SingleQuoted [VarLiteral "bar"])]
 
   it "does not parse variable names beginning with a digit" $
     parseConfig `shouldFailOn` "45FOO_BAR=foobar"
 
   it "strips unquoted values" $
-    parseConfig "foo=bar " `shouldParse` [("foo", "bar")]
+    parseConfig "foo=bar "
+      `shouldParse` [ParsedVariable "foo" (Unquoted [VarLiteral "bar"])]
 
   it "ignores empty lines" $
     parseConfig "\n \t  \nfoo=bar\n \nfizz=buzz"
-      `shouldParse` [("foo", "bar"), ("fizz", "buzz")]
+      `shouldParse` [ParsedVariable "foo" (Unquoted [VarLiteral "bar"]),
+                     ParsedVariable "fizz" (Unquoted [VarLiteral "buzz"])]
 
   it "ignores inline comments after unquoted arguments" $
-    parseConfig "FOO=bar # this is foo" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO=bar # this is foo"
+      `shouldParse` [ParsedVariable "FOO" (Unquoted [VarLiteral "bar"])]
 
   it "ignores inline comments after quoted arguments" $
-    parseConfig "FOO=\"bar\" # this is foo" `shouldParse` [("FOO", "bar")]
+    parseConfig "FOO=\"bar\" # this is foo"
+      `shouldParse` [ParsedVariable "FOO" (DoubleQuoted [VarLiteral "bar"])]
 
   it "allows \"#\" in quoted values" $
     parseConfig "foo=\"bar#baz\" # comment"
-      `shouldParse` [("foo", "bar#baz")]
+      `shouldParse` [ParsedVariable "foo" (DoubleQuoted [VarLiteral "bar#baz"])]
 
   it "ignores comment lines" $
     parseConfig "\n\t \n\n # HERE GOES FOO \nfoo=bar"
-      `shouldParse` [("foo", "bar")]
+      `shouldParse` [ParsedVariable "foo" (Unquoted [VarLiteral "bar"])]
 
   it "doesn't allow more configuration options after a quoted value" $
     parseConfig `shouldFailOn` "foo='bar'baz='buz'"
 
-parseConfig :: String -> Either (ParseError Char Dec) [(String, String)]
+parseConfig :: String -> Either (ParseError Char Dec) [ParsedVariable]
 parseConfig = parse configParser ""

spec/Configuration/DotenvSpec.hs

@@ -8,6 +8,10 @@ import Test.Hspec
 
 import System.Environment (lookupEnv)
 import Control.Monad (liftM)
+import Data.Maybe (fromMaybe)
+#if !MIN_VERSION_base(4,8,0)
+import Data.Functor ((<$>))
+#endif
 
 #if !MIN_VERSION_base(4,8,0)
 import Control.Applicative ((<$))
@@ -83,6 +87,15 @@ spec = do
       liftM (!! 1) (parseFile "spec/fixtures/.dotenv") `shouldReturn`
         ("UNICODE_TEST", "Manabí")
 
+    it "recognises environment variables" $ do
+      home <- fromMaybe "" <$> lookupEnv "HOME"
+      liftM (!! 2) (parseFile "spec/fixtures/.dotenv") `shouldReturn`
+        ("ENVIRONMENT", home)
+
+    it "recognises previous variables" $
+      liftM (!! 3) (parseFile "spec/fixtures/.dotenv") `shouldReturn`
+        ("PREVIOUS", "true")
+
   describe "onMissingFile" $ after_ (unsetEnv "DOTENV") $ do
     context "when target file is present" $
       it "loading works as usual" $ do

spec/fixtures/.dotenv

@@ -1,2 +1,4 @@
 DOTENV=true
 UNICODE_TEST=Manabí
+ENVIRONMENT="$HOME"
+PREVIOUS="$DOTENV"

src/Configuration/Dotenv.hs

@@ -19,6 +19,7 @@ module Configuration.Dotenv
  where
 
 import Configuration.Dotenv.Parse (configParser)
+import Configuration.Dotenv.ParsedVariable (interpolateParsedVariables)
 import Control.Monad.Catch
 import Control.Monad.IO.Class (MonadIO(..))
 import System.Environment (lookupEnv)
@@ -60,7 +61,7 @@ parseFile f = do
 
   case parse configParser f contents of
     Left e        -> error $ "Failed to read file" ++ show e
-    Right options -> return options
+    Right options -> liftIO $ interpolateParsedVariables options
 
 applySetting :: MonadIO m => Bool -> (String, String) -> m ()
 applySetting override (key, value) =

src/Configuration/Dotenv/Parse.hs

@@ -16,43 +16,60 @@
 
 module Configuration.Dotenv.Parse (configParser) where
 
+import Configuration.Dotenv.ParsedVariable
 import Control.Applicative
 import Control.Monad
 import Text.Megaparsec
 import Text.Megaparsec.String (Parser)
 import qualified Text.Megaparsec.Lexer as L
 
+data QuoteType = SingleQuote | DoubleQuote
+
 -- | Returns a parser for a Dotenv configuration file. Accepts key and value
 -- arguments separated by @=@. Comments in all positions are handled
 -- appropriately.
-configParser :: Parser [(String, String)]
+configParser :: Parser [ParsedVariable]
 configParser = between scn eof (sepEndBy1 envLine (eol <* scn))
 
 -- | Parse a single environment variable assignment.
-envLine :: Parser (String, String)
-envLine = (,) <$> (lexeme variableName <* lexeme (char '=')) <*> lexeme value
+envLine :: Parser ParsedVariable
+envLine = ParsedVariable <$> (lexeme variableName <* lexeme (char '=')) <*> lexeme value
 
 -- | Variables must start with a letter or underscore, and may contain
 -- letters, digits or '_' character after the first character.
-variableName :: Parser String
+variableName :: Parser VarName
 variableName = ((:) <$> firstChar <*> many otherChar) <?> "variable name"
   where
     firstChar = char '_'  <|> letterChar
     otherChar = firstChar <|> digitChar
 
 -- | Value: quoted or unquoted.
-value :: Parser String
+value :: Parser VarValue
 value = (quotedValue <|> unquotedValue) <?> "variable value"
   where
-    quotedValue   = quotedWith '\'' <|> quotedWith '\"'
-    unquotedValue = many (noneOf "\'\" \t\n\r")
+    quotedValue   = quotedWith SingleQuote <|> quotedWith DoubleQuote
+    unquotedValue = Unquoted <$> (many $ fragment "\'\" \t\n\r")
 
 -- | Parse a value quoted with given character.
-quotedWith :: Char -> Parser String
-quotedWith q = between (char q) (char q) (many $ escapedChar <|> normalChar)
+quotedWith :: QuoteType -> Parser VarValue
+quotedWith SingleQuote = SingleQuoted <$> (between (char '\'') (char '\'') $ many (literalValueFragment "\'\\"))
+quotedWith DoubleQuote = DoubleQuoted <$> (between (char '\"') (char '\"') $ many (fragment "\""))
+
+fragment :: [Char] -> Parser VarFragment
+fragment charsToEscape = interpolatedValueFragment <|> literalValueFragment ('$' : '\\' : charsToEscape)
+
+interpolatedValueFragment :: Parser VarFragment
+interpolatedValueFragment = VarInterpolation <$>
+                            ((between (symbol "${") (symbol "}") variableName) <|>
+                            (char '$' >> variableName))
+  where
+    symbol                = L.symbol sc
+
+literalValueFragment :: [Char] -> Parser VarFragment
+literalValueFragment charsToEscape = VarLiteral <$> (some $ escapedChar <|> normalChar)
   where
     escapedChar = (char '\\' *> anyChar) <?> "escaped character"
-    normalChar  = noneOf (q : "\\") <?> "unescaped character"
+    normalChar  = noneOf charsToEscape <?> "unescaped character"
 
 ----------------------------------------------------------------------------
 -- Boilerplate and whitespace setup

src/Configuration/Dotenv/ParsedVariable.hs

@@ -0,0 +1,57 @@
+{-# LANGUAGE CPP #-}
+
+module Configuration.Dotenv.ParsedVariable (ParsedVariable(..),
+                                            VarName,
+                                            VarValue(..),
+                                            VarContents,
+                                            VarFragment(..),
+                                            interpolateParsedVariables) where
+
+import Control.Monad (foldM)
+#if !MIN_VERSION_base(4,8,0)
+import Data.Functor ((<$>))
+#endif
+import Control.Applicative ((<|>))
+import System.Environment (lookupEnv)
+
+data ParsedVariable
+  = ParsedVariable VarName VarValue deriving (Show, Eq)
+
+type VarName = String
+
+data VarValue
+  = Unquoted VarContents
+  | SingleQuoted VarContents
+  | DoubleQuoted VarContents deriving (Show, Eq)
+
+type VarContents = [VarFragment]
+
+data VarFragment
+  = VarInterpolation String
+  | VarLiteral String deriving (Show, Eq)
+
+interpolateParsedVariables :: [ParsedVariable] -> IO [(String, String)]
+interpolateParsedVariables = fmap reverse . foldM addInterpolated []
+
+addInterpolated :: [(String, String)] -> ParsedVariable -> IO [(String, String)]
+addInterpolated previous (ParsedVariable name value) = (: previous) <$> ((,) name <$> interpolate previous value)
+
+interpolate :: [(String, String)] -> VarValue -> IO String
+interpolate _        (SingleQuoted contents) = return $ joinContents contents
+interpolate previous (DoubleQuoted contents) = interpolateContents previous contents
+interpolate previous (Unquoted     contents) = interpolateContents previous contents
+
+interpolateContents :: [(String, String)] -> VarContents -> IO String
+interpolateContents previous contents = concat <$> mapM (interpolateFragment previous) contents
+
+interpolateFragment :: [(String, String)] -> VarFragment -> IO String
+interpolateFragment _        (VarLiteral       value  ) = return value
+interpolateFragment previous (VarInterpolation varname) = fromPreviousOrEnv >>= maybe (return "") return
+  where
+    fromPreviousOrEnv = (lookup varname previous <|>) <$> lookupEnv varname
+
+joinContents :: VarContents -> String
+joinContents = concatMap fragmentToString
+  where
+    fragmentToString (VarInterpolation value) = value
+    fragmentToString (VarLiteral value)       = value