Merge pull request #970 from bobeff/feature/link-files

Add `develop --global` option

Author: Araq

readme.markdown

@@ -290,6 +290,9 @@ packages for which the develop command is executed.
 * `--develop-file` - Changes the name of the develop file which to be
 manipulated. It is useful for creating a free develop file which is not 
 associated with any project intended for inclusion in some other develop file.
+* `-g, --global` - Creates an old style link file in the special `links`
+directory. It is read by Nim to be able to use global develop mode packages.
+Nimble uses it as a global develop file if a local one does not exist.
 
 The options for manipulation of the develop files could be given only when
 executing `develop` command from some package's directory unless

src/nimble.nim

@@ -1166,6 +1166,21 @@ proc listTasks(options: Options) =
 
 proc developAllDependencies(pkgInfo: PackageInfo, options: var Options)
 
+proc saveLinkFile(pkgInfo: PackageInfo, options: Options) =
+  let
+    pkgName = pkgInfo.basicInfo.name
+    pkgLinkDir = options.getPkgsLinksDir / pkgName.getLinkFileDir
+    pkgLinkFilePath = pkgLinkDir / pkgName.getLinkFileName
+    pkgLinkFileContent = pkgInfo.myPath & "\n" & pkgInfo.getNimbleFileDir
+
+  if pkgLinkDir.dirExists and not options.prompt(
+    &"The link file for {pkgName} already exists. Overwrite?"):
+    return
+
+  pkgLinkDir.createDir
+  writeFile(pkgLinkFilePath, pkgLinkFileContent)
+  displaySuccess(pkgLinkFileSavedMsg(pkgLinkFilePath))
+
 proc developFromDir(pkgInfo: PackageInfo, options: var Options) =
   assert options.action.typ == actionDevelop,
     "This procedure should be called only when executing develop sub-command."
@@ -1201,6 +1216,9 @@ proc developFromDir(pkgInfo: PackageInfo, options: var Options) =
       # Dependencies need to be processed before the creation of the pkg dir.
       discard processAllDependencies(pkgInfo, options)
 
+  if options.action.global:
+    saveLinkFile(pkgInfo, options)
+
   displaySuccess(pkgSetupInDevModeMsg(pkgInfo.basicInfo.name, dir))
 
   # Execute the post-develop hook.
@@ -1302,7 +1320,7 @@ proc updateSyncFile(dependentPkg: PackageInfo, options: Options)
 
 proc updatePathsFile(pkgInfo: PackageInfo, options: Options) =
   let paths = pkgInfo.getDependenciesPaths(options)
-  var pathsFileContent: string
+  var pathsFileContent = "--noNimblePath\n"
   for path in paths:
     pathsFileContent &= &"--path:{path.escape}\n"
   var action = if fileExists(nimblePathsFileName): "updated" else: "generated"
@@ -1459,10 +1477,9 @@ proc check(options: Options) =
   try:
     let currentDir = getCurrentDir()
     let pkgInfo = getPkgInfo(currentDir, options, true)
-    if currentDir.developFileExists:
-      validateDevelopFile(pkgInfo, options)
-      let dependencies = pkgInfo.processAllDependencies(options).toSeq
-      validateDevelopDependenciesVersionRanges(pkgInfo, dependencies, options)
+    validateDevelopFile(pkgInfo, options)
+    let dependencies = pkgInfo.processAllDependencies(options).toSeq
+    validateDevelopDependenciesVersionRanges(pkgInfo, dependencies, options)
     displaySuccess(&"The package \"{pkgInfo.basicInfo.name}\" is valid.")
   except CatchableError as error:
     displayError(error)

src/nimblepkg/common.nim

@@ -24,6 +24,7 @@ type
 const
   nimbleVersion* = "0.14.0"
   nimblePackagesDirName* = "pkgs2"
+  nimblePackagesLinksDirName* ="links"
   nimbleBinariesDirName* = "bin"
 
 proc newNimbleError*[ErrorType](msg: string, hint = "",
@@ -77,3 +78,9 @@ template cdNewDir*(dir: string, body: untyped) =
   createNewDir dir
   cd dir:
     body
+
+proc getLinkFileDir*(pkgName: string): string =
+  pkgName & "-#head"
+
+proc getLinkFileName*(pkgName: string): string =
+  pkgName & ".nimble-link"

src/nimblepkg/developfile.nim

@@ -307,7 +307,7 @@ proc mergeFollowedDevFileData(lhs: var DevelopFileData, rhs: DevelopFileData,
                   errors.collidingNames)
 
 proc load(path: Path, dependentPkg: PackageInfo, options: Options,
-          silentIfFileNotExists, raiseOnValidationErrors: bool):
+          silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool):
     DevelopFileData
 
 template load(dependentPkg: PackageInfo, args: varargs[untyped]):
@@ -318,8 +318,37 @@ template load(dependentPkg: PackageInfo, args: varargs[untyped]):
   dependentPkg.assertIsLoaded
   load(dependentPkg.getPkgDevFilePath, dependentPkg, args)
 
+proc loadGlobalDependencies(result: var DevelopFileData,
+                            collidingNames: var CollidingNames,
+                            options: Options) =
+  ## Loads data from the `links` subdirectory in the Nimble cache. The links
+  ## in the cache are treated as paths in a global develop file used when a
+  ## local one does not exist.
+
+  for (kind, path) in walkDir(options.getPkgsLinksDir):
+    if kind != pcDir:
+      continue
+    let (pkgName, _, _) = getNameVersionChecksum(path)
+    let linkFilePath = path / pkgName.getLinkFileName
+    if not linkFilePath.fileExists:
+      displayWarning(&"Not found link file in \"{path}\".")
+      continue
+    let lines = linkFilePath.readFile.split("\n")
+    if lines.len != 2:
+      displayWarning(&"Invalid link file \"{linkFilePath}\".")
+      continue
+    let pkgPath = lines[1]
+    let (pkgInfo, error) = validatePackage(pkgPath, options)
+    if error == nil:
+      let path = path.Path
+      result.addPackage(pkgInfo, path, [path].toHashSet, collidingNames)
+    else:
+      displayWarning(
+        &"Package \"{pkgName}\" at path \"{pkgPath}\" is invalid. Skipping it.")
+      displayDetails(error.msg)
+
 proc load(path: Path, dependentPkg: PackageInfo, options: Options,
-          silentIfFileNotExists, raiseOnValidationErrors: bool):
+          silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool):
     DevelopFileData =
   ## Loads data from a develop file at path `path`.
   ##
@@ -328,6 +357,10 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   ##
   ## If `raiseOnValidationErrors` raises a `NimbleError` in the case some of the
   ## contents of the develop file are invalid.
+  ## 
+  ## If `loadGlobalDeps` then load the packages pointed by the link files in the
+  ## `links` directory in the Nimble cache instead of the once pointed by the
+  ## local develop file.
   ##
   ## Raises if the develop file or some of the included develop files:
   ##   - cannot be read.
@@ -343,9 +376,6 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   result.path = path
   result.dependentPkg = dependentPkg
 
-  if silentIfFileNotExists and not path.fileExists:
-    return
-
   var
     errors {.global.}: ErrorsCollection
     visitedFiles {.global.}: HashSet[Path]
@@ -355,31 +385,38 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   if dependentPkg.isLoaded:
     visitedPkgs.incl dependentPkg.getNimbleFileDir
 
-  try:
-    fromJson(result.jsonData, parseFile(path), Joptions(allowExtraKeys: true))
-  except ValueError as error:
-    raise nimbleError(notAValidDevFileJsonMsg($path), details = error)
-
-  for depPath in result.jsonData.dependencies:
-    let depPath = if depPath.isAbsolute:
-      depPath.normalizedPath else: (path.splitFile.dir / depPath).normalizedPath
-    let (pkgInfo, error) = validatePackage(depPath, options)
-    if error == nil:
-      result.addPackage(pkgInfo, path, [path].toHashSet, errors.collidingNames)
-    else:
-      errors.invalidPackages[depPath] = error
+  if loadGlobalDeps:
+    loadGlobalDependencies(result, errors.collidingNames, options)
+  else:
+    if silentIfFileNotExists and not path.fileExists:
+      return
 
-  for inclPath in result.jsonData.includes:
-    let inclPath = inclPath.normalizedPath
-    if visitedFiles.contains(inclPath):
-      continue
-    var inclDevFileData = initDevelopFileData()
     try:
-      inclDevFileData = load(inclPath, initPackageInfo(), options, false, false)
-    except CatchableError as error:
-      errors.invalidIncludeFiles[path] = error
-      continue
-    result.mergeIncludedDevFileData(inclDevFileData, errors)
+      fromJson(result.jsonData, parseFile(path), Joptions(allowExtraKeys: true))
+    except ValueError as error:
+      raise nimbleError(notAValidDevFileJsonMsg($path), details = error)
+
+    for depPath in result.jsonData.dependencies:
+      let depPath = if depPath.isAbsolute:
+        depPath.normalizedPath else: (path.splitFile.dir / depPath).normalizedPath
+      let (pkgInfo, error) = validatePackage(depPath, options)
+      if error == nil:
+        result.addPackage(pkgInfo, path, [path].toHashSet, errors.collidingNames)
+      else:
+        errors.invalidPackages[depPath] = error
+
+    for inclPath in result.jsonData.includes:
+      let inclPath = inclPath.normalizedPath
+      if visitedFiles.contains(inclPath):
+        continue
+      var inclDevFileData = initDevelopFileData()
+      try:
+        inclDevFileData = load(
+          inclPath, initPackageInfo(), options, false, false, false)
+      except CatchableError as error:
+        errors.invalidIncludeFiles[path] = error
+        continue
+      result.mergeIncludedDevFileData(inclDevFileData, errors)
 
   if result.dependentPkg.isLoaded and path.splitPath.tail == developFileName:
     # If this is a package develop file, but not a free one, for each of the
@@ -390,7 +427,7 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
         continue
       var followedPkgDevFileData = initDevelopFileData()
       try:
-        followedPkgDevFileData = load(pkg[], options, true, false)
+        followedPkgDevFileData = load(pkg[], options, true, false, false)
       except:
         # The errors will be accumulated in `errors` global variable and
         # reported by the `load` call which initiated the recursive process.
@@ -580,7 +617,7 @@ proc includeDevelopFile(data: var DevelopFileData, path: Path,
 
   var inclFileData = initDevelopFileData()
   try:
-    inclFileData = load(path, initPackageInfo(), options, false, true)
+    inclFileData = load(path, initPackageInfo(), options, false, true, false)
   except CatchableError as error:
     displayError(failedToLoadFileMsg($path))
     displayDetails(error)
@@ -660,7 +697,7 @@ proc updateDevelopFile*(dependentPkg: PackageInfo, options: Options): bool =
   var
     hasError = false
     hasSuccessfulRemoves = false
-    data = load(developFile, dependentPkg, options, true, true)
+    data = load(developFile, dependentPkg, options, true, true, false)
 
   defer:
     let writeEmpty = hasSuccessfulRemoves or
@@ -691,7 +728,8 @@ proc processDevelopDependencies*(dependentPkg: PackageInfo, options: Options):
   ## Returns a sequence with the develop mode dependencies of the `dependentPkg`
   ## and recursively all of their develop mode dependencies.
 
-  let data = load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  let data = load(dependentPkg, options, true, true, loadGlobalDeps)
   result = newSeqOfCap[PackageInfo](data.nameToPkg.len)
   for _, pkg in data.nameToPkg:
     result.add pkg[]
@@ -702,7 +740,8 @@ proc getDevelopDependencies*(dependentPkg: PackageInfo, options: Options):
   ## mode dependencies of package `dependentPkg` and recursively all of their
   ## develop mode dependencies.
 
-  let data = load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  let data = load(dependentPkg, options, true, true, loadGlobalDeps)
   return data.nameToPkg
 
 type
@@ -908,6 +947,7 @@ proc validateDevelopFile*(dependentPkg: PackageInfo, options: Options) =
   ## The procedure is used in the Nimble's `check` command to transitively
   ## validate the contents of the develop files.
 
-  discard load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  discard load(dependentPkg, options, true, true, loadGlobalDeps)
   if dependentPkg.areLockedDepsLoaded:
     validateDevelopFileAgainstLockFile(dependentPkg, options)

src/nimblepkg/displaymessages.nim

@@ -157,3 +157,6 @@ proc pkgAlreadyExistsInTheCacheMsg*(pkgInfo: PackageInfo): string =
 proc skipDownloadingInAlreadyExistingDirectoryMsg*(dir, name: string): string =
   &"The download directory \"{dir}\" already exists.\n" &
   &"Skipping the download of \"{name}\"."
+
+proc pkgLinkFileSavedMsg*(path: string): string =
+  &"Package link file \"{path}\" is saved."

src/nimblepkg/options.nim

@@ -75,6 +75,7 @@ type
         ## Whether to put in develop mode also the dependencies of the packages
         ## listed in the develop command.
       developFile*: string
+      global*: bool
     of actionSearch:
       search*: seq[string] # Search string.
     of actionInit, actionDump:
@@ -109,7 +110,7 @@ Commands:
                                   executed in package's directory.
          [--with-dependencies]    Puts in develop mode also the dependencies
                                   of the packages in the list or of the current
-                                  directory package if the list is
+                                  directory package if the list is empty.
          [--develop-file]         Specifies the name of the develop file which
                                   to be manipulated. If not present creates it.
          [-p, --path path]        Specifies the path whether the packages should
@@ -131,6 +132,11 @@ Commands:
          [-e, --exclude file]     Excludes a develop file from a specified
                                   develop file or from `nimble.develop` if not
                                   specified and executed in package's directory.
+         [-g, --global]           Creates an old style link file in the special
+                                  `links` directory. It is read by Nim to be
+                                  able to use global develop mode packages.
+                                  Nimble uses it as a global develop file if a
+                                  local one does not exist.
   check                           Verifies the validity of a package in the
                                   current working directory.
   init         [pkgname]          Initializes a new Nimble project in the
@@ -315,6 +321,9 @@ proc getNimbleDir*(options: Options): string =
 proc getPkgsDir*(options: Options): string =
   options.getNimbleDir() / nimblePackagesDirName
 
+proc getPkgsLinksDir*(options: Options): string =
+  options.getNimbleDir() / nimblePackagesLinksDirName
+
 proc getBinDir*(options: Options): string =
   options.getNimbleDir() / nimbleBinariesDirName
 
@@ -563,6 +572,8 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
         raise nimbleError(multiplePathOptionsGivenMsg)
     of "with-dependencies":
       result.action.withDependencies = true
+    of "global":
+      result.action.global = true
     of "develop-file":
       if result.action.developFile.len == 0:
         result.action.developFile = val.normalizedPath

tests/tdevelopfeature.nim

@@ -6,7 +6,7 @@
 import unittest, os, strutils, strformat, json, sets
 import testscommon, nimblepkg/displaymessages, nimblepkg/paths
 
-from nimblepkg/common import cd
+from nimblepkg/common import cd, getLinkFileName, getLinkFileDir
 from nimblepkg/developfile import developFileName, pkgFoundMoreThanOnceMsg
 from nimblepkg/version import newVersion, parseVersionRange
 from nimblepkg/nimbledatafile import nimbleDataFileName, NimbleDataJsonKeys
@@ -81,6 +81,33 @@ suite "develop feature":
         check lines.inLinesOrdered(pkgSetupInDevModeMsg(
           pkgBName, installDir / pkgBName))
 
+  test "can develop global":
+    cleanDir installDir
+    usePackageListFile &"develop/{pkgListFileName}":
+      let dependencyPath = getCurrentDir() / "develop" / "dependency"
+
+      # Check that a link file can be created for a global develop dependency.
+      cd dependencyPath:
+        let (output, exitCode) = execNimble("develop", "--global")
+        check exitCode == QuitSuccess
+        var lines = output.processOutput
+        let linkFilePath = installDir / "links" / depName.getLinkFileDir /
+                           depName.getLinkFileName
+        check lines.inLinesOrdered(pkgLinkFileSavedMsg(linkFilePath))
+        check lines.inLinesOrdered(pkgSetupInDevModeMsg(
+          depName, dependencyPath))
+        check linkFilePath.fileExists
+        let linkFileLines = linkFilePath.readFile.split('\n')
+        let expectedLinkNimbleFilePath = dependencyPath / depName & ".nimble"
+        check linkFileLines[0] == expectedLinkNimbleFilePath
+        check linkFileLines[1] == dependencyPath
+
+      # Check that a link file can be used for finding the dependency.
+      cd "develop/dependent":
+        cleanFile dependentPkgName.addFileExt(ExeExt)
+        let (_, exitCode) = execNimble("run")
+        check exitCode == QuitSuccess
+
   test "cannot remove package with develop reverse dependency":
     cdCleanDir installDir:
       usePackageListFile &"../develop/{pkgListFileName}":