Merge branch 'master' into feature/run-package-executable

Author: Araq

readme.markdown

@@ -228,11 +228,11 @@ query parameter. For example:
 ### nimble develop
 
 The develop command is used for putting packages in a development mode. When
-executed with a list of packages it clones their repository. If it is
-executed in a package directory it adds cloned packages to the special
-`nimble.develop` file. This is a special file which is used for holding the
-paths to development mode dependencies of the current directory package. It has
-the following structure:
+executed with a list of packages it clones their repository. If it is executed
+in a package directory it adds cloned packages to the special `nimble.develop`
+file. This is a special file which is used for holding the paths to development
+mode dependencies of the current directory package. It has the following
+structure:
 
 ```json
 {
@@ -247,17 +247,15 @@ the following structure:
 * `dependencies` - JSON array of paths to Nimble packages directories.
 
 The format for included develop files is the same as the project's develop
-file, but their validation works slightly different.
+file.
 
-Validation rules:
+Develop files validation rules:
 
 * The included develop files must be valid.
-* The packages listed in `dependencies` section must be dependencies required
-by the package's `.nimble` file and to be in the required by its version range.
-Transitive dependencies are not allowed, but this may be changed in the future.
-* The packages listed in the included develop files are required to be valid
-**Nimble** packages, but they are not required to be valid dependencies of the
-current project. In the latter case, they are simply ignored.
+* The packages listed in the `dependencies` section and in the included develop
+files are required to be valid **Nimble** packages, but they are not required
+to be valid dependencies of the current project. In the latter case, they are
+simply ignored.
 * The develop files of the develop mode dependencies of a package are being
 followed and processed recursively. Finally, only one common set of develop
 mode dependencies is created.
@@ -292,6 +290,9 @@ packages for which the develop command is executed.
 * `--develop-file` - Changes the name of the develop file which to be
 manipulated. It is useful for creating a free develop file which is not 
 associated with any project intended for inclusion in some other develop file.
+* `-g, --global` - Creates an old style link file in the special `links`
+directory. It is read by Nim to be able to use global develop mode packages.
+Nimble uses it as a global develop file if a local one does not exist.
 
 The options for manipulation of the develop files could be given only when
 executing `develop` command from some package's directory unless
@@ -300,16 +301,6 @@ executing `develop` command from some package's directory unless
 Because the develop files are user-specific and they contain local file system
 paths they **MUST NOT** be committed.
 
-**Current limitations:**
-
-* Currently transitive dependencies in the `dependencies` section of the
-develop file are not allowed. In the future, they should be allowed because
-this will allow using in develop mode some transitive package dependencies
-without having in develop mode the full dependency tree path to them. It was a
-design mistake that was not allowed at the beginning. The current workaround is
-to add the transitive dependency as a dependency in the project's `.nimble`
-file.
-
 ### nimble lock
 
 The `nimble lock` command will generate or update a package lock file named

src/nimble.nim

@@ -1170,6 +1170,21 @@ proc listTasks(options: Options) =
 
 proc developAllDependencies(pkgInfo: PackageInfo, options: var Options)
 
+proc saveLinkFile(pkgInfo: PackageInfo, options: Options) =
+  let
+    pkgName = pkgInfo.basicInfo.name
+    pkgLinkDir = options.getPkgsLinksDir / pkgName.getLinkFileDir
+    pkgLinkFilePath = pkgLinkDir / pkgName.getLinkFileName
+    pkgLinkFileContent = pkgInfo.myPath & "\n" & pkgInfo.getNimbleFileDir
+
+  if pkgLinkDir.dirExists and not options.prompt(
+    &"The link file for {pkgName} already exists. Overwrite?"):
+    return
+
+  pkgLinkDir.createDir
+  writeFile(pkgLinkFilePath, pkgLinkFileContent)
+  displaySuccess(pkgLinkFileSavedMsg(pkgLinkFilePath))
+
 proc developFromDir(pkgInfo: PackageInfo, options: var Options) =
   assert options.action.typ == actionDevelop,
     "This procedure should be called only when executing develop sub-command."
@@ -1205,6 +1220,9 @@ proc developFromDir(pkgInfo: PackageInfo, options: var Options) =
       # Dependencies need to be processed before the creation of the pkg dir.
       discard processAllDependencies(pkgInfo, options)
 
+  if options.action.global:
+    saveLinkFile(pkgInfo, options)
+
   displaySuccess(pkgSetupInDevModeMsg(pkgInfo.basicInfo.name, dir))
 
   # Execute the post-develop hook.
@@ -1306,7 +1324,7 @@ proc updateSyncFile(dependentPkg: PackageInfo, options: Options)
 
 proc updatePathsFile(pkgInfo: PackageInfo, options: Options) =
   let paths = pkgInfo.getDependenciesPaths(options)
-  var pathsFileContent: string
+  var pathsFileContent = "--noNimblePath\n"
   for path in paths:
     pathsFileContent &= &"--path:{path.escape}\n"
   var action = if fileExists(nimblePathsFileName): "updated" else: "generated"
@@ -1463,10 +1481,9 @@ proc check(options: Options) =
   try:
     let currentDir = getCurrentDir()
     let pkgInfo = getPkgInfo(currentDir, options, true)
-    if currentDir.developFileExists:
-      validateDevelopFile(pkgInfo, options)
-      let dependencies = pkgInfo.processAllDependencies(options).toSeq
-      validateDevelopDependenciesVersionRanges(pkgInfo, dependencies, options)
+    validateDevelopFile(pkgInfo, options)
+    let dependencies = pkgInfo.processAllDependencies(options).toSeq
+    validateDevelopDependenciesVersionRanges(pkgInfo, dependencies, options)
     displaySuccess(&"The package \"{pkgInfo.basicInfo.name}\" is valid.")
   except CatchableError as error:
     displayError(error)

src/nimblepkg/common.nim

@@ -24,6 +24,7 @@ type
 const
   nimbleVersion* = "0.14.0"
   nimblePackagesDirName* = "pkgs2"
+  nimblePackagesLinksDirName* ="links"
   nimbleBinariesDirName* = "bin"
 
 proc newNimbleError*[ErrorType](msg: string, hint = "",
@@ -77,3 +78,9 @@ template cdNewDir*(dir: string, body: untyped) =
   createNewDir dir
   cd dir:
     body
+
+proc getLinkFileDir*(pkgName: string): string =
+  pkgName & "-#head"
+
+proc getLinkFileName*(pkgName: string): string =
+  pkgName & ".nimble-link"

src/nimblepkg/developfile.nim

@@ -307,7 +307,7 @@ proc mergeFollowedDevFileData(lhs: var DevelopFileData, rhs: DevelopFileData,
                   errors.collidingNames)
 
 proc load(path: Path, dependentPkg: PackageInfo, options: Options,
-          silentIfFileNotExists, raiseOnValidationErrors: bool):
+          silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool):
     DevelopFileData
 
 template load(dependentPkg: PackageInfo, args: varargs[untyped]):
@@ -318,8 +318,37 @@ template load(dependentPkg: PackageInfo, args: varargs[untyped]):
   dependentPkg.assertIsLoaded
   load(dependentPkg.getPkgDevFilePath, dependentPkg, args)
 
+proc loadGlobalDependencies(result: var DevelopFileData,
+                            collidingNames: var CollidingNames,
+                            options: Options) =
+  ## Loads data from the `links` subdirectory in the Nimble cache. The links
+  ## in the cache are treated as paths in a global develop file used when a
+  ## local one does not exist.
+
+  for (kind, path) in walkDir(options.getPkgsLinksDir):
+    if kind != pcDir:
+      continue
+    let (pkgName, _, _) = getNameVersionChecksum(path)
+    let linkFilePath = path / pkgName.getLinkFileName
+    if not linkFilePath.fileExists:
+      displayWarning(&"Not found link file in \"{path}\".")
+      continue
+    let lines = linkFilePath.readFile.split("\n")
+    if lines.len != 2:
+      displayWarning(&"Invalid link file \"{linkFilePath}\".")
+      continue
+    let pkgPath = lines[1]
+    let (pkgInfo, error) = validatePackage(pkgPath, options)
+    if error == nil:
+      let path = path.Path
+      result.addPackage(pkgInfo, path, [path].toHashSet, collidingNames)
+    else:
+      displayWarning(
+        &"Package \"{pkgName}\" at path \"{pkgPath}\" is invalid. Skipping it.")
+      displayDetails(error.msg)
+
 proc load(path: Path, dependentPkg: PackageInfo, options: Options,
-          silentIfFileNotExists, raiseOnValidationErrors: bool):
+          silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool):
     DevelopFileData =
   ## Loads data from a develop file at path `path`.
   ##
@@ -328,6 +357,10 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   ##
   ## If `raiseOnValidationErrors` raises a `NimbleError` in the case some of the
   ## contents of the develop file are invalid.
+  ## 
+  ## If `loadGlobalDeps` then load the packages pointed by the link files in the
+  ## `links` directory in the Nimble cache instead of the once pointed by the
+  ## local develop file.
   ##
   ## Raises if the develop file or some of the included develop files:
   ##   - cannot be read.
@@ -343,9 +376,6 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   result.path = path
   result.dependentPkg = dependentPkg
 
-  if silentIfFileNotExists and not path.fileExists:
-    return
-
   var
     errors {.global.}: ErrorsCollection
     visitedFiles {.global.}: HashSet[Path]
@@ -355,31 +385,38 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
   if dependentPkg.isLoaded:
     visitedPkgs.incl dependentPkg.getNimbleFileDir
 
-  try:
-    fromJson(result.jsonData, parseFile(path), Joptions(allowExtraKeys: true))
-  except ValueError as error:
-    raise nimbleError(notAValidDevFileJsonMsg($path), details = error)
-
-  for depPath in result.jsonData.dependencies:
-    let depPath = if depPath.isAbsolute:
-      depPath.normalizedPath else: (path.splitFile.dir / depPath).normalizedPath
-    let (pkgInfo, error) = validatePackage(depPath, options)
-    if error == nil:
-      result.addPackage(pkgInfo, path, [path].toHashSet, errors.collidingNames)
-    else:
-      errors.invalidPackages[depPath] = error
+  if loadGlobalDeps:
+    loadGlobalDependencies(result, errors.collidingNames, options)
+  else:
+    if silentIfFileNotExists and not path.fileExists:
+      return
 
-  for inclPath in result.jsonData.includes:
-    let inclPath = inclPath.normalizedPath
-    if visitedFiles.contains(inclPath):
-      continue
-    var inclDevFileData = initDevelopFileData()
     try:
-      inclDevFileData = load(inclPath, initPackageInfo(), options, false, false)
-    except CatchableError as error:
-      errors.invalidIncludeFiles[path] = error
-      continue
-    result.mergeIncludedDevFileData(inclDevFileData, errors)
+      fromJson(result.jsonData, parseFile(path), Joptions(allowExtraKeys: true))
+    except ValueError as error:
+      raise nimbleError(notAValidDevFileJsonMsg($path), details = error)
+
+    for depPath in result.jsonData.dependencies:
+      let depPath = if depPath.isAbsolute:
+        depPath.normalizedPath else: (path.splitFile.dir / depPath).normalizedPath
+      let (pkgInfo, error) = validatePackage(depPath, options)
+      if error == nil:
+        result.addPackage(pkgInfo, path, [path].toHashSet, errors.collidingNames)
+      else:
+        errors.invalidPackages[depPath] = error
+
+    for inclPath in result.jsonData.includes:
+      let inclPath = inclPath.normalizedPath
+      if visitedFiles.contains(inclPath):
+        continue
+      var inclDevFileData = initDevelopFileData()
+      try:
+        inclDevFileData = load(
+          inclPath, initPackageInfo(), options, false, false, false)
+      except CatchableError as error:
+        errors.invalidIncludeFiles[path] = error
+        continue
+      result.mergeIncludedDevFileData(inclDevFileData, errors)
 
   if result.dependentPkg.isLoaded and path.splitPath.tail == developFileName:
     # If this is a package develop file, but not a free one, for each of the
@@ -390,7 +427,7 @@ proc load(path: Path, dependentPkg: PackageInfo, options: Options,
         continue
       var followedPkgDevFileData = initDevelopFileData()
       try:
-        followedPkgDevFileData = load(pkg[], options, true, false)
+        followedPkgDevFileData = load(pkg[], options, true, false, false)
       except:
         # The errors will be accumulated in `errors` global variable and
         # reported by the `load` call which initiated the recursive process.
@@ -580,7 +617,7 @@ proc includeDevelopFile(data: var DevelopFileData, path: Path,
 
   var inclFileData = initDevelopFileData()
   try:
-    inclFileData = load(path, initPackageInfo(), options, false, true)
+    inclFileData = load(path, initPackageInfo(), options, false, true, false)
   except CatchableError as error:
     displayError(failedToLoadFileMsg($path))
     displayDetails(error)
@@ -660,7 +697,7 @@ proc updateDevelopFile*(dependentPkg: PackageInfo, options: Options): bool =
   var
     hasError = false
     hasSuccessfulRemoves = false
-    data = load(developFile, dependentPkg, options, true, true)
+    data = load(developFile, dependentPkg, options, true, true, false)
 
   defer:
     let writeEmpty = hasSuccessfulRemoves or
@@ -691,7 +728,8 @@ proc processDevelopDependencies*(dependentPkg: PackageInfo, options: Options):
   ## Returns a sequence with the develop mode dependencies of the `dependentPkg`
   ## and recursively all of their develop mode dependencies.
 
-  let data = load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  let data = load(dependentPkg, options, true, true, loadGlobalDeps)
   result = newSeqOfCap[PackageInfo](data.nameToPkg.len)
   for _, pkg in data.nameToPkg:
     result.add pkg[]
@@ -702,7 +740,8 @@ proc getDevelopDependencies*(dependentPkg: PackageInfo, options: Options):
   ## mode dependencies of package `dependentPkg` and recursively all of their
   ## develop mode dependencies.
 
-  let data = load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  let data = load(dependentPkg, options, true, true, loadGlobalDeps)
   return data.nameToPkg
 
 type
@@ -908,6 +947,7 @@ proc validateDevelopFile*(dependentPkg: PackageInfo, options: Options) =
   ## The procedure is used in the Nimble's `check` command to transitively
   ## validate the contents of the develop files.
 
-  discard load(dependentPkg, options, true, true)
+  let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists
+  discard load(dependentPkg, options, true, true, loadGlobalDeps)
   if dependentPkg.areLockedDepsLoaded:
     validateDevelopFileAgainstLockFile(dependentPkg, options)

src/nimblepkg/displaymessages.nim

@@ -167,3 +167,6 @@ proc binaryNotDefinedInPkgMsg*(binaryName, pkgName: string): string =
 proc notFoundPkgWithNameInPkgDepTree*(pkgName: string): string =
   &"Not found package with name '{pkgName}' in the current package's " &
    "dependency tree."
+
+proc pkgLinkFileSavedMsg*(path: string): string =
+  &"Package link file \"{path}\" is saved."

src/nimblepkg/download.nim

@@ -15,15 +15,18 @@ type
     version: Version
     vcsRevision: Sha1Hash
 
+proc updateSubmodules(dir: string) =
+  discard tryDoCmdEx(
+    &"git -C {dir} submodule update --init --recursive --depth 1")
+
 proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) =
   case meth
   of DownloadMethod.git:
     # Force is used here because local changes may appear straight after a clone
     # has happened. Like in the case of git on Windows where it messes up the
     # damn line endings.
-    discard tryDoCmdEx(&"git -C {downloadDir} checkout --force {branch}")
-    discard tryDoCmdEx(
-      &"git -C {downloadDir} submodule update --recursive --depth 1")
+    discard tryDoCmdEx(&"git -C {downloadDir} checkout --force {branch}") 
+    downloadDir.updateSubmodules
   of DownloadMethod.hg:
     discard tryDoCmdEx(&"hg --cwd {downloadDir} checkout {branch}")
 
@@ -35,7 +38,8 @@ proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "",
       depthArg = if onlyTip: "--depth 1" else: ""
       branchArg = if branch == "": "" else: &"-b {branch}"
     discard tryDoCmdEx(
-      &"git clone --recursive {depthArg} {branchArg} {url} {downloadDir}")
+       "git clone --config core.autocrlf=false --recursive " &
+      &"{depthArg} {branchArg} {url} {downloadDir}")
   of DownloadMethod.hg:
     let
       tipArg = if onlyTip: "-r tip " else: ""
@@ -143,10 +147,12 @@ proc cloneSpecificRevision(downloadMethod: DownloadMethod,
     let downloadDir = downloadDir.quoteShell
     createDir(downloadDir)
     discard tryDoCmdEx(&"git -C {downloadDir} init")
+    discard tryDoCmdEx(&"git -C {downloadDir} config core.autocrlf false")
     discard tryDoCmdEx(&"git -C {downloadDir} remote add origin {url}")
     discard tryDoCmdEx(
       &"git -C {downloadDir} fetch --depth 1 origin {vcsRevision}")
     discard tryDoCmdEx(&"git -C {downloadDir} reset --hard FETCH_HEAD")
+    downloadDir.updateSubmodules
   of DownloadMethod.hg:
     discard tryDoCmdEx(&"hg clone {url} -r {vcsRevision}")