diff --git a/.gitmodules b/.gitmodules index c5b29ce6..a403c890 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,9 @@ [submodule "vendor/chronos"] path = vendor/chronos url = https://github.com/status-im/nim-chronos.git +[submodule "vendor/results"] + path = vendor/results + url = https://github.com/arnetheduck/nim-results.git +[submodule "vendor/stew"] + path = vendor/stew + url = https://github.com/status-im/nim-stew.git diff --git a/config.nims b/config.nims index 773f7d9f..7768532b 100644 --- a/config.nims +++ b/config.nims @@ -6,5 +6,8 @@ switch("define", "ssl") switch("path", "vendor" / "zippy" / "src") switch("path", "vendor" / "sat" / "src") switch("path", "vendor" / "checksums" / "src") +switch("path", "vendor" / "chronos") +switch("path", "vendor" / "results") +switch("path", "vendor" / "stew") switch("define", "zippyNoSimd") diff --git a/src/nimble.nim b/src/nimble.nim index 18721bbf..1475ddc5 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -5,7 +5,7 @@ import os, tables, strtabs, json, browsers, algorithm, sets, uri, sugar, sequtil strformat import std/options as std_opt - +import chronos import strutils except toLower from unicode import toLower import sat/sat @@ -660,10 +660,13 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, # Copy this package's files based on the preferences specified in PkgInfo. var filesInstalled: HashSet[string] iterInstallFiles(realDir, pkgInfo, options, - proc (file: string) = - createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) - let dest = changeRoot(realDir, pkgDestDir, file) - filesInstalled.incl copyFileD(file, dest) + proc (file: string) {.raises: [].} = + try: + createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) + let dest = changeRoot(realDir, pkgDestDir, file) + filesInstalled.incl copyFileD(file, dest) + except Exception: + discard ) # Copy the .nimble file. @@ -2566,9 +2569,22 @@ proc run(options: Options, nimBin: string) = # In vnext path, build develop mode packages (similar to old code path) if pkgInfo.isLink: # Use vnext buildPkg for develop mode packages - let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and + let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and options.satResult.rootPackage.basicInfo.name == pkgInfo.basicInfo.name - buildPkg(nimBin, pkgInfo, isInRootDir, options) + let buildTasks = waitFor buildPkg(nimBin, pkgInfo, isInRootDir, options) + # Execute the build tasks immediately for run command + for task in buildTasks: + let future = task.startBuild() + let response = waitFor future + if response.stdOutput.len > 0: + display("Output", response.stdOutput, priority = HighPriority) + if response.stdError.len > 0: + displayWarning(response.stdError) + if response.status != 0: + raise buildFailed(&"Build failed for binary: {task.bin}", details = nil) + # Create symlinks after build completes + if buildTasks.len > 0: + vnext.createBinSymlink(pkgInfo, options) if options.getCompilationFlags.len > 0: displayWarning(ignoringCompilationFlagsMsg) diff --git a/src/nimblepkg/asyncfileops.nim b/src/nimblepkg/asyncfileops.nim new file mode 100644 index 00000000..24307c20 --- /dev/null +++ b/src/nimblepkg/asyncfileops.nim @@ -0,0 +1,39 @@ +# Async file operations using chronos async processes +# Similar to Node.js - uses external commands for file I/O + +import std/os +import chronos except Duration +import chronos/asyncproc + +export chronos except Duration +export asyncproc + +proc copyFileAsync*(source, dest: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} = + ## Async file copy using chronos async processes + when defined(windows): + # Windows: use xcopy for better handling + let cmd = "xcopy /Y /Q " & quoteShell(source) & " " & quoteShell(dest) & "*" + else: + # Unix: use cp command with preserve permissions and recursive for dirs + let cmd = "cp -f -p -r " & quoteShell(source) & " " & quoteShell(dest) + + let exitCode = await execCommand(cmd) + if exitCode != 0: + raise newException(IOError, "Failed to copy file from " & source & " to " & dest & " (exit code: " & $exitCode & ")") + +proc copyDirAsync*(sourceDir, destDir: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} = + ## Async directory copy using chronos async processes - copies entire directory tree + when defined(windows): + # Windows: use robocopy for robust directory copying + # /E = copy subdirs including empty, /NFL = no file list, /NDL = no dir list, /NJH = no job header, /NJS = no job summary, /NC = no class, /NS = no size, /NP = no progress + let cmd = "robocopy " & quoteShell(sourceDir) & " " & quoteShell(destDir) & " /E /NFL /NDL /NJH /NJS /NC /NS /NP" + let exitCode = await execCommand(cmd) + # robocopy exit codes: 0-7 are success (0=no files, 1=files copied, 2=extra files, etc.) + if exitCode > 7: + raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")") + else: + # Unix: use cp -r to copy entire directory recursively + let cmd = "cp -r -p " & quoteShell(sourceDir) & "/. " & quoteShell(destDir) + let exitCode = await execCommand(cmd) + if exitCode != 0: + raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")") diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim index de67b921..6cf7ab54 100644 --- a/src/nimblepkg/cli.nim +++ b/src/nimblepkg/cli.nim @@ -76,22 +76,23 @@ proc displayInfoLine*(field, msg: string) = proc displayCategory(category: string, displayType: DisplayType, priority: Priority) = - if isSuppressed(displayType): - return + {.cast(gcsafe).}: + if isSuppressed(displayType): + return - # Calculate how much the `category` must be offset to align along a center - # line. - let offset = calculateCategoryOffset(category) - - # Display the category. - let text = "$1$2 " % [spaces(offset), category] - if globalCLI.showColor: - if priority != DebugPriority: - setForegroundColor(stdout, foregrounds[displayType]) - writeStyled(text, styles[priority]) - resetAttributes() - else: - stdout.write(text) + # Calculate how much the `category` must be offset to align along a center + # line. + let offset = calculateCategoryOffset(category) + + # Display the category. + let text = "$1$2 " % [spaces(offset), category] + if globalCLI.showColor: + if priority != DebugPriority: + setForegroundColor(stdout, foregrounds[displayType]) + writeStyled(text, styles[priority]) + resetAttributes() + else: + stdout.write(text) proc displayLine(category, line: string, displayType: DisplayType, @@ -106,27 +107,28 @@ proc displayLine(category, line: string, displayType: DisplayType, proc display*(category, msg: string, displayType = Message, priority = MediumPriority) = - # Multiple warnings containing the same messages should not be shown. - let warningPair = (category, msg) - if displayType == Warning: - if warningPair in globalCLI.warnings: - return - else: - globalCLI.warnings.incl(warningPair) + {.cast(gcsafe).}: + # Multiple warnings containing the same messages should not be shown. + let warningPair = (category, msg) + if displayType == Warning: + if warningPair in globalCLI.warnings: + return + else: + globalCLI.warnings.incl(warningPair) - # Suppress this message if its priority isn't high enough. - # TODO: Per-priority suppression counts? - if priority < globalCLI.level: - if priority != DebugPriority: - globalCLI.suppressionCount.inc - return + # Suppress this message if its priority isn't high enough. + # TODO: Per-priority suppression counts? + if priority < globalCLI.level: + if priority != DebugPriority: + globalCLI.suppressionCount.inc + return - # Display each line in the message. - var i = 0 - for line in msg.splitLines(): - if len(line) == 0: continue - displayLine(if i == 0: category else: "...", line, displayType, priority) - i.inc + # Display each line in the message. + var i = 0 + for line in msg.splitLines(): + if len(line) == 0: continue + displayLine(if i == 0: category else: "...", line, displayType, priority) + i.inc proc displayWarning*(message: string, priority = HighPriority) = display("Warning: ", message, Warning, priority) diff --git a/src/nimblepkg/nimscriptexecutor.nim b/src/nimblepkg/nimscriptexecutor.nim index 293d1018..4a5f3545 100644 --- a/src/nimblepkg/nimscriptexecutor.nim +++ b/src/nimblepkg/nimscriptexecutor.nim @@ -5,30 +5,32 @@ import os, strutils, sets import packageparser, common, options, nimscriptwrapper, cli -proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool = +proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool {. raises: [].} = ## Returns whether to continue. result = true + {.cast(gcsafe).}: + # For certain commands hooks should not be evaluated. + if hookAction in noHookActions: + return - # For certain commands hooks should not be evaluated. - if hookAction in noHookActions: - return + var nimbleFile = "" + try: + nimbleFile = findNimbleFile(getCurrentDir(), true, options) - var nimbleFile = "" - try: - nimbleFile = findNimbleFile(getCurrentDir(), true, options) - except NimbleError: return true - # PackageInfos are cached so we can read them as many times as we want. - let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) - let actionName = - if hookAction == actionCustom: options.action.command - else: ($hookAction)[6 .. ^1] - let hookExists = - if before: actionName.normalize in pkgInfo.preHooks - else: actionName.normalize in pkgInfo.postHooks - if pkgInfo.isNimScript and hookExists: - let res = execHook(nimBin, nimbleFile, actionName, before, options) - if res.success: - result = res.retVal + # PackageInfos are cached so we can read them as many times as we want. + let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) + let actionName = + if hookAction == actionCustom: options.action.command + else: ($hookAction)[6 .. ^1] + let hookExists = + if before: actionName.normalize in pkgInfo.preHooks + else: actionName.normalize in pkgInfo.postHooks + if pkgInfo.isNimScript and hookExists: + let res = execHook(nimBin, nimbleFile, actionName, before, options) + if res.success: + result = res.retVal + except NimbleError: return true + except Exception: return false #TODO fix the propagation of Exception proc execCustom*(nimBin: string, nimbleFile: string, options: Options, execResult: var ExecutionResult[bool]): bool = diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 42fcc63d..97266338 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -79,6 +79,7 @@ type filePathPkgs*: seq[PackageInfo] #Packages loaded from file:// requires. Top level is always included. isFilePathDiscovering*: bool # Whether we are discovering file:// requires to fill up filePathPkgs. If true, it wont validate file:// requires. visitedHooks*: seq[VisitedHook] # Whether we are executing hooks. + jobs*: int # Number of build jobs to run in parallel. 0 means unlimited. ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade @@ -293,6 +294,8 @@ Nimble Options: --features Activate features. Only used when using the declarative parser. --ignoreSubmodules Ignore submodules when cloning a repository. --legacy Use the legacy code path (pre nimble 1.0.0) + --jobs Number of build jobs to run in parallel. 0 means unlimited. Default is 1. + For more information read the GitHub readme: https://github.com/nim-lang/nimble#readme """ @@ -784,6 +787,13 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = result.features = val.split(";").mapIt(it.strip) of "ignoresubmodules": result.ignoreSubmodules = true + of "jobs": + try: + result.jobs = parseInt(val) + except ValueError: + raise nimbleError(&"{val} is not a valid value") + if result.jobs < 0: + raise nimbleError("Number of jobs must be greater than or equal to 0") else: isGlobalFlag = false var wasFlagHandled = true @@ -918,7 +928,8 @@ proc initOptions*(): Options = useDeclarativeParser: false, legacy: false, #default to legacy code path for nimble < 1.0.0 satResult: SatResult(), - localDeps: true + localDeps: true, + jobs: 1 ) # Load visited hooks from environment variable to prevent recursive hook execution diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 7928eb0c..e47f947d 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -444,7 +444,7 @@ proc checkInstallDir(pkgInfo: PackageInfo, if thisDir == "nimcache": result = true proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, - action: proc (f: string)) = + action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each filename of the files that have a whitelisted ## file extension. for kind, path in walkDir(dir): @@ -454,7 +454,7 @@ proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, if path.splitFile.ext.substr(1) in pkgInfo.installExt: action(path) -proc iterFilesInDir(dir: string, action: proc (f: string)) = +proc iterFilesInDir(dir: string, action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each file in ``dir`` and any ## subdirectories that are in it. for kind, path in walkDir(dir): @@ -520,7 +520,7 @@ proc iterInstallFilesSimple*(realDir: string, pkgInfo: PackageInfo, action(file) proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo, - options: Options, action: proc (f: string)) = + options: Options, action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each file within the ``realDir`` that should be ## installed. # Get the package root directory for skipDirs comparison @@ -670,9 +670,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = - let srcTimestamp = getFileInfo(file).lastWriteTime - if binTimestamp < srcTimestamp: - rebuild = true + try: + let srcTimestamp = getFileInfo(file).lastWriteTime + if binTimestamp < srcTimestamp: + rebuild = true + except OSError: + discard ) return rebuild else: @@ -685,9 +688,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = - let srcTimestamp = getFileInfo(file).lastWriteTime - if binTimestamp < srcTimestamp: - rebuild = true + try: + let srcTimestamp = getFileInfo(file).lastWriteTime + if binTimestamp < srcTimestamp: + rebuild = true + except OSError: + discard ) return rebuild diff --git a/src/nimblepkg/packageinfotypes.nim b/src/nimblepkg/packageinfotypes.nim index a73bc6c2..aabcd925 100644 --- a/src/nimblepkg/packageinfotypes.nim +++ b/src/nimblepkg/packageinfotypes.nim @@ -169,9 +169,10 @@ proc appendGloballyActiveFeatures*(pkgName: string, features: seq[string]) = proc getGloballyActiveFeatures*(): seq[string] = #returns features.{pkgName}.{feature} - for pkgName, features in globallyActiveFeatures: - for feature in features: - result.add(&"features.{pkgName}.{feature}") + {.cast(gcsafe).}: + for pkgName, features in globallyActiveFeatures: + for feature in features: + result.add(&"features.{pkgName}.{feature}") proc initSATResult*(pass: SATPass): SATResult = SATResult(pkgsToInstall: @[], solvedPkgs: @[], output: "", pkgs: initHashSet[PackageInfo](), diff --git a/src/nimblepkg/vcstools.nim b/src/nimblepkg/vcstools.nim index e2b921ed..474f45bd 100644 --- a/src/nimblepkg/vcstools.nim +++ b/src/nimblepkg/vcstools.nim @@ -192,13 +192,13 @@ proc getVcsRevision*(dir: Path): Sha1Hash = ## - the external command fails. ## - the directory does not exist. ## - there is no vcsRevisions in the repository. + {.cast(gcsafe).}: + let vcsRevision = tryDoVcsCmd(dir, + gitCmd = "rev-parse HEAD", + hgCmd = "id -i --debug", + noVcsAction = $notSetSha1Hash) - let vcsRevision = tryDoVcsCmd(dir, - gitCmd = "rev-parse HEAD", - hgCmd = "id -i --debug", - noVcsAction = $notSetSha1Hash) - - return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) + return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) proc getVcsRevisions*(dir: Path): Sha1Hash = ## Returns current revision number if the directory `dir` is under version diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index 9034524c..31d8998e 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -16,10 +16,17 @@ import std/[sequtils, sets, options, os, strutils, tables, strformat, algorithm] import nimblesat, packageinfotypes, options, version, declarativeparser, packageinfo, common, nimenv, lockfile, cli, downloadnim, packageparser, tools, nimscriptexecutor, packagemetadatafile, displaymessages, packageinstaller, reversedeps, developfile, urls - +import chronos, chronos/asyncproc when defined(windows): import std/strscans + +type BuildTask* = object + bin*: string + cmd*: string + pkgName*: string + startBuild*: proc(): Future[CommandExResponse] {.closure.} + proc debugSATResult*(options: Options, calledFrom: string) = let satResult = options.satResult let color = "\e[32m" @@ -755,10 +762,11 @@ proc getPathsToBuildFor*(satResult: SATResult, pkgInfo: PackageInfo, recursive: result.incl(pkgInfo.expandPaths(nimBin, options)) proc getPathsAllPkgs*(options: Options): HashSet[string] = - let satResult = options.satResult - for pkg in satResult.pkgs: - for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): - result.incl(path) + {.cast(gcsafe).}: + let satResult = options.satResult + for pkg in satResult.pkgs: + for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): + result.incl(path) proc getNimBin(satResult: SATResult): string = #TODO change this so nim is passed as a parameter but we also need to change getPkgInfo so for the time being its also in options @@ -772,8 +780,8 @@ proc getNimBin(satResult: SATResult): string = raise newNimbleError[NimbleError]("No Nim found") proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], - args: seq[string], options: Options, nimBin: string) = - ## Builds a package as specified by ``pkgInfo``. + args: seq[string], options: Options, nimBin: string): Future[seq[BuildTask]] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = + ## Collects build tasks for all binaries in the package. Returns futures to be batched at a higher level. # Handle pre-`build` hook. let realDir = pkgInfo.getRealDir() @@ -787,9 +795,7 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], "Nothing to build. Did you specify a module to build using the" & " `bin` key in your .nimble file?") - var - binariesBuilt = 0 - args = args + var args = args args.add "-d:NimblePkgVersion=" & $pkgInfo.basicInfo.version for path in paths: args.add("--path:" & path.quoteShell) @@ -822,6 +828,9 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], options.getCompilationBinary(pkgInfo).get("") else: "" + # Collect all binary build tasks (will be batched at installPkgs level) + var buildTasks: seq[BuildTask] + for bin, src in pkgInfo.bin: # Check if this is the only binary that we want to build. if binToBuild.len != 0 and binToBuild != bin: @@ -831,43 +840,43 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], let outputDir = pkgInfo.getOutputDir("") if dirExists(outputDir): if fileExists(outputDir / bin): - if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - binariesBuilt.inc() - continue + {.cast(gcsafe).}: + if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + continue else: createDir(outputDir) # Check if we can copy an existing binary from source directory when --noRebuild is used - if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop, actionUpgrade, actionLock, actionAdd} and + if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop, actionUpgrade, actionLock, actionAdd} and options.action.noRebuild: # When installing from a local directory, check for binary in the original directory - let sourceBinary = + let sourceBinary = if options.startDir != pkgDir: options.startDir / bin else: pkgDir / bin - + if fileExists(sourceBinary): # Check if the source binary is up-to-date - if not pkgInfo.needsRebuild(sourceBinary, realDir, options): - let targetBinary = outputDir / bin - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - copyFile(sourceBinary, targetBinary) - when not defined(windows): - # Preserve executable permissions - setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) - binariesBuilt.inc() - continue + {.cast(gcsafe).}: + if not pkgInfo.needsRebuild(sourceBinary, realDir, options): + let targetBinary = outputDir / bin + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + copyFile(sourceBinary, targetBinary) + when not defined(windows): + # Preserve executable permissions + setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) + continue let outputOpt = "-o:" & pkgInfo.getOutputDir(bin).quoteShell display("Building", "$1/$2 using $3 backend" % [pkginfo.basicInfo.name, bin, pkgInfo.backend], priority = HighPriority) # For installed packages, we need to handle srcDir correctly - let input = + let input = if pkgInfo.isInstalled and not pkgInfo.isLink and pkgInfo.srcDir != "": # For installed packages with srcDir, the source file is in srcDir realDir / pkgInfo.srcDir / src.changeFileExt("nim") @@ -878,28 +887,25 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], let cmd = "$# $# --colors:$# --noNimblePath $# $# $#" % [ options.satResult.getNimBin().quoteShell, pkgInfo.backend, if options.noColor: "off" else: "on", join(args, " "), outputOpt, input.quoteShell] - try: - # echo "***Executing cmd: ", cmd - doCmd(cmd) - binariesBuilt.inc() - except CatchableError as error: - raise buildFailed( - &"Build failed for the package: {pkgInfo.basicInfo.name}", details = error) - if binariesBuilt == 0: - let binary = options.getCompilationBinary(pkgInfo).get("") - if binary != "": - raise nimbleError(binaryNotDefinedInPkgMsg(binary, pkgInfo.basicInfo.name)) + display("Executing", cmd, priority = MediumPriority) - raise nimbleError( - "No binaries built, did you specify a valid binary name?" - ) + # Create build task (don't start yet - will be started based on jobs limit) + var task: BuildTask + task.bin = bin + task.cmd = cmd + task.pkgName = pkgInfo.basicInfo.name + task.startBuild = proc(): Future[CommandExResponse] = execCommandEx(cmd) + buildTasks.add(task) - # Handle post-`build` hook. + # Handle post-`build` hook before returning tasks cd pkgDir: # Make sure `execHook` executes the correct .nimble file. discard execHook(nimBin, options, actionBuild, false) -proc createBinSymlink(pkgInfo: PackageInfo, options: Options) = + # Return build tasks to be batched at installPkgs level based on jobs setting + return buildTasks + +proc createBinSymlink*(pkgInfo: PackageInfo, options: Options) = var binariesInstalled: HashSet[string] let binDir = options.getBinDir() let pkgDestDir = pkgInfo.getPkgDest(options) @@ -960,9 +966,12 @@ proc solutionToFullInfo*(satResult: SATResult, options: var Options) {.instrumen proc isRoot(pkgInfo: PackageInfo, satResult: SATResult): bool = pkgInfo.basicInfo.name == satResult.rootPackage.basicInfo.name and pkgInfo.basicInfo.version == satResult.rootPackage.basicInfo.version -proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options) {.instrument.} = +proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options): Future[seq[BuildTask]] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = + ## Collects build tasks for the package. Symlinks are created after builds complete. # let paths = getPathsToBuildFor(options.satResult, pkgToBuild, recursive = true, options) - let paths = getPathsAllPkgs(options) + let paths = try: getPathsAllPkgs(options) + except Exception: + initHashSet[string]() # echo "Paths ", paths # echo "Requires ", pkgToBuild.requires # echo "Package ", pkgToBuild.basicInfo.name @@ -975,12 +984,12 @@ proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, o var pkgToBuild = pkgToBuild if isRootInRootDir: pkgToBuild.isInstalled = false - buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) - # For globally installed packages, always create symlinks - # Only skip symlinks if we're building the root package in its own directory - let shouldCreateSymlinks = not isRootInRootDir or options.action.typ == actionInstall - if shouldCreateSymlinks: - createBinSymlink(pkgToBuild, options) + + # Collect build tasks, don't wait for them yet + let buildTasks = await buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) + + # Note: createBinSymlink will be called after all builds complete in installPkgs + return buildTasks proc getVersionRangeFoPkgToInstall(satResult: SATResult, name: string, ver: Version): VersionRange = if satResult.rootPackage.basicInfo.name == name and satResult.rootPackage.basicInfo.version == ver: @@ -1125,12 +1134,16 @@ proc installPkgs*(satResult: var SATResult, options: var Options) {.instrument.} satResult.installedPkgs = installedPkgs.toSeq() for pkgInfo in satResult.installedPkgs: - # Run before-install hook now that package before the build step but after the package is copied over to the + # Run before-install hook now that package before the build step but after the package is copied over to the #install dir. let hookDir = pkgInfo.myPath.splitFile.dir if dirExists(hookDir): executeHook(nimBin, hookDir, options, actionInstall, before = true) + # Collect ALL build tasks from all packages (flattened list) + var allBuildTasks: seq[BuildTask] + var pkgsWithBuilds: seq[PackageInfo] # Track which packages have builds for symlink creation + for pkgToBuild in pkgsToBuild: if pkgToBuild.bin.len == 0: if options.action.typ == actionBuild: @@ -1142,12 +1155,79 @@ proc installPkgs*(satResult: var SATResult, options: var Options) {.instrument.} # echo "Building package: ", pkgToBuild.basicInfo.name, " at ", pkgToBuild.myPath, " binaries: ", pkgToBuild.bin let isRoot = pkgToBuild.isRoot(options.satResult) and isInRootDir if isRoot and options.action.typ in rootBuildActions: - buildPkg(nimBin, pkgToBuild, isRoot, options) + let buildTasks = waitFor buildPkg(nimBin, pkgToBuild, isRoot, options) + allBuildTasks.add(buildTasks) satResult.buildPkgs.add(pkgToBuild) + pkgsWithBuilds.add(pkgToBuild) elif not isRoot: #Build non root package for all actions that requires the package as a dependency - buildPkg(nimBin, pkgToBuild, isRoot, options) + let buildTasks = waitFor buildPkg(nimBin, pkgToBuild, isRoot, options) + allBuildTasks.add(buildTasks) satResult.buildPkgs.add(pkgToBuild) + pkgsWithBuilds.add(pkgToBuild) + + # Now batch all build tasks according to jobs limit + if allBuildTasks.len > 0: + # echo "Building ", allBuildTasks.len, " binaries across ", pkgsWithBuilds.len, " packages" + # Start builds and collect futures + var buildFutures: seq[tuple[task: BuildTask, future: Future[CommandExResponse]]] + + measureTime "Packages built in ", false: + if options.jobs <= 0: + # jobs=0 means unlimited parallelism - start all at once + for task in allBuildTasks: + let future = task.startBuild() + buildFutures.add((task, future)) + + let allFutures = buildFutures.mapIt(it.future) + waitFor allFutures(allFutures) + else: + # Process in batches of size options.jobs - start only what we can run + # echo "Using job limit: ", options.jobs + for i in countup(0, allBuildTasks.len - 1, options.jobs): + let batchEnd = min(i + options.jobs, allBuildTasks.len) + + # Start this batch of builds + var batchFutures: seq[Future[CommandExResponse]] + for j in i ..< batchEnd: + let future = allBuildTasks[j].startBuild() + buildFutures.add((allBuildTasks[j], future)) + batchFutures.add(future) + + # Wait for this batch to complete before starting next batch + waitFor allFutures(batchFutures) + + # Process results and create symlinks + for item in buildFutures: + let task = item.task + let future = item.future + try: + let response = future.read() + + # Display output + if response.stdOutput.len > 0: + display("Nim Output", response.stdOutput, priority = HighPriority) + if response.stdError.len > 0: + display("Nim Stderr", response.stdError, priority = HighPriority) + + # Check exit code + if response.status != QuitSuccess: + raise nimbleError( + "Execution failed with exit code $1\nCommand: $2" % + [$response.status, task.cmd]) + except CatchableError as error: + raise buildFailed( + &"Build failed for package: {task.pkgName}, binary: {task.bin}", details = error) + + # Create symlinks after all builds complete + for pkgToBuild in pkgsWithBuilds: + let isRoot = pkgToBuild.isRoot(options.satResult) and isInRootDir + let shouldCreateSymlinks = not isRoot or options.action.typ == actionInstall + if shouldCreateSymlinks: + try: + createBinSymlink(pkgToBuild, options) + except Exception: + display("Error creating bin symlink", "Error creating bin symlink: " & getCurrentExceptionMsg(), Error, HighPriority) for pkg in satResult.installedPkgs.mitems: satResult.pkgs.incl pkg diff --git a/vendor/results b/vendor/results new file mode 160000 index 00000000..df8113dd --- /dev/null +++ b/vendor/results @@ -0,0 +1 @@ +Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27 diff --git a/vendor/stew b/vendor/stew new file mode 160000 index 00000000..b6616873 --- /dev/null +++ b/vendor/stew @@ -0,0 +1 @@ +Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257