fix: allow node_modules removal in build phase

buildkit/build_llb/build_graph.go

@@ -1,3 +1,5 @@
+// Converts the internal build plan graph to a BuildKit LLB
+
 package build_llb
 
 import (
@@ -96,7 +98,7 @@ func NewBuildGraph(plan *plan.BuildPlan, localState *llb.State, cacheStore *Buil
 	return g, nil
 }
 
-// GenerateLLB generates the LLB state for the build graph
+// generate the LLB state for the build graph
 func (g *BuildGraph) GenerateLLB() (*BuildGraphOutput, error) {
 	// Get processing order using topological sort
 	order, err := g.graph.ComputeProcessingOrder()
@@ -180,7 +182,7 @@ func (g *BuildGraph) processNode(node *StepNode) error {
 	return nil
 }
 
-// convertNodeToLLB converts a step node to an LLB state
+// converts a step node to an LLB state
 func (g *BuildGraph) convertNodeToLLB(node *StepNode) (*llb.State, error) {
 	state, err := g.getNodeStartingState(node)
 	if err != nil {
@@ -391,7 +393,7 @@ func (g *BuildGraph) getSecretInvalidationMountOptions(node *StepNode, secretOpt
 	return opts
 }
 
-// getCacheMountOptions returns the llb.RunOption slice for the given cache keys
+// returns the llb.RunOption slice for the given cache keys
 func (g *BuildGraph) getCacheMountOptions(cacheKeys []string) ([]llb.RunOption, error) {
 	var opts []llb.RunOption
 

cli/common.go

@@ -88,6 +88,7 @@ func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App,
 	return buildResult, app, env, nil
 }
 
+// add $schema link to resulting map JSON for improved IDE experience when manually editing
 func addSchemaToPlanMap(p *plan.BuildPlan) (map[string]any, error) {
 	if p == nil {
 		return map[string]any{"$schema": config.SchemaUrl}, nil

core/__snapshots__/TestGenerateBuildPlanForExamples_node-npm-install-in-build_1.snap.json

@@ -0,0 +1,155 @@
+{
+ "caches": {
+  "node-modules": {
+   "directory": "/app/node_modules/.cache",
+   "type": "shared"
+  },
+  "npm-install": {
+   "directory": "/root/.npm",
+   "type": "shared"
+  }
+ },
+ "deploy": {
+  "base": {
+   "image": "ghcr.io/railwayapp/railpack-runtime:latest"
+  },
+  "inputs": [
+   {
+    "include": [
+     "/mise/shims",
+     "/mise/installs",
+     "/usr/local/bin/mise",
+     "/etc/mise/config.toml",
+     "/root/.local/state/mise"
+    ],
+    "step": "packages:mise"
+   },
+   {
+    "include": [
+     "/app/node_modules"
+    ],
+    "step": "build"
+   },
+   {
+    "exclude": [
+     "node_modules",
+     ".yarn"
+    ],
+    "include": [
+     "/root/.cache",
+     "."
+    ],
+    "step": "build"
+   },
+   {
+    "include": [
+     "."
+    ],
+    "step": "build"
+   }
+  ],
+  "startCommand": "npm run start",
+  "variables": {
+   "CI": "true",
+   "NODE_ENV": "production",
+   "NPM_CONFIG_FUND": "false",
+   "NPM_CONFIG_PRODUCTION": "false",
+   "NPM_CONFIG_UPDATE_NOTIFIER": "false"
+  }
+ },
+ "steps": [
+  {
+   "assets": {
+    "mise.toml": "[mise.toml]"
+   },
+   "commands": [
+    {
+     "path": "/mise/shims"
+    },
+    {
+     "customName": "create mise config",
+     "name": "mise.toml",
+     "path": "/etc/mise/config.toml"
+    },
+    {
+     "cmd": "sh -c 'mise trust -a \u0026\u0026 mise install'",
+     "customName": "install mise packages: node"
+    }
+   ],
+   "inputs": [
+    {
+     "image": "ghcr.io/railwayapp/railpack-builder:latest"
+    }
+   ],
+   "name": "packages:mise",
+   "variables": {
+    "MISE_CACHE_DIR": "/mise/cache",
+    "MISE_CONFIG_DIR": "/mise",
+    "MISE_DATA_DIR": "/mise",
+    "MISE_INSTALLS_DIR": "/mise/installs",
+    "MISE_NODE_VERIFY": "false",
+    "MISE_SHIMS_DIR": "/mise/shims"
+   }
+  },
+  {
+   "caches": [
+    "npm-install"
+   ],
+   "commands": [
+    {
+     "path": "/app/node_modules/.bin"
+    },
+    {
+     "cmd": "mkdir -p /app/node_modules/.cache"
+    },
+    {
+     "dest": "package.json",
+     "src": "package.json"
+    },
+    {
+     "dest": "package-lock.json",
+     "src": "package-lock.json"
+    },
+    {
+     "cmd": "npm ci"
+    }
+   ],
+   "inputs": [
+    {
+     "step": "packages:mise"
+    }
+   ],
+   "name": "install",
+   "variables": {
+    "CI": "true",
+    "NODE_ENV": "production",
+    "NPM_CONFIG_FUND": "false",
+    "NPM_CONFIG_PRODUCTION": "false",
+    "NPM_CONFIG_UPDATE_NOTIFIER": "false"
+   }
+  },
+  {
+   "commands": [
+    {
+     "cmd": "sh -c 'npm ci'",
+     "customName": "npm ci"
+    }
+   ],
+   "inputs": [
+    {
+     "step": "install"
+    },
+    {
+     "include": [
+      "."
+     ],
+     "local": true
+    }
+   ],
+   "name": "build",
+   "secrets": [
+    "*"
+   ]
+  }
+ ]
+}

core/cleanse.go

@@ -0,0 +1,77 @@
+package core
+
+import (
+	"regexp"
+
+	"github.com/railwayapp/railpack/core/logger"
+	"github.com/railwayapp/railpack/core/plan"
+	"github.com/railwayapp/railpack/core/providers/node"
+)
+
+// Regexes for matching commands that intentionally remove node_modules or perform
+// clean installs (which implicitly delete the directory) so we can avoid mounting
+// the node_modules cache in those steps.
+var (
+	// Matches "npm ci" with flexible whitespace, using word boundaries
+	npmCiCommandRegex = regexp.MustCompile(`(?i)\bnpm\s+ci\b`)
+
+	// Matches common delete commands targeting node_modules
+	removeNodeModulesRegex = regexp.MustCompile(`(?i)\b(?:rm\s+-r[f]?|rmdir|rimraf)\s+(?:\S*\/)?node_modules\b`)
+)
+
+// willRemoveNodeModules determines if any command in the provided slice removes
+// the node_modules directory either directly (rm/rimraf) or indirectly (npm ci).
+// this is brittle & imperfect: https://github.com/railwayapp/railpack/pull/259
+func willRemoveNodeModules(commands []plan.Command) bool {
+	for _, cmd := range commands {
+		if execCmd, ok := cmd.(plan.ExecCommand); ok {
+			if npmCiCommandRegex.MatchString(execCmd.Cmd) || removeNodeModulesRegex.MatchString(execCmd.Cmd) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// cleansePlanStructure applies mutations to the build plan structure after it
+// is generated but before validation / serialization. Today this focuses on
+// detaching the node_modules cache from steps that explicitly remove
+// node_modules so the global cache isn't invalidated unintentionally.
+func cleansePlanStructure(buildPlan *plan.BuildPlan, logger *logger.Logger) {
+	// let's get the cache key name that has a Directory of NODE_MODULES_CACHE
+	var nodeModulesCacheKey string
+	for cacheName, cacheDef := range buildPlan.Caches {
+		if cacheDef.Directory == node.NODE_MODULES_CACHE {
+			nodeModulesCacheKey = cacheName
+			break
+		}
+	}
+
+	if nodeModulesCacheKey == "" {
+		// no node_modules cache defined, nothing to do
+		return
+	}
+
+	// Only detach the node modules cache from steps that remove node_modules themselves.
+	// Keep the global cache definition so earlier steps (like install) can still mount it.
+	for i, step := range buildPlan.Steps {
+		if step.Name == "install" || !willRemoveNodeModules(step.Commands) {
+			continue
+		}
+
+		before := len(step.Caches)
+		if before == 0 {
+			continue
+		}
+
+		// It's important that we do not result in an array with a zeroed string, which is why we are using this ugly loop
+		var newCaches []string
+		for _, name := range step.Caches {
+			if name != "" && name != nodeModulesCacheKey {
+				newCaches = append(newCaches, name)
+			}
+		}
+
+		buildPlan.Steps[i].Caches = newCaches
+	}
+}

core/cleanse_test.go

@@ -0,0 +1,61 @@
+package core
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/railwayapp/railpack/core/logger"
+	"github.com/railwayapp/railpack/core/plan"
+	"github.com/railwayapp/railpack/core/providers/node"
+)
+
+func newTestLogger() *logger.Logger { return logger.NewLogger() }
+
+// helper to create a basic build plan with a node_modules cache (when withCache true)
+func buildPlan(withCache bool) *plan.BuildPlan {
+	p := plan.NewBuildPlan()
+	if withCache {
+		p.Caches["node_modules"] = &plan.Cache{Directory: node.NODE_MODULES_CACHE, Type: plan.CacheTypeShared}
+	}
+	return p
+}
+
+func TestCleanse_CachePresent_StepDoesNotRemoveNodeModules(t *testing.T) {
+	p := buildPlan(true)
+	step := plan.Step{Name: "build", Caches: []string{"node_modules"}}
+	step.Commands = []plan.Command{plan.NewExecShellCommand("echo 'nothing to see'")}
+	p.Steps = append(p.Steps, step)
+
+	cleansePlanStructure(p, newTestLogger())
+
+	// should remain mounted
+	if !reflect.DeepEqual(p.Steps[0].Caches, []string{"node_modules"}) {
+		t.Fatalf("expected cache to remain since step doesn't remove node_modules, got %#v", p.Steps[0].Caches)
+	}
+}
+
+func TestCleanse_CachePresent_StepRemovesNodeModules(t *testing.T) {
+	p := buildPlan(true)
+	step := plan.Step{Name: "build", Caches: []string{"node_modules"}}
+	step.Commands = []plan.Command{plan.NewExecShellCommand("rm -rf node_modules && echo done")}
+	p.Steps = append(p.Steps, step)
+
+	cleansePlanStructure(p, newTestLogger())
+
+	if len(p.Steps[0].Caches) != 0 { // should be removed (allow nil or empty)
+		t.Fatalf("expected cache to be removed (nil or empty), got %#v", p.Steps[0].Caches)
+	}
+}
+
+func TestCleanse_InstallStepAlwaysKeepsCache(t *testing.T) {
+	p := buildPlan(true)
+	install := plan.Step{Name: "install", Caches: []string{"node_modules"}}
+	install.Commands = []plan.Command{plan.NewExecShellCommand("npm ci")}
+	p.Steps = append(p.Steps, install)
+
+	cleansePlanStructure(p, newTestLogger())
+
+	if !reflect.DeepEqual(p.Steps[0].Caches, []string{"node_modules"}) { // should remain even though npm ci matches removal heuristic
+		t.Fatalf("expected install step cache to remain, got %#v", p.Steps[0].Caches)
+	}
+}

core/core.go

@@ -109,12 +109,16 @@ func GenerateBuildPlan(app *app.App, env *app.Environment, options *GenerateBuil
 		return &BuildResult{Success: false, Logs: logger.Logs}
 	}
 
+	// before `Generate()` any commands provided by railpack.json are *not* merged into the provider-generated
+	// buildPlan. This means providers can't view any of the custom structure provided by the user via a railpack.json
 	buildPlan, resolvedPackages, err := ctx.Generate()
 	if err != nil {
 		logger.LogError("%s", err.Error())
 		return &BuildResult{Success: false, Logs: logger.Logs}
 	}
 
+	cleansePlanStructure(buildPlan, logger)
+
 	if !ValidatePlan(buildPlan, app, logger, &ValidatePlanOptions{
 		ErrorMissingStartCommand: options.ErrorMissingStartCommand,
 		ProviderToUse:            providerToUse,
@@ -135,6 +139,8 @@ func GenerateBuildPlan(app *app.App, env *app.Environment, options *GenerateBuil
 	return buildResult
 }
 
+// cleansing logic moved to cleanse.go
+
 // GetConfig merges the options, environment, and file config into a single config
 func GetConfig(app *app.App, env *app.Environment, options *GenerateBuildPlanOptions, logger *logger.Logger) (*c.Config, error) {
 	optionsConfig := GenerateConfigFromOptions(options)
@@ -151,7 +157,6 @@ func GetConfig(app *app.App, env *app.Environment, options *GenerateBuildPlanOpt
 	return mergedConfig, nil
 }
 
-// GenerateConfigFromFile generates a config from the config file
 func GenerateConfigFromFile(app *app.App, env *app.Environment, options *GenerateBuildPlanOptions, logger *logger.Logger) (*c.Config, error) {
 	config := c.EmptyConfig()
 
@@ -233,7 +238,7 @@ func GenerateConfigFromEnvironment(env *app.Environment) *c.Config {
 	return config
 }
 
-// GenerateConfigFromOptions generates a config from the CLI options
+// generates a config from the CLI options
 func GenerateConfigFromOptions(options *GenerateBuildPlanOptions) *c.Config {
 	config := c.EmptyConfig()
 

core/plan/cache.go

@@ -6,11 +6,8 @@ const (
 )
 
 type Cache struct {
-	// The directory to cache
 	Directory string `json:"directory,omitempty" jsonschema:"description=The directory to cache"`
-
-	// The type of cache (either "shared" or "locked")
-	Type string `json:"type,omitempty" jsonschema:"enum=shared,enum=locked,default=shared,description=The type of cache (either 'shared' or 'locked')"`
+	Type      string `json:"type,omitempty" jsonschema:"enum=shared,enum=locked,default=shared,description=The type of cache (either 'shared' or 'locked')"`
 }
 
 func NewCache(directory string) *Cache {