Optimize code generation with parallel file writing

tl;dr: make codegen much faster through parallel writing of files

We have a _very large_ API design, with over 100 services, which
generates almost 3k files with `goa generate`. As our API has grown,
running `goa generate` has got really slow.

This commit adds timing data to the `--debug` output.

Then, the lowest-hanging-fruit optimisation has been applied: writing
all the output files in parallel.

---

As a recap, the generation process has several stages:

1. **Compile temporary binary**: Goa creates a temporary Go program that
   imports the design package (which triggers package initialization and
   DSL execution via blank import)
2. **Execute binary**: The binary runs through multiple phases:
   - Package initialization (runs DSL definitions)
   - `eval.RunDSL()` - processes the DSL in 4 phases (execute, prepare,
     validate, finalize)
   - `generator.Generate()` - produces the actual Go files

Measuring codegen for the incident-io codebase on 3,036 generated files:

**Total time: 61 seconds**

Breakdown:
- build.Import: 117ms
- NewGenerator (packages.Load): 52ms
- Write (generate main.go): 14ms
- Compile (go get + go build): 3.6s
  - packages.Load: 47ms
  - go get: 514ms
  - go build: 3.0s
- **Run (execute binary): 52.2s** ⚠️ 85% of total time
  - Check eval.Context.Errors: <1ms
  - eval.RunDSL(): 105ms
  - **generator.Generate(): 51.6s** ⚠️
    - Stage 1 (Compute design roots): <1ms
    - Stage 2 (Compute gen package): 33ms
    - Stage 3 (Retrieve generators): <1ms
    - Stage 4 (Pre-generation plugins): <1ms
    - **Stage 5 (Generate files): 26.2s** (3 generators, sequential)
      - Generator 0: 7.2s → 1,438 files
      - Generator 1: 18.8s → 1,594 files
      - Generator 2: 0.2s → 4 files
    - Stage 6 (Post-generation plugins): <1ms
    - **Stage 7 (Write files): 32.1s** ⚠️ Biggest bottleneck (52% of generation)
    - Stage 8 (Compute filenames): 2ms

This commit tries optimising the biggest stage of this process and
changes file rendering from sequential loop to parallel worker pool:

```go
// Before: Sequential (32.1s for 3,036 files)
for _, f := range genfiles {
    filename, err := f.Render(dir)
    // ...
}

// After: Parallel with runtime.NumCPU() workers
numWorkers := runtime.NumCPU()
// Worker pool processes files concurrently
```

Which changed execution time from **61 seconds total** to **34 seconds
total**, for an overall speedup of 1.8x.

While here, I also tried parallelising Stage 5 (generator functions) but
hit infinite recursion in `AsObject()` when handling circular type
references concurrently. This is where I'd go for the next biggest
speed-up.

Author: lawrencejones

cmd/goa/gen.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"goa.design/goa/v3/codegen"
 	"golang.org/x/tools/go/packages"
@@ -44,7 +45,7 @@ type Generator struct {
 }
 
 // NewGenerator creates a Generator.
-func NewGenerator(cmd, path, output string) *Generator {
+func NewGenerator(cmd, path, output string, debug bool) *Generator {
 	bin := "goa"
 	if runtime.GOOS == "windows" {
 		bin += ".exe"
@@ -55,7 +56,11 @@ func NewGenerator(cmd, path, output string) *Generator {
 	{
 		version = 2
 		matched := false
+		startPkgLoad := time.Now()
 		pkgs, _ := packages.Load(&packages.Config{Mode: packages.NeedFiles | packages.NeedModule}, path)
+		if debug {
+			fmt.Fprintf(os.Stderr, "[TIMING]   packages.Load (design files) took %v\n", time.Since(startPkgLoad))
+		}
 		fset := token.NewFileSet()
 		p := regexp.MustCompile(`goa.design/goa/v(\d+)/dsl`)
 		for _, pkg := range pkgs {
@@ -132,6 +137,7 @@ func (g *Generator) Write(_ bool) error {
 			codegen.SimpleImport("sort"),
 			codegen.SimpleImport("strconv"),
 			codegen.SimpleImport("strings"),
+			codegen.SimpleImport("time"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "codegen"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "codegen/generator"),
 			codegen.SimpleImport("goa.design/goa/" + ver + "eval"),
@@ -154,23 +160,36 @@ func (g *Generator) Write(_ bool) error {
 }
 
 // Compile compiles the generator.
-func (g *Generator) Compile() error {
+func (g *Generator) Compile(debug bool) error {
 	// We first need to go get the generated package to make sure that all
 	// dependencies are added to go.sum prior to compiling.
+	startLoad := time.Now()
 	pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName}, fmt.Sprintf(".%c%s", filepath.Separator, g.tmpDir))
 	if err != nil {
 		return err
 	}
 	if len(pkgs) != 1 {
 		return fmt.Errorf("expected to find one package in %s", g.tmpDir)
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   packages.Load (temp dir) took %v\n", time.Since(startLoad))
+	}
+
 	if !g.hasVendorDirectory {
+		startGet := time.Now()
 		if err := g.runGoCmd("get", pkgs[0].PkgPath); err != nil {
 			return err
 		}
+		if debug {
+			fmt.Fprintf(os.Stderr, "[TIMING]   go get took %v\n", time.Since(startGet))
+		}
 	}
 
+	startBuild := time.Now()
 	err = g.runGoCmd("build", "-o", g.bin)
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   go build took %v\n", time.Since(startBuild))
+	}
 
 	// If we're in vendor context we check the error string to see if it's an issue of unsatisfied dependencies
 	if err != nil && g.hasVendorDirectory {
@@ -183,7 +202,7 @@ func (g *Generator) Compile() error {
 }
 
 // Run runs the compiled binary and return the output lines.
-func (g *Generator) Run() ([]string, error) {
+func (g *Generator) Run(debug bool) ([]string, error) {
 	var cmdl string
 	{
 		args := make([]string, len(os.Args)-1)
@@ -210,7 +229,7 @@ func (g *Generator) Run() ([]string, error) {
 		cmdl = fmt.Sprintf("$ %s%s", rawcmd, cmdl)
 	}
 
-	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl}
+	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl, "--debug=" + strconv.FormatBool(debug)}
 	cmd := exec.Command(filepath.Join(g.tmpDir, g.bin), args...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -291,6 +310,7 @@ const mainT = `func main() {
 		out     = flag.String("output", "", "")
 		version = flag.String("version", "", "")
 		cmdl    = flag.String("cmd", "", "")
+		debug   = flag.Bool("debug", false, "")
 		ver int
 	)
 	{
@@ -311,15 +331,31 @@ const mainT = `func main() {
 		ver = v
 	}
 
+	startBinary := time.Now()
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Starting generated binary execution\n")
+	}
+
 	if ver > goa.Major {
 		fail("cannot run goa %s on design using goa v%s\n", goa.Version(), *version)
 	}
+
+	startCheckErrors := time.Now()
 	if err := eval.Context.Errors; err != nil {
 		fail(err.Error())
 	}
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Check eval.Context.Errors took %v\n", time.Since(startCheckErrors))
+	}
+
+	startRunDSL := time.Now()
 	if err := eval.RunDSL(); err != nil {
 		fail(err.Error())
 	}
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] eval.RunDSL() took %v\n", time.Since(startRunDSL))
+	}
+
 {{- range .CleanupDirs }}
 	if err := os.RemoveAll({{ printf "%q" . }}); err != nil {
 		fail(err.Error())
@@ -328,11 +364,16 @@ const mainT = `func main() {
 {{- if gt .DesignVersion 2 }}
 	codegen.DesignVersion = ver
 {{- end }}
-	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }})
+
+	startGenerate := time.Now()
+	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }}, *debug)
 	if err != nil {
 		fail(err.Error())
 	}
-
+	if *debug {
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] generator.Generate() took %v\n", time.Since(startGenerate))
+		fmt.Fprintf(os.Stderr, "[TIMING]   [binary] Total binary execution took %v\n", time.Since(startBinary))
+	}
 	fmt.Println(strings.Join(outputs, "\n"))
 }
 

cmd/goa/main.go

@@ -5,6 +5,7 @@ import (
 	"go/build"
 	"os"
 	"strings"
+	"time"
 
 	"flag"
 
@@ -77,29 +78,55 @@ var (
 
 func generate(cmd, path, output string, debug bool) error {
 	var (
-		files []string
-		err   error
-		tmp   *Generator
+		files                                                            []string
+		err                                                              error
+		tmp                                                              *Generator
+		startTotal, startImport, startNewGen, startWrite, startCompile, startRun time.Time
 	)
 
+	startTotal = time.Now()
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Starting goa generation\n")
+	}
+
+	startImport = time.Now()
 	if _, err = build.Import(path, ".", 0); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] build.Import took %v\n", time.Since(startImport))
+	}
 
-	tmp = NewGenerator(cmd, path, output)
+	startNewGen = time.Now()
+	tmp = NewGenerator(cmd, path, output, debug)
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] NewGenerator took %v\n", time.Since(startNewGen))
+	}
 
+	startWrite = time.Now()
 	if err = tmp.Write(debug); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Write (generate main.go) took %v\n", time.Since(startWrite))
+	}
 
-	if err = tmp.Compile(); err != nil {
+	startCompile = time.Now()
+	if err = tmp.Compile(debug); err != nil {
 		goto fail
 	}
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Compile (go get + go build) took %v\n", time.Since(startCompile))
+	}
 
-	if files, err = tmp.Run(); err != nil {
+	startRun = time.Now()
+	if files, err = tmp.Run(debug); err != nil {
 		goto fail
 	}
-
+	if debug {
+		fmt.Fprintf(os.Stderr, "[TIMING] Run (execute binary) took %v\n", time.Since(startRun))
+		fmt.Fprintf(os.Stderr, "[TIMING] Total generation time: %v\n", time.Since(startTotal))
+	}
 	fmt.Println(strings.Join(files, "\n"))
 	if !debug {
 		tmp.Remove()