From 396a04630d9f0fc2209285adc60145fda28f0196 Mon Sep 17 00:00:00 2001 From: Denis Isaev Date: Sun, 31 Mar 2019 22:09:22 +0300 Subject: [PATCH] dev: improve memory tracking --- .gitignore | 1 + README.md | 16 ++++++++++++ README.tmpl.md | 16 ++++++++++++ pkg/commands/root.go | 39 +++++++++++++++++++++++++++++- pkg/commands/run.go | 13 +++++++++- pkg/result/processors/skip_dirs.go | 10 ++++---- 6 files changed, 88 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e2ace38b..bb381fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /*.txt +/*.pdf /*.pprof /dist/ /.idea/ diff --git a/README.md b/README.md index 0cd75632..4d677a8f 100644 --- a/README.md +++ b/README.md @@ -341,6 +341,22 @@ On average golangci-lint consumes 26% less memory. Golangci-lint directly calls linters (no forking) and reuses 80% of work by parsing program only once. Read [this section](#internals) for details. +### Memory Usage of Golangci-lint + +A trade-off between memory usage and execution time can be controlled by [`GOCC`](https://golang.org/pkg/runtime/#hdr-Environment_Variables) environment variable. +Less `GOGC` values trigger garbage collection more frequently and golangci-lint consumes less memory and more CPU. Below is the trade-off table for running on this repo: + +|`GOGC`|Peak Memory, GB|Executon Time, s| +|------|---------------|----------------| +|`5` |1.1 |60 | +|`10` |1.1 |34 | +|`20` |1.3 |25 | +|`30` |1.6 |20.2 | +|`50` |2.0 |17.1 | +|`80` |2.2 |14.1 | +|`100` (default)|2.2 |13.8 | +|`off` |3.2 |9.3 | + ## Internals 1. Work sharing diff --git a/README.tmpl.md b/README.tmpl.md index f57d8a45..9fa3a8f0 100644 --- a/README.tmpl.md +++ b/README.tmpl.md @@ -310,6 +310,22 @@ On average golangci-lint consumes 26% less memory. Golangci-lint directly calls linters (no forking) and reuses 80% of work by parsing program only once. Read [this section](#internals) for details. +### Memory Usage of Golangci-lint + +A trade-off between memory usage and execution time can be controlled by [`GOCC`](https://golang.org/pkg/runtime/#hdr-Environment_Variables) environment variable. +Less `GOGC` values trigger garbage collection more frequently and golangci-lint consumes less memory and more CPU. Below is the trade-off table for running on this repo: + +|`GOGC`|Peak Memory, GB|Executon Time, s| +|------|---------------|----------------| +|`5` |1.1 |60 | +|`10` |1.1 |34 | +|`20` |1.3 |25 | +|`30` |1.6 |20.2 | +|`50` |2.0 |17.1 | +|`80` |2.2 |14.1 | +|`100` (default)|2.2 |13.8 | +|`off` |3.2 |9.3 | + ## Internals 1. Work sharing diff --git a/pkg/commands/root.go b/pkg/commands/root.go index 539b8dff..48a53328 100644 --- a/pkg/commands/root.go +++ b/pkg/commands/root.go @@ -5,6 +5,7 @@ import ( "os" "runtime" "runtime/pprof" + "strconv" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -30,6 +31,12 @@ func (e *Executor) persistentPreRun(_ *cobra.Command, _ []string) { e.log.Fatalf("Can't start CPU profiling: %s", err) } } + + if e.cfg.Run.MemProfilePath != "" { + if rate := os.Getenv("GL_MEMPROFILE_RATE"); rate != "" { + runtime.MemProfileRate, _ = strconv.Atoi(rate) + } + } } func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) { @@ -41,15 +48,45 @@ func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) { if err != nil { e.log.Fatalf("Can't create file %s: %s", e.cfg.Run.MemProfilePath, err) } - runtime.GC() // get up-to-date statistics + + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + printMemStats(&ms, e.log) + if err := pprof.WriteHeapProfile(f); err != nil { e.log.Fatalf("Can't write heap profile: %s", err) } + f.Close() } os.Exit(e.exitCode) } +func printMemStats(ms *runtime.MemStats, logger logutils.Log) { + logger.Infof("Mem stats: alloc=%s total_alloc=%s sys=%s "+ + "heap_alloc=%s heap_sys=%s heap_idle=%s heap_released=%s heap_in_use=%s "+ + "stack_in_use=%s stack_sys=%s "+ + "mspan_sys=%s mcache_sys=%s buck_hash_sys=%s gc_sys=%s other_sys=%s "+ + "mallocs_n=%d frees_n=%d heap_objects_n=%d gc_cpu_fraction=%.2f", + formatMemory(ms.Alloc), formatMemory(ms.TotalAlloc), formatMemory(ms.Sys), + formatMemory(ms.HeapAlloc), formatMemory(ms.HeapSys), + formatMemory(ms.HeapIdle), formatMemory(ms.HeapReleased), formatMemory(ms.HeapInuse), + formatMemory(ms.StackInuse), formatMemory(ms.StackSys), + formatMemory(ms.MSpanSys), formatMemory(ms.MCacheSys), formatMemory(ms.BuckHashSys), + formatMemory(ms.GCSys), formatMemory(ms.OtherSys), + ms.Mallocs, ms.Frees, ms.HeapObjects, ms.GCCPUFraction) +} + +func formatMemory(memBytes uint64) string { + if memBytes < 1024 { + return fmt.Sprintf("%db", memBytes) + } + if memBytes < 1024*1024 { + return fmt.Sprintf("%dkb", memBytes/1024) + } + return fmt.Sprintf("%dmb", memBytes/1024/1024) +} + func getDefaultConcurrency() int { if os.Getenv("HELP_RUN") == "1" { return 8 // to make stable concurrency for README help generating builds diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 085ec9ca..42faf7e2 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -430,11 +430,21 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() - for { + logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1" + + track := func() { var m runtime.MemStats runtime.ReadMemStats(&m) + if logEveryRecord { + printMemStats(&m, logger) + } + rssValues = append(rssValues, m.Sys) + } + + for { + track() stop := false select { @@ -447,6 +457,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log break } } + track() var avg, max uint64 for _, v := range rssValues { diff --git a/pkg/result/processors/skip_dirs.go b/pkg/result/processors/skip_dirs.go index 3dc43bf4..e7cd2c80 100644 --- a/pkg/result/processors/skip_dirs.go +++ b/pkg/result/processors/skip_dirs.go @@ -14,7 +14,7 @@ import ( type SkipDirs struct { patterns []*regexp.Regexp log logutils.Log - skippedDirs map[string][]string // regexp to dir mapping + skippedDirs map[string]string // dir to the last regexp mapping absArgsDirs []string } @@ -52,7 +52,7 @@ func NewSkipDirs(patterns []string, log logutils.Log, runArgs []string) (*SkipDi return &SkipDirs{ patterns: patternsRe, log: log, - skippedDirs: map[string][]string{}, + skippedDirs: map[string]string{}, absArgsDirs: absArgsDirs, }, nil } @@ -99,7 +99,7 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool { for _, pattern := range p.patterns { if pattern.MatchString(issueRelDir) { ps := pattern.String() - p.skippedDirs[ps] = append(p.skippedDirs[ps], issueRelDir) + p.skippedDirs[issueRelDir] = ps return false } } @@ -108,7 +108,7 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool { } func (p SkipDirs) Finish() { - for pattern, dirs := range p.skippedDirs { - p.log.Infof("Skipped by pattern %s dirs: %s", pattern, dirs) + for dir, pattern := range p.skippedDirs { + p.log.Infof("Skipped dir %s by pattern %s", dir, pattern) } }