eeafj/golangci-lint
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

dev: improve memory tracking

Browse Source
This commit is contained in:
Denis Isaev 2019-03-31 22:09:22 +03:00 committed by Isaev Denis
parent 9ce2182d05
commit 396a04630d
6 changed files with 88 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
/*.txt
/*.pdf
/*.pprof
/dist/
/.idea/

16
README.md
View File

@ -341,6 +341,22 @@ On average golangci-lint consumes 26% less memory.
Golangci-lint directly calls linters (no forking) and reuses 80% of work by parsing program only once.
Read [this section](#internals) for details.
### Memory Usage of Golangci-lint
A trade-off between memory usage and execution time can be controlled by [`GOCC`](https://golang.org/pkg/runtime/#hdr-Environment_Variables) environment variable.
Less `GOGC` values trigger garbage collection more frequently and golangci-lint consumes less memory and more CPU. Below is the trade-off table for running on this repo:
|`GOGC`|Peak Memory, GB|Executon Time, s|
|------|---------------|----------------|
|`5` |1.1 |60 |
|`10` |1.1 |34 |
|`20` |1.3 |25 |
|`30` |1.6 |20.2 |
|`50` |2.0 |17.1 |
|`80` |2.2 |14.1 |
|`100` (default)|2.2 |13.8 |
|`off` |3.2 |9.3 |
## Internals
1. Work sharing

16
README.tmpl.md
View File

@ -310,6 +310,22 @@ On average golangci-lint consumes 26% less memory.
Golangci-lint directly calls linters (no forking) and reuses 80% of work by parsing program only once.
Read [this section](#internals) for details.
### Memory Usage of Golangci-lint
A trade-off between memory usage and execution time can be controlled by [`GOCC`](https://golang.org/pkg/runtime/#hdr-Environment_Variables) environment variable.
Less `GOGC` values trigger garbage collection more frequently and golangci-lint consumes less memory and more CPU. Below is the trade-off table for running on this repo:
|`GOGC`|Peak Memory, GB|Executon Time, s|
|------|---------------|----------------|
|`5` |1.1 |60 |
|`10` |1.1 |34 |
|`20` |1.3 |25 |
|`30` |1.6 |20.2 |
|`50` |2.0 |17.1 |
|`80` |2.2 |14.1 |
|`100` (default)|2.2 |13.8 |
|`off` |3.2 |9.3 |
## Internals
1. Work sharing

39
pkg/commands/root.go
View File

@ -5,6 +5,7 @@ import (
"os"
"runtime"
"runtime/pprof"
"strconv"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -30,6 +31,12 @@ func (e *Executor) persistentPreRun(_ *cobra.Command, _ []string) {
e.log.Fatalf("Can't start CPU profiling: %s", err)
}
}
if e.cfg.Run.MemProfilePath != "" {
if rate := os.Getenv("GL_MEMPROFILE_RATE"); rate != "" {
runtime.MemProfileRate, _ = strconv.Atoi(rate)
}
}
}
func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
@ -41,15 +48,45 @@ func (e *Executor) persistentPostRun(_ *cobra.Command, _ []string) {
if err != nil {
e.log.Fatalf("Can't create file %s: %s", e.cfg.Run.MemProfilePath, err)
}
runtime.GC() // get up-to-date statistics
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
printMemStats(&ms, e.log)
if err := pprof.WriteHeapProfile(f); err != nil {
e.log.Fatalf("Can't write heap profile: %s", err)
}
f.Close()
}
os.Exit(e.exitCode)
}
func printMemStats(ms *runtime.MemStats, logger logutils.Log) {
logger.Infof("Mem stats: alloc=%s total_alloc=%s sys=%s "+
"heap_alloc=%s heap_sys=%s heap_idle=%s heap_released=%s heap_in_use=%s "+
"stack_in_use=%s stack_sys=%s "+
"mspan_sys=%s mcache_sys=%s buck_hash_sys=%s gc_sys=%s other_sys=%s "+
"mallocs_n=%d frees_n=%d heap_objects_n=%d gc_cpu_fraction=%.2f",
formatMemory(ms.Alloc), formatMemory(ms.TotalAlloc), formatMemory(ms.Sys),
formatMemory(ms.HeapAlloc), formatMemory(ms.HeapSys),
formatMemory(ms.HeapIdle), formatMemory(ms.HeapReleased), formatMemory(ms.HeapInuse),
formatMemory(ms.StackInuse), formatMemory(ms.StackSys),
formatMemory(ms.MSpanSys), formatMemory(ms.MCacheSys), formatMemory(ms.BuckHashSys),
formatMemory(ms.GCSys), formatMemory(ms.OtherSys),
ms.Mallocs, ms.Frees, ms.HeapObjects, ms.GCCPUFraction)
}
func formatMemory(memBytes uint64) string {
if memBytes < 1024 {
return fmt.Sprintf("%db", memBytes)
}
if memBytes < 1024*1024 {
return fmt.Sprintf("%dkb", memBytes/1024)
}
return fmt.Sprintf("%dmb", memBytes/1024/1024)
}
func getDefaultConcurrency() int {
if os.Getenv("HELP_RUN") == "1" {
return 8 // to make stable concurrency for README help generating builds

13
pkg/commands/run.go
View File

@ -430,11 +430,21 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1"
track := func() {
var m runtime.MemStats
runtime.ReadMemStats(&m)
if logEveryRecord {
printMemStats(&m, logger)
}
rssValues = append(rssValues, m.Sys)
}
for {
track()
stop := false
select {
@ -447,6 +457,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
break
}
}
track()
var avg, max uint64
for _, v := range rssValues {

10
pkg/result/processors/skip_dirs.go
View File

@ -14,7 +14,7 @@ import (
type SkipDirs struct {
patterns []*regexp.Regexp
log logutils.Log
skippedDirs map[string][]string // regexp to dir mapping
skippedDirs map[string]string // dir to the last regexp mapping
absArgsDirs []string
}
@ -52,7 +52,7 @@ func NewSkipDirs(patterns []string, log logutils.Log, runArgs []string) (*SkipDi
return &SkipDirs{
patterns: patternsRe,
log: log,
skippedDirs: map[string][]string{},
skippedDirs: map[string]string{},
absArgsDirs: absArgsDirs,
}, nil
}
@ -99,7 +99,7 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool {
for _, pattern := range p.patterns {
if pattern.MatchString(issueRelDir) {
ps := pattern.String()
p.skippedDirs[ps] = append(p.skippedDirs[ps], issueRelDir)
p.skippedDirs[issueRelDir] = ps
return false
}
}
@ -108,7 +108,7 @@ func (p *SkipDirs) shouldPassIssue(i *result.Issue) bool {
}
func (p SkipDirs) Finish() {
for pattern, dirs := range p.skippedDirs {
p.log.Infof("Skipped by pattern %s dirs: %s", pattern, dirs)
for dir, pattern := range p.skippedDirs {
p.log.Infof("Skipped dir %s by pattern %s", dir, pattern)
}
}