golangci-lint/internal/pkgcache/pkgcache.go

package pkgcache

import (
	"bytes"
	"encoding/gob"
	"encoding/hex"
	"fmt"
	"runtime"
	"sort"
	"sync"

	"github.com/pkg/errors"
	"golang.org/x/tools/go/packages"

	"github.com/golangci/golangci-lint/internal/cache"
	"github.com/golangci/golangci-lint/pkg/logutils"
	"github.com/golangci/golangci-lint/pkg/timeutils"
)

type HashMode int

const (
	HashModeNeedOnlySelf HashMode = iota
	HashModeNeedDirectDeps
	HashModeNeedAllDeps
)

// Cache is a per-package data cache. A cached data is invalidated when
// package or it's dependencies change.
type Cache struct {
	lowLevelCache *cache.Cache
	pkgHashes     sync.Map
	sw            *timeutils.Stopwatch
	log           logutils.Log  // not used now, but may be needed for future debugging purposes
	ioSem         chan struct{} // semaphore limiting parallel IO
}

func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) {
	c, err := cache.Default()
	if err != nil {
		return nil, err
	}
	return &Cache{
		lowLevelCache: c,
		sw:            sw,
		log:           log,
		ioSem:         make(chan struct{}, runtime.GOMAXPROCS(-1)),
	}, nil
}

func (c *Cache) Trim() {
	c.sw.TrackStage("trim", func() {
		c.lowLevelCache.Trim()
	})
}

func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data interface{}) error {
	var err error
	buf := &bytes.Buffer{}
	c.sw.TrackStage("gob", func() {
		err = gob.NewEncoder(buf).Encode(data)
	})
	if err != nil {
		return errors.Wrap(err, "failed to gob encode")
	}

	var aID cache.ActionID

	c.sw.TrackStage("key build", func() {
		aID, err = c.pkgActionID(pkg, mode)
		if err == nil {
			subkey, subkeyErr := cache.Subkey(aID, key)
			if subkeyErr != nil {
				err = errors.Wrap(subkeyErr, "failed to build subkey")
			}
			aID = subkey
		}
	})
	if err != nil {
		return errors.Wrapf(err, "failed to calculate package %s action id", pkg.Name)
	}
	c.ioSem <- struct{}{}
	c.sw.TrackStage("cache io", func() {
		err = c.lowLevelCache.PutBytes(aID, buf.Bytes())
	})
	<-c.ioSem
	if err != nil {
		return errors.Wrapf(err, "failed to save data to low-level cache by key %s for package %s", key, pkg.Name)
	}

	return nil
}

var ErrMissing = errors.New("missing data")

func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data interface{}) error {
	var aID cache.ActionID
	var err error
	c.sw.TrackStage("key build", func() {
		aID, err = c.pkgActionID(pkg, mode)
		if err == nil {
			subkey, subkeyErr := cache.Subkey(aID, key)
			if subkeyErr != nil {
				err = errors.Wrap(subkeyErr, "failed to build subkey")
			}
			aID = subkey
		}
	})
	if err != nil {
		return errors.Wrapf(err, "failed to calculate package %s action id", pkg.Name)
	}

	var b []byte
	c.ioSem <- struct{}{}
	c.sw.TrackStage("cache io", func() {
		b, _, err = c.lowLevelCache.GetBytes(aID)
	})
	<-c.ioSem
	if err != nil {
		if cache.IsErrMissing(err) {
			return ErrMissing
		}
		return errors.Wrapf(err, "failed to get data from low-level cache by key %s for package %s", key, pkg.Name)
	}

	c.sw.TrackStage("gob", func() {
		err = gob.NewDecoder(bytes.NewReader(b)).Decode(data)
	})
	if err != nil {
		return errors.Wrap(err, "failed to gob decode")
	}

	return nil
}

func (c *Cache) pkgActionID(pkg *packages.Package, mode HashMode) (cache.ActionID, error) {
	hash, err := c.packageHash(pkg, mode)
	if err != nil {
		return cache.ActionID{}, errors.Wrap(err, "failed to get package hash")
	}

	key, err := cache.NewHash("action ID")
	if err != nil {
		return cache.ActionID{}, errors.Wrap(err, "failed to make a hash")
	}
	fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
	fmt.Fprintf(key, "pkghash %s\n", hash)

	return key.Sum(), nil
}

// packageHash computes a package's hash. The hash is based on all Go
// files that make up the package, as well as the hashes of imported
// packages.
func (c *Cache) packageHash(pkg *packages.Package, mode HashMode) (string, error) {
	type hashResults map[HashMode]string
	hashResI, ok := c.pkgHashes.Load(pkg)
	if ok {
		hashRes := hashResI.(hashResults)
		if _, ok := hashRes[mode]; !ok {
			return "", fmt.Errorf("no mode %d in hash result", mode)
		}
		return hashRes[mode], nil
	}

	hashRes := hashResults{}

	key, err := cache.NewHash("package hash")
	if err != nil {
		return "", errors.Wrap(err, "failed to make a hash")
	}

	fmt.Fprintf(key, "pkgpath %s\n", pkg.PkgPath)
	for _, f := range pkg.CompiledGoFiles {
		c.ioSem <- struct{}{}
		h, fErr := cache.FileHash(f)
		<-c.ioSem
		if fErr != nil {
			return "", errors.Wrapf(fErr, "failed to calculate file %s hash", f)
		}
		fmt.Fprintf(key, "file %s %x\n", f, h)
	}
	curSum := key.Sum()
	hashRes[HashModeNeedOnlySelf] = hex.EncodeToString(curSum[:])

	imps := make([]*packages.Package, 0, len(pkg.Imports))
	for _, imp := range pkg.Imports {
		imps = append(imps, imp)
	}
	sort.Slice(imps, func(i, j int) bool {
		return imps[i].PkgPath < imps[j].PkgPath
	})

	calcDepsHash := func(depMode HashMode) error {
		for _, dep := range imps {
			if dep.PkgPath == "unsafe" {
				continue
			}

			depHash, depErr := c.packageHash(dep, depMode)
			if depErr != nil {
				return errors.Wrapf(depErr, "failed to calculate hash for dependency %s with mode %d", dep.Name, depMode)
			}

			fmt.Fprintf(key, "import %s %s\n", dep.PkgPath, depHash)
		}
		return nil
	}

	if err := calcDepsHash(HashModeNeedOnlySelf); err != nil {
		return "", err
	}

	curSum = key.Sum()
	hashRes[HashModeNeedDirectDeps] = hex.EncodeToString(curSum[:])

	if err := calcDepsHash(HashModeNeedAllDeps); err != nil {
		return "", err
	}
	curSum = key.Sum()
	hashRes[HashModeNeedAllDeps] = hex.EncodeToString(curSum[:])

	if _, ok := hashRes[mode]; !ok {
		return "", fmt.Errorf("invalid mode %d", mode)
	}

	c.pkgHashes.Store(pkg, hashRes)
	return hashRes[mode], nil
}