postgres_exporter/magefile.go

// +build mage
// Self-contained go-project magefile.

// nolint: deadcode
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/magefile/mage/mg"
	"github.com/magefile/mage/sh"
	"github.com/magefile/mage/target"

	"errors"
	"math/bits"
	"strconv"

	"github.com/mholt/archiver"
)

var curDir = func() string {
	name, _ := os.Getwd()
	return name
}()

const constCoverageDir = ".coverage"
const constToolDir = "tools"
const constBinDir = "bin"
const constReleaseDir = "release"
const constCmdDir = "cmd"
const constCoverFile = "cover.out"
const constAssets = "assets"
const constAssetsGenerated = "assets/generated"

var coverageDir = mustStr(filepath.Abs(path.Join(curDir, constCoverageDir)))
var toolDir = mustStr(filepath.Abs(path.Join(curDir, constToolDir)))
var binDir = mustStr(filepath.Abs(path.Join(curDir, constBinDir)))
var releaseDir = mustStr(filepath.Abs(path.Join(curDir, constReleaseDir)))
var cmdDir = mustStr(filepath.Abs(path.Join(curDir, constCmdDir)))
var assetsGenerated = mustStr(filepath.Abs(path.Join(curDir, constAssetsGenerated)))

// Calculate file paths
var toolsGoPath = toolDir
var toolsSrcDir = mustStr(filepath.Abs(path.Join(toolDir, "src")))
var toolsBinDir = mustStr(filepath.Abs(path.Join(toolDir, "bin")))
var toolsVendorDir = mustStr(filepath.Abs(path.Join(toolDir, "vendor")))

var outputDirs = []string{binDir, releaseDir, toolsGoPath, toolsBinDir,
	toolsVendorDir, assetsGenerated, coverageDir}

var toolsEnv = map[string]string{"GOPATH": toolsGoPath}

var containerName = func() string {
	if name := os.Getenv("CONTAINER_NAME"); name != "" {
		return name
	}
	return "wrouesnel/postgres_exporter:latest"
}()

type Platform struct {
	OS        string
	Arch      string
	BinSuffix string
}

func (p *Platform) String() string {
	return fmt.Sprintf("%s-%s", p.OS, p.Arch)
}

func (p *Platform) PlatformDir() string {
	platformDir := path.Join(binDir, fmt.Sprintf("%s_%s_%s", productName, versionShort, p.String()))
	return platformDir
}

func (p *Platform) PlatformBin(cmd string) string {
	platformBin := fmt.Sprintf("%s%s", cmd, p.BinSuffix)
	return path.Join(p.PlatformDir(), platformBin)
}

func (p *Platform) ArchiveDir() string {
	return fmt.Sprintf("%s_%s_%s", productName, versionShort, p.String())
}

func (p *Platform) ReleaseBase() string {
	return path.Join(releaseDir, fmt.Sprintf("%s_%s_%s", productName, versionShort, p.String()))
}

// Supported platforms
var platforms []Platform = []Platform{
	{"linux", "amd64", ""},
	{"linux", "386", ""},
	{"darwin", "amd64", ""},
	{"darwin", "386", ""},
	{"windows", "amd64", ".exe"},
	{"windows", "386", ".exe"},
}

// productName can be overridden by environ product name
var productName = func() string {
	if name := os.Getenv("PRODUCT_NAME"); name != "" {
		return name
	}
	name, _ := os.Getwd()
	return path.Base(name)
}()

// Source files
var goSrc []string
var goDirs []string
var goPkgs []string
var goCmds []string

var version = func() string {
	if v := os.Getenv("VERSION"); v != "" {
		return v
	}
	out, _ := sh.Output("git", "describe", "--dirty")

	if out == "" {
		return "v0.0.0"
	}

	return out
}()

var versionShort = func() string {
	if v := os.Getenv("VERSION_SHORT"); v != "" {
		return v
	}
	out, _ := sh.Output("git", "describe", "--abbrev=0")

	if out == "" {
		return "v0.0.0"
	}

	return out
}()

var concurrency = func() int {
	if v := os.Getenv("CONCURRENCY"); v != "" {
		pv, err := strconv.ParseUint(v, 10, bits.UintSize)
		if err != nil {
			panic(err)
		}
		return int(pv)
	}
	return runtime.NumCPU()
}()

var linterDeadline = func() time.Duration {
	if v := os.Getenv("LINTER_DEADLINE"); v != "" {
		d, _ := time.ParseDuration(v)
		if d != 0 {
			return d
		}
	}
	return time.Second * 60
}()

func Log(args ...interface{}) {
	if mg.Verbose() {
		fmt.Println(args...)
	}
}

func init() {
	// Set environment
	os.Setenv("PATH", fmt.Sprintf("%s:%s", toolsBinDir, os.Getenv("PATH")))
	Log("Build PATH: ", os.Getenv("PATH"))
	Log("Concurrency:", concurrency)
	goSrc = func() []string {
		results := new([]string)
		filepath.Walk(".", func(relpath string, info os.FileInfo, err error) error {
			// Ensure absolute path so globs work
			path, err := filepath.Abs(relpath)
			if err != nil {
				panic(err)
			}

			// Look for files
			if info.IsDir() {
				return nil
			}

			// Exclusions
			for _, exclusion := range []string{toolDir, binDir, releaseDir, coverageDir} {
				if strings.HasPrefix(path, exclusion) {
					if info.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}
			}

			if strings.Contains(path, "/vendor/") {
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if strings.Contains(path, ".git") {
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if !strings.HasSuffix(path, ".go") {
				return nil
			}

			*results = append(*results, path)
			return nil
		})
		return *results
	}()
	goDirs = func() []string {
		resultMap := make(map[string]struct{})
		for _, path := range goSrc {
			absDir, err := filepath.Abs(filepath.Dir(path))
			if err != nil {
				panic(err)
			}
			resultMap[absDir] = struct{}{}
		}
		results := []string{}
		for k := range resultMap {
			results = append(results, k)
		}
		return results
	}()
	goPkgs = func() []string {
		results := []string{}
		out, err := sh.Output("go", "list", "./...")
		if err != nil {
			panic(err)
		}
		for _, line := range strings.Split(out, "\n") {
			if !strings.Contains(line, "/vendor/") {
				results = append(results, line)
			}
		}
		return results
	}()
	goCmds = func() []string {
		results := []string{}

		finfos, err := ioutil.ReadDir(cmdDir)
		if err != nil {
			panic(err)
		}
		for _, finfo := range finfos {
			results = append(results, finfo.Name())
		}
		return results
	}()

	// Ensure output dirs exist
	for _, dir := range outputDirs {
		os.MkdirAll(dir, os.FileMode(0777))
	}
}

func mustStr(r string, err error) string {
	if err != nil {
		panic(err)
	}
	return r
}

func getCoreTools() []string {
	staticTools := []string{
		"github.com/kardianos/govendor",
		"github.com/wadey/gocovmerge",
		"github.com/mattn/goveralls",
		"github.com/tmthrgd/go-bindata/go-bindata",
		"github.com/GoASTScanner/gas/cmd/gas", // workaround for Ast scanner
		"github.com/alecthomas/gometalinter",
	}
	return staticTools
}

func getMetalinters() []string {
	// Gometalinter should now be on the command line
	dynamicTools := []string{}

	goMetalinterHelp, _ := sh.Output("gometalinter", "--help")
	linterRx := regexp.MustCompile(`\s+\w+:\s*\((.+)\)`)
	for _, l := range strings.Split(goMetalinterHelp, "\n") {
		linter := linterRx.FindStringSubmatch(l)
		if len(linter) > 1 {
			dynamicTools = append(dynamicTools, linter[1])
		}
	}
	return dynamicTools
}

func ensureVendorSrcLink() error {
	Log("Symlink vendor to tools dir")
	if err := sh.Rm(toolsSrcDir); err != nil {
		return err
	}
	if err := os.Symlink(toolsVendorDir, toolsSrcDir); err != nil {
		return err
	}
	return nil
}

// concurrencyLimitedBuild executes a certain number of commands limited by concurrency
func concurrencyLimitedBuild(buildCmds ...interface{}) error {
	resultsCh := make(chan error, len(buildCmds))
	concurrencyControl := make(chan struct{}, concurrency)
	for _, buildCmd := range buildCmds {
		go func(buildCmd interface{}) {
			concurrencyControl <- struct{}{}
			resultsCh <- buildCmd.(func() error)()
			<-concurrencyControl

		}(buildCmd)
	}
	// Doesn't work at the moment
	//	mg.Deps(buildCmds...)
	results := []error{}
	var resultErr error = nil
	for len(results) < len(buildCmds) {
		err := <-resultsCh
		results = append(results, err)
		if err != nil {
			fmt.Println(err)
			resultErr = errors.New("parallel build failed")
		}
		fmt.Printf("Finished %v of %v\n", len(results), len(buildCmds))
	}

	return resultErr
}

// Tools builds build tools of the project and is depended on by all other build targets.
func Tools() (err error) {
	// Catch panics and convert to errors
	defer func() {
		if perr := recover(); perr != nil {
			err = perr.(error)
		}
	}()

	if err := ensureVendorSrcLink(); err != nil {
		return err
	}

	toolBuild := func(toolType string, tools ...string) error {
		toolTargets := []interface{}{}
		for _, toolImport := range tools {
			toolParts := strings.Split(toolImport, "/")
			toolBin := path.Join(toolsBinDir, toolParts[len(toolParts)-1])
			Log("Check for changes:", toolBin, toolsVendorDir)
			changed, terr := target.Dir(toolBin, toolsVendorDir)
			if terr != nil {
				if !os.IsNotExist(terr) {
					panic(terr)
				}
				changed = true
			}
			if changed {
				localToolImport := toolImport
				f := func() error { return sh.RunWith(toolsEnv, "go", "install", "-v", localToolImport) }
				toolTargets = append(toolTargets, f)
			}
		}

		Log("Build", toolType, "tools")
		if berr := concurrencyLimitedBuild(toolTargets...); berr != nil {
			return berr
		}
		return nil
	}

	if berr := toolBuild("static", getCoreTools()...); berr != nil {
		return berr
	}

	if berr := toolBuild("static", getMetalinters()...); berr != nil {
		return berr
	}

	return nil
}

// UpdateTools automatically updates tool dependencies to the latest version.
func UpdateTools() error {
	if err := ensureVendorSrcLink(); err != nil {
		return err
	}

	// Ensure govendor is up to date without doing anything
	govendorPkg := "github.com/kardianos/govendor"
	govendorParts := strings.Split(govendorPkg, "/")
	govendorBin := path.Join(toolsBinDir, govendorParts[len(govendorParts)-1])

	sh.RunWith(toolsEnv, "go", "get", "-v", "-u", govendorPkg)

	if changed, cerr := target.Dir(govendorBin, toolsSrcDir); changed || os.IsNotExist(cerr) {
		if err := sh.RunWith(toolsEnv, "go", "install", "-v", govendorPkg); err != nil {
			return err
		}
	} else if cerr != nil {
		panic(cerr)
	}

	// Set current directory so govendor has the right path
	previousPwd, wderr := os.Getwd()
	if wderr != nil {
		return wderr
	}
	if err := os.Chdir(toolDir); err != nil {
		return err
	}

	// govendor fetch core tools
	for _, toolImport := range append(getCoreTools(), getMetalinters()...) {
		sh.RunV("govendor", "fetch", "-v", toolImport)
	}

	// change back to original working directory
	if err := os.Chdir(previousPwd); err != nil {
		return err
	}
	return nil
}

// Assets builds binary assets to be bundled into the binary.
func Assets() error {
	mg.Deps(Tools)

	if err := os.MkdirAll("assets/generated", os.FileMode(0777)); err != nil {
		return err
	}

	return sh.RunV("go-bindata", "-pkg=assets", "-o", "assets/bindata.go", "-ignore=bindata.go",
		"-ignore=.*.map$", "-prefix=assets/generated", "assets/generated/...")
}

// Lint runs gometalinter for code quality. CI will run this before accepting PRs.
func Lint() error {
	mg.Deps(Tools)
	args := []string{"-j", fmt.Sprintf("%v", concurrency), fmt.Sprintf("--deadline=%s",
		linterDeadline.String()), "--enable-all", "--line-length=120",
		"--disable=gocyclo", "--disable=testify", "--disable=test", "--exclude=assets/bindata.go"}
	return sh.RunV("gometalinter", append(args, goDirs...)...)
}

// Style checks formatting of the file. CI will run this before acceptiing PRs.
func Style() error {
	mg.Deps(Tools)
	args := []string{"--disable-all", "--enable=gofmt", "--enable=goimports"}
	return sh.RunV("gometalinter", append(args, goSrc...)...)
}

// Fmt automatically formats all source code files
func Fmt() error {
	mg.Deps(Tools)
	fmtErr := sh.RunV("gofmt", append([]string{"-s", "-w"}, goSrc...)...)
	if fmtErr != nil {
		return fmtErr
	}
	impErr := sh.RunV("goimports", append([]string{"-w"}, goSrc...)...)
	if impErr != nil {
		return fmtErr
	}
	return nil
}

func listCoverageFiles() ([]string, error) {
	result := []string{}
	finfos, derr := ioutil.ReadDir(coverageDir)
	if derr != nil {
		return result, derr
	}
	for _, finfo := range finfos {
		result = append(result, path.Join(coverageDir, finfo.Name()))
	}
	return result, nil
}

// Test run test suite
func Test() error {
	mg.Deps(Tools)

	// Ensure coverage directory exists
	if err := os.MkdirAll(coverageDir, os.FileMode(0777)); err != nil {
		return err
	}

	// Clean up coverage directory
	coverFiles, derr := listCoverageFiles()
	if derr != nil {
		return derr
	}
	for _, coverFile := range coverFiles {
		if err := sh.Rm(coverFile); err != nil {
			return err
		}
	}

	// Run tests
	coverProfiles := []string{}
	for _, pkg := range goPkgs {
		coverProfile := path.Join(coverageDir, fmt.Sprintf("%s%s", strings.Replace(pkg, "/", "-", -1), ".out"))
		testErr := sh.Run("go", "test", "-v", "-covermode", "count", fmt.Sprintf("-coverprofile=%s", coverProfile),
			pkg)
		if testErr != nil {
			return testErr
		}
		coverProfiles = append(coverProfiles, coverProfile)
	}

	return nil
}

// Build the intgration test binary
func IntegrationTestBinary() error {
	changed, err := target.Path("postgres_exporter_integration_test", goSrc...)
	if (changed && (err == nil)) || os.IsNotExist(err) {
		return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "go", "test", "./cmd/postgres_exporter",
			"-c", "-tags", "integration",
			"-a", "-ldflags", "-extldflags '-static'", "-X", fmt.Sprintf("main.Version=%s", version),
			"-o", "postgres_exporter_integration_test", "-cover", "-covermode", "count")
	}
	return err
}

// TestIntegration runs integration tests
func TestIntegration() error {
	mg.Deps(Binary, IntegrationTestBinary)

	exporterPath := mustStr(filepath.Abs("postgres_exporter"))
	testBinaryPath := mustStr(filepath.Abs("postgres_exporter_integration_test"))
	testScriptPath := mustStr(filepath.Abs("postgres_exporter_integration_test_script"))

	integrationCoverageProfile := path.Join(coverageDir, "cover.integration.out")

	return sh.RunV("cmd/postgres_exporter/tests/test-smoke", exporterPath,
		fmt.Sprintf("%s %s %s", testScriptPath, testBinaryPath, integrationCoverageProfile))
}

// Coverage sums up the coverage profiles in .coverage. It does not clean up after itself or before.
func Coverage() error {
	// Clean up coverage directory
	coverFiles, derr := listCoverageFiles()
	if derr != nil {
		return derr
	}

	mergedCoverage, err := sh.Output("gocovmerge", coverFiles...)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(constCoverFile, []byte(mergedCoverage), os.FileMode(0777))
}

// All runs a full suite suitable for CI
func All() error {
	mg.SerialDeps(Style, Lint, Test, TestIntegration, Coverage, Release)
	return nil
}

// Release builds release archives under the release/ directory
func Release() error {
	mg.Deps(ReleaseBin)

	for _, platform := range platforms {
		owd, wderr := os.Getwd()
		if wderr != nil {
			return wderr
		}
		os.Chdir(binDir)

		if platform.OS == "windows" {
			// build a zip binary as well
			err := archiver.Zip.Make(fmt.Sprintf("%s.zip", platform.ReleaseBase()), []string{platform.ArchiveDir()})
			if err != nil {
				return err
			}
		}
		// build tar gz
		err := archiver.TarGz.Make(fmt.Sprintf("%s.tar.gz", platform.ReleaseBase()), []string{platform.ArchiveDir()})
		if err != nil {
			return err
		}
		os.Chdir(owd)
	}

	return nil
}

func makeBuilder(cmd string, platform Platform) func() error {
	f := func() error {
		// Depend on assets
		mg.Deps(Assets)

		cmdSrc := fmt.Sprintf("./%s/%s", mustStr(filepath.Rel(curDir, cmdDir)), cmd)

		Log("Make platform binary directory:", platform.PlatformDir())
		if err := os.MkdirAll(platform.PlatformDir(), os.FileMode(0777)); err != nil {
			return err
		}

		Log("Checking for changes:", platform.PlatformBin(cmd))
		if changed, err := target.Path(platform.PlatformBin(cmd), goSrc...); !changed {
			if err != nil {
				if !os.IsNotExist(err) {
					return err
				}
			} else {
				return nil
			}
		}

		fmt.Println("Building", platform.PlatformBin(cmd))
		return sh.RunWith(map[string]string{"CGO_ENABLED": "0", "GOOS": platform.OS, "GOARCH": platform.Arch},
			"go", "build", "-a", "-ldflags", fmt.Sprintf("-extldflags '-static' -X version.Version=%s", version),
			"-o", platform.PlatformBin(cmd), cmdSrc)
	}
	return f
}

func getCurrentPlatform() *Platform {
	var curPlatform *Platform
	for _, p := range platforms {
		if p.OS == runtime.GOOS && p.Arch == runtime.GOARCH {
			storedP := p
			curPlatform = &storedP
		}
	}
	Log("Determined current platform:", curPlatform)
	return curPlatform
}

// Binary build a binary for the current platform
func Binary() error {
	curPlatform := getCurrentPlatform()
	if curPlatform == nil {
		return errors.New("current platform is not supported")
	}

	for _, cmd := range goCmds {
		err := makeBuilder(cmd, *curPlatform)()
		if err != nil {
			return err
		}
		// Make a root symlink to the build
		cmdPath := path.Join(curDir, cmd)
		os.Remove(cmdPath)
		if err := os.Symlink(curPlatform.PlatformBin(cmd), cmdPath); err != nil {
			return err
		}
	}

	return nil
}

// ReleaseBin builds cross-platform release binaries under the bin/ directory
func ReleaseBin() error {
	buildCmds := []interface{}{}

	for _, cmd := range goCmds {
		for _, platform := range platforms {
			buildCmds = append(buildCmds, makeBuilder(cmd, platform))
		}
	}

	resultsCh := make(chan error, len(buildCmds))
	concurrencyControl := make(chan struct{}, concurrency)
	for _, buildCmd := range buildCmds {
		go func(buildCmd interface{}) {
			concurrencyControl <- struct{}{}
			resultsCh <- buildCmd.(func() error)()
			<-concurrencyControl

		}(buildCmd)
	}
	// Doesn't work at the moment
	//	mg.Deps(buildCmds...)
	results := []error{}
	var resultErr error = nil
	for len(results) < len(buildCmds) {
		err := <-resultsCh
		results = append(results, err)
		if err != nil {
			fmt.Println(err)
			resultErr = errors.New("parallel build failed")
		}
		fmt.Printf("Finished %v of %v\n", len(results), len(buildCmds))
	}

	return resultErr
}

// Docker builds the docker image
func Docker() error {
	mg.Deps(Binary)
	p := getCurrentPlatform()
	if p == nil {
		return errors.New("current platform is not supported")
	}

	return sh.RunV("docker", "build",
		fmt.Sprintf("--build-arg=binary=%s",
			mustStr(filepath.Rel(curDir, p.PlatformBin("postgres_exporter")))),
		"-t", containerName, ".")
}

// Clean deletes build output and cleans up the working directory
func Clean() error {
	for _, name := range goCmds {
		if err := sh.Rm(path.Join(binDir, name)); err != nil {
			return err
		}
	}

	for _, name := range outputDirs {
		if err := sh.Rm(name); err != nil {
			return err
		}
	}
	return nil
}

// Debug prints the value of internal state variables
func Debug() error {
	fmt.Println("Source Files:", goSrc)
	fmt.Println("Packages:", goPkgs)
	fmt.Println("Directories:", goDirs)
	fmt.Println("Command Paths:", goCmds)
	fmt.Println("Output Dirs:", outputDirs)
	fmt.Println("Tool Src Dir:", toolsSrcDir)
	fmt.Println("Tool Vendor Dir:", toolsVendorDir)
	fmt.Println("Tool GOPATH:", toolsGoPath)
	fmt.Println("PATH:", os.Getenv("PATH"))
	return nil
}

// Autogen configure local git repository with commit hooks
func Autogen() error {
	fmt.Println("Installing git hooks in local repository...")
	return os.Link(path.Join(curDir, toolDir, "pre-commit"), ".git/hooks/pre-commit")
}