vikunja/build/magefile.go

758 lines
22 KiB
Go

// Vikunja is a to-do list application to facilitate your life.
// Copyright 2018-present Vikunja and contributors. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//go:build mage
// Centralized release pipeline for every Go binary in this monorepo.
//
// Both vikunja and veans cross-compile through the same code: xgo for the full
// OS/arch matrix, upx where the binary supports it, sha256 alongside each
// artifact, per-target zip bundle, and nfpm.yaml templating for deb/rpm/apk/
// archlinux packaging. Repository-metadata targets (apt/rpm/pacman) consume
// the merged ../dist/repo-work/incoming/ tree the CI populates from both
// projects' packages.
//
// The module is intentionally separate from the project magefiles so the
// release tooling can evolve without touching them. The small filesystem
// helpers (copyFile, moveFile, sha256File) are duplicated rather than
// imported — this magefile depends on nothing but stdlib + mage.
package main
import (
"context"
"crypto/sha256"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
// -----------------------------------------------------------------------------
// project definitions
// project describes one releasable Go binary in this monorepo. Adding a new
// project means adding an entry to projectByName plus a constructor below.
type project struct {
// Name is the short identifier used on the CLI: `mage release:build <name>`.
Name string
// Root is the project root, relative to this build/ directory.
Root string
// BuildPath is the Go package to build, relative to Root (e.g. "." or "./cmd/foo").
BuildPath string
// Executable is the output binary name (sans -<os>-<arch> suffix).
Executable string
// BuildTags are the base build tags applied to every cross-compile.
BuildTags string
// Ldflags returns the full -X flag string for the given version.
Ldflags func(version string) string
// NfpmConfigPath is the nfpm.yaml location, relative to Root.
NfpmConfigPath string
// NfpmBinPathDefault is the default <binlocation> substitution. Empty
// means use the Executable name as-is.
NfpmBinPathDefault string
// OsPackageExtras hook copies any extra files (LICENSE, sample config…)
// into each per-target bundle folder. Called once per binary.
OsPackageExtras func(folder string, p *project) error
}
func projectByName(name string) (*project, error) {
switch name {
case "vikunja":
return vikunjaProject(), nil
case "veans":
return veansProject(), nil
default:
return nil, fmt.Errorf("unknown project %q (known: vikunja, veans)", name)
}
}
func vikunjaProject() *project {
return &project{
Name: "vikunja",
Root: "../",
BuildPath: ".",
Executable: "vikunja",
BuildTags: "osusergo netgo",
Ldflags: func(v string) string {
// Matches the parent magefile's pre-refactor ldflags. The
// main.Tags value is the literal build-tag string baked in
// for `vikunja info` to report.
return fmt.Sprintf(`-X "code.vikunja.io/api/pkg/version.Version=%s" -X "main.Tags=osusergo netgo"`, v)
},
NfpmConfigPath: "nfpm.yaml",
NfpmBinPathDefault: "vikunja",
OsPackageExtras: func(folder string, p *project) error {
// config.yml.sample must be generated by the CI (or local dev)
// before this runs — we don't want to vendor the
// config-raw.json→YAML logic. The workflow does
// `mage generate:config-yaml 1` in the project root before
// invoking release:build.
if err := copyFile(filepath.Join(p.Root, "config.yml.sample"), filepath.Join(folder, "config.yml.sample")); err != nil {
return fmt.Errorf("copy config.yml.sample (run `mage generate:config-yaml 1` first): %w", err)
}
return copyFile(filepath.Join(p.Root, "LICENSE"), filepath.Join(folder, "LICENSE"))
},
}
}
func veansProject() *project {
return &project{
Name: "veans",
Root: "../veans/",
BuildPath: "./cmd/veans",
Executable: "veans",
BuildTags: "osusergo netgo",
Ldflags: func(v string) string {
return fmt.Sprintf(`-X main.version=%s`, v)
},
NfpmConfigPath: "nfpm.yaml",
NfpmBinPathDefault: "./veans",
OsPackageExtras: func(folder string, _ *project) error {
// veans intentionally doesn't carry its own LICENSE — the
// AGPLv3 at the repo root applies to both.
return copyFile("../LICENSE", filepath.Join(folder, "LICENSE"))
},
}
}
// -----------------------------------------------------------------------------
// version resolution
func releaseVersion(ctx context.Context) (string, error) {
if v := os.Getenv("RELEASE_VERSION"); v != "" {
return v, nil
}
out, err := exec.CommandContext(ctx, "git", "describe", "--tags", "--always", "--abbrev=10").Output()
if err != nil {
return "", fmt.Errorf("git describe: %w", err)
}
return strings.Replace(strings.TrimSpace(string(out)), "-g", "-", 1), nil
}
func versionTagOrUnstable(v string) string {
switch v {
case "", "main":
return "unstable"
default:
return v
}
}
// -----------------------------------------------------------------------------
// Release namespace
type Release mg.Namespace
// Build runs the full release pipeline for the named project: dirs → xgo
// (windows/linux/darwin in parallel) → upx → copy → sha256 → per-target
// bundle dir → zip.
func (Release) Build(ctx context.Context, name string) error {
p, err := projectByName(name)
if err != nil {
return err
}
version, err := releaseVersion(ctx)
if err != nil {
return err
}
if err := releaseDirs(p); err != nil {
return err
}
if err := prepareXgo(ctx); err != nil {
return err
}
if err := xgoAllOS(ctx, p, version); err != nil {
return err
}
if err := compressBinaries(p); err != nil {
return err
}
if err := copyBinaries(p); err != nil {
return err
}
if err := writeChecksums(p); err != nil {
return err
}
if err := bundleOsPackages(p); err != nil {
return err
}
return zipBundles(ctx, p)
}
// Xgo cross-compiles a single os/arch[/variant] target for the named project.
// Variant follows the parent magefile convention: `linux/arm/7` → arm-7.
//
// Unlike Release.Build, this skips prepareXgo on purpose: the only caller
// that hits this path in CI is the Dockerfile, which runs inside the xgo
// image (xgo binary already present, docker daemon not available). Local
// users invoking `mage release:xgo` need to install xgo themselves.
func (Release) Xgo(ctx context.Context, name, target string) error {
p, err := projectByName(name)
if err != nil {
return err
}
version, err := releaseVersion(ctx)
if err != nil {
return err
}
parts := strings.Split(target, "/")
if len(parts) < 2 {
return fmt.Errorf("invalid target %q (expected os/arch[/variant])", target)
}
variant := ""
if len(parts) > 2 && parts[2] != "" {
variant = "-" + strings.ReplaceAll(parts[2], "v", "")
}
return runXgo(ctx, p, version, parts[0]+"/"+parts[1]+variant)
}
// PrepareNFPMConfig templates the named project's nfpm.yaml in place for the
// given nfpm arch (amd64|arm64|arm7|386). Destructive — CI checks out a fresh
// copy per matrix shard so the trampling is fine.
func (Release) PrepareNFPMConfig(ctx context.Context, name, arch string) error {
p, err := projectByName(name)
if err != nil {
return err
}
version, err := releaseVersion(ctx)
if err != nil {
return err
}
cfgPath := filepath.Join(p.Root, p.NfpmConfigPath)
raw, err := os.ReadFile(cfgPath)
if err != nil {
return err
}
binLocation := os.Getenv("NFPM_BIN_PATH")
if binLocation == "" {
binLocation = p.NfpmBinPathDefault
if binLocation == "" {
binLocation = p.Executable
}
}
out := strings.ReplaceAll(string(raw), "<version>", version)
out = strings.ReplaceAll(out, "<arch>", arch)
out = strings.ReplaceAll(out, "<binlocation>", binLocation)
return os.WriteFile(cfgPath, []byte(out), 0o600)
}
// -----------------------------------------------------------------------------
// Repo-metadata targets — project-agnostic; operate on the merged tree at
// ../dist/repo-work/incoming and ../dist/repo-output.
// RepoApt generates an APT repository (reprepro) for every .deb in the
// incoming tree. REPO_SUITE (stable|unstable) selects the target suite;
// RELEASE_GPG_KEY + RELEASE_GPG_PASSPHRASE drive the Release file signing.
func (Release) RepoApt(ctx context.Context) error {
suite := repoSuite()
incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming")
outputBase := filepath.Join(repoRootDist, "repo-output", "apt")
confDir := filepath.Join(outputBase, "conf")
if err := os.MkdirAll(confDir, 0o755); err != nil {
return fmt.Errorf("creating reprepro conf dir: %w", err)
}
distConf, err := os.ReadFile("reprepro-dist-conf")
if err != nil {
return fmt.Errorf("reading reprepro-dist-conf: %w", err)
}
if err := os.WriteFile(filepath.Join(confDir, "distributions"), distConf, 0o600); err != nil {
return fmt.Errorf("writing distributions config: %w", err)
}
debs, err := filepath.Glob(filepath.Join(incomingDir, "*.deb"))
if err != nil {
return err
}
for _, deb := range debs {
abs, _ := filepath.Abs(deb)
if err := sh.RunV("reprepro", "-b", outputBase, "includedeb", suite, abs); err != nil {
return fmt.Errorf("reprepro includedeb %s: %w", filepath.Base(deb), err)
}
}
gpgKey := os.Getenv("RELEASE_GPG_KEY")
gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE")
releaseFile := filepath.Join(outputBase, "dists", suite, "Release")
if _, err := os.Stat(releaseFile); err == nil {
if err := sh.RunV("gpg",
"--default-key", gpgKey,
"--batch", "--yes",
"--passphrase", gpgPassphrase,
"--pinentry-mode", "loopback",
"--detach-sign", "--armor",
"-o", releaseFile+".gpg",
releaseFile,
); err != nil {
return fmt.Errorf("signing Release (detached): %w", err)
}
if err := sh.RunV("gpg",
"--default-key", gpgKey,
"--batch", "--yes",
"--passphrase", gpgPassphrase,
"--pinentry-mode", "loopback",
"--clearsign",
"-o", filepath.Join(filepath.Dir(releaseFile), "InRelease"),
releaseFile,
); err != nil {
return fmt.Errorf("signing Release (clearsign): %w", err)
}
}
fmt.Println("APT repo metadata generated in", outputBase)
return nil
}
// RepoRpm generates an RPM repository (createrepo_c) per arch in
// ../dist/repo-work/incoming/.
func (Release) RepoRpm(ctx context.Context) error {
suite := repoSuite()
incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming")
outputBase := filepath.Join(repoRootDist, "repo-output", "rpm", suite)
gpgKey := os.Getenv("RELEASE_GPG_KEY")
gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE")
for _, arch := range []string{"x86_64", "aarch64", "armv7"} {
repoDir := filepath.Join(outputBase, arch)
if err := os.MkdirAll(repoDir, 0o755); err != nil {
return err
}
rpms, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".rpm"))
if len(rpms) == 0 {
continue
}
for _, rpm := range rpms {
abs, _ := filepath.Abs(rpm)
dst := filepath.Join(repoDir, filepath.Base(rpm))
_ = os.Remove(dst)
if err := os.Symlink(abs, dst); err != nil {
return err
}
}
args := []string{repoDir}
if _, err := os.Stat(filepath.Join(repoDir, "repodata")); err == nil {
args = []string{"--update", repoDir}
}
if err := sh.RunV("createrepo_c", args...); err != nil {
return fmt.Errorf("createrepo_c for %s: %w", arch, err)
}
if err := sh.RunV("gpg",
"--default-key", gpgKey,
"--batch", "--yes",
"--passphrase", gpgPassphrase,
"--pinentry-mode", "loopback",
"--detach-sign", "--armor",
"-o", filepath.Join(repoDir, "repodata", "repomd.xml.asc"),
filepath.Join(repoDir, "repodata", "repomd.xml"),
); err != nil {
return fmt.Errorf("signing repomd.xml for %s: %w", arch, err)
}
}
fmt.Println("RPM repo metadata generated in", outputBase)
return nil
}
// RepoPacman generates a Pacman repository (repo-add) per arch.
func (Release) RepoPacman(ctx context.Context) error {
suite := repoSuite()
incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming")
outputBase := filepath.Join(repoRootDist, "repo-output", "pacman", suite)
gpgKey := os.Getenv("RELEASE_GPG_KEY")
gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE")
for _, arch := range []string{"x86_64", "aarch64", "armv7"} {
repoDir := filepath.Join(outputBase, arch)
if err := os.MkdirAll(repoDir, 0o755); err != nil {
return err
}
pkgs, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".archlinux"))
if len(pkgs) == 0 {
continue
}
for _, pkg := range pkgs {
abs, _ := filepath.Abs(pkg)
dst := filepath.Join(repoDir, filepath.Base(pkg))
_ = os.Remove(dst)
if err := os.Symlink(abs, dst); err != nil {
return err
}
}
dbPath := filepath.Join(repoDir, "vikunja.db.tar.gz")
repoPkgs, _ := filepath.Glob(filepath.Join(repoDir, "*.archlinux"))
repoAddArgs := append([]string{dbPath}, repoPkgs...)
if err := sh.RunV("repo-add", repoAddArgs...); err != nil {
return fmt.Errorf("repo-add for %s: %w", arch, err)
}
for _, name := range []string{"vikunja.db", "vikunja.files"} {
link := filepath.Join(repoDir, name)
_ = os.Remove(link)
if err := os.Symlink(name+".tar.gz", link); err != nil {
return fmt.Errorf("creating symlink %s: %w", name, err)
}
}
if err := sh.RunV("gpg",
"--default-key", gpgKey,
"--batch", "--yes",
"--passphrase", gpgPassphrase,
"--pinentry-mode", "loopback",
"--detach-sign",
"-o", filepath.Join(repoDir, "vikunja.db.sig"),
dbPath,
); err != nil {
return fmt.Errorf("signing db for %s: %w", arch, err)
}
}
fmt.Println("Pacman repo metadata generated in", outputBase)
return nil
}
// -----------------------------------------------------------------------------
// pipeline internals
const (
distSubdir = "dist"
subBin = "binaries"
subRelease = "release"
subZip = "zip"
// repoRootDist is where the repo-publish targets read and write — it's
// the dist/ directory at the repo root, not under build/. The CI
// populates dist/repo-work/incoming with packages from every project.
repoRootDist = "../dist"
)
func projectDist(p *project, sub string) string {
return filepath.Join(p.Root, distSubdir, sub)
}
func releaseDirs(p *project) error {
for _, d := range []string{subBin, subRelease, subZip} {
if err := os.MkdirAll(projectDist(p, d), 0o755); err != nil {
return err
}
}
return nil
}
func prepareXgo(_ context.Context) error {
if _, err := exec.LookPath("xgo"); err != nil {
fmt.Println("xgo not found, installing src.techknowlogick.com/xgo...")
if err := sh.RunV("go", "install", "src.techknowlogick.com/xgo@latest"); err != nil {
return fmt.Errorf("installing xgo: %w", err)
}
}
fmt.Println("Pulling latest xgo docker image...")
return sh.RunV("docker", "pull", "ghcr.io/techknowlogick/xgo:latest")
}
func xgoOutName(p *project, version string) string {
if v := os.Getenv("XGO_OUT_NAME"); v != "" {
return v
}
return p.Executable + "-" + versionTagOrUnstable(version)
}
func runXgo(ctx context.Context, p *project, version, targets string) error {
extraLdflags := `-linkmode external -extldflags "-static" `
// xgo's darwin builds can't use the static external linker.
if strings.HasPrefix(targets, "darwin") {
extraLdflags = ""
}
// xgo resolves its last arg as a Go package path. Running it from build/
// with `../` confuses the module resolution (it tries to find a package
// inside this build module). Invoke xgo from the project root so we can
// pass p.BuildPath ("." or "./cmd/veans") just like the original
// per-project magefiles did.
absRoot, err := filepath.Abs(p.Root)
if err != nil {
return fmt.Errorf("resolve project root: %w", err)
}
absDest, err := filepath.Abs(projectDist(p, subBin))
if err != nil {
return fmt.Errorf("resolve dest dir: %w", err)
}
//nolint:gosec // mage helper; args are derived from the static project table above.
cmd := exec.CommandContext(ctx, "xgo",
"-dest", absDest,
"-tags", p.BuildTags,
"-ldflags", extraLdflags+p.Ldflags(version),
"-targets", targets,
"-out", xgoOutName(p, version),
p.BuildPath,
)
cmd.Dir = absRoot
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
func xgoAllOS(ctx context.Context, p *project, version string) error {
groups := []string{
"windows/*",
strings.Join([]string{
"linux/amd64",
"linux/arm-5",
"linux/arm-6",
"linux/arm-7",
"linux/arm64",
"linux/mips",
"linux/mipsle",
"linux/mips64",
"linux/mips64le",
"linux/riscv64",
}, ","),
"darwin-10.15/*",
}
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
)
record := func(err error) {
if err == nil {
return
}
mu.Lock()
if firstErr == nil {
firstErr = err
}
mu.Unlock()
}
for _, targets := range groups {
wg.Add(1)
go func(t string) {
defer wg.Done()
record(runXgo(ctx, p, version, t))
}(targets)
}
wg.Wait()
return firstErr
}
// compressBinaries runs upx -9 over each binary that upx can handle. The skip
// list matches the parent magefile's behavior.
func compressBinaries(p *project) error {
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
)
record := func(err error) {
if err == nil {
return
}
mu.Lock()
if firstErr == nil {
firstErr = err
}
mu.Unlock()
}
walkErr := filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
name := info.Name()
if !strings.Contains(name, p.Executable) {
return nil
}
if strings.Contains(name, "mips") ||
strings.Contains(name, "s390x") ||
strings.Contains(name, "riscv64") ||
strings.Contains(name, "darwin") ||
(strings.Contains(name, "windows") && strings.Contains(name, "arm64")) {
return nil
}
wg.Add(1)
go func(pp string) {
defer wg.Done()
if err := sh.RunV("chmod", "+x", pp); err != nil {
record(err)
return
}
record(sh.RunV("upx", "-9", pp))
}(path)
return nil
})
if walkErr != nil {
return walkErr
}
wg.Wait()
return firstErr
}
func copyBinaries(p *project) error {
return filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if !strings.Contains(info.Name(), p.Executable) {
return nil
}
return copyFile(path, filepath.Join(projectDist(p, subRelease), info.Name()))
})
}
func writeChecksums(p *project) error {
release := projectDist(p, subRelease)
return filepath.Walk(release, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if strings.HasSuffix(info.Name(), ".sha256") {
return nil
}
sum, err := sha256File(path)
if err != nil {
return err
}
return os.WriteFile(path+".sha256", []byte(sum+" "+info.Name()+"\n"), 0o644)
})
}
func bundleOsPackages(p *project) error {
release := projectDist(p, subRelease)
bins := map[string]os.FileInfo{}
if err := filepath.Walk(release, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
if strings.HasSuffix(info.Name(), ".sha256") {
return nil
}
bins[path] = info
return nil
}); err != nil {
return err
}
for binPath, info := range bins {
folder := filepath.Join(release, info.Name()+"-full")
if err := os.MkdirAll(folder, 0o755); err != nil {
return err
}
if err := moveFile(binPath+".sha256", filepath.Join(folder, info.Name()+".sha256")); err != nil {
return err
}
if err := moveFile(binPath, filepath.Join(folder, info.Name())); err != nil {
return err
}
if p.OsPackageExtras != nil {
if err := p.OsPackageExtras(folder, p); err != nil {
return err
}
}
}
return nil
}
func zipBundles(ctx context.Context, p *project) error {
zipDirAbs, err := filepath.Abs(projectDist(p, subZip))
if err != nil {
return err
}
release := projectDist(p, subRelease)
return filepath.Walk(release, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() || filepath.Base(path) == subRelease {
return nil
}
fmt.Printf("Zipping %s...\n", info.Name())
zipFile := filepath.Join(zipDirAbs, info.Name()+".zip")
//nolint:gosec // mage helper; args derive from the local filesystem walk above.
c := exec.CommandContext(ctx, "zip", "-r", zipFile, ".", "-i", "*")
c.Dir = path
c.Stdout, c.Stderr = os.Stdout, os.Stderr
return c.Run()
})
}
// repoSuite validates the REPO_SUITE env var; defaults to "stable". Limiting
// the values prevents path traversal via the suite name flowing into a
// filesystem path.
func repoSuite() string {
switch os.Getenv("REPO_SUITE") {
case "stable", "unstable":
return os.Getenv("REPO_SUITE")
default:
return "stable"
}
}
// -----------------------------------------------------------------------------
// helpers — duplicated from the project magefiles so this module depends on
// nothing but stdlib + mage. Don't import these from elsewhere; rewrite them
// here if they need to change.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
if _, err := io.Copy(out, in); err != nil {
return err
}
si, err := os.Stat(src)
if err != nil {
return err
}
if err := os.Chmod(dst, si.Mode()); err != nil {
return err
}
return out.Close()
}
func moveFile(src, dst string) error {
if err := copyFile(src, dst); err != nil {
return err
}
return os.Remove(src)
}
func sha256File(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
// Aliases for kebab-case spelling at the CLI.
var Aliases = map[string]any{
"release": Release.Build,
"release:build": Release.Build,
"release:xgo": Release.Xgo,
"release:prepare-nfpm-config": Release.PrepareNFPMConfig,
"release:repo-apt": Release.RepoApt,
"release:repo-rpm": Release.RepoRpm,
"release:repo-pacman": Release.RepoPacman,
}