diff --git a/build/go.mod b/build/go.mod new file mode 100644 index 000000000..17999956b --- /dev/null +++ b/build/go.mod @@ -0,0 +1,5 @@ +module code.vikunja.io/build + +go 1.25.0 + +require github.com/magefile/mage v1.17.2 diff --git a/build/go.sum b/build/go.sum new file mode 100644 index 000000000..a4324314b --- /dev/null +++ b/build/go.sum @@ -0,0 +1,2 @@ +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= diff --git a/build/magefile.go b/build/magefile.go new file mode 100644 index 000000000..d76990d32 --- /dev/null +++ b/build/magefile.go @@ -0,0 +1,757 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build mage + +// Centralized release pipeline for every Go binary in this monorepo. +// +// Both vikunja and veans cross-compile through the same code: xgo for the full +// OS/arch matrix, upx where the binary supports it, sha256 alongside each +// artifact, per-target zip bundle, and nfpm.yaml templating for deb/rpm/apk/ +// archlinux packaging. Repository-metadata targets (apt/rpm/pacman) consume +// the merged ../dist/repo-work/incoming/ tree the CI populates from both +// projects' packages. +// +// The module is intentionally separate from the project magefiles so the +// release tooling can evolve without touching them. The small filesystem +// helpers (copyFile, moveFile, sha256File) are duplicated rather than +// imported — this magefile depends on nothing but stdlib + mage. +package main + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// ----------------------------------------------------------------------------- +// project definitions + +// project describes one releasable Go binary in this monorepo. Adding a new +// project means adding an entry to projectByName plus a constructor below. +type project struct { + // Name is the short identifier used on the CLI: `mage release:build `. + Name string + // Root is the project root, relative to this build/ directory. + Root string + // BuildPath is the Go package to build, relative to Root (e.g. "." or "./cmd/foo"). + BuildPath string + // Executable is the output binary name (sans -- suffix). + Executable string + // BuildTags are the base build tags applied to every cross-compile. + BuildTags string + // Ldflags returns the full -X flag string for the given version. + Ldflags func(version string) string + // NfpmConfigPath is the nfpm.yaml location, relative to Root. + NfpmConfigPath string + // NfpmBinPathDefault is the default substitution. Empty + // means use the Executable name as-is. + NfpmBinPathDefault string + // OsPackageExtras hook copies any extra files (LICENSE, sample config…) + // into each per-target bundle folder. Called once per binary. + OsPackageExtras func(folder string, p *project) error +} + +func projectByName(name string) (*project, error) { + switch name { + case "vikunja": + return vikunjaProject(), nil + case "veans": + return veansProject(), nil + default: + return nil, fmt.Errorf("unknown project %q (known: vikunja, veans)", name) + } +} + +func vikunjaProject() *project { + return &project{ + Name: "vikunja", + Root: "../", + BuildPath: ".", + Executable: "vikunja", + BuildTags: "osusergo netgo", + Ldflags: func(v string) string { + // Matches the parent magefile's pre-refactor ldflags. The + // main.Tags value is the literal build-tag string baked in + // for `vikunja info` to report. + return fmt.Sprintf(`-X "code.vikunja.io/api/pkg/version.Version=%s" -X "main.Tags=osusergo netgo"`, v) + }, + NfpmConfigPath: "nfpm.yaml", + NfpmBinPathDefault: "vikunja", + OsPackageExtras: func(folder string, p *project) error { + // config.yml.sample must be generated by the CI (or local dev) + // before this runs — we don't want to vendor the + // config-raw.json→YAML logic. The workflow does + // `mage generate:config-yaml 1` in the project root before + // invoking release:build. + if err := copyFile(filepath.Join(p.Root, "config.yml.sample"), filepath.Join(folder, "config.yml.sample")); err != nil { + return fmt.Errorf("copy config.yml.sample (run `mage generate:config-yaml 1` first): %w", err) + } + return copyFile(filepath.Join(p.Root, "LICENSE"), filepath.Join(folder, "LICENSE")) + }, + } +} + +func veansProject() *project { + return &project{ + Name: "veans", + Root: "../veans/", + BuildPath: "./cmd/veans", + Executable: "veans", + BuildTags: "osusergo netgo", + Ldflags: func(v string) string { + return fmt.Sprintf(`-X main.version=%s`, v) + }, + NfpmConfigPath: "nfpm.yaml", + NfpmBinPathDefault: "./veans", + OsPackageExtras: func(folder string, _ *project) error { + // veans intentionally doesn't carry its own LICENSE — the + // AGPLv3 at the repo root applies to both. + return copyFile("../LICENSE", filepath.Join(folder, "LICENSE")) + }, + } +} + +// ----------------------------------------------------------------------------- +// version resolution + +func releaseVersion(ctx context.Context) (string, error) { + if v := os.Getenv("RELEASE_VERSION"); v != "" { + return v, nil + } + out, err := exec.CommandContext(ctx, "git", "describe", "--tags", "--always", "--abbrev=10").Output() + if err != nil { + return "", fmt.Errorf("git describe: %w", err) + } + return strings.Replace(strings.TrimSpace(string(out)), "-g", "-", 1), nil +} + +func versionTagOrUnstable(v string) string { + switch v { + case "", "main": + return "unstable" + default: + return v + } +} + +// ----------------------------------------------------------------------------- +// Release namespace + +type Release mg.Namespace + +// Build runs the full release pipeline for the named project: dirs → xgo +// (windows/linux/darwin in parallel) → upx → copy → sha256 → per-target +// bundle dir → zip. +func (Release) Build(ctx context.Context, name string) error { + p, err := projectByName(name) + if err != nil { + return err + } + version, err := releaseVersion(ctx) + if err != nil { + return err + } + if err := releaseDirs(p); err != nil { + return err + } + if err := prepareXgo(ctx); err != nil { + return err + } + if err := xgoAllOS(ctx, p, version); err != nil { + return err + } + if err := compressBinaries(p); err != nil { + return err + } + if err := copyBinaries(p); err != nil { + return err + } + if err := writeChecksums(p); err != nil { + return err + } + if err := bundleOsPackages(p); err != nil { + return err + } + return zipBundles(ctx, p) +} + +// Xgo cross-compiles a single os/arch[/variant] target for the named project. +// Variant follows the parent magefile convention: `linux/arm/7` → arm-7. +// +// Unlike Release.Build, this skips prepareXgo on purpose: the only caller +// that hits this path in CI is the Dockerfile, which runs inside the xgo +// image (xgo binary already present, docker daemon not available). Local +// users invoking `mage release:xgo` need to install xgo themselves. +func (Release) Xgo(ctx context.Context, name, target string) error { + p, err := projectByName(name) + if err != nil { + return err + } + version, err := releaseVersion(ctx) + if err != nil { + return err + } + parts := strings.Split(target, "/") + if len(parts) < 2 { + return fmt.Errorf("invalid target %q (expected os/arch[/variant])", target) + } + variant := "" + if len(parts) > 2 && parts[2] != "" { + variant = "-" + strings.ReplaceAll(parts[2], "v", "") + } + return runXgo(ctx, p, version, parts[0]+"/"+parts[1]+variant) +} + +// PrepareNFPMConfig templates the named project's nfpm.yaml in place for the +// given nfpm arch (amd64|arm64|arm7|386). Destructive — CI checks out a fresh +// copy per matrix shard so the trampling is fine. +func (Release) PrepareNFPMConfig(ctx context.Context, name, arch string) error { + p, err := projectByName(name) + if err != nil { + return err + } + version, err := releaseVersion(ctx) + if err != nil { + return err + } + cfgPath := filepath.Join(p.Root, p.NfpmConfigPath) + raw, err := os.ReadFile(cfgPath) + if err != nil { + return err + } + binLocation := os.Getenv("NFPM_BIN_PATH") + if binLocation == "" { + binLocation = p.NfpmBinPathDefault + if binLocation == "" { + binLocation = p.Executable + } + } + out := strings.ReplaceAll(string(raw), "", version) + out = strings.ReplaceAll(out, "", arch) + out = strings.ReplaceAll(out, "", binLocation) + return os.WriteFile(cfgPath, []byte(out), 0o600) +} + +// ----------------------------------------------------------------------------- +// Repo-metadata targets — project-agnostic; operate on the merged tree at +// ../dist/repo-work/incoming and ../dist/repo-output. + +// RepoApt generates an APT repository (reprepro) for every .deb in the +// incoming tree. REPO_SUITE (stable|unstable) selects the target suite; +// RELEASE_GPG_KEY + RELEASE_GPG_PASSPHRASE drive the Release file signing. +func (Release) RepoApt(ctx context.Context) error { + suite := repoSuite() + incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") + outputBase := filepath.Join(repoRootDist, "repo-output", "apt") + confDir := filepath.Join(outputBase, "conf") + if err := os.MkdirAll(confDir, 0o755); err != nil { + return fmt.Errorf("creating reprepro conf dir: %w", err) + } + distConf, err := os.ReadFile("reprepro-dist-conf") + if err != nil { + return fmt.Errorf("reading reprepro-dist-conf: %w", err) + } + if err := os.WriteFile(filepath.Join(confDir, "distributions"), distConf, 0o600); err != nil { + return fmt.Errorf("writing distributions config: %w", err) + } + + debs, err := filepath.Glob(filepath.Join(incomingDir, "*.deb")) + if err != nil { + return err + } + for _, deb := range debs { + abs, _ := filepath.Abs(deb) + if err := sh.RunV("reprepro", "-b", outputBase, "includedeb", suite, abs); err != nil { + return fmt.Errorf("reprepro includedeb %s: %w", filepath.Base(deb), err) + } + } + + gpgKey := os.Getenv("RELEASE_GPG_KEY") + gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") + releaseFile := filepath.Join(outputBase, "dists", suite, "Release") + if _, err := os.Stat(releaseFile); err == nil { + if err := sh.RunV("gpg", + "--default-key", gpgKey, + "--batch", "--yes", + "--passphrase", gpgPassphrase, + "--pinentry-mode", "loopback", + "--detach-sign", "--armor", + "-o", releaseFile+".gpg", + releaseFile, + ); err != nil { + return fmt.Errorf("signing Release (detached): %w", err) + } + if err := sh.RunV("gpg", + "--default-key", gpgKey, + "--batch", "--yes", + "--passphrase", gpgPassphrase, + "--pinentry-mode", "loopback", + "--clearsign", + "-o", filepath.Join(filepath.Dir(releaseFile), "InRelease"), + releaseFile, + ); err != nil { + return fmt.Errorf("signing Release (clearsign): %w", err) + } + } + fmt.Println("APT repo metadata generated in", outputBase) + return nil +} + +// RepoRpm generates an RPM repository (createrepo_c) per arch in +// ../dist/repo-work/incoming/. +func (Release) RepoRpm(ctx context.Context) error { + suite := repoSuite() + incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") + outputBase := filepath.Join(repoRootDist, "repo-output", "rpm", suite) + gpgKey := os.Getenv("RELEASE_GPG_KEY") + gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") + + for _, arch := range []string{"x86_64", "aarch64", "armv7"} { + repoDir := filepath.Join(outputBase, arch) + if err := os.MkdirAll(repoDir, 0o755); err != nil { + return err + } + rpms, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".rpm")) + if len(rpms) == 0 { + continue + } + for _, rpm := range rpms { + abs, _ := filepath.Abs(rpm) + dst := filepath.Join(repoDir, filepath.Base(rpm)) + _ = os.Remove(dst) + if err := os.Symlink(abs, dst); err != nil { + return err + } + } + args := []string{repoDir} + if _, err := os.Stat(filepath.Join(repoDir, "repodata")); err == nil { + args = []string{"--update", repoDir} + } + if err := sh.RunV("createrepo_c", args...); err != nil { + return fmt.Errorf("createrepo_c for %s: %w", arch, err) + } + if err := sh.RunV("gpg", + "--default-key", gpgKey, + "--batch", "--yes", + "--passphrase", gpgPassphrase, + "--pinentry-mode", "loopback", + "--detach-sign", "--armor", + "-o", filepath.Join(repoDir, "repodata", "repomd.xml.asc"), + filepath.Join(repoDir, "repodata", "repomd.xml"), + ); err != nil { + return fmt.Errorf("signing repomd.xml for %s: %w", arch, err) + } + } + fmt.Println("RPM repo metadata generated in", outputBase) + return nil +} + +// RepoPacman generates a Pacman repository (repo-add) per arch. +func (Release) RepoPacman(ctx context.Context) error { + suite := repoSuite() + incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") + outputBase := filepath.Join(repoRootDist, "repo-output", "pacman", suite) + gpgKey := os.Getenv("RELEASE_GPG_KEY") + gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") + + for _, arch := range []string{"x86_64", "aarch64", "armv7"} { + repoDir := filepath.Join(outputBase, arch) + if err := os.MkdirAll(repoDir, 0o755); err != nil { + return err + } + pkgs, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".archlinux")) + if len(pkgs) == 0 { + continue + } + for _, pkg := range pkgs { + abs, _ := filepath.Abs(pkg) + dst := filepath.Join(repoDir, filepath.Base(pkg)) + _ = os.Remove(dst) + if err := os.Symlink(abs, dst); err != nil { + return err + } + } + dbPath := filepath.Join(repoDir, "vikunja.db.tar.gz") + repoPkgs, _ := filepath.Glob(filepath.Join(repoDir, "*.archlinux")) + repoAddArgs := append([]string{dbPath}, repoPkgs...) + if err := sh.RunV("repo-add", repoAddArgs...); err != nil { + return fmt.Errorf("repo-add for %s: %w", arch, err) + } + for _, name := range []string{"vikunja.db", "vikunja.files"} { + link := filepath.Join(repoDir, name) + _ = os.Remove(link) + if err := os.Symlink(name+".tar.gz", link); err != nil { + return fmt.Errorf("creating symlink %s: %w", name, err) + } + } + if err := sh.RunV("gpg", + "--default-key", gpgKey, + "--batch", "--yes", + "--passphrase", gpgPassphrase, + "--pinentry-mode", "loopback", + "--detach-sign", + "-o", filepath.Join(repoDir, "vikunja.db.sig"), + dbPath, + ); err != nil { + return fmt.Errorf("signing db for %s: %w", arch, err) + } + } + fmt.Println("Pacman repo metadata generated in", outputBase) + return nil +} + +// ----------------------------------------------------------------------------- +// pipeline internals + +const ( + distSubdir = "dist" + subBin = "binaries" + subRelease = "release" + subZip = "zip" + + // repoRootDist is where the repo-publish targets read and write — it's + // the dist/ directory at the repo root, not under build/. The CI + // populates dist/repo-work/incoming with packages from every project. + repoRootDist = "../dist" +) + +func projectDist(p *project, sub string) string { + return filepath.Join(p.Root, distSubdir, sub) +} + +func releaseDirs(p *project) error { + for _, d := range []string{subBin, subRelease, subZip} { + if err := os.MkdirAll(projectDist(p, d), 0o755); err != nil { + return err + } + } + return nil +} + +func prepareXgo(_ context.Context) error { + if _, err := exec.LookPath("xgo"); err != nil { + fmt.Println("xgo not found, installing src.techknowlogick.com/xgo...") + if err := sh.RunV("go", "install", "src.techknowlogick.com/xgo@latest"); err != nil { + return fmt.Errorf("installing xgo: %w", err) + } + } + fmt.Println("Pulling latest xgo docker image...") + return sh.RunV("docker", "pull", "ghcr.io/techknowlogick/xgo:latest") +} + +func xgoOutName(p *project, version string) string { + if v := os.Getenv("XGO_OUT_NAME"); v != "" { + return v + } + return p.Executable + "-" + versionTagOrUnstable(version) +} + +func runXgo(ctx context.Context, p *project, version, targets string) error { + extraLdflags := `-linkmode external -extldflags "-static" ` + // xgo's darwin builds can't use the static external linker. + if strings.HasPrefix(targets, "darwin") { + extraLdflags = "" + } + // xgo resolves its last arg as a Go package path. Running it from build/ + // with `../` confuses the module resolution (it tries to find a package + // inside this build module). Invoke xgo from the project root so we can + // pass p.BuildPath ("." or "./cmd/veans") just like the original + // per-project magefiles did. + absRoot, err := filepath.Abs(p.Root) + if err != nil { + return fmt.Errorf("resolve project root: %w", err) + } + absDest, err := filepath.Abs(projectDist(p, subBin)) + if err != nil { + return fmt.Errorf("resolve dest dir: %w", err) + } + //nolint:gosec // mage helper; args are derived from the static project table above. + cmd := exec.CommandContext(ctx, "xgo", + "-dest", absDest, + "-tags", p.BuildTags, + "-ldflags", extraLdflags+p.Ldflags(version), + "-targets", targets, + "-out", xgoOutName(p, version), + p.BuildPath, + ) + cmd.Dir = absRoot + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func xgoAllOS(ctx context.Context, p *project, version string) error { + groups := []string{ + "windows/*", + strings.Join([]string{ + "linux/amd64", + "linux/arm-5", + "linux/arm-6", + "linux/arm-7", + "linux/arm64", + "linux/mips", + "linux/mipsle", + "linux/mips64", + "linux/mips64le", + "linux/riscv64", + }, ","), + "darwin-10.15/*", + } + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + ) + record := func(err error) { + if err == nil { + return + } + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + for _, targets := range groups { + wg.Add(1) + go func(t string) { + defer wg.Done() + record(runXgo(ctx, p, version, t)) + }(targets) + } + wg.Wait() + return firstErr +} + +// compressBinaries runs upx -9 over each binary that upx can handle. The skip +// list matches the parent magefile's behavior. +func compressBinaries(p *project) error { + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + ) + record := func(err error) { + if err == nil { + return + } + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + walkErr := filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + name := info.Name() + if !strings.Contains(name, p.Executable) { + return nil + } + if strings.Contains(name, "mips") || + strings.Contains(name, "s390x") || + strings.Contains(name, "riscv64") || + strings.Contains(name, "darwin") || + (strings.Contains(name, "windows") && strings.Contains(name, "arm64")) { + return nil + } + wg.Add(1) + go func(pp string) { + defer wg.Done() + if err := sh.RunV("chmod", "+x", pp); err != nil { + record(err) + return + } + record(sh.RunV("upx", "-9", pp)) + }(path) + return nil + }) + if walkErr != nil { + return walkErr + } + wg.Wait() + return firstErr +} + +func copyBinaries(p *project) error { + return filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + if !strings.Contains(info.Name(), p.Executable) { + return nil + } + return copyFile(path, filepath.Join(projectDist(p, subRelease), info.Name())) + }) +} + +func writeChecksums(p *project) error { + release := projectDist(p, subRelease) + return filepath.Walk(release, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + if strings.HasSuffix(info.Name(), ".sha256") { + return nil + } + sum, err := sha256File(path) + if err != nil { + return err + } + return os.WriteFile(path+".sha256", []byte(sum+" "+info.Name()+"\n"), 0o644) + }) +} + +func bundleOsPackages(p *project) error { + release := projectDist(p, subRelease) + bins := map[string]os.FileInfo{} + if err := filepath.Walk(release, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + if strings.HasSuffix(info.Name(), ".sha256") { + return nil + } + bins[path] = info + return nil + }); err != nil { + return err + } + for binPath, info := range bins { + folder := filepath.Join(release, info.Name()+"-full") + if err := os.MkdirAll(folder, 0o755); err != nil { + return err + } + if err := moveFile(binPath+".sha256", filepath.Join(folder, info.Name()+".sha256")); err != nil { + return err + } + if err := moveFile(binPath, filepath.Join(folder, info.Name())); err != nil { + return err + } + if p.OsPackageExtras != nil { + if err := p.OsPackageExtras(folder, p); err != nil { + return err + } + } + } + return nil +} + +func zipBundles(ctx context.Context, p *project) error { + zipDirAbs, err := filepath.Abs(projectDist(p, subZip)) + if err != nil { + return err + } + release := projectDist(p, subRelease) + return filepath.Walk(release, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() || filepath.Base(path) == subRelease { + return nil + } + fmt.Printf("Zipping %s...\n", info.Name()) + zipFile := filepath.Join(zipDirAbs, info.Name()+".zip") + //nolint:gosec // mage helper; args derive from the local filesystem walk above. + c := exec.CommandContext(ctx, "zip", "-r", zipFile, ".", "-i", "*") + c.Dir = path + c.Stdout, c.Stderr = os.Stdout, os.Stderr + return c.Run() + }) +} + +// repoSuite validates the REPO_SUITE env var; defaults to "stable". Limiting +// the values prevents path traversal via the suite name flowing into a +// filesystem path. +func repoSuite() string { + switch os.Getenv("REPO_SUITE") { + case "stable", "unstable": + return os.Getenv("REPO_SUITE") + default: + return "stable" + } +} + +// ----------------------------------------------------------------------------- +// helpers — duplicated from the project magefiles so this module depends on +// nothing but stdlib + mage. Don't import these from elsewhere; rewrite them +// here if they need to change. + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + if _, err := io.Copy(out, in); err != nil { + return err + } + si, err := os.Stat(src) + if err != nil { + return err + } + if err := os.Chmod(dst, si.Mode()); err != nil { + return err + } + return out.Close() +} + +func moveFile(src, dst string) error { + if err := copyFile(src, dst); err != nil { + return err + } + return os.Remove(src) +} + +func sha256File(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// Aliases for kebab-case spelling at the CLI. +var Aliases = map[string]any{ + "release": Release.Build, + "release:build": Release.Build, + "release:xgo": Release.Xgo, + "release:prepare-nfpm-config": Release.PrepareNFPMConfig, + "release:repo-apt": Release.RepoApt, + "release:repo-rpm": Release.RepoRpm, + "release:repo-pacman": Release.RepoPacman, +}