// Vikunja is a to-do list application to facilitate your life. // Copyright 2018-present Vikunja and contributors. All rights reserved. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . //go:build mage // Centralized release pipeline for every Go binary in this monorepo. // // Both vikunja and veans cross-compile through the same code: xgo for the full // OS/arch matrix, upx where the binary supports it, sha256 alongside each // artifact, per-target zip bundle, and nfpm.yaml templating for deb/rpm/apk/ // archlinux packaging. Repository-metadata targets (apt/rpm/pacman) consume // the merged ../dist/repo-work/incoming/ tree the CI populates from both // projects' packages. // // The module is intentionally separate from the project magefiles so the // release tooling can evolve without touching them. The small filesystem // helpers (copyFile, moveFile, sha256File) are duplicated rather than // imported — this magefile depends on nothing but stdlib + mage. package main import ( "context" "crypto/sha256" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "sync" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" ) // ----------------------------------------------------------------------------- // project definitions // project describes one releasable Go binary in this monorepo. Adding a new // project means adding an entry to projectByName plus a constructor below. type project struct { // Name is the short identifier used on the CLI: `mage release:build `. Name string // Root is the project root, relative to this build/ directory. Root string // BuildPath is the Go package to build, relative to Root (e.g. "." or "./cmd/foo"). BuildPath string // Executable is the output binary name (sans -- suffix). Executable string // BuildTags are the base build tags applied to every cross-compile. BuildTags string // Ldflags returns the full -X flag string for the given version. Ldflags func(version string) string // NfpmConfigPath is the nfpm.yaml location, relative to Root. NfpmConfigPath string // NfpmBinPathDefault is the default substitution. Empty // means use the Executable name as-is. NfpmBinPathDefault string // OsPackageExtras hook copies any extra files (LICENSE, sample config…) // into each per-target bundle folder. Called once per binary. OsPackageExtras func(folder string, p *project) error } func projectByName(name string) (*project, error) { switch name { case "vikunja": return vikunjaProject(), nil case "veans": return veansProject(), nil default: return nil, fmt.Errorf("unknown project %q (known: vikunja, veans)", name) } } func vikunjaProject() *project { return &project{ Name: "vikunja", Root: "../", BuildPath: ".", Executable: "vikunja", BuildTags: "osusergo netgo", Ldflags: func(v string) string { // Matches the parent magefile's pre-refactor ldflags. The // main.Tags value is the literal build-tag string baked in // for `vikunja info` to report. return fmt.Sprintf(`-X "code.vikunja.io/api/pkg/version.Version=%s" -X "main.Tags=osusergo netgo"`, v) }, NfpmConfigPath: "nfpm.yaml", NfpmBinPathDefault: "vikunja", OsPackageExtras: func(folder string, p *project) error { // config.yml.sample must be generated by the CI (or local dev) // before this runs — we don't want to vendor the // config-raw.json→YAML logic. The workflow does // `mage generate:config-yaml 1` in the project root before // invoking release:build. if err := copyFile(filepath.Join(p.Root, "config.yml.sample"), filepath.Join(folder, "config.yml.sample")); err != nil { return fmt.Errorf("copy config.yml.sample (run `mage generate:config-yaml 1` first): %w", err) } return copyFile(filepath.Join(p.Root, "LICENSE"), filepath.Join(folder, "LICENSE")) }, } } func veansProject() *project { return &project{ Name: "veans", Root: "../veans/", BuildPath: "./cmd/veans", Executable: "veans", BuildTags: "osusergo netgo", Ldflags: func(v string) string { return fmt.Sprintf(`-X main.version=%s`, v) }, NfpmConfigPath: "nfpm.yaml", NfpmBinPathDefault: "./veans", OsPackageExtras: func(folder string, _ *project) error { // veans intentionally doesn't carry its own LICENSE — the // AGPLv3 at the repo root applies to both. return copyFile("../LICENSE", filepath.Join(folder, "LICENSE")) }, } } // ----------------------------------------------------------------------------- // version resolution func releaseVersion(ctx context.Context) (string, error) { if v := os.Getenv("RELEASE_VERSION"); v != "" { return v, nil } out, err := exec.CommandContext(ctx, "git", "describe", "--tags", "--always", "--abbrev=10").Output() if err != nil { return "", fmt.Errorf("git describe: %w", err) } return strings.Replace(strings.TrimSpace(string(out)), "-g", "-", 1), nil } func versionTagOrUnstable(v string) string { switch v { case "", "main": return "unstable" default: return v } } // ----------------------------------------------------------------------------- // Release namespace type Release mg.Namespace // Build runs the full release pipeline for the named project: dirs → xgo // (windows/linux/darwin in parallel) → upx → copy → sha256 → per-target // bundle dir → zip. func (Release) Build(ctx context.Context, name string) error { p, err := projectByName(name) if err != nil { return err } version, err := releaseVersion(ctx) if err != nil { return err } if err := releaseDirs(p); err != nil { return err } if err := prepareXgo(ctx); err != nil { return err } if err := xgoAllOS(ctx, p, version); err != nil { return err } if err := compressBinaries(p); err != nil { return err } if err := copyBinaries(p); err != nil { return err } if err := writeChecksums(p); err != nil { return err } if err := bundleOsPackages(p); err != nil { return err } return zipBundles(ctx, p) } // Xgo cross-compiles a single os/arch[/variant] target for the named project. // Variant follows the parent magefile convention: `linux/arm/7` → arm-7. // // Unlike Release.Build, this skips prepareXgo on purpose: the only caller // that hits this path in CI is the Dockerfile, which runs inside the xgo // image (xgo binary already present, docker daemon not available). Local // users invoking `mage release:xgo` need to install xgo themselves. func (Release) Xgo(ctx context.Context, name, target string) error { p, err := projectByName(name) if err != nil { return err } version, err := releaseVersion(ctx) if err != nil { return err } parts := strings.Split(target, "/") if len(parts) < 2 { return fmt.Errorf("invalid target %q (expected os/arch[/variant])", target) } variant := "" if len(parts) > 2 && parts[2] != "" { variant = "-" + strings.ReplaceAll(parts[2], "v", "") } return runXgo(ctx, p, version, parts[0]+"/"+parts[1]+variant) } // PrepareNFPMConfig templates the named project's nfpm.yaml in place for the // given nfpm arch (amd64|arm64|arm7|386). Destructive — CI checks out a fresh // copy per matrix shard so the trampling is fine. func (Release) PrepareNFPMConfig(ctx context.Context, name, arch string) error { p, err := projectByName(name) if err != nil { return err } version, err := releaseVersion(ctx) if err != nil { return err } cfgPath := filepath.Join(p.Root, p.NfpmConfigPath) raw, err := os.ReadFile(cfgPath) if err != nil { return err } binLocation := os.Getenv("NFPM_BIN_PATH") if binLocation == "" { binLocation = p.NfpmBinPathDefault if binLocation == "" { binLocation = p.Executable } } out := strings.ReplaceAll(string(raw), "", version) out = strings.ReplaceAll(out, "", arch) out = strings.ReplaceAll(out, "", binLocation) return os.WriteFile(cfgPath, []byte(out), 0o600) } // ----------------------------------------------------------------------------- // Repo-metadata targets — project-agnostic; operate on the merged tree at // ../dist/repo-work/incoming and ../dist/repo-output. // RepoApt generates an APT repository (reprepro) for every .deb in the // incoming tree. REPO_SUITE (stable|unstable) selects the target suite; // RELEASE_GPG_KEY + RELEASE_GPG_PASSPHRASE drive the Release file signing. func (Release) RepoApt(ctx context.Context) error { suite := repoSuite() incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") outputBase := filepath.Join(repoRootDist, "repo-output", "apt") confDir := filepath.Join(outputBase, "conf") if err := os.MkdirAll(confDir, 0o755); err != nil { return fmt.Errorf("creating reprepro conf dir: %w", err) } distConf, err := os.ReadFile("reprepro-dist-conf") if err != nil { return fmt.Errorf("reading reprepro-dist-conf: %w", err) } if err := os.WriteFile(filepath.Join(confDir, "distributions"), distConf, 0o600); err != nil { return fmt.Errorf("writing distributions config: %w", err) } debs, err := filepath.Glob(filepath.Join(incomingDir, "*.deb")) if err != nil { return err } for _, deb := range debs { abs, _ := filepath.Abs(deb) if err := sh.RunV("reprepro", "-b", outputBase, "includedeb", suite, abs); err != nil { return fmt.Errorf("reprepro includedeb %s: %w", filepath.Base(deb), err) } } gpgKey := os.Getenv("RELEASE_GPG_KEY") gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") releaseFile := filepath.Join(outputBase, "dists", suite, "Release") if _, err := os.Stat(releaseFile); err == nil { if err := sh.RunV("gpg", "--default-key", gpgKey, "--batch", "--yes", "--passphrase", gpgPassphrase, "--pinentry-mode", "loopback", "--detach-sign", "--armor", "-o", releaseFile+".gpg", releaseFile, ); err != nil { return fmt.Errorf("signing Release (detached): %w", err) } if err := sh.RunV("gpg", "--default-key", gpgKey, "--batch", "--yes", "--passphrase", gpgPassphrase, "--pinentry-mode", "loopback", "--clearsign", "-o", filepath.Join(filepath.Dir(releaseFile), "InRelease"), releaseFile, ); err != nil { return fmt.Errorf("signing Release (clearsign): %w", err) } } fmt.Println("APT repo metadata generated in", outputBase) return nil } // RepoRpm generates an RPM repository (createrepo_c) per arch in // ../dist/repo-work/incoming/. func (Release) RepoRpm(ctx context.Context) error { suite := repoSuite() incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") outputBase := filepath.Join(repoRootDist, "repo-output", "rpm", suite) gpgKey := os.Getenv("RELEASE_GPG_KEY") gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") for _, arch := range []string{"x86_64", "aarch64", "armv7"} { repoDir := filepath.Join(outputBase, arch) if err := os.MkdirAll(repoDir, 0o755); err != nil { return err } rpms, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".rpm")) if len(rpms) == 0 { continue } for _, rpm := range rpms { abs, _ := filepath.Abs(rpm) dst := filepath.Join(repoDir, filepath.Base(rpm)) _ = os.Remove(dst) if err := os.Symlink(abs, dst); err != nil { return err } } args := []string{repoDir} if _, err := os.Stat(filepath.Join(repoDir, "repodata")); err == nil { args = []string{"--update", repoDir} } if err := sh.RunV("createrepo_c", args...); err != nil { return fmt.Errorf("createrepo_c for %s: %w", arch, err) } if err := sh.RunV("gpg", "--default-key", gpgKey, "--batch", "--yes", "--passphrase", gpgPassphrase, "--pinentry-mode", "loopback", "--detach-sign", "--armor", "-o", filepath.Join(repoDir, "repodata", "repomd.xml.asc"), filepath.Join(repoDir, "repodata", "repomd.xml"), ); err != nil { return fmt.Errorf("signing repomd.xml for %s: %w", arch, err) } } fmt.Println("RPM repo metadata generated in", outputBase) return nil } // RepoPacman generates a Pacman repository (repo-add) per arch. func (Release) RepoPacman(ctx context.Context) error { suite := repoSuite() incomingDir := filepath.Join(repoRootDist, "repo-work", "incoming") outputBase := filepath.Join(repoRootDist, "repo-output", "pacman", suite) gpgKey := os.Getenv("RELEASE_GPG_KEY") gpgPassphrase := os.Getenv("RELEASE_GPG_PASSPHRASE") for _, arch := range []string{"x86_64", "aarch64", "armv7"} { repoDir := filepath.Join(outputBase, arch) if err := os.MkdirAll(repoDir, 0o755); err != nil { return err } pkgs, _ := filepath.Glob(filepath.Join(incomingDir, "*-"+arch+".archlinux")) if len(pkgs) == 0 { continue } for _, pkg := range pkgs { abs, _ := filepath.Abs(pkg) dst := filepath.Join(repoDir, filepath.Base(pkg)) _ = os.Remove(dst) if err := os.Symlink(abs, dst); err != nil { return err } } dbPath := filepath.Join(repoDir, "vikunja.db.tar.gz") repoPkgs, _ := filepath.Glob(filepath.Join(repoDir, "*.archlinux")) repoAddArgs := append([]string{dbPath}, repoPkgs...) if err := sh.RunV("repo-add", repoAddArgs...); err != nil { return fmt.Errorf("repo-add for %s: %w", arch, err) } for _, name := range []string{"vikunja.db", "vikunja.files"} { link := filepath.Join(repoDir, name) _ = os.Remove(link) if err := os.Symlink(name+".tar.gz", link); err != nil { return fmt.Errorf("creating symlink %s: %w", name, err) } } if err := sh.RunV("gpg", "--default-key", gpgKey, "--batch", "--yes", "--passphrase", gpgPassphrase, "--pinentry-mode", "loopback", "--detach-sign", "-o", filepath.Join(repoDir, "vikunja.db.sig"), dbPath, ); err != nil { return fmt.Errorf("signing db for %s: %w", arch, err) } } fmt.Println("Pacman repo metadata generated in", outputBase) return nil } // ----------------------------------------------------------------------------- // pipeline internals const ( distSubdir = "dist" subBin = "binaries" subRelease = "release" subZip = "zip" // repoRootDist is where the repo-publish targets read and write — it's // the dist/ directory at the repo root, not under build/. The CI // populates dist/repo-work/incoming with packages from every project. repoRootDist = "../dist" ) func projectDist(p *project, sub string) string { return filepath.Join(p.Root, distSubdir, sub) } func releaseDirs(p *project) error { for _, d := range []string{subBin, subRelease, subZip} { if err := os.MkdirAll(projectDist(p, d), 0o755); err != nil { return err } } return nil } func prepareXgo(_ context.Context) error { if _, err := exec.LookPath("xgo"); err != nil { fmt.Println("xgo not found, installing src.techknowlogick.com/xgo...") if err := sh.RunV("go", "install", "src.techknowlogick.com/xgo@latest"); err != nil { return fmt.Errorf("installing xgo: %w", err) } } fmt.Println("Pulling latest xgo docker image...") return sh.RunV("docker", "pull", "ghcr.io/techknowlogick/xgo:latest") } func xgoOutName(p *project, version string) string { if v := os.Getenv("XGO_OUT_NAME"); v != "" { return v } return p.Executable + "-" + versionTagOrUnstable(version) } func runXgo(ctx context.Context, p *project, version, targets string) error { extraLdflags := `-linkmode external -extldflags "-static" ` // xgo's darwin builds can't use the static external linker. if strings.HasPrefix(targets, "darwin") { extraLdflags = "" } // xgo resolves its last arg as a Go package path. Running it from build/ // with `../` confuses the module resolution (it tries to find a package // inside this build module). Invoke xgo from the project root so we can // pass p.BuildPath ("." or "./cmd/veans") just like the original // per-project magefiles did. absRoot, err := filepath.Abs(p.Root) if err != nil { return fmt.Errorf("resolve project root: %w", err) } absDest, err := filepath.Abs(projectDist(p, subBin)) if err != nil { return fmt.Errorf("resolve dest dir: %w", err) } //nolint:gosec // mage helper; args are derived from the static project table above. cmd := exec.CommandContext(ctx, "xgo", "-dest", absDest, "-tags", p.BuildTags, "-ldflags", extraLdflags+p.Ldflags(version), "-targets", targets, "-out", xgoOutName(p, version), p.BuildPath, ) cmd.Dir = absRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func xgoAllOS(ctx context.Context, p *project, version string) error { groups := []string{ "windows/*", strings.Join([]string{ "linux/amd64", "linux/arm-5", "linux/arm-6", "linux/arm-7", "linux/arm64", "linux/mips", "linux/mipsle", "linux/mips64", "linux/mips64le", "linux/riscv64", }, ","), "darwin-10.15/*", } var ( wg sync.WaitGroup mu sync.Mutex firstErr error ) record := func(err error) { if err == nil { return } mu.Lock() if firstErr == nil { firstErr = err } mu.Unlock() } for _, targets := range groups { wg.Add(1) go func(t string) { defer wg.Done() record(runXgo(ctx, p, version, t)) }(targets) } wg.Wait() return firstErr } // compressBinaries runs upx -9 over each binary that upx can handle. The skip // list matches the parent magefile's behavior. func compressBinaries(p *project) error { var ( wg sync.WaitGroup mu sync.Mutex firstErr error ) record := func(err error) { if err == nil { return } mu.Lock() if firstErr == nil { firstErr = err } mu.Unlock() } walkErr := filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } name := info.Name() if !strings.Contains(name, p.Executable) { return nil } if strings.Contains(name, "mips") || strings.Contains(name, "s390x") || strings.Contains(name, "riscv64") || strings.Contains(name, "darwin") || (strings.Contains(name, "windows") && strings.Contains(name, "arm64")) { return nil } wg.Add(1) go func(pp string) { defer wg.Done() if err := sh.RunV("chmod", "+x", pp); err != nil { record(err) return } record(sh.RunV("upx", "-9", pp)) }(path) return nil }) if walkErr != nil { return walkErr } wg.Wait() return firstErr } func copyBinaries(p *project) error { return filepath.Walk(projectDist(p, subBin), func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } if !strings.Contains(info.Name(), p.Executable) { return nil } return copyFile(path, filepath.Join(projectDist(p, subRelease), info.Name())) }) } func writeChecksums(p *project) error { release := projectDist(p, subRelease) return filepath.Walk(release, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } if strings.HasSuffix(info.Name(), ".sha256") { return nil } sum, err := sha256File(path) if err != nil { return err } return os.WriteFile(path+".sha256", []byte(sum+" "+info.Name()+"\n"), 0o644) }) } func bundleOsPackages(p *project) error { release := projectDist(p, subRelease) bins := map[string]os.FileInfo{} if err := filepath.Walk(release, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } if strings.HasSuffix(info.Name(), ".sha256") { return nil } bins[path] = info return nil }); err != nil { return err } for binPath, info := range bins { folder := filepath.Join(release, info.Name()+"-full") if err := os.MkdirAll(folder, 0o755); err != nil { return err } if err := moveFile(binPath+".sha256", filepath.Join(folder, info.Name()+".sha256")); err != nil { return err } if err := moveFile(binPath, filepath.Join(folder, info.Name())); err != nil { return err } if p.OsPackageExtras != nil { if err := p.OsPackageExtras(folder, p); err != nil { return err } } } return nil } func zipBundles(ctx context.Context, p *project) error { zipDirAbs, err := filepath.Abs(projectDist(p, subZip)) if err != nil { return err } release := projectDist(p, subRelease) return filepath.Walk(release, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() || filepath.Base(path) == subRelease { return nil } fmt.Printf("Zipping %s...\n", info.Name()) zipFile := filepath.Join(zipDirAbs, info.Name()+".zip") //nolint:gosec // mage helper; args derive from the local filesystem walk above. c := exec.CommandContext(ctx, "zip", "-r", zipFile, ".", "-i", "*") c.Dir = path c.Stdout, c.Stderr = os.Stdout, os.Stderr return c.Run() }) } // repoSuite validates the REPO_SUITE env var; defaults to "stable". Limiting // the values prevents path traversal via the suite name flowing into a // filesystem path. func repoSuite() string { switch os.Getenv("REPO_SUITE") { case "stable", "unstable": return os.Getenv("REPO_SUITE") default: return "stable" } } // ----------------------------------------------------------------------------- // helpers — duplicated from the project magefiles so this module depends on // nothing but stdlib + mage. Don't import these from elsewhere; rewrite them // here if they need to change. func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() out, err := os.Create(dst) if err != nil { return err } defer out.Close() if _, err := io.Copy(out, in); err != nil { return err } si, err := os.Stat(src) if err != nil { return err } if err := os.Chmod(dst, si.Mode()); err != nil { return err } return out.Close() } func moveFile(src, dst string) error { if err := copyFile(src, dst); err != nil { return err } return os.Remove(src) } func sha256File(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return fmt.Sprintf("%x", h.Sum(nil)), nil } // Aliases for kebab-case spelling at the CLI. var Aliases = map[string]any{ "release": Release.Build, "release:build": Release.Build, "release:xgo": Release.Xgo, "release:prepare-nfpm-config": Release.PrepareNFPMConfig, "release:repo-apt": Release.RepoApt, "release:repo-rpm": Release.RepoRpm, "release:repo-pacman": Release.RepoPacman, }