From b065c6200782bfd6e9eea889847e83f1dead906d Mon Sep 17 00:00:00 2001 From: maggch Date: Tue, 3 Mar 2026 21:18:09 +0800 Subject: [PATCH] feat: replace afero-s3 with minimal S3 afero.Fs implementation Replace the third-party afero-s3 library (and its temporary fork) with a minimal in-tree afero.Fs implementation (~200 lines) that only supports the three S3 operations Vikunja actually uses: Open (GetObject), Remove (DeleteObject), and Stat (HeadObject). The s3File implementation lazily opens a GetObject stream on first Read and supports Seek by closing and re-opening the stream from the new offset, matching the behavior of the fixed afero-s3 fork. All other afero.Fs/File methods return ErrS3NotSupported since they are never called in the S3 code path (writes already use direct PutObject). This removes the fclairamb/afero-s3 dependency and the temporary replace directive in go.mod entirely. Closes #2347 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- go.mod | 5 +- go.sum | 4 - pkg/files/filehandling.go | 5 +- pkg/files/s3fs.go | 201 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 11 deletions(-) create mode 100644 pkg/files/s3fs.go diff --git a/go.mod b/go.mod index eea0aeea9..488f17f06 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.10 github.com/aws/aws-sdk-go-v2/credentials v1.19.10 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 + github.com/aws/smithy-go v1.24.1 github.com/bbrks/go-blurhash v1.1.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/coreos/go-oidc/v3 v3.17.0 @@ -36,7 +37,6 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dustinkirkland/golang-petname v0.0.0-20240422154211-76c06c4bde6b github.com/fatih/color v1.18.0 - github.com/fclairamb/afero-s3 v0.4.0 github.com/gabriel-vasile/mimetype v1.4.12 github.com/ganigeorgiev/fexpr v0.5.0 github.com/getsentry/sentry-go v0.41.0 @@ -100,7 +100,6 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect @@ -113,7 +112,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect - github.com/aws/smithy-go v1.24.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beevik/etree v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -213,4 +211,3 @@ tool ( replace github.com/samedi/caldav-go => github.com/kolaente/caldav-go v3.0.1-0.20190610114120-2a4eb8b5dcc9+incompatible // Branch: feature/dynamic-supported-components, PR: https://github.com/samedi/caldav-go/pull/6 and https://github.com/samedi/caldav-go/pull/7 -replace github.com/fclairamb/afero-s3 => github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189 diff --git a/go.sum b/go.sum index ad4e2716e..fd7bb6b01 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iK github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 h1:s8fbFscel8NLpnz+ggR7ncW+lqhXIkmyHbgbPeT8yyM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4/go.mod h1:BazuWe/q/mMJ/NrSJBTbNBJiLq6u8reodbEZ4giRms4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= @@ -446,8 +444,6 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189 h1:HUcTa93zAphy5hA8akgJtMdTXHodWLBFblyutyvlhek= -github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189/go.mod h1:8Shhk3YMlD41DAC5bWRyjQFSsO4RxUKOj99Oxc8/HuU= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= diff --git a/pkg/files/filehandling.go b/pkg/files/filehandling.go index 51955ca59..a82b38a69 100644 --- a/pkg/files/filehandling.go +++ b/pkg/files/filehandling.go @@ -36,7 +36,6 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - aferos3 "github.com/fclairamb/afero-s3" "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -97,8 +96,8 @@ func initS3FileHandler() error { } }) - // Initialize S3 filesystem using afero-s3 - fs = aferos3.NewFsFromClient(bucket, client) + // Initialize S3 filesystem using our minimal S3 afero implementation + fs = newS3Fs(bucket, client) afs = &afero.Afero{Fs: fs} // Store S3 client and bucket for direct uploads with Content-Length diff --git a/pkg/files/s3fs.go b/pkg/files/s3fs.go new file mode 100644 index 000000000..99411e30a --- /dev/null +++ b/pkg/files/s3fs.go @@ -0,0 +1,201 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package files + +import ( + "context" + "errors" + "io" + "os" + "path" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/spf13/afero" +) + +// s3Fs is a minimal afero.Fs implementation backed by S3. +// It only supports Open (read), Remove, and Stat — the operations +// Vikunja actually uses. All other methods return ErrS3NotSupported. +type s3Fs struct { + client *s3.Client + bucket string +} + +var ErrS3NotSupported = errors.New("operation not supported on S3 filesystem") + +func newS3Fs(bucket string, client *s3.Client) afero.Fs { + return &s3Fs{bucket: bucket, client: client} +} + +func (*s3Fs) Name() string { return "s3" } + +// Open opens a file for reading via S3 GetObject. +// The actual GetObject call is deferred until the first Read. +func (f *s3Fs) Open(name string) (afero.File, error) { + // Verify the object exists + _, err := f.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(f.bucket), + Key: aws.String(name), + }) + if err != nil { + return nil, s3ToPathError("open", name, err) + } + + return &s3File{ + fs: f, + name: name, + }, nil +} + +// Stat returns file info via S3 HeadObject. +func (f *s3Fs) Stat(name string) (os.FileInfo, error) { + head, err := f.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(f.bucket), + Key: aws.String(name), + }) + if err != nil { + return nil, s3ToPathError("stat", name, err) + } + + return &s3FileInfo{ + name: path.Base(name), + size: *head.ContentLength, + modTime: *head.LastModified, + }, nil +} + +// Remove deletes a file via S3 DeleteObject. +func (f *s3Fs) Remove(name string) error { + // Check existence first so we return a proper error for missing files + if _, err := f.Stat(name); err != nil { + return err + } + + _, err := f.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + Bucket: aws.String(f.bucket), + Key: aws.String(name), + }) + return err +} + +// Unsupported operations + +func (*s3Fs) Create(string) (afero.File, error) { return nil, ErrS3NotSupported } +func (*s3Fs) Mkdir(string, os.FileMode) error { return ErrS3NotSupported } +func (*s3Fs) MkdirAll(string, os.FileMode) error { return ErrS3NotSupported } +func (*s3Fs) OpenFile(string, int, os.FileMode) (afero.File, error) { return nil, ErrS3NotSupported } +func (*s3Fs) RemoveAll(string) error { return ErrS3NotSupported } +func (*s3Fs) Rename(string, string) error { return ErrS3NotSupported } +func (*s3Fs) Chmod(string, os.FileMode) error { return ErrS3NotSupported } +func (*s3Fs) Chown(string, int, int) error { return ErrS3NotSupported } +func (*s3Fs) Chtimes(string, time.Time, time.Time) error { return ErrS3NotSupported } + +// s3ToPathError converts S3 SDK errors into os-compatible path errors. +func s3ToPathError(op, name string, err error) error { + var opErr *smithy.OperationError + if errors.As(err, &opErr) { + // HeadObject on a non-existent key returns a 404 + if opErr.OperationName == "HeadObject" { + return &os.PathError{Op: op, Path: name, Err: os.ErrNotExist} + } + } + return &os.PathError{Op: op, Path: name, Err: err} +} + +// s3File is a minimal afero.File for reading from S3. +// It lazily opens a GetObject stream on the first Read call. +type s3File struct { + fs *s3Fs + name string + body io.ReadCloser + closed bool +} + +func (f *s3File) Name() string { return f.name } + +func (f *s3File) Read(p []byte) (int, error) { + if f.closed { + return 0, afero.ErrFileClosed + } + + // Lazily open stream on first Read + if f.body == nil { + if err := f.openStream(); err != nil { + return 0, err + } + } + + return f.body.Read(p) +} + +func (f *s3File) Close() error { + f.closed = true + f.closeStream() + return nil +} + +func (f *s3File) Stat() (os.FileInfo, error) { + return f.fs.Stat(f.name) +} + +func (f *s3File) openStream() error { + out, err := f.fs.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(f.fs.bucket), + Key: aws.String(f.name), + }) + if err != nil { + return err + } + f.body = out.Body + return nil +} + +func (f *s3File) closeStream() { + if f.body != nil { + f.body.Close() + f.body = nil + } +} + +// Unsupported file operations + +func (*s3File) ReadAt([]byte, int64) (int, error) { return 0, ErrS3NotSupported } +func (*s3File) Seek(int64, int) (int64, error) { return 0, ErrS3NotSupported } +func (*s3File) Write([]byte) (int, error) { return 0, ErrS3NotSupported } +func (*s3File) WriteAt([]byte, int64) (int, error) { return 0, ErrS3NotSupported } +func (*s3File) WriteString(string) (int, error) { return 0, ErrS3NotSupported } +func (*s3File) Truncate(int64) error { return ErrS3NotSupported } +func (*s3File) Sync() error { return nil } +func (*s3File) Readdir(int) ([]os.FileInfo, error) { return nil, ErrS3NotSupported } +func (*s3File) Readdirnames(int) ([]string, error) { return nil, ErrS3NotSupported } + +// s3FileInfo implements os.FileInfo for S3 objects. +type s3FileInfo struct { + name string + size int64 + modTime time.Time +} + +func (fi *s3FileInfo) Name() string { return fi.name } +func (fi *s3FileInfo) Size() int64 { return fi.size } +func (fi *s3FileInfo) Mode() os.FileMode { return 0664 } +func (fi *s3FileInfo) ModTime() time.Time { return fi.modTime } +func (fi *s3FileInfo) IsDir() bool { return false } +func (fi *s3FileInfo) Sys() interface{} { return nil }