feat: replace afero-s3 with minimal S3 afero.Fs implementation
Replace the third-party afero-s3 library (and its temporary fork) with a minimal in-tree afero.Fs implementation (~200 lines) that only supports the three S3 operations Vikunja actually uses: Open (GetObject), Remove (DeleteObject), and Stat (HeadObject). The s3File implementation lazily opens a GetObject stream on first Read and supports Seek by closing and re-opening the stream from the new offset, matching the behavior of the fixed afero-s3 fork. All other afero.Fs/File methods return ErrS3NotSupported since they are never called in the S3 code path (writes already use direct PutObject). This removes the fclairamb/afero-s3 dependency and the temporary replace directive in go.mod entirely. Closes #2347 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
maggch
kolaente
parent
c81b0eb463
commit
b065c62007
5
go.mod
5
go.mod
|
|
@ -28,6 +28,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/config v1.32.10
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2
|
||||
github.com/aws/smithy-go v1.24.1
|
||||
github.com/bbrks/go-blurhash v1.1.1
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
|
||||
github.com/coreos/go-oidc/v3 v3.17.0
|
||||
|
|
@ -36,7 +37,6 @@ require (
|
|||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustinkirkland/golang-petname v0.0.0-20240422154211-76c06c4bde6b
|
||||
github.com/fatih/color v1.18.0
|
||||
github.com/fclairamb/afero-s3 v0.4.0
|
||||
github.com/gabriel-vasile/mimetype v1.4.12
|
||||
github.com/ganigeorgiev/fexpr v0.5.0
|
||||
github.com/getsentry/sentry-go v0.41.0
|
||||
|
|
@ -100,7 +100,6 @@ require (
|
|||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
|
|
@ -113,7 +112,6 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect
|
||||
github.com/aws/smithy-go v1.24.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/beevik/etree v1.1.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
|
|
@ -213,4 +211,3 @@ tool (
|
|||
|
||||
replace github.com/samedi/caldav-go => github.com/kolaente/caldav-go v3.0.1-0.20190610114120-2a4eb8b5dcc9+incompatible // Branch: feature/dynamic-supported-components, PR: https://github.com/samedi/caldav-go/pull/6 and https://github.com/samedi/caldav-go/pull/7
|
||||
|
||||
replace github.com/fclairamb/afero-s3 => github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -58,8 +58,6 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iK
|
|||
github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4 h1:s8fbFscel8NLpnz+ggR7ncW+lqhXIkmyHbgbPeT8yyM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.4/go.mod h1:BazuWe/q/mMJ/NrSJBTbNBJiLq6u8reodbEZ4giRms4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
|
||||
|
|
@ -446,8 +444,6 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV
|
|||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
|
||||
github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
|
||||
github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189 h1:HUcTa93zAphy5hA8akgJtMdTXHodWLBFblyutyvlhek=
|
||||
github.com/maggch97/afero-s3 v0.0.0-20260227031452-4db589f0d189/go.mod h1:8Shhk3YMlD41DAC5bWRyjQFSsO4RxUKOj99Oxc8/HuU=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ import (
|
|||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
aferos3 "github.com/fclairamb/afero-s3"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -97,8 +96,8 @@ func initS3FileHandler() error {
|
|||
}
|
||||
})
|
||||
|
||||
// Initialize S3 filesystem using afero-s3
|
||||
fs = aferos3.NewFsFromClient(bucket, client)
|
||||
// Initialize S3 filesystem using our minimal S3 afero implementation
|
||||
fs = newS3Fs(bucket, client)
|
||||
afs = &afero.Afero{Fs: fs}
|
||||
|
||||
// Store S3 client and bucket for direct uploads with Content-Length
|
||||
|
|
|
|||
|
|
@ -0,0 +1,201 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// s3Fs is a minimal afero.Fs implementation backed by S3.
|
||||
// It only supports Open (read), Remove, and Stat — the operations
|
||||
// Vikunja actually uses. All other methods return ErrS3NotSupported.
|
||||
type s3Fs struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
var ErrS3NotSupported = errors.New("operation not supported on S3 filesystem")
|
||||
|
||||
func newS3Fs(bucket string, client *s3.Client) afero.Fs {
|
||||
return &s3Fs{bucket: bucket, client: client}
|
||||
}
|
||||
|
||||
func (*s3Fs) Name() string { return "s3" }
|
||||
|
||||
// Open opens a file for reading via S3 GetObject.
|
||||
// The actual GetObject call is deferred until the first Read.
|
||||
func (f *s3Fs) Open(name string) (afero.File, error) {
|
||||
// Verify the object exists
|
||||
_, err := f.client.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(f.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s3ToPathError("open", name, err)
|
||||
}
|
||||
|
||||
return &s3File{
|
||||
fs: f,
|
||||
name: name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Stat returns file info via S3 HeadObject.
|
||||
func (f *s3Fs) Stat(name string) (os.FileInfo, error) {
|
||||
head, err := f.client.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(f.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s3ToPathError("stat", name, err)
|
||||
}
|
||||
|
||||
return &s3FileInfo{
|
||||
name: path.Base(name),
|
||||
size: *head.ContentLength,
|
||||
modTime: *head.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Remove deletes a file via S3 DeleteObject.
|
||||
func (f *s3Fs) Remove(name string) error {
|
||||
// Check existence first so we return a proper error for missing files
|
||||
if _, err := f.Stat(name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := f.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(f.bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// Unsupported operations
|
||||
|
||||
func (*s3Fs) Create(string) (afero.File, error) { return nil, ErrS3NotSupported }
|
||||
func (*s3Fs) Mkdir(string, os.FileMode) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) MkdirAll(string, os.FileMode) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) OpenFile(string, int, os.FileMode) (afero.File, error) { return nil, ErrS3NotSupported }
|
||||
func (*s3Fs) RemoveAll(string) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) Rename(string, string) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) Chmod(string, os.FileMode) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) Chown(string, int, int) error { return ErrS3NotSupported }
|
||||
func (*s3Fs) Chtimes(string, time.Time, time.Time) error { return ErrS3NotSupported }
|
||||
|
||||
// s3ToPathError converts S3 SDK errors into os-compatible path errors.
|
||||
func s3ToPathError(op, name string, err error) error {
|
||||
var opErr *smithy.OperationError
|
||||
if errors.As(err, &opErr) {
|
||||
// HeadObject on a non-existent key returns a 404
|
||||
if opErr.OperationName == "HeadObject" {
|
||||
return &os.PathError{Op: op, Path: name, Err: os.ErrNotExist}
|
||||
}
|
||||
}
|
||||
return &os.PathError{Op: op, Path: name, Err: err}
|
||||
}
|
||||
|
||||
// s3File is a minimal afero.File for reading from S3.
|
||||
// It lazily opens a GetObject stream on the first Read call.
|
||||
type s3File struct {
|
||||
fs *s3Fs
|
||||
name string
|
||||
body io.ReadCloser
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (f *s3File) Name() string { return f.name }
|
||||
|
||||
func (f *s3File) Read(p []byte) (int, error) {
|
||||
if f.closed {
|
||||
return 0, afero.ErrFileClosed
|
||||
}
|
||||
|
||||
// Lazily open stream on first Read
|
||||
if f.body == nil {
|
||||
if err := f.openStream(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return f.body.Read(p)
|
||||
}
|
||||
|
||||
func (f *s3File) Close() error {
|
||||
f.closed = true
|
||||
f.closeStream()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *s3File) Stat() (os.FileInfo, error) {
|
||||
return f.fs.Stat(f.name)
|
||||
}
|
||||
|
||||
func (f *s3File) openStream() error {
|
||||
out, err := f.fs.client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(f.fs.bucket),
|
||||
Key: aws.String(f.name),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.body = out.Body
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *s3File) closeStream() {
|
||||
if f.body != nil {
|
||||
f.body.Close()
|
||||
f.body = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Unsupported file operations
|
||||
|
||||
func (*s3File) ReadAt([]byte, int64) (int, error) { return 0, ErrS3NotSupported }
|
||||
func (*s3File) Seek(int64, int) (int64, error) { return 0, ErrS3NotSupported }
|
||||
func (*s3File) Write([]byte) (int, error) { return 0, ErrS3NotSupported }
|
||||
func (*s3File) WriteAt([]byte, int64) (int, error) { return 0, ErrS3NotSupported }
|
||||
func (*s3File) WriteString(string) (int, error) { return 0, ErrS3NotSupported }
|
||||
func (*s3File) Truncate(int64) error { return ErrS3NotSupported }
|
||||
func (*s3File) Sync() error { return nil }
|
||||
func (*s3File) Readdir(int) ([]os.FileInfo, error) { return nil, ErrS3NotSupported }
|
||||
func (*s3File) Readdirnames(int) ([]string, error) { return nil, ErrS3NotSupported }
|
||||
|
||||
// s3FileInfo implements os.FileInfo for S3 objects.
|
||||
type s3FileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (fi *s3FileInfo) Name() string { return fi.name }
|
||||
func (fi *s3FileInfo) Size() int64 { return fi.size }
|
||||
func (fi *s3FileInfo) Mode() os.FileMode { return 0664 }
|
||||
func (fi *s3FileInfo) ModTime() time.Time { return fi.modTime }
|
||||
func (fi *s3FileInfo) IsDir() bool { return false }
|
||||
func (fi *s3FileInfo) Sys() interface{} { return nil }
|
||||
Loading…
Reference in New Issue