fix(backgrounds): stream unsplash download to temp file instead of memory
Use a temp file instead of io.ReadAll to avoid buffering the entire Unsplash image in RAM, which could cause OOM with large images or high maxsize configuration.
This commit is contained in:
parent
0d395a9e5d
commit
bcfde14b14
|
|
@ -367,7 +367,7 @@ func TestFileSave_S3_ReturnsErrorOnPutObjectFailure(t *testing.T) {
|
|||
assert.Contains(t, err.Error(), "failed to upload file to S3")
|
||||
}
|
||||
|
||||
func TestFileSave_S3_LogsWarnOnSizeMismatch(t *testing.T) {
|
||||
func TestFileSave_S3_UsesActualReaderSizeOnMismatch(t *testing.T) {
|
||||
originalClient := s3Client
|
||||
originalBucket := s3Bucket
|
||||
t.Cleanup(func() {
|
||||
|
|
|
|||
|
|
@ -20,9 +20,11 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -286,19 +288,31 @@ func (p *Provider) Set(s *xorm.Session, image *background.Image, project *models
|
|||
return files.ErrFileIsTooLarge{Size: uint64(resp.ContentLength)}
|
||||
}
|
||||
|
||||
// Buffer the response body so we have a seekable reader for S3 uploads.
|
||||
// Stream the response body into a temp file so we have a seekable reader
|
||||
// for S3 uploads without buffering the entire image in memory.
|
||||
tmpFile, err := os.CreateTemp("", "vikunja-unsplash-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create temp file for unsplash download: %w", err)
|
||||
}
|
||||
defer os.Remove(tmpFile.Name())
|
||||
defer tmpFile.Close()
|
||||
|
||||
// Use LimitReader as a safety net in case Content-Length was missing or inaccurate.
|
||||
limitedReader := io.LimitReader(resp.Body, int64(maxSize)+1) // #nosec G115 -- maxSize is configured, not user input
|
||||
bodyBytes, err := io.ReadAll(limitedReader)
|
||||
written, err := io.Copy(tmpFile, limitedReader)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("could not write unsplash download to temp file: %w", err)
|
||||
}
|
||||
if uint64(len(bodyBytes)) > maxSize {
|
||||
return files.ErrFileIsTooLarge{Size: uint64(len(bodyBytes))}
|
||||
if uint64(written) > maxSize {
|
||||
return files.ErrFileIsTooLarge{Size: uint64(written)}
|
||||
}
|
||||
|
||||
if _, err = tmpFile.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("could not seek temp file to start: %w", err)
|
||||
}
|
||||
|
||||
// Save it as a file in vikunja
|
||||
file, err := files.Create(bytes.NewReader(bodyBytes), "", uint64(len(bodyBytes)), auth)
|
||||
file, err := files.Create(tmpFile, "", uint64(written), auth)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue