From 55c122fb42f4a8946b0a532fb98dcdbd152a973c Mon Sep 17 00:00:00 2001 From: kolaente Date: Sat, 21 Feb 2026 23:35:28 +0100 Subject: [PATCH] feat: add repair-file-mime-types CLI command Add a CLI command that finds all files in the database with no MIME type set, detects it from the stored file content, and updates the database. This is useful for backfilling MIME types on files created before MIME detection was added on upload. --- go.mod | 3 + go.sum | 8 +++ pkg/cmd/repair_file_mime_types.go | 81 +++++++++++++++++++++++++++ pkg/files/repair.go | 92 +++++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+) create mode 100644 pkg/cmd/repair_file_mime_types.go create mode 100644 pkg/files/repair.go diff --git a/go.mod b/go.mod index 2e46bf99f..13dd4b5d6 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/redis/go-redis/v9 v9.17.3 github.com/robfig/cron/v3 v3.0.1 github.com/samedi/caldav-go v3.0.0+incompatible + github.com/schollz/progressbar/v3 v3.19.0 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 @@ -144,6 +145,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -158,6 +160,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.17.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sony/gobreaker v1.0.0 // indirect diff --git a/go.sum b/go.sum index 987cfc3ce..79b023fcf 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -368,6 +370,8 @@ github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K7 github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= @@ -428,6 +432,8 @@ github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -442,6 +448,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= +github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= diff --git a/pkg/cmd/repair_file_mime_types.go b/pkg/cmd/repair_file_mime_types.go new file mode 100644 index 000000000..897a4974c --- /dev/null +++ b/pkg/cmd/repair_file_mime_types.go @@ -0,0 +1,81 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(repairFileMimeTypesCmd) +} + +var repairFileMimeTypesCmd = &cobra.Command{ + Use: "repair-file-mime-types", + Short: "Detect and set MIME types for all files that have none", + Long: `Scans all files in the database that have no MIME type set, +detects the type from the stored file content, and updates the database. + +This is useful after upgrading from a version that did not store MIME types +on file creation. Only files with an empty or NULL mime column are affected.`, + PreRun: func(_ *cobra.Command, _ []string) { + initialize.FullInitWithoutAsync() + }, + Run: func(_ *cobra.Command, _ []string) { + s := db.NewSession() + defer s.Close() + + if err := s.Begin(); err != nil { + log.Errorf("Failed to start transaction: %s", err) + return + } + defer func() { + _ = s.Rollback() + }() + + result, err := files.RepairFileMimeTypes(s) + if err != nil { + log.Errorf("Failed to repair file MIME types: %s", err) + return + } + + if err := s.Commit(); err != nil { + log.Errorf("Failed to commit changes: %s", err) + return + } + + log.Infof("Repair complete:") + log.Infof(" Files scanned: %d", result.Total) + log.Infof(" Files updated: %d", result.Updated) + + if len(result.Errors) > 0 { + log.Errorf("Errors encountered (%d):", len(result.Errors)) + for _, e := range result.Errors { + log.Errorf(" - %s", e) + } + } + + if result.Total == 0 { + log.Infof("No files with missing MIME types found - all files are healthy!") + } + }, +} diff --git a/pkg/files/repair.go b/pkg/files/repair.go new file mode 100644 index 000000000..caee024ab --- /dev/null +++ b/pkg/files/repair.go @@ -0,0 +1,92 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package files + +import ( + "fmt" + + "code.vikunja.io/api/pkg/log" + + "github.com/gabriel-vasile/mimetype" + "github.com/schollz/progressbar/v3" + "xorm.io/xorm" +) + +// RepairMimeTypesResult holds the summary of a MIME type repair run. +type RepairMimeTypesResult struct { + Total int + Updated int + Errors []string +} + +// RepairFileMimeTypes finds all files with no MIME type set, detects it from +// the stored file content, and updates the database. +func RepairFileMimeTypes(s *xorm.Session) (*RepairMimeTypesResult, error) { + var files []*File + err := s.Where("mime = '' OR mime IS NULL").Find(&files) + if err != nil { + return nil, fmt.Errorf("failed to query files with missing mime type: %w", err) + } + + result := &RepairMimeTypesResult{ + Total: len(files), + } + + if len(files) == 0 { + return result, nil + } + + bar := progressbar.Default(int64(len(files)), "Detecting MIME types") + + for _, f := range files { + file, err := afs.Open(f.getAbsoluteFilePath()) + if err != nil { + msg := fmt.Sprintf("file %d: failed to open: %s", f.ID, err) + log.Errorf("file %d: failed to open: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + mime, err := mimetype.DetectReader(file) + _ = file.Close() + if err != nil { + msg := fmt.Sprintf("file %d: failed to detect mime type: %s", f.ID, err) + log.Errorf("file %d: failed to detect mime type: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + f.Mime = mime.String() + _, err = s.ID(f.ID).Cols("mime").Update(f) + if err != nil { + msg := fmt.Sprintf("file %d: failed to update mime type: %s", f.ID, err) + log.Errorf("file %d: failed to update mime type: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + result.Updated++ + _ = bar.Add(1) + } + + _ = bar.Finish() + + return result, nil +}