feat: add repair-file-mime-types CLI command
Add a CLI command that finds all files in the database with no MIME type set, detects it from the stored file content, and updates the database. This is useful for backfilling MIME types on files created before MIME detection was added on upload.
This commit is contained in:
kolaente
parent
4915f535d0
commit
55c122fb42
3
go.mod
3
go.mod
|
|
@ -63,6 +63,7 @@ require (
|
|||
github.com/redis/go-redis/v9 v9.17.3
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/samedi/caldav-go v3.0.0+incompatible
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
|
|
@ -144,6 +145,7 @@ require (
|
|||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/oapi-codegen/runtime v1.1.1 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
|
|
@ -158,6 +160,7 @@ require (
|
|||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.17.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -91,6 +91,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
|
|||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
|
||||
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
|
||||
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
|
|
@ -368,6 +370,8 @@ github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K7
|
|||
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
|
||||
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
|
|
@ -428,6 +432,8 @@ github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0
|
|||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
|
|
@ -442,6 +448,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
|
|||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"code.vikunja.io/api/pkg/db"
|
||||
"code.vikunja.io/api/pkg/files"
|
||||
"code.vikunja.io/api/pkg/initialize"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(repairFileMimeTypesCmd)
|
||||
}
|
||||
|
||||
var repairFileMimeTypesCmd = &cobra.Command{
|
||||
Use: "repair-file-mime-types",
|
||||
Short: "Detect and set MIME types for all files that have none",
|
||||
Long: `Scans all files in the database that have no MIME type set,
|
||||
detects the type from the stored file content, and updates the database.
|
||||
|
||||
This is useful after upgrading from a version that did not store MIME types
|
||||
on file creation. Only files with an empty or NULL mime column are affected.`,
|
||||
PreRun: func(_ *cobra.Command, _ []string) {
|
||||
initialize.FullInitWithoutAsync()
|
||||
},
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
if err := s.Begin(); err != nil {
|
||||
log.Errorf("Failed to start transaction: %s", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = s.Rollback()
|
||||
}()
|
||||
|
||||
result, err := files.RepairFileMimeTypes(s)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to repair file MIME types: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.Commit(); err != nil {
|
||||
log.Errorf("Failed to commit changes: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Repair complete:")
|
||||
log.Infof(" Files scanned: %d", result.Total)
|
||||
log.Infof(" Files updated: %d", result.Updated)
|
||||
|
||||
if len(result.Errors) > 0 {
|
||||
log.Errorf("Errors encountered (%d):", len(result.Errors))
|
||||
for _, e := range result.Errors {
|
||||
log.Errorf(" - %s", e)
|
||||
}
|
||||
}
|
||||
|
||||
if result.Total == 0 {
|
||||
log.Infof("No files with missing MIME types found - all files are healthy!")
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package files
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
// RepairMimeTypesResult holds the summary of a MIME type repair run.
|
||||
type RepairMimeTypesResult struct {
|
||||
Total int
|
||||
Updated int
|
||||
Errors []string
|
||||
}
|
||||
|
||||
// RepairFileMimeTypes finds all files with no MIME type set, detects it from
|
||||
// the stored file content, and updates the database.
|
||||
func RepairFileMimeTypes(s *xorm.Session) (*RepairMimeTypesResult, error) {
|
||||
var files []*File
|
||||
err := s.Where("mime = '' OR mime IS NULL").Find(&files)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query files with missing mime type: %w", err)
|
||||
}
|
||||
|
||||
result := &RepairMimeTypesResult{
|
||||
Total: len(files),
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
bar := progressbar.Default(int64(len(files)), "Detecting MIME types")
|
||||
|
||||
for _, f := range files {
|
||||
file, err := afs.Open(f.getAbsoluteFilePath())
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("file %d: failed to open: %s", f.ID, err)
|
||||
log.Errorf("file %d: failed to open: %s", f.ID, err)
|
||||
result.Errors = append(result.Errors, msg)
|
||||
_ = bar.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
mime, err := mimetype.DetectReader(file)
|
||||
_ = file.Close()
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("file %d: failed to detect mime type: %s", f.ID, err)
|
||||
log.Errorf("file %d: failed to detect mime type: %s", f.ID, err)
|
||||
result.Errors = append(result.Errors, msg)
|
||||
_ = bar.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
f.Mime = mime.String()
|
||||
_, err = s.ID(f.ID).Cols("mime").Update(f)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("file %d: failed to update mime type: %s", f.ID, err)
|
||||
log.Errorf("file %d: failed to update mime type: %s", f.ID, err)
|
||||
result.Errors = append(result.Errors, msg)
|
||||
_ = bar.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
result.Updated++
|
||||
_ = bar.Add(1)
|
||||
}
|
||||
|
||||
_ = bar.Finish()
|
||||
|
||||
return result, nil
|
||||
}
|
||||
Loading…
Reference in New Issue