diff --git a/go.mod b/go.mod index 2e46bf99f..13dd4b5d6 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/redis/go-redis/v9 v9.17.3 github.com/robfig/cron/v3 v3.0.1 github.com/samedi/caldav-go v3.0.0+incompatible + github.com/schollz/progressbar/v3 v3.19.0 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 @@ -144,6 +145,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -158,6 +160,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.17.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sony/gobreaker v1.0.0 // indirect diff --git a/go.sum b/go.sum index 987cfc3ce..79b023fcf 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -368,6 +370,8 @@ github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K7 github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= @@ -428,6 +432,8 @@ github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -442,6 +448,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= +github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= diff --git a/pkg/cmd/repair_file_mime_types.go b/pkg/cmd/repair_file_mime_types.go new file mode 100644 index 000000000..897a4974c --- /dev/null +++ b/pkg/cmd/repair_file_mime_types.go @@ -0,0 +1,81 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "code.vikunja.io/api/pkg/db" + "code.vikunja.io/api/pkg/files" + "code.vikunja.io/api/pkg/initialize" + "code.vikunja.io/api/pkg/log" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(repairFileMimeTypesCmd) +} + +var repairFileMimeTypesCmd = &cobra.Command{ + Use: "repair-file-mime-types", + Short: "Detect and set MIME types for all files that have none", + Long: `Scans all files in the database that have no MIME type set, +detects the type from the stored file content, and updates the database. + +This is useful after upgrading from a version that did not store MIME types +on file creation. Only files with an empty or NULL mime column are affected.`, + PreRun: func(_ *cobra.Command, _ []string) { + initialize.FullInitWithoutAsync() + }, + Run: func(_ *cobra.Command, _ []string) { + s := db.NewSession() + defer s.Close() + + if err := s.Begin(); err != nil { + log.Errorf("Failed to start transaction: %s", err) + return + } + defer func() { + _ = s.Rollback() + }() + + result, err := files.RepairFileMimeTypes(s) + if err != nil { + log.Errorf("Failed to repair file MIME types: %s", err) + return + } + + if err := s.Commit(); err != nil { + log.Errorf("Failed to commit changes: %s", err) + return + } + + log.Infof("Repair complete:") + log.Infof(" Files scanned: %d", result.Total) + log.Infof(" Files updated: %d", result.Updated) + + if len(result.Errors) > 0 { + log.Errorf("Errors encountered (%d):", len(result.Errors)) + for _, e := range result.Errors { + log.Errorf(" - %s", e) + } + } + + if result.Total == 0 { + log.Infof("No files with missing MIME types found - all files are healthy!") + } + }, +} diff --git a/pkg/files/repair.go b/pkg/files/repair.go new file mode 100644 index 000000000..caee024ab --- /dev/null +++ b/pkg/files/repair.go @@ -0,0 +1,92 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package files + +import ( + "fmt" + + "code.vikunja.io/api/pkg/log" + + "github.com/gabriel-vasile/mimetype" + "github.com/schollz/progressbar/v3" + "xorm.io/xorm" +) + +// RepairMimeTypesResult holds the summary of a MIME type repair run. +type RepairMimeTypesResult struct { + Total int + Updated int + Errors []string +} + +// RepairFileMimeTypes finds all files with no MIME type set, detects it from +// the stored file content, and updates the database. +func RepairFileMimeTypes(s *xorm.Session) (*RepairMimeTypesResult, error) { + var files []*File + err := s.Where("mime = '' OR mime IS NULL").Find(&files) + if err != nil { + return nil, fmt.Errorf("failed to query files with missing mime type: %w", err) + } + + result := &RepairMimeTypesResult{ + Total: len(files), + } + + if len(files) == 0 { + return result, nil + } + + bar := progressbar.Default(int64(len(files)), "Detecting MIME types") + + for _, f := range files { + file, err := afs.Open(f.getAbsoluteFilePath()) + if err != nil { + msg := fmt.Sprintf("file %d: failed to open: %s", f.ID, err) + log.Errorf("file %d: failed to open: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + mime, err := mimetype.DetectReader(file) + _ = file.Close() + if err != nil { + msg := fmt.Sprintf("file %d: failed to detect mime type: %s", f.ID, err) + log.Errorf("file %d: failed to detect mime type: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + f.Mime = mime.String() + _, err = s.ID(f.ID).Cols("mime").Update(f) + if err != nil { + msg := fmt.Sprintf("file %d: failed to update mime type: %s", f.ID, err) + log.Errorf("file %d: failed to update mime type: %s", f.ID, err) + result.Errors = append(result.Errors, msg) + _ = bar.Add(1) + continue + } + + result.Updated++ + _ = bar.Add(1) + } + + _ = bar.Finish() + + return result, nil +}