From a5b1a90c428ec21d7095e374c899f8f953b8a557 Mon Sep 17 00:00:00 2001 From: kolaente Date: Wed, 25 Feb 2026 11:59:57 +0100 Subject: [PATCH] refactor: remove typesense support Typesense was an optional external search backend. This commit fully removes the integration, leaving the database searcher as the only search implementation. Changes: - Delete pkg/models/typesense.go (core integration) - Delete pkg/cmd/index.go (CLI command for indexing) - Simplify task search to always use database searcher - Remove Typesense event listeners for task sync - Remove TypesenseSync model registration - Remove Typesense config keys and defaults - Remove Typesense doctor health check - Remove Typesense initialization from startup - Clean up benchmark test - Add migration to drop typesense_sync table - Remove golangci-lint suppression for typesense.go - Remove typesense-go dependency --- .golangci.yml | 4 - config-raw.json | 20 - go.mod | 4 - go.sum | 12 - pkg/cmd/doctor.go | 2 +- pkg/cmd/index.go | 73 ---- pkg/config/config.go | 7 - pkg/doctor/services.go | 68 --- pkg/initialize/init.go | 3 - pkg/migration/20260225114726.go | 35 ++ pkg/models/listeners.go | 130 ------ pkg/models/models.go | 1 - pkg/models/saved_filters.go | 30 +- pkg/models/task_position.go | 5 - pkg/models/task_search.go | 271 ------------ pkg/models/task_search_bench_test.go | 13 - pkg/models/tasks.go | 19 +- pkg/models/typesense.go | 597 --------------------------- 18 files changed, 39 insertions(+), 1255 deletions(-) delete mode 100644 pkg/cmd/index.go create mode 100644 pkg/migration/20260225114726.go delete mode 100644 pkg/models/typesense.go diff --git a/.golangci.yml b/.golangci.yml index eb225f272..d4afbf288 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -123,10 +123,6 @@ linters: - revive path: pkg/migration/* text: parameter 'tx' seems to be unused, consider removing or renaming it as - - linters: - - govet - path: pkg/models/typesense.go - text: 'structtag: struct field Position repeats json tag "position" also at' - linters: - gosec path: pkg/cmd/user.go diff --git a/config-raw.json b/config-raw.json index c81a7ca89..277da7ae4 100644 --- a/config-raw.json +++ b/config-raw.json @@ -250,26 +250,6 @@ } ] }, - { - "key": "typesense", - "children": [ - { - "key": "enabled", - "default_value": "false", - "comment": "Whether to enable the Typesense integration. If true, all tasks will be synced to the configured Typesense\ninstance and all search and filtering will run through Typesense instead of only through the database.\nTypesense allows fast fulltext search including fuzzy matching support. It may return different results than\nwhat you'd get with a database-only search." - }, - { - "key": "url", - "default_value": "", - "comment": "The url to the Typesense instance you want to use. Can be hosted locally or in Typesense Cloud as long as Vikunja is able to reach it. Must be a http(s) url." - }, - { - "key": "apikey", - "default_value": "", - "comment": "The Typesense API key you want to use." - } - ] - }, { "key": "redis", "children": [ diff --git a/go.mod b/go.mod index 13dd4b5d6..341d36dc3 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,6 @@ require ( github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 github.com/tkuchiki/go-timezone v0.2.3 - github.com/typesense/typesense-go/v2 v2.0.0 github.com/ulule/limiter/v3 v3.11.2 github.com/wneessen/go-mail v0.7.2 github.com/yuin/goldmark v1.7.16 @@ -94,7 +93,6 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19 // indirect @@ -147,7 +145,6 @@ require ( github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect @@ -172,7 +169,6 @@ require ( github.com/tj/assert v0.0.3 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect - go.uber.org/mock v0.5.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.31.0 // indirect diff --git a/go.sum b/go.sum index 79b023fcf..dc55c9aa5 100644 --- a/go.sum +++ b/go.sum @@ -16,15 +16,12 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY= github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4= github.com/adlio/trello v1.12.0 h1:JqOE2GFHQ9YtEviRRRSnicSxPbt4WFOxhqXzjMOw8lw= github.com/adlio/trello v1.12.0/go.mod h1:I4Lti4jf2KxjTNgTqs5W3lLuE78QZZdYbbPnQQGwjOo= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= -github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= -github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/arran4/golang-ical v0.3.2 h1:MGNjcXJFSuCXmYX/RpZhR2HDCYoFuK8vTPFLEdFC3JY= github.com/arran4/golang-ical v0.3.2/go.mod h1:xblDGxxIUMWwFZk9dlECUlc1iXNV65LJZOTHLVwu8bo= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -77,7 +74,6 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -308,7 +304,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 h1:SwcnSwBR7X/5EHJQlXBockkJVIMRVt5yKaesBPMtyZQ= github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6/go.mod h1:WrYiIuiXUMIvTDAQw97C+9l0CnBmCcvosPjN3XDqS/o= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -384,8 +379,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= -github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= @@ -471,7 +464,6 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= -github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -499,8 +491,6 @@ github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tkuchiki/go-timezone v0.2.3 h1:D3TVdIPrFsu9lxGxqNX2wsZwn1MZtTqTW0mdevMozHc= github.com/tkuchiki/go-timezone v0.2.3/go.mod h1:oFweWxYl35C/s7HMVZXiA19Jr9Y0qJHMaG/J2TES4LY= -github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I= -github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao= github.com/ulule/limiter/v3 v3.11.2 h1:P4yOrxoEMJbOTfRJR2OzjL90oflzYPPmWg+dvwN2tHA= github.com/ulule/limiter/v3 v3.11.2/go.mod h1:QG5GnFOCV+k7lrL5Y8kgEeeflPH3+Cviqlqa8SVSQxI= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= @@ -521,8 +511,6 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= diff --git a/pkg/cmd/doctor.go b/pkg/cmd/doctor.go index ac2b72230..407b99011 100644 --- a/pkg/cmd/doctor.go +++ b/pkg/cmd/doctor.go @@ -41,7 +41,7 @@ issues with your Vikunja installation. It checks: - Configuration (config file, public URL, JWT secret, CORS) - Database connectivity and version - File storage (local or S3) -- Optional services (Redis, Typesense, Mailer, LDAP, OpenID) +- Optional services (Redis, Mailer, LDAP, OpenID) Exit codes: 0 - All checks passed diff --git a/pkg/cmd/index.go b/pkg/cmd/index.go deleted file mode 100644 index 4b8684cb2..000000000 --- a/pkg/cmd/index.go +++ /dev/null @@ -1,73 +0,0 @@ -// Vikunja is a to-do list application to facilitate your life. -// Copyright 2018-present Vikunja and contributors. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package cmd - -import ( - "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/initialize" - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/models" - - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(indexCmd) -} - -var indexPartialFlag bool - -var indexCmd = &cobra.Command{ - Use: "index", - Short: "Reindex all of Vikunja's data into Typesense. This will remove any existing index.", - PreRun: func(_ *cobra.Command, _ []string) { - initialize.FullInitWithoutAsync() - }, - Run: func(_ *cobra.Command, _ []string) { - if config.TypesenseURL.GetString() == "" { - log.Error("Typesense not configured") - return - } - - err := models.CreateTypesenseCollections() - if err != nil { - log.Criticalf("Could not create Typesense collections: %s", err.Error()) - return - } - if indexPartialFlag { - log.Infof("Indexing changed tasks… This may take a while.") - err = models.SyncUpdatedTasksIntoTypesense() - if err != nil { - log.Criticalf("Could not reindex all changed tasks into Typesense: %s", err.Error()) - return - } - } else { - log.Infof("Indexing all tasks… This may take a while.") - err = models.ReindexAllTasks() - if err != nil { - log.Criticalf("Could not reindex all tasks into Typesense: %s", err.Error()) - return - } - } - - log.Infof("Done!") - }, -} - -func init() { - indexCmd.Flags().BoolVarP(&indexPartialFlag, "partial", "p", false, "If provided, Vikunja will only index those tasks which are not present in the index. It will not remove any existing tasks.") -} diff --git a/pkg/config/config.go b/pkg/config/config.go index edeff9564..cc61ec51b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -117,10 +117,6 @@ const ( DatabaseTLS Key = `database.tls` DatabaseSchema Key = `database.schema` - TypesenseEnabled Key = `typesense.enabled` - TypesenseURL Key = `typesense.url` - TypesenseAPIKey Key = `typesense.apikey` - MailerEnabled Key = `mailer.enabled` MailerHost Key = `mailer.host` MailerPort Key = `mailer.port` @@ -398,9 +394,6 @@ func InitDefaultConfig() { DatabaseTLS.setDefault("false") DatabaseSchema.setDefault("public") - // Typesense - TypesenseEnabled.setDefault(false) - // Mailer MailerEnabled.setDefault(false) MailerHost.setDefault("") diff --git a/pkg/doctor/services.go b/pkg/doctor/services.go index 082f61baa..acc7d0b05 100644 --- a/pkg/doctor/services.go +++ b/pkg/doctor/services.go @@ -36,10 +36,6 @@ func CheckOptionalServices() []CheckGroup { groups = append(groups, checkRedis()) } - if config.TypesenseEnabled.GetBool() { - groups = append(groups, checkTypesense()) - } - if config.MailerEnabled.GetBool() { groups = append(groups, checkMailer()) } @@ -101,70 +97,6 @@ func checkRedis() CheckGroup { } } -func checkTypesense() CheckGroup { - url := config.TypesenseURL.GetString() - healthURL := url + "/health" - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) - if err != nil { - return CheckGroup{ - Name: "Typesense", - Results: []CheckResult{ - { - Name: "Connection", - Passed: false, - Error: err.Error(), - }, - }, - } - } - - req.Header.Set("X-TYPESENSE-API-KEY", config.TypesenseAPIKey.GetString()) - - client := &http.Client{Timeout: 5 * time.Second} - resp, err := client.Do(req) - if err != nil { - return CheckGroup{ - Name: "Typesense", - Results: []CheckResult{ - { - Name: "Connection", - Passed: false, - Error: err.Error(), - }, - }, - } - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return CheckGroup{ - Name: "Typesense", - Results: []CheckResult{ - { - Name: "Connection", - Passed: false, - Error: fmt.Sprintf("health check returned status %d", resp.StatusCode), - }, - }, - } - } - - return CheckGroup{ - Name: "Typesense", - Results: []CheckResult{ - { - Name: "Connection", - Passed: true, - Value: fmt.Sprintf("OK (%s)", url), - }, - }, - } -} - func checkMailer() CheckGroup { host := config.MailerHost.GetString() port := config.MailerPort.GetInt() diff --git a/pkg/initialize/init.go b/pkg/initialize/init.go index 9595e1705..0ec1c9fd7 100644 --- a/pkg/initialize/init.go +++ b/pkg/initialize/init.go @@ -91,9 +91,6 @@ func FullInitWithoutAsync() { // Set Engine InitEngines() - // Init Typesense - models.InitTypesense() - // Start the mail daemon mail.StartMailDaemon() diff --git a/pkg/migration/20260225114726.go b/pkg/migration/20260225114726.go new file mode 100644 index 000000000..0f3719fde --- /dev/null +++ b/pkg/migration/20260225114726.go @@ -0,0 +1,35 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "xorm.io/xorm" +) + +func init() { + migrations = append(migrations, &xormigrate.Migration{ + ID: "20260225114726", + Description: "Drop the typesense_sync table", + Migrate: func(tx *xorm.Engine) error { + return tx.DropTables("typesense_sync") + }, + Rollback: func(tx *xorm.Engine) error { + return nil + }, + }) +} diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index a3674b36e..05ddec492 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -17,7 +17,6 @@ package models import ( - "context" "encoding/json" "strconv" "time" @@ -70,12 +69,6 @@ func RegisterListeners() { events.RegisterListener((&TaskCreatedEvent{}).Name(), &UpdateTaskInSavedFilterViews{}) events.RegisterListener((&TaskUpdatedEvent{}).Name(), &UpdateTaskInSavedFilterViews{}) events.RegisterListener((&TaskCommentCreatedEvent{}).Name(), &MarkTaskUnreadOnComment{}) - if config.TypesenseEnabled.GetBool() { - events.RegisterListener((&TaskDeletedEvent{}).Name(), &RemoveTaskFromTypesense{}) - events.RegisterListener((&TaskCreatedEvent{}).Name(), &AddTaskToTypesense{}) - events.RegisterListener((&TaskUpdatedEvent{}).Name(), &UpdateTaskInTypesense{}) - events.RegisterListener((&TaskPositionsRecalculatedEvent{}).Name(), &UpdateTaskPositionsInTypesense{}) - } if config.WebhooksEnabled.GetBool() { RegisterEventForWebhook(&TaskCreatedEvent{}) RegisterEventForWebhook(&TaskUpdatedEvent{}) @@ -503,121 +496,6 @@ func (s *HandleTaskUpdateLastUpdated) Handle(msg *message.Message) (err error) { return sess.Commit() } -// RemoveTaskFromTypesense represents a listener -type RemoveTaskFromTypesense struct { -} - -// Name defines the name for the RemoveTaskFromTypesense listener -func (s *RemoveTaskFromTypesense) Name() string { - return "typesense.task.remove" -} - -// Handle is executed when the event RemoveTaskFromTypesense listens on is fired -func (s *RemoveTaskFromTypesense) Handle(msg *message.Message) (err error) { - event := &TaskDeletedEvent{} - err = json.Unmarshal(msg.Payload, event) - if err != nil { - return err - } - - log.Debugf("[Typesense Sync] Removing task %d from Typesense", event.Task.ID) - - _, err = typesenseClient. - Collection("tasks"). - Document(strconv.FormatInt(event.Task.ID, 10)). - Delete(context.Background()) - return err -} - -// AddTaskToTypesense represents a listener -type AddTaskToTypesense struct { -} - -// Name defines the name for the AddTaskToTypesense listener -func (l *AddTaskToTypesense) Name() string { - return "typesense.task.add" -} - -// Handle is executed when the event AddTaskToTypesense listens on is fired -func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) { - event := &TaskCreatedEvent{} - err = json.Unmarshal(msg.Payload, event) - if err != nil { - return err - } - - log.Debugf("New task %d created, adding to typesense…", event.Task.ID) - - s := db.NewSession() - defer s.Close() - - task := make(map[int64]*Task, 1) - task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector - - return reindexTasksInTypesense(s, task) -} - -// UpdateTaskInTypesense represents a listener -type UpdateTaskInTypesense struct { -} - -// Name defines the name for the UpdateTaskInTypesense listener -func (l *UpdateTaskInTypesense) Name() string { - return "typesense.task.update" -} - -// Handle is executed when the event UpdateTaskInTypesense listens on is fired -func (l *UpdateTaskInTypesense) Handle(msg *message.Message) (err error) { - event := &TaskUpdatedEvent{} - err = json.Unmarshal(msg.Payload, event) - if err != nil { - return err - } - - s := db.NewSession() - defer s.Close() - - task := make(map[int64]*Task, 1) - task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector - - return reindexTasksInTypesense(s, task) -} - -// UpdateTaskPositionsInTypesense represents a listener -type UpdateTaskPositionsInTypesense struct { -} - -// Name defines the name for the UpdateTaskPositionsInTypesense listener -func (l *UpdateTaskPositionsInTypesense) Name() string { - return "typesense.task.position.update" -} - -// Handle is executed when the event UpdateTaskPositionsInTypesense listens on is fired -func (l *UpdateTaskPositionsInTypesense) Handle(msg *message.Message) (err error) { - event := &TaskPositionsRecalculatedEvent{} - err = json.Unmarshal(msg.Payload, event) - if err != nil { - return err - } - - taskIDs := []int64{} - for _, position := range event.NewTaskPositions { - taskIDs = append(taskIDs, position.TaskID) - } - - s := db.NewSession() - defer s.Close() - - tasks, err := GetTasksSimpleByIDs(s, taskIDs) - - taskMap := make(map[int64]*Task, 1) - for _, task := range tasks { - taskMap[task.ID] = task - } - - return reindexTasksInTypesense(s, taskMap) -} - // IncreaseAttachmentCounter represents a listener type IncreaseAttachmentCounter struct { } @@ -761,14 +639,6 @@ func (l *UpdateTaskInSavedFilterViews) Handle(msg *message.Message) (err error) if err != nil { return } - - task := make(map[int64]*Task, 1) - task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector - - err = reindexTasksInTypesense(s, task) - if err != nil { - return err - } } return s.Commit() diff --git a/pkg/models/models.go b/pkg/models/models.go index b97f81c31..17a30913a 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -59,7 +59,6 @@ func GetTables() []interface{} { &Subscription{}, &Favorite{}, &APIToken{}, - &TypesenseSync{}, &Webhook{}, &Reaction{}, &ProjectView{}, diff --git a/pkg/models/saved_filters.go b/pkg/models/saved_filters.go index 87c401c40..872f75588 100644 --- a/pkg/models/saved_filters.go +++ b/pkg/models/saved_filters.go @@ -19,7 +19,6 @@ package models import ( "time" - "code.vikunja.io/api/pkg/config" "code.vikunja.io/api/pkg/cron" "code.vikunja.io/api/pkg/db" "code.vikunja.io/api/pkg/log" @@ -431,7 +430,6 @@ func RegisterAddTaskToFilterViewCron() { newTaskBuckets := []*TaskBucket{} newTaskPositions := []*TaskPosition{} deleteCond := []builder.Cond{} - taskIDsToRemove := []int64{} viewsToRecalc := map[int64]struct { view *ProjectView ownerID int64 @@ -522,12 +520,11 @@ func RegisterAddTaskToFilterViewCron() { builder.Eq{"task_id": taskID}, builder.Eq{"project_view_id": view.ID}, )) - taskIDsToRemove = append(taskIDsToRemove, taskID) } } } - upsertRelatedTaskProperties(s, logPrefix, newTaskBuckets, newTaskPositions, deleteCond, taskIDsToRemove) + upsertRelatedTaskProperties(s, logPrefix, newTaskBuckets, newTaskPositions, deleteCond) for _, data := range viewsToRecalc { if err := RecalculateTaskPositions(s, data.view, &user.User{ID: data.ownerID}); err != nil { @@ -544,7 +541,7 @@ func RegisterAddTaskToFilterViewCron() { } } -func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBuckets []*TaskBucket, newTaskPositions []*TaskPosition, deleteCond []builder.Cond, taskIDsToRemove []int64) { +func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBuckets []*TaskBucket, newTaskPositions []*TaskPosition, deleteCond []builder.Cond) { var err error if len(newTaskBuckets) > 0 { _, err = s.Insert(newTaskBuckets) @@ -568,27 +565,4 @@ func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBucke log.Errorf("%sError deleting task positions: %s", logPrefix, err) } } - - if config.TypesenseEnabled.GetBool() && (len(newTaskPositions) > 0 || len(taskIDsToRemove) > 0) { - taskIDs := []int64{} - for _, position := range newTaskPositions { - taskIDs = append(taskIDs, position.TaskID) - } - taskIDs = append(taskIDs, taskIDsToRemove...) - tasks, err := GetTasksSimpleByIDs(s, taskIDs) - if err != nil { - log.Errorf("%sError fetching tasks: %s", logPrefix, err) - return - } - taskMap := make(map[int64]*Task) - for _, t := range tasks { - taskMap[t.ID] = t - } - - err = reindexTasksInTypesense(s, taskMap) - if err != nil { - log.Errorf("%sError reindexing tasks into Typesense: %s", logPrefix, err) - return - } - } } diff --git a/pkg/models/task_position.go b/pkg/models/task_position.go index 7881afe03..96c5997e3 100644 --- a/pkg/models/task_position.go +++ b/pkg/models/task_position.go @@ -202,11 +202,6 @@ func RecalculateTaskPositions(s *xorm.Session, view *ProjectView, a web.Auth) (e a: a, } - // We're directly using the db here, even if Typesense is configured, because in some edge cases Typesense - // does not know about all tasks. These tasks then won't have their position recalculated, which means they will - // seemingly jump around after reloading their project. - // The real fix here is of course to make sure all tasks are indexed in Typesense, but until that's fixed, - // this solves the issue of task positions not being saved. allTasks, _, err := dbSearcher.Search(opts) if err != nil { return diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 44cf9c33d..d4a39197c 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -17,18 +17,12 @@ package models import ( - "context" "fmt" - "strconv" "strings" - "time" "code.vikunja.io/api/pkg/db" - "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/web" - "github.com/typesense/typesense-go/v2/typesense/api" - "github.com/typesense/typesense-go/v2/typesense/api/pointer" "xorm.io/builder" "xorm.io/xorm" "xorm.io/xorm/schemas" @@ -490,268 +484,3 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo } return } - -type typesenseTaskSearcher struct { - s *xorm.Session -} - -func convertFilterValues(value interface{}) string { - if _, is := value.([]interface{}); is { - filter := []string{} - for _, v := range value.([]interface{}) { - filter = append(filter, convertFilterValues(v)) - } - - return strings.Join(filter, ",") - } - - if stringSlice, is := value.([]string); is { - filter := []string{} - for _, v := range stringSlice { - filter = append(filter, convertFilterValues(v)) - } - - return strings.Join(filter, ",") - } - - switch v := value.(type) { - case string: - return v - case int: - return strconv.Itoa(v) - case int64: - return strconv.FormatInt(v, 10) - case bool: - if v { - return "true" - } - - return "false" - case time.Time: - return strconv.FormatInt(v.Unix(), 10) - default: - log.Errorf("Unknown search type for value %v of type %T", value, value) - } - - return "" -} - -// Parsing and rebuilding the filter for Typesense has the advantage that we have more control over -// what Typesense finally gets to see. -func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string, err error) { - - filters := []string{} - - for _, f := range rawFilters { - - if nested, is := f.value.([]*taskFilter); is { - nestedDBFilters, err := convertParsedFilterToTypesense(nested) - if err != nil { - return "", err - } - filters = append(filters, "("+nestedDBFilters+")") - continue - } - - if f.field == "reminders" { - f.field = "reminders.reminder" - } - - if f.field == "assignees" { - f.field = "assignees.username" - } - - if f.field == "labels" || f.field == "label_id" { - f.field = "labels.id" - } - - if f.field == "project" { - f.field = "project_id" - } - - if f.field == taskPropertyBucketID { - f.field = "buckets" - } - - filter := f.field - - switch f.comparator { - case taskFilterComparatorEquals: - filter += ":=" - case taskFilterComparatorNotEquals: - filter += ":!=" - case taskFilterComparatorGreater: - filter += ":>" - case taskFilterComparatorGreateEquals: - filter += ":>=" - case taskFilterComparatorLess: - filter += ":<" - case taskFilterComparatorLessEquals: - filter += ":<=" - case taskFilterComparatorLike: - filter += ":" - case taskFilterComparatorIn: - filter += ":[" - case taskFilterComparatorNotIn: - filter += ":!=[" - case taskFilterComparatorInvalid: - // Nothing to do - default: - filter += ":=" - } - - filter += convertFilterValues(f.value) - - if f.comparator == taskFilterComparatorIn || f.comparator == taskFilterComparatorNotIn { - filter += "]" - } - - filters = append(filters, filter) - } - - if len(filters) > 0 { - if len(filters) == 1 { - filterBy = filters[0] - } else { - for i, f := range filters { - if len(filters) > i+1 { - switch rawFilters[i+1].join { - case filterConcatOr: - filterBy = f + " || " + filters[i+1] - case filterConcatAnd: - filterBy = f + " && " + filters[i+1] - } - } - } - } - } - - return -} - -func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { - - projectIDStrings := []string{} - for _, id := range opts.projectIDs { - projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) - } - - filter, err := convertParsedFilterToTypesense(opts.parsedFilters) - if err != nil { - return nil, 0, err - } - - filterBy := []string{"project_id: [" + strings.Join(projectIDStrings, ", ") + "]"} - - if filter != "" { - filterBy = append(filterBy, "("+filter+")") - } - - var sortbyFields []string - var usedParams int - for _, param := range opts.sortby { - - if opts.isSavedFilter && param.sortBy == taskPropertyPosition { - continue - } - - // Validate the params - if err := param.validate(); err != nil { - return nil, totalCount, err - } - - sortBy := param.sortBy - - // Typesense does not allow sorting by ID, so we sort by created timestamp instead - if param.sortBy == taskPropertyID { - sortBy = taskPropertyCreated - } - - if param.sortBy == taskPropertyPosition { - sortBy = "positions.view_" + strconv.FormatInt(param.projectViewID, 10) - } - - sortbyFields = append(sortbyFields, sortBy+"(missing_values:last):"+param.orderBy.String()) - - if usedParams == 2 { - // Typesense supports up to 3 sorting parameters - // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters - break - } - - usedParams++ - } - - sortby := strings.Join(sortbyFields, ",") - - //////////////// - // Actual search - - if opts.search == "" { - opts.search = "*" - } - - params := &api.SearchCollectionParams{ - Q: pointer.String(opts.search), - QueryBy: pointer.String("title, identifier, description, comments.comment"), - Page: pointer.Int(opts.page), - ExhaustiveSearch: pointer.True(), - FilterBy: pointer.String(strings.Join(filterBy, " && ")), - } - - if opts.perPage > 0 { - if opts.perPage > 250 { - log.Warningf("Typesense only supports up to 250 results per page, requested %d.", opts.perPage) - opts.perPage = 250 - } - params.PerPage = pointer.Int(opts.perPage) - } - - if sortby != "" { - params.SortBy = pointer.String(sortby) - } - - result, err := typesenseClient.Collection("tasks"). - Documents(). - Search(context.Background(), params) - if err != nil { - return - } - - taskIDs := []int64{} - for _, h := range *result.Hits { - hit := *h.Document - taskID, err := strconv.ParseInt(hit["id"].(string), 10, 64) - if err != nil { - return nil, 0, err - } - taskIDs = append(taskIDs, taskID) - } - - tasks = []*Task{} - - orderby, err := getOrderByDBStatement(opts) - if err != nil { - return nil, 0, err - } - - var distinct = "tasks.*" - if strings.Contains(orderby, "task_positions.") { - distinct += ", task_positions.position" - } - - query := t.s. - Distinct(distinct). - In("id", taskIDs). - OrderBy(orderby) - - for _, param := range opts.sortby { - if param.sortBy == taskPropertyPosition { - query = query.Join("LEFT", "task_positions", "task_positions.task_id = tasks.id AND task_positions.project_view_id = ?", param.projectViewID) - break - } - } - - err = query.Find(&tasks) - return tasks, int64(*result.Found), err -} diff --git a/pkg/models/task_search_bench_test.go b/pkg/models/task_search_bench_test.go index 9031e902a..142a58e41 100644 --- a/pkg/models/task_search_bench_test.go +++ b/pkg/models/task_search_bench_test.go @@ -107,22 +107,9 @@ func BenchmarkTaskSearch(b *testing.B) { // Log database configuration b.Logf("Database Type: %s", config.DatabaseType.GetString()) - if config.TypesenseEnabled.GetBool() { - b.Log("Typesense is enabled") - } auth := createBenchmarkData(b, needle) - if config.TypesenseEnabled.GetBool() { - InitTypesense() - if err := CreateTypesenseCollections(); err != nil { - b.Skipf("typesense server not available: %v", err) - } - if err := ReindexAllTasks(); err != nil { - b.Skipf("typesense server not available: %v", err) - } - } - // Get all projects for the user s := db.NewSession() projects, _, _, err := getRawProjectsForUser( diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index f264b0e1f..084d26399 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -17,7 +17,6 @@ package models import ( - "errors" "math" "regexp" "sort" @@ -36,7 +35,6 @@ import ( "github.com/google/uuid" clone "github.com/huandu/go-clone/generic" "github.com/jinzhu/copier" - "github.com/typesense/typesense-go/v2/typesense" "xorm.io/builder" "xorm.io/xorm" ) @@ -309,22 +307,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op a: a, hasFavoritesProject: hasFavoritesProject, } - if config.TypesenseEnabled.GetBool() { - var tsSearcher taskSearcher = &typesenseTaskSearcher{ - s: s, - } - origOpts := clone.Clone(opts) - tasks, totalItems, err = tsSearcher.Search(opts) - // It is possible that project views are not yet in Typesense's index. This causes the query here to fail. - // To avoid crashing everything, we fall back to the db search in that case. - var tsErr = &typesense.HTTPError{} - if err != nil && errors.As(err, &tsErr) && tsErr.Status == 404 { - log.Warningf("Unable to fetch tasks from Typesense, error was '%v'. Falling back to db.", err) - tasks, totalItems, err = dbSearcher.Search(origOpts) - } - } else { - tasks, totalItems, err = dbSearcher.Search(opts) - } + tasks, totalItems, err = dbSearcher.Search(opts) return tasks, len(tasks), totalItems, err } diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go deleted file mode 100644 index 2938fdca7..000000000 --- a/pkg/models/typesense.go +++ /dev/null @@ -1,597 +0,0 @@ -// Vikunja is a to-do list application to facilitate your life. -// Copyright 2018-present Vikunja and contributors. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package models - -import ( - "context" - "fmt" - "strconv" - "time" - - "code.vikunja.io/api/pkg/config" - "code.vikunja.io/api/pkg/db" - "code.vikunja.io/api/pkg/log" - "code.vikunja.io/api/pkg/user" - - "github.com/typesense/typesense-go/v2/typesense" - "github.com/typesense/typesense-go/v2/typesense/api" - "github.com/typesense/typesense-go/v2/typesense/api/pointer" - "xorm.io/xorm" -) - -type TypesenseSync struct { - Collection string `xorm:"not null"` - SyncStartedAt time.Time `xorm:"not null"` - SyncFinishedAt time.Time `xorm:"null"` -} - -var typesenseClient *typesense.Client - -func InitTypesense() { - if !config.TypesenseEnabled.GetBool() { - return - } - - typesenseClient = typesense.NewClient( - typesense.WithServer(config.TypesenseURL.GetString()), - typesense.WithAPIKey(config.TypesenseAPIKey.GetString())) -} - -func CreateTypesenseCollections() error { - taskSchema := &api.CollectionSchema{ - Name: "tasks", - EnableNestedFields: pointer.True(), - Fields: []api.Field{ - { - Name: "id", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "title", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "description", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "done", - Type: "bool", - }, - { - Name: "done_at", - Type: "int64", // unix timestamp - Optional: pointer.True(), - }, - { - Name: "due_date", - Type: "int64", // unix timestamp - Optional: pointer.True(), - }, - { - Name: "project_id", - Type: "int64", - }, - { - Name: "repeat_after", - Type: "int64", - }, - { - Name: "repeat_mode", - Type: "int32", - }, - { - Name: "priority", - Type: "int64", - }, - { - Name: "start_date", - Type: "int64", // unix timestamp - Optional: pointer.True(), - }, - { - Name: "end_date", - Type: "int64", // unix timestamp - Optional: pointer.True(), - }, - { - Name: "hex_color", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "percent_done", - Type: "float", - }, - { - Name: "identifier", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "index", - Type: "int64", - }, - { - Name: "uid", - Type: "string", - Sort: pointer.True(), - }, - { - Name: "cover_image_attachment_id", - Type: "int64", - }, - { - Name: "created", - Type: "int64", // unix timestamp - }, - { - Name: "updated", - Type: "int64", // unix timestamp - }, - { - Name: "created_by_id", - Type: "int64", - }, - { - Name: "reminders", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "assignees", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "labels", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "related_tasks", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "attachments", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "comments", - Type: "object[]", // TODO - Optional: pointer.True(), - }, - { - Name: "positions", - Type: "object", - }, - { - Name: "positions.view_.*", - Type: "float", - }, - { - Name: "buckets", - Type: "int64[]", - }, - }, - } - - // delete any collection which might exist - _, _ = typesenseClient.Collection("tasks").Delete(context.Background()) - - _, err := typesenseClient.Collections().Create(context.Background(), taskSchema) - return err -} - -func ReindexAllTasks() (err error) { - s := db.NewSession() - defer s.Close() - - _, err = s.Where("collection = ?", "tasks").Delete(&TypesenseSync{}) - if err != nil { - return fmt.Errorf("could not delete old sync status: %s", err.Error()) - } - - currentSync := &TypesenseSync{ - Collection: "tasks", - SyncStartedAt: time.Now(), - } - _, err = s.Insert(currentSync) - if err != nil { - return fmt.Errorf("could not update last sync: %s", err.Error()) - } - - tasks := make(map[int64]*Task) - err = s.Find(tasks) - if err != nil { - return fmt.Errorf("could not get all tasks: %s", err.Error()) - } - - err = indexDummyTask() - if err != nil { - return fmt.Errorf("could not index dummy task: %w", err) - } - - err = reindexTasksInTypesense(s, tasks) - if err != nil { - return fmt.Errorf("could not reindex all tasks: %s", err.Error()) - } - - currentSync.SyncFinishedAt = time.Now() - _, err = s.Where("collection = ?", "tasks"). - Cols("sync_finished_at"). - Update(currentSync) - if err != nil { - return fmt.Errorf("could update last sync state: %s", err.Error()) - } - - return s.Commit() -} - -func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error) { - - if !config.TypesenseEnabled.GetBool() { - return - } - - if len(tasks) == 0 { - log.Infof("No tasks to index") - return - } - - err = addMoreInfoToTasks(s, tasks, &user.User{ID: 1}, nil, []TaskCollectionExpandable{ - TaskCollectionExpandReactions, - TaskCollectionExpandComments, - }) - if err != nil { - return fmt.Errorf("could not fetch more task info: %s", err.Error()) - } - - typesenseTasks := []interface{}{} - - positionsByTask, err := getPositionsByTask(s) - if err != nil { - return err - } - - bucketsByTask, err := getBucketsByTask(s) - if err != nil { - return err - } - - for _, task := range tasks { - ttask := convertTaskToTypesenseTask(task, positionsByTask[task.ID], bucketsByTask[task.ID]) - if ttask == nil { - log.Debugf("Converted typesense task %d is nil, not indexing", task.ID) - continue - } - - typesenseTasks = append(typesenseTasks, ttask) - } - - response, err := typesenseClient.Collection("tasks"). - Documents(). - Import(context.Background(), typesenseTasks, &api.ImportDocumentsParams{ - Action: pointer.String("upsert"), - BatchSize: pointer.Int(100), - }) - if err != nil { - log.Errorf("Could not upsert tasks into Typesense: %s", err) - return err - } - for _, r := range response { - if r.Success { - continue - } - log.Errorf("Errors during index: [error=%s, document=%s]", r.Error, r.Document) - } - - log.Debugf("Indexed %d tasks into Typesense", len(typesenseTasks)) - - return nil -} - -type TaskPositionWithView struct { - ProjectView `xorm:"extends"` - TaskPosition `xorm:"extends"` -} - -func getPositionsByTask(s *xorm.Session) (positionsByTask map[int64][]*TaskPositionWithView, err error) { - rawPositions := []*TaskPositionWithView{} - err = s. - Table("project_views"). - Join("LEFT", "task_positions", "project_views.id = task_positions.project_view_id"). - Find(&rawPositions) - if err != nil { - return - } - - positionsByTask = make(map[int64][]*TaskPositionWithView, len(rawPositions)) - for _, p := range rawPositions { - _, has := positionsByTask[p.TaskID] - if !has { - positionsByTask[p.TaskID] = []*TaskPositionWithView{} - } - positionsByTask[p.TaskID] = append(positionsByTask[p.TaskID], p) - } - return positionsByTask, nil -} - -func getBucketsByTask(s *xorm.Session) (positionsByTask map[int64][]*TaskBucket, err error) { - rawBuckets := []*TaskBucket{} - err = s.Find(&rawBuckets) - if err != nil { - return - } - - positionsByTask = make(map[int64][]*TaskBucket, len(rawBuckets)) - for _, p := range rawBuckets { - _, has := positionsByTask[p.TaskID] - if !has { - positionsByTask[p.TaskID] = []*TaskBucket{} - } - positionsByTask[p.TaskID] = append(positionsByTask[p.TaskID], p) - } - return positionsByTask, nil -} - -func indexDummyTask() (err error) { - // The initial sync should contain one dummy task with all related fields populated so that typesense - // creates the indexes properly. A little hacky, but gets the job done. - dummyTask := &typesenseTask{ - ID: "-100", - Title: "Dummytask", - Created: time.Now().Unix(), - Updated: time.Now().Unix(), - Reminders: []*TaskReminder{ - { - ID: -10, - TaskID: -100, - Reminder: time.Now(), - RelativePeriod: 10, - RelativeTo: ReminderRelationDueDate, - Created: time.Now(), - }, - }, - Assignees: []*user.User{ - { - ID: -100, - Username: "dummy", - Name: "dummy", - Email: "dummy@vikunja", - Created: time.Now(), - Updated: time.Now(), - }, - }, - Labels: []*Label{ - { - ID: -110, - Title: "dummylabel", - Description: "Lorem Ipsum Dummy", - HexColor: "000000", - Created: time.Now(), - Updated: time.Now(), - }, - }, - Attachments: []*TaskAttachment{ - { - ID: -120, - TaskID: -100, - Created: time.Now(), - }, - }, - Comments: []*TaskComment{ - { - ID: -220, - Comment: "Lorem Ipsum Dummy", - Created: time.Now(), - Updated: time.Now(), - Author: &user.User{ - ID: -100, - Username: "dummy", - Name: "dummy", - Email: "dummy@vikunja", - Created: time.Now(), - Updated: time.Now(), - }, - }, - }, - Positions: map[string]float64{ - "view_1": 10, - "view_2": 30, - "view_3": 5450, - "view_4": 42, - }, - Buckets: []int64{42}, - } - - _, err = typesenseClient.Collection("tasks"). - Documents(). - Create(context.Background(), dummyTask) - if err != nil { - return - } - - _, err = typesenseClient.Collection("tasks"). - Document(dummyTask.ID). - Delete(context.Background()) - return -} - -type typesenseTask struct { - ID string `json:"id"` - Title string `json:"title"` - Description string `json:"description"` - Done bool `json:"done"` - DoneAt *int64 `json:"done_at"` - DueDate *int64 `json:"due_date"` - ProjectID int64 `json:"project_id"` - RepeatAfter int64 `json:"repeat_after"` - RepeatMode int `json:"repeat_mode"` - Priority int64 `json:"priority"` - StartDate *int64 `json:"start_date"` - EndDate *int64 `json:"end_date"` - HexColor string `json:"hex_color"` - PercentDone float64 `json:"percent_done"` - Identifier string `json:"identifier"` - Index int64 `json:"index"` - UID string `json:"uid"` - CoverImageAttachmentID int64 `json:"cover_image_attachment_id"` - Created int64 `json:"created"` - Updated int64 `json:"updated"` - CreatedByID int64 `json:"created_by_id"` - Reminders interface{} `json:"reminders"` - Assignees interface{} `json:"assignees"` - Labels interface{} `json:"labels"` - //RelatedTasks interface{} `json:"related_tasks"` // TODO - Attachments interface{} `json:"attachments"` - Comments interface{} `json:"comments"` - Positions map[string]float64 `json:"positions"` - Buckets []int64 `json:"buckets"` -} - -func convertTaskToTypesenseTask(task *Task, positions []*TaskPositionWithView, buckets []*TaskBucket) *typesenseTask { - - tt := &typesenseTask{ - ID: fmt.Sprintf("%d", task.ID), - Title: task.Title, - Description: task.Description, - Done: task.Done, - DoneAt: pointer.Int64(task.DoneAt.UTC().Unix()), - DueDate: pointer.Int64(task.DueDate.UTC().Unix()), - ProjectID: task.ProjectID, - RepeatAfter: task.RepeatAfter, - RepeatMode: int(task.RepeatMode), - Priority: task.Priority, - StartDate: pointer.Int64(task.StartDate.UTC().Unix()), - EndDate: pointer.Int64(task.EndDate.UTC().Unix()), - HexColor: task.HexColor, - PercentDone: task.PercentDone, - Identifier: task.Identifier, - Index: task.Index, - UID: task.UID, - CoverImageAttachmentID: task.CoverImageAttachmentID, - Created: task.Created.UTC().Unix(), - Updated: task.Updated.UTC().Unix(), - CreatedByID: task.CreatedByID, - Reminders: task.Reminders, - Assignees: task.Assignees, - Labels: task.Labels, - //RelatedTasks: task.RelatedTasks, - Attachments: task.Attachments, - Positions: make(map[string]float64, len(positions)), - Buckets: make([]int64, 0, len(buckets)), - } - - if task.DoneAt.IsZero() { - tt.DoneAt = nil - } - if task.DueDate.IsZero() { - tt.DueDate = nil - } - if task.StartDate.IsZero() { - tt.StartDate = nil - } - if task.EndDate.IsZero() { - tt.EndDate = nil - } - - for _, position := range positions { - pos := position.TaskPosition.Position - if pos == 0 { - pos = float64(task.ID) - } - tt.Positions["view_"+strconv.FormatInt(position.ID, 10)] = pos - } - - for _, bucket := range buckets { - tt.Buckets = append(tt.Buckets, bucket.BucketID) - } - - return tt -} - -// This function is only used to catch up with the Typesense Sync when it didn't index for some reason - -func SyncUpdatedTasksIntoTypesense() (err error) { - tasks := make(map[int64]*Task) - - s := db.NewSession() - defer s.Close() - - lastSync := &TypesenseSync{} - has, err := s.Where("collection = ?", "tasks"). - Get(lastSync) - if err != nil { - _ = s.Rollback() - return err - } - - if !has { - log.Errorf("[Typesense Sync] No typesense sync stats yet, please run a full index via the CLI first") - _ = s.Rollback() - return - } - - currentSync := &TypesenseSync{SyncStartedAt: time.Now()} - _, err = s.Where("collection = ?", "tasks"). - Cols("sync_started_at", "sync_finished_at"). - Update(currentSync) - if err != nil { - _ = s.Rollback() - return - } - - err = s. - Where("updated >= ?", lastSync.SyncStartedAt). - And("updated != created"). // new tasks are already indexed via the event handler - Find(tasks) - if err != nil { - _ = s.Rollback() - return - } - - if len(tasks) > 0 { - log.Debugf("[Typesense Sync] Updating %d tasks", len(tasks)) - - err = reindexTasksInTypesense(s, tasks) - if err != nil { - _ = s.Rollback() - return - } - } - - if len(tasks) == 0 { - log.Debugf("[Typesense Sync] No tasks changed since the last sync, not syncing") - } - - currentSync.SyncFinishedAt = time.Now() - _, err = s.Where("collection = ?", "tasks"). - Cols("sync_finished_at"). - Update(currentSync) - if err != nil { - _ = s.Rollback() - return err - } - - return s.Commit() -}