refactor: remove typesense support
Typesense was an optional external search backend. This commit fully removes the integration, leaving the database searcher as the only search implementation. Changes: - Delete pkg/models/typesense.go (core integration) - Delete pkg/cmd/index.go (CLI command for indexing) - Simplify task search to always use database searcher - Remove Typesense event listeners for task sync - Remove TypesenseSync model registration - Remove Typesense config keys and defaults - Remove Typesense doctor health check - Remove Typesense initialization from startup - Clean up benchmark test - Add migration to drop typesense_sync table - Remove golangci-lint suppression for typesense.go - Remove typesense-go dependency
This commit is contained in:
kolaente
parent
111ac9c726
commit
a5b1a90c42
|
|
@ -123,10 +123,6 @@ linters:
|
|||
- revive
|
||||
path: pkg/migration/*
|
||||
text: parameter 'tx' seems to be unused, consider removing or renaming it as
|
||||
- linters:
|
||||
- govet
|
||||
path: pkg/models/typesense.go
|
||||
text: 'structtag: struct field Position repeats json tag "position" also at'
|
||||
- linters:
|
||||
- gosec
|
||||
path: pkg/cmd/user.go
|
||||
|
|
|
|||
|
|
@ -250,26 +250,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "typesense",
|
||||
"children": [
|
||||
{
|
||||
"key": "enabled",
|
||||
"default_value": "false",
|
||||
"comment": "Whether to enable the Typesense integration. If true, all tasks will be synced to the configured Typesense\ninstance and all search and filtering will run through Typesense instead of only through the database.\nTypesense allows fast fulltext search including fuzzy matching support. It may return different results than\nwhat you'd get with a database-only search."
|
||||
},
|
||||
{
|
||||
"key": "url",
|
||||
"default_value": "",
|
||||
"comment": "The url to the Typesense instance you want to use. Can be hosted locally or in Typesense Cloud as long as Vikunja is able to reach it. Must be a http(s) url."
|
||||
},
|
||||
{
|
||||
"key": "apikey",
|
||||
"default_value": "",
|
||||
"comment": "The Typesense API key you want to use."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "redis",
|
||||
"children": [
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -70,7 +70,6 @@ require (
|
|||
github.com/stretchr/testify v1.11.1
|
||||
github.com/swaggo/swag v1.16.6
|
||||
github.com/tkuchiki/go-timezone v0.2.3
|
||||
github.com/typesense/typesense-go/v2 v2.0.0
|
||||
github.com/ulule/limiter/v3 v3.11.2
|
||||
github.com/wneessen/go-mail v0.7.2
|
||||
github.com/yuin/goldmark v1.7.16
|
||||
|
|
@ -94,7 +93,6 @@ require (
|
|||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19 // indirect
|
||||
|
|
@ -147,7 +145,6 @@ require (
|
|||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/oapi-codegen/runtime v1.1.1 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.1.0 // indirect
|
||||
|
|
@ -172,7 +169,6 @@ require (
|
|||
github.com/tj/assert v0.0.3 // indirect
|
||||
github.com/urfave/cli/v2 v2.3.0 // indirect
|
||||
github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -16,15 +16,12 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge
|
|||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY=
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
|
||||
github.com/adlio/trello v1.12.0 h1:JqOE2GFHQ9YtEviRRRSnicSxPbt4WFOxhqXzjMOw8lw=
|
||||
github.com/adlio/trello v1.12.0/go.mod h1:I4Lti4jf2KxjTNgTqs5W3lLuE78QZZdYbbPnQQGwjOo=
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
|
||||
github.com/arran4/golang-ical v0.3.2 h1:MGNjcXJFSuCXmYX/RpZhR2HDCYoFuK8vTPFLEdFC3JY=
|
||||
github.com/arran4/golang-ical v0.3.2/go.mod h1:xblDGxxIUMWwFZk9dlECUlc1iXNV65LJZOTHLVwu8bo=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
|
|
@ -77,7 +74,6 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
|
|||
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs=
|
||||
github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
|
|
@ -308,7 +304,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
|
|||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6 h1:SwcnSwBR7X/5EHJQlXBockkJVIMRVt5yKaesBPMtyZQ=
|
||||
github.com/jszwedko/go-datemath v0.1.1-0.20230526204004-640a500621d6/go.mod h1:WrYiIuiXUMIvTDAQw97C+9l0CnBmCcvosPjN3XDqS/o=
|
||||
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
|
|
@ -384,8 +379,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA
|
|||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
|
||||
github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
||||
|
|
@ -471,7 +464,6 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
|||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
|
|
@ -499,8 +491,6 @@ github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
|||
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
|
||||
github.com/tkuchiki/go-timezone v0.2.3 h1:D3TVdIPrFsu9lxGxqNX2wsZwn1MZtTqTW0mdevMozHc=
|
||||
github.com/tkuchiki/go-timezone v0.2.3/go.mod h1:oFweWxYl35C/s7HMVZXiA19Jr9Y0qJHMaG/J2TES4LY=
|
||||
github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I=
|
||||
github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao=
|
||||
github.com/ulule/limiter/v3 v3.11.2 h1:P4yOrxoEMJbOTfRJR2OzjL90oflzYPPmWg+dvwN2tHA=
|
||||
github.com/ulule/limiter/v3 v3.11.2/go.mod h1:QG5GnFOCV+k7lrL5Y8kgEeeflPH3+Cviqlqa8SVSQxI=
|
||||
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
|
||||
|
|
@ -521,8 +511,6 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
|||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ issues with your Vikunja installation. It checks:
|
|||
- Configuration (config file, public URL, JWT secret, CORS)
|
||||
- Database connectivity and version
|
||||
- File storage (local or S3)
|
||||
- Optional services (Redis, Typesense, Mailer, LDAP, OpenID)
|
||||
- Optional services (Redis, Mailer, LDAP, OpenID)
|
||||
|
||||
Exit codes:
|
||||
0 - All checks passed
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/initialize"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(indexCmd)
|
||||
}
|
||||
|
||||
var indexPartialFlag bool
|
||||
|
||||
var indexCmd = &cobra.Command{
|
||||
Use: "index",
|
||||
Short: "Reindex all of Vikunja's data into Typesense. This will remove any existing index.",
|
||||
PreRun: func(_ *cobra.Command, _ []string) {
|
||||
initialize.FullInitWithoutAsync()
|
||||
},
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
if config.TypesenseURL.GetString() == "" {
|
||||
log.Error("Typesense not configured")
|
||||
return
|
||||
}
|
||||
|
||||
err := models.CreateTypesenseCollections()
|
||||
if err != nil {
|
||||
log.Criticalf("Could not create Typesense collections: %s", err.Error())
|
||||
return
|
||||
}
|
||||
if indexPartialFlag {
|
||||
log.Infof("Indexing changed tasks… This may take a while.")
|
||||
err = models.SyncUpdatedTasksIntoTypesense()
|
||||
if err != nil {
|
||||
log.Criticalf("Could not reindex all changed tasks into Typesense: %s", err.Error())
|
||||
return
|
||||
}
|
||||
} else {
|
||||
log.Infof("Indexing all tasks… This may take a while.")
|
||||
err = models.ReindexAllTasks()
|
||||
if err != nil {
|
||||
log.Criticalf("Could not reindex all tasks into Typesense: %s", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Done!")
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
indexCmd.Flags().BoolVarP(&indexPartialFlag, "partial", "p", false, "If provided, Vikunja will only index those tasks which are not present in the index. It will not remove any existing tasks.")
|
||||
}
|
||||
|
|
@ -117,10 +117,6 @@ const (
|
|||
DatabaseTLS Key = `database.tls`
|
||||
DatabaseSchema Key = `database.schema`
|
||||
|
||||
TypesenseEnabled Key = `typesense.enabled`
|
||||
TypesenseURL Key = `typesense.url`
|
||||
TypesenseAPIKey Key = `typesense.apikey`
|
||||
|
||||
MailerEnabled Key = `mailer.enabled`
|
||||
MailerHost Key = `mailer.host`
|
||||
MailerPort Key = `mailer.port`
|
||||
|
|
@ -398,9 +394,6 @@ func InitDefaultConfig() {
|
|||
DatabaseTLS.setDefault("false")
|
||||
DatabaseSchema.setDefault("public")
|
||||
|
||||
// Typesense
|
||||
TypesenseEnabled.setDefault(false)
|
||||
|
||||
// Mailer
|
||||
MailerEnabled.setDefault(false)
|
||||
MailerHost.setDefault("")
|
||||
|
|
|
|||
|
|
@ -36,10 +36,6 @@ func CheckOptionalServices() []CheckGroup {
|
|||
groups = append(groups, checkRedis())
|
||||
}
|
||||
|
||||
if config.TypesenseEnabled.GetBool() {
|
||||
groups = append(groups, checkTypesense())
|
||||
}
|
||||
|
||||
if config.MailerEnabled.GetBool() {
|
||||
groups = append(groups, checkMailer())
|
||||
}
|
||||
|
|
@ -101,70 +97,6 @@ func checkRedis() CheckGroup {
|
|||
}
|
||||
}
|
||||
|
||||
func checkTypesense() CheckGroup {
|
||||
url := config.TypesenseURL.GetString()
|
||||
healthURL := url + "/health"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil)
|
||||
if err != nil {
|
||||
return CheckGroup{
|
||||
Name: "Typesense",
|
||||
Results: []CheckResult{
|
||||
{
|
||||
Name: "Connection",
|
||||
Passed: false,
|
||||
Error: err.Error(),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
req.Header.Set("X-TYPESENSE-API-KEY", config.TypesenseAPIKey.GetString())
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return CheckGroup{
|
||||
Name: "Typesense",
|
||||
Results: []CheckResult{
|
||||
{
|
||||
Name: "Connection",
|
||||
Passed: false,
|
||||
Error: err.Error(),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return CheckGroup{
|
||||
Name: "Typesense",
|
||||
Results: []CheckResult{
|
||||
{
|
||||
Name: "Connection",
|
||||
Passed: false,
|
||||
Error: fmt.Sprintf("health check returned status %d", resp.StatusCode),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return CheckGroup{
|
||||
Name: "Typesense",
|
||||
Results: []CheckResult{
|
||||
{
|
||||
Name: "Connection",
|
||||
Passed: true,
|
||||
Value: fmt.Sprintf("OK (%s)", url),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func checkMailer() CheckGroup {
|
||||
host := config.MailerHost.GetString()
|
||||
port := config.MailerPort.GetInt()
|
||||
|
|
|
|||
|
|
@ -91,9 +91,6 @@ func FullInitWithoutAsync() {
|
|||
// Set Engine
|
||||
InitEngines()
|
||||
|
||||
// Init Typesense
|
||||
models.InitTypesense()
|
||||
|
||||
// Start the mail daemon
|
||||
mail.StartMailDaemon()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package migration
|
||||
|
||||
import (
|
||||
"src.techknowlogick.com/xormigrate"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
func init() {
|
||||
migrations = append(migrations, &xormigrate.Migration{
|
||||
ID: "20260225114726",
|
||||
Description: "Drop the typesense_sync table",
|
||||
Migrate: func(tx *xorm.Engine) error {
|
||||
return tx.DropTables("typesense_sync")
|
||||
},
|
||||
Rollback: func(tx *xorm.Engine) error {
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -17,7 +17,6 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
|
@ -70,12 +69,6 @@ func RegisterListeners() {
|
|||
events.RegisterListener((&TaskCreatedEvent{}).Name(), &UpdateTaskInSavedFilterViews{})
|
||||
events.RegisterListener((&TaskUpdatedEvent{}).Name(), &UpdateTaskInSavedFilterViews{})
|
||||
events.RegisterListener((&TaskCommentCreatedEvent{}).Name(), &MarkTaskUnreadOnComment{})
|
||||
if config.TypesenseEnabled.GetBool() {
|
||||
events.RegisterListener((&TaskDeletedEvent{}).Name(), &RemoveTaskFromTypesense{})
|
||||
events.RegisterListener((&TaskCreatedEvent{}).Name(), &AddTaskToTypesense{})
|
||||
events.RegisterListener((&TaskUpdatedEvent{}).Name(), &UpdateTaskInTypesense{})
|
||||
events.RegisterListener((&TaskPositionsRecalculatedEvent{}).Name(), &UpdateTaskPositionsInTypesense{})
|
||||
}
|
||||
if config.WebhooksEnabled.GetBool() {
|
||||
RegisterEventForWebhook(&TaskCreatedEvent{})
|
||||
RegisterEventForWebhook(&TaskUpdatedEvent{})
|
||||
|
|
@ -503,121 +496,6 @@ func (s *HandleTaskUpdateLastUpdated) Handle(msg *message.Message) (err error) {
|
|||
return sess.Commit()
|
||||
}
|
||||
|
||||
// RemoveTaskFromTypesense represents a listener
|
||||
type RemoveTaskFromTypesense struct {
|
||||
}
|
||||
|
||||
// Name defines the name for the RemoveTaskFromTypesense listener
|
||||
func (s *RemoveTaskFromTypesense) Name() string {
|
||||
return "typesense.task.remove"
|
||||
}
|
||||
|
||||
// Handle is executed when the event RemoveTaskFromTypesense listens on is fired
|
||||
func (s *RemoveTaskFromTypesense) Handle(msg *message.Message) (err error) {
|
||||
event := &TaskDeletedEvent{}
|
||||
err = json.Unmarshal(msg.Payload, event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("[Typesense Sync] Removing task %d from Typesense", event.Task.ID)
|
||||
|
||||
_, err = typesenseClient.
|
||||
Collection("tasks").
|
||||
Document(strconv.FormatInt(event.Task.ID, 10)).
|
||||
Delete(context.Background())
|
||||
return err
|
||||
}
|
||||
|
||||
// AddTaskToTypesense represents a listener
|
||||
type AddTaskToTypesense struct {
|
||||
}
|
||||
|
||||
// Name defines the name for the AddTaskToTypesense listener
|
||||
func (l *AddTaskToTypesense) Name() string {
|
||||
return "typesense.task.add"
|
||||
}
|
||||
|
||||
// Handle is executed when the event AddTaskToTypesense listens on is fired
|
||||
func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) {
|
||||
event := &TaskCreatedEvent{}
|
||||
err = json.Unmarshal(msg.Payload, event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("New task %d created, adding to typesense…", event.Task.ID)
|
||||
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
task := make(map[int64]*Task, 1)
|
||||
task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
|
||||
|
||||
return reindexTasksInTypesense(s, task)
|
||||
}
|
||||
|
||||
// UpdateTaskInTypesense represents a listener
|
||||
type UpdateTaskInTypesense struct {
|
||||
}
|
||||
|
||||
// Name defines the name for the UpdateTaskInTypesense listener
|
||||
func (l *UpdateTaskInTypesense) Name() string {
|
||||
return "typesense.task.update"
|
||||
}
|
||||
|
||||
// Handle is executed when the event UpdateTaskInTypesense listens on is fired
|
||||
func (l *UpdateTaskInTypesense) Handle(msg *message.Message) (err error) {
|
||||
event := &TaskUpdatedEvent{}
|
||||
err = json.Unmarshal(msg.Payload, event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
task := make(map[int64]*Task, 1)
|
||||
task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
|
||||
|
||||
return reindexTasksInTypesense(s, task)
|
||||
}
|
||||
|
||||
// UpdateTaskPositionsInTypesense represents a listener
|
||||
type UpdateTaskPositionsInTypesense struct {
|
||||
}
|
||||
|
||||
// Name defines the name for the UpdateTaskPositionsInTypesense listener
|
||||
func (l *UpdateTaskPositionsInTypesense) Name() string {
|
||||
return "typesense.task.position.update"
|
||||
}
|
||||
|
||||
// Handle is executed when the event UpdateTaskPositionsInTypesense listens on is fired
|
||||
func (l *UpdateTaskPositionsInTypesense) Handle(msg *message.Message) (err error) {
|
||||
event := &TaskPositionsRecalculatedEvent{}
|
||||
err = json.Unmarshal(msg.Payload, event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
taskIDs := []int64{}
|
||||
for _, position := range event.NewTaskPositions {
|
||||
taskIDs = append(taskIDs, position.TaskID)
|
||||
}
|
||||
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
tasks, err := GetTasksSimpleByIDs(s, taskIDs)
|
||||
|
||||
taskMap := make(map[int64]*Task, 1)
|
||||
for _, task := range tasks {
|
||||
taskMap[task.ID] = task
|
||||
}
|
||||
|
||||
return reindexTasksInTypesense(s, taskMap)
|
||||
}
|
||||
|
||||
// IncreaseAttachmentCounter represents a listener
|
||||
type IncreaseAttachmentCounter struct {
|
||||
}
|
||||
|
|
@ -761,14 +639,6 @@ func (l *UpdateTaskInSavedFilterViews) Handle(msg *message.Message) (err error)
|
|||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
task := make(map[int64]*Task, 1)
|
||||
task[event.Task.ID] = event.Task // Will be filled with all data by the Typesense connector
|
||||
|
||||
err = reindexTasksInTypesense(s, task)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return s.Commit()
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ func GetTables() []interface{} {
|
|||
&Subscription{},
|
||||
&Favorite{},
|
||||
&APIToken{},
|
||||
&TypesenseSync{},
|
||||
&Webhook{},
|
||||
&Reaction{},
|
||||
&ProjectView{},
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package models
|
|||
import (
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/cron"
|
||||
"code.vikunja.io/api/pkg/db"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
|
|
@ -431,7 +430,6 @@ func RegisterAddTaskToFilterViewCron() {
|
|||
newTaskBuckets := []*TaskBucket{}
|
||||
newTaskPositions := []*TaskPosition{}
|
||||
deleteCond := []builder.Cond{}
|
||||
taskIDsToRemove := []int64{}
|
||||
viewsToRecalc := map[int64]struct {
|
||||
view *ProjectView
|
||||
ownerID int64
|
||||
|
|
@ -522,12 +520,11 @@ func RegisterAddTaskToFilterViewCron() {
|
|||
builder.Eq{"task_id": taskID},
|
||||
builder.Eq{"project_view_id": view.ID},
|
||||
))
|
||||
taskIDsToRemove = append(taskIDsToRemove, taskID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
upsertRelatedTaskProperties(s, logPrefix, newTaskBuckets, newTaskPositions, deleteCond, taskIDsToRemove)
|
||||
upsertRelatedTaskProperties(s, logPrefix, newTaskBuckets, newTaskPositions, deleteCond)
|
||||
|
||||
for _, data := range viewsToRecalc {
|
||||
if err := RecalculateTaskPositions(s, data.view, &user.User{ID: data.ownerID}); err != nil {
|
||||
|
|
@ -544,7 +541,7 @@ func RegisterAddTaskToFilterViewCron() {
|
|||
}
|
||||
}
|
||||
|
||||
func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBuckets []*TaskBucket, newTaskPositions []*TaskPosition, deleteCond []builder.Cond, taskIDsToRemove []int64) {
|
||||
func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBuckets []*TaskBucket, newTaskPositions []*TaskPosition, deleteCond []builder.Cond) {
|
||||
var err error
|
||||
if len(newTaskBuckets) > 0 {
|
||||
_, err = s.Insert(newTaskBuckets)
|
||||
|
|
@ -568,27 +565,4 @@ func upsertRelatedTaskProperties(s *xorm.Session, logPrefix string, newTaskBucke
|
|||
log.Errorf("%sError deleting task positions: %s", logPrefix, err)
|
||||
}
|
||||
}
|
||||
|
||||
if config.TypesenseEnabled.GetBool() && (len(newTaskPositions) > 0 || len(taskIDsToRemove) > 0) {
|
||||
taskIDs := []int64{}
|
||||
for _, position := range newTaskPositions {
|
||||
taskIDs = append(taskIDs, position.TaskID)
|
||||
}
|
||||
taskIDs = append(taskIDs, taskIDsToRemove...)
|
||||
tasks, err := GetTasksSimpleByIDs(s, taskIDs)
|
||||
if err != nil {
|
||||
log.Errorf("%sError fetching tasks: %s", logPrefix, err)
|
||||
return
|
||||
}
|
||||
taskMap := make(map[int64]*Task)
|
||||
for _, t := range tasks {
|
||||
taskMap[t.ID] = t
|
||||
}
|
||||
|
||||
err = reindexTasksInTypesense(s, taskMap)
|
||||
if err != nil {
|
||||
log.Errorf("%sError reindexing tasks into Typesense: %s", logPrefix, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -202,11 +202,6 @@ func RecalculateTaskPositions(s *xorm.Session, view *ProjectView, a web.Auth) (e
|
|||
a: a,
|
||||
}
|
||||
|
||||
// We're directly using the db here, even if Typesense is configured, because in some edge cases Typesense
|
||||
// does not know about all tasks. These tasks then won't have their position recalculated, which means they will
|
||||
// seemingly jump around after reloading their project.
|
||||
// The real fix here is of course to make sure all tasks are indexed in Typesense, but until that's fixed,
|
||||
// this solves the issue of task positions not being saved.
|
||||
allTasks, _, err := dbSearcher.Search(opts)
|
||||
if err != nil {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -17,18 +17,12 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/db"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/web"
|
||||
|
||||
"github.com/typesense/typesense-go/v2/typesense/api"
|
||||
"github.com/typesense/typesense-go/v2/typesense/api/pointer"
|
||||
"xorm.io/builder"
|
||||
"xorm.io/xorm"
|
||||
"xorm.io/xorm/schemas"
|
||||
|
|
@ -490,268 +484,3 @@ func (d *dbTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCo
|
|||
}
|
||||
return
|
||||
}
|
||||
|
||||
type typesenseTaskSearcher struct {
|
||||
s *xorm.Session
|
||||
}
|
||||
|
||||
func convertFilterValues(value interface{}) string {
|
||||
if _, is := value.([]interface{}); is {
|
||||
filter := []string{}
|
||||
for _, v := range value.([]interface{}) {
|
||||
filter = append(filter, convertFilterValues(v))
|
||||
}
|
||||
|
||||
return strings.Join(filter, ",")
|
||||
}
|
||||
|
||||
if stringSlice, is := value.([]string); is {
|
||||
filter := []string{}
|
||||
for _, v := range stringSlice {
|
||||
filter = append(filter, convertFilterValues(v))
|
||||
}
|
||||
|
||||
return strings.Join(filter, ",")
|
||||
}
|
||||
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return v
|
||||
case int:
|
||||
return strconv.Itoa(v)
|
||||
case int64:
|
||||
return strconv.FormatInt(v, 10)
|
||||
case bool:
|
||||
if v {
|
||||
return "true"
|
||||
}
|
||||
|
||||
return "false"
|
||||
case time.Time:
|
||||
return strconv.FormatInt(v.Unix(), 10)
|
||||
default:
|
||||
log.Errorf("Unknown search type for value %v of type %T", value, value)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// Parsing and rebuilding the filter for Typesense has the advantage that we have more control over
|
||||
// what Typesense finally gets to see.
|
||||
func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string, err error) {
|
||||
|
||||
filters := []string{}
|
||||
|
||||
for _, f := range rawFilters {
|
||||
|
||||
if nested, is := f.value.([]*taskFilter); is {
|
||||
nestedDBFilters, err := convertParsedFilterToTypesense(nested)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
filters = append(filters, "("+nestedDBFilters+")")
|
||||
continue
|
||||
}
|
||||
|
||||
if f.field == "reminders" {
|
||||
f.field = "reminders.reminder"
|
||||
}
|
||||
|
||||
if f.field == "assignees" {
|
||||
f.field = "assignees.username"
|
||||
}
|
||||
|
||||
if f.field == "labels" || f.field == "label_id" {
|
||||
f.field = "labels.id"
|
||||
}
|
||||
|
||||
if f.field == "project" {
|
||||
f.field = "project_id"
|
||||
}
|
||||
|
||||
if f.field == taskPropertyBucketID {
|
||||
f.field = "buckets"
|
||||
}
|
||||
|
||||
filter := f.field
|
||||
|
||||
switch f.comparator {
|
||||
case taskFilterComparatorEquals:
|
||||
filter += ":="
|
||||
case taskFilterComparatorNotEquals:
|
||||
filter += ":!="
|
||||
case taskFilterComparatorGreater:
|
||||
filter += ":>"
|
||||
case taskFilterComparatorGreateEquals:
|
||||
filter += ":>="
|
||||
case taskFilterComparatorLess:
|
||||
filter += ":<"
|
||||
case taskFilterComparatorLessEquals:
|
||||
filter += ":<="
|
||||
case taskFilterComparatorLike:
|
||||
filter += ":"
|
||||
case taskFilterComparatorIn:
|
||||
filter += ":["
|
||||
case taskFilterComparatorNotIn:
|
||||
filter += ":!=["
|
||||
case taskFilterComparatorInvalid:
|
||||
// Nothing to do
|
||||
default:
|
||||
filter += ":="
|
||||
}
|
||||
|
||||
filter += convertFilterValues(f.value)
|
||||
|
||||
if f.comparator == taskFilterComparatorIn || f.comparator == taskFilterComparatorNotIn {
|
||||
filter += "]"
|
||||
}
|
||||
|
||||
filters = append(filters, filter)
|
||||
}
|
||||
|
||||
if len(filters) > 0 {
|
||||
if len(filters) == 1 {
|
||||
filterBy = filters[0]
|
||||
} else {
|
||||
for i, f := range filters {
|
||||
if len(filters) > i+1 {
|
||||
switch rawFilters[i+1].join {
|
||||
case filterConcatOr:
|
||||
filterBy = f + " || " + filters[i+1]
|
||||
case filterConcatAnd:
|
||||
filterBy = f + " && " + filters[i+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) {
|
||||
|
||||
projectIDStrings := []string{}
|
||||
for _, id := range opts.projectIDs {
|
||||
projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10))
|
||||
}
|
||||
|
||||
filter, err := convertParsedFilterToTypesense(opts.parsedFilters)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
filterBy := []string{"project_id: [" + strings.Join(projectIDStrings, ", ") + "]"}
|
||||
|
||||
if filter != "" {
|
||||
filterBy = append(filterBy, "("+filter+")")
|
||||
}
|
||||
|
||||
var sortbyFields []string
|
||||
var usedParams int
|
||||
for _, param := range opts.sortby {
|
||||
|
||||
if opts.isSavedFilter && param.sortBy == taskPropertyPosition {
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate the params
|
||||
if err := param.validate(); err != nil {
|
||||
return nil, totalCount, err
|
||||
}
|
||||
|
||||
sortBy := param.sortBy
|
||||
|
||||
// Typesense does not allow sorting by ID, so we sort by created timestamp instead
|
||||
if param.sortBy == taskPropertyID {
|
||||
sortBy = taskPropertyCreated
|
||||
}
|
||||
|
||||
if param.sortBy == taskPropertyPosition {
|
||||
sortBy = "positions.view_" + strconv.FormatInt(param.projectViewID, 10)
|
||||
}
|
||||
|
||||
sortbyFields = append(sortbyFields, sortBy+"(missing_values:last):"+param.orderBy.String())
|
||||
|
||||
if usedParams == 2 {
|
||||
// Typesense supports up to 3 sorting parameters
|
||||
// https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters
|
||||
break
|
||||
}
|
||||
|
||||
usedParams++
|
||||
}
|
||||
|
||||
sortby := strings.Join(sortbyFields, ",")
|
||||
|
||||
////////////////
|
||||
// Actual search
|
||||
|
||||
if opts.search == "" {
|
||||
opts.search = "*"
|
||||
}
|
||||
|
||||
params := &api.SearchCollectionParams{
|
||||
Q: pointer.String(opts.search),
|
||||
QueryBy: pointer.String("title, identifier, description, comments.comment"),
|
||||
Page: pointer.Int(opts.page),
|
||||
ExhaustiveSearch: pointer.True(),
|
||||
FilterBy: pointer.String(strings.Join(filterBy, " && ")),
|
||||
}
|
||||
|
||||
if opts.perPage > 0 {
|
||||
if opts.perPage > 250 {
|
||||
log.Warningf("Typesense only supports up to 250 results per page, requested %d.", opts.perPage)
|
||||
opts.perPage = 250
|
||||
}
|
||||
params.PerPage = pointer.Int(opts.perPage)
|
||||
}
|
||||
|
||||
if sortby != "" {
|
||||
params.SortBy = pointer.String(sortby)
|
||||
}
|
||||
|
||||
result, err := typesenseClient.Collection("tasks").
|
||||
Documents().
|
||||
Search(context.Background(), params)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
taskIDs := []int64{}
|
||||
for _, h := range *result.Hits {
|
||||
hit := *h.Document
|
||||
taskID, err := strconv.ParseInt(hit["id"].(string), 10, 64)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
taskIDs = append(taskIDs, taskID)
|
||||
}
|
||||
|
||||
tasks = []*Task{}
|
||||
|
||||
orderby, err := getOrderByDBStatement(opts)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var distinct = "tasks.*"
|
||||
if strings.Contains(orderby, "task_positions.") {
|
||||
distinct += ", task_positions.position"
|
||||
}
|
||||
|
||||
query := t.s.
|
||||
Distinct(distinct).
|
||||
In("id", taskIDs).
|
||||
OrderBy(orderby)
|
||||
|
||||
for _, param := range opts.sortby {
|
||||
if param.sortBy == taskPropertyPosition {
|
||||
query = query.Join("LEFT", "task_positions", "task_positions.task_id = tasks.id AND task_positions.project_view_id = ?", param.projectViewID)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
err = query.Find(&tasks)
|
||||
return tasks, int64(*result.Found), err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,22 +107,9 @@ func BenchmarkTaskSearch(b *testing.B) {
|
|||
|
||||
// Log database configuration
|
||||
b.Logf("Database Type: %s", config.DatabaseType.GetString())
|
||||
if config.TypesenseEnabled.GetBool() {
|
||||
b.Log("Typesense is enabled")
|
||||
}
|
||||
|
||||
auth := createBenchmarkData(b, needle)
|
||||
|
||||
if config.TypesenseEnabled.GetBool() {
|
||||
InitTypesense()
|
||||
if err := CreateTypesenseCollections(); err != nil {
|
||||
b.Skipf("typesense server not available: %v", err)
|
||||
}
|
||||
if err := ReindexAllTasks(); err != nil {
|
||||
b.Skipf("typesense server not available: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get all projects for the user
|
||||
s := db.NewSession()
|
||||
projects, _, _, err := getRawProjectsForUser(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
|
@ -36,7 +35,6 @@ import (
|
|||
"github.com/google/uuid"
|
||||
clone "github.com/huandu/go-clone/generic"
|
||||
"github.com/jinzhu/copier"
|
||||
"github.com/typesense/typesense-go/v2/typesense"
|
||||
"xorm.io/builder"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
|
@ -309,22 +307,7 @@ func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, op
|
|||
a: a,
|
||||
hasFavoritesProject: hasFavoritesProject,
|
||||
}
|
||||
if config.TypesenseEnabled.GetBool() {
|
||||
var tsSearcher taskSearcher = &typesenseTaskSearcher{
|
||||
s: s,
|
||||
}
|
||||
origOpts := clone.Clone(opts)
|
||||
tasks, totalItems, err = tsSearcher.Search(opts)
|
||||
// It is possible that project views are not yet in Typesense's index. This causes the query here to fail.
|
||||
// To avoid crashing everything, we fall back to the db search in that case.
|
||||
var tsErr = &typesense.HTTPError{}
|
||||
if err != nil && errors.As(err, &tsErr) && tsErr.Status == 404 {
|
||||
log.Warningf("Unable to fetch tasks from Typesense, error was '%v'. Falling back to db.", err)
|
||||
tasks, totalItems, err = dbSearcher.Search(origOpts)
|
||||
}
|
||||
} else {
|
||||
tasks, totalItems, err = dbSearcher.Search(opts)
|
||||
}
|
||||
|
||||
return tasks, len(tasks), totalItems, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,597 +0,0 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/db"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
|
||||
"github.com/typesense/typesense-go/v2/typesense"
|
||||
"github.com/typesense/typesense-go/v2/typesense/api"
|
||||
"github.com/typesense/typesense-go/v2/typesense/api/pointer"
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
type TypesenseSync struct {
|
||||
Collection string `xorm:"not null"`
|
||||
SyncStartedAt time.Time `xorm:"not null"`
|
||||
SyncFinishedAt time.Time `xorm:"null"`
|
||||
}
|
||||
|
||||
var typesenseClient *typesense.Client
|
||||
|
||||
func InitTypesense() {
|
||||
if !config.TypesenseEnabled.GetBool() {
|
||||
return
|
||||
}
|
||||
|
||||
typesenseClient = typesense.NewClient(
|
||||
typesense.WithServer(config.TypesenseURL.GetString()),
|
||||
typesense.WithAPIKey(config.TypesenseAPIKey.GetString()))
|
||||
}
|
||||
|
||||
func CreateTypesenseCollections() error {
|
||||
taskSchema := &api.CollectionSchema{
|
||||
Name: "tasks",
|
||||
EnableNestedFields: pointer.True(),
|
||||
Fields: []api.Field{
|
||||
{
|
||||
Name: "id",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "title",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "description",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "done",
|
||||
Type: "bool",
|
||||
},
|
||||
{
|
||||
Name: "done_at",
|
||||
Type: "int64", // unix timestamp
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "due_date",
|
||||
Type: "int64", // unix timestamp
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "project_id",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "repeat_after",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "repeat_mode",
|
||||
Type: "int32",
|
||||
},
|
||||
{
|
||||
Name: "priority",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "start_date",
|
||||
Type: "int64", // unix timestamp
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "end_date",
|
||||
Type: "int64", // unix timestamp
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "hex_color",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "percent_done",
|
||||
Type: "float",
|
||||
},
|
||||
{
|
||||
Name: "identifier",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "index",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "uid",
|
||||
Type: "string",
|
||||
Sort: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "cover_image_attachment_id",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "created",
|
||||
Type: "int64", // unix timestamp
|
||||
},
|
||||
{
|
||||
Name: "updated",
|
||||
Type: "int64", // unix timestamp
|
||||
},
|
||||
{
|
||||
Name: "created_by_id",
|
||||
Type: "int64",
|
||||
},
|
||||
{
|
||||
Name: "reminders",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "assignees",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "labels",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "related_tasks",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "attachments",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "comments",
|
||||
Type: "object[]", // TODO
|
||||
Optional: pointer.True(),
|
||||
},
|
||||
{
|
||||
Name: "positions",
|
||||
Type: "object",
|
||||
},
|
||||
{
|
||||
Name: "positions.view_.*",
|
||||
Type: "float",
|
||||
},
|
||||
{
|
||||
Name: "buckets",
|
||||
Type: "int64[]",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// delete any collection which might exist
|
||||
_, _ = typesenseClient.Collection("tasks").Delete(context.Background())
|
||||
|
||||
_, err := typesenseClient.Collections().Create(context.Background(), taskSchema)
|
||||
return err
|
||||
}
|
||||
|
||||
func ReindexAllTasks() (err error) {
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
_, err = s.Where("collection = ?", "tasks").Delete(&TypesenseSync{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not delete old sync status: %s", err.Error())
|
||||
}
|
||||
|
||||
currentSync := &TypesenseSync{
|
||||
Collection: "tasks",
|
||||
SyncStartedAt: time.Now(),
|
||||
}
|
||||
_, err = s.Insert(currentSync)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not update last sync: %s", err.Error())
|
||||
}
|
||||
|
||||
tasks := make(map[int64]*Task)
|
||||
err = s.Find(tasks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get all tasks: %s", err.Error())
|
||||
}
|
||||
|
||||
err = indexDummyTask()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not index dummy task: %w", err)
|
||||
}
|
||||
|
||||
err = reindexTasksInTypesense(s, tasks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not reindex all tasks: %s", err.Error())
|
||||
}
|
||||
|
||||
currentSync.SyncFinishedAt = time.Now()
|
||||
_, err = s.Where("collection = ?", "tasks").
|
||||
Cols("sync_finished_at").
|
||||
Update(currentSync)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could update last sync state: %s", err.Error())
|
||||
}
|
||||
|
||||
return s.Commit()
|
||||
}
|
||||
|
||||
func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error) {
|
||||
|
||||
if !config.TypesenseEnabled.GetBool() {
|
||||
return
|
||||
}
|
||||
|
||||
if len(tasks) == 0 {
|
||||
log.Infof("No tasks to index")
|
||||
return
|
||||
}
|
||||
|
||||
err = addMoreInfoToTasks(s, tasks, &user.User{ID: 1}, nil, []TaskCollectionExpandable{
|
||||
TaskCollectionExpandReactions,
|
||||
TaskCollectionExpandComments,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not fetch more task info: %s", err.Error())
|
||||
}
|
||||
|
||||
typesenseTasks := []interface{}{}
|
||||
|
||||
positionsByTask, err := getPositionsByTask(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bucketsByTask, err := getBucketsByTask(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, task := range tasks {
|
||||
ttask := convertTaskToTypesenseTask(task, positionsByTask[task.ID], bucketsByTask[task.ID])
|
||||
if ttask == nil {
|
||||
log.Debugf("Converted typesense task %d is nil, not indexing", task.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
typesenseTasks = append(typesenseTasks, ttask)
|
||||
}
|
||||
|
||||
response, err := typesenseClient.Collection("tasks").
|
||||
Documents().
|
||||
Import(context.Background(), typesenseTasks, &api.ImportDocumentsParams{
|
||||
Action: pointer.String("upsert"),
|
||||
BatchSize: pointer.Int(100),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("Could not upsert tasks into Typesense: %s", err)
|
||||
return err
|
||||
}
|
||||
for _, r := range response {
|
||||
if r.Success {
|
||||
continue
|
||||
}
|
||||
log.Errorf("Errors during index: [error=%s, document=%s]", r.Error, r.Document)
|
||||
}
|
||||
|
||||
log.Debugf("Indexed %d tasks into Typesense", len(typesenseTasks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type TaskPositionWithView struct {
|
||||
ProjectView `xorm:"extends"`
|
||||
TaskPosition `xorm:"extends"`
|
||||
}
|
||||
|
||||
func getPositionsByTask(s *xorm.Session) (positionsByTask map[int64][]*TaskPositionWithView, err error) {
|
||||
rawPositions := []*TaskPositionWithView{}
|
||||
err = s.
|
||||
Table("project_views").
|
||||
Join("LEFT", "task_positions", "project_views.id = task_positions.project_view_id").
|
||||
Find(&rawPositions)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
positionsByTask = make(map[int64][]*TaskPositionWithView, len(rawPositions))
|
||||
for _, p := range rawPositions {
|
||||
_, has := positionsByTask[p.TaskID]
|
||||
if !has {
|
||||
positionsByTask[p.TaskID] = []*TaskPositionWithView{}
|
||||
}
|
||||
positionsByTask[p.TaskID] = append(positionsByTask[p.TaskID], p)
|
||||
}
|
||||
return positionsByTask, nil
|
||||
}
|
||||
|
||||
func getBucketsByTask(s *xorm.Session) (positionsByTask map[int64][]*TaskBucket, err error) {
|
||||
rawBuckets := []*TaskBucket{}
|
||||
err = s.Find(&rawBuckets)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
positionsByTask = make(map[int64][]*TaskBucket, len(rawBuckets))
|
||||
for _, p := range rawBuckets {
|
||||
_, has := positionsByTask[p.TaskID]
|
||||
if !has {
|
||||
positionsByTask[p.TaskID] = []*TaskBucket{}
|
||||
}
|
||||
positionsByTask[p.TaskID] = append(positionsByTask[p.TaskID], p)
|
||||
}
|
||||
return positionsByTask, nil
|
||||
}
|
||||
|
||||
func indexDummyTask() (err error) {
|
||||
// The initial sync should contain one dummy task with all related fields populated so that typesense
|
||||
// creates the indexes properly. A little hacky, but gets the job done.
|
||||
dummyTask := &typesenseTask{
|
||||
ID: "-100",
|
||||
Title: "Dummytask",
|
||||
Created: time.Now().Unix(),
|
||||
Updated: time.Now().Unix(),
|
||||
Reminders: []*TaskReminder{
|
||||
{
|
||||
ID: -10,
|
||||
TaskID: -100,
|
||||
Reminder: time.Now(),
|
||||
RelativePeriod: 10,
|
||||
RelativeTo: ReminderRelationDueDate,
|
||||
Created: time.Now(),
|
||||
},
|
||||
},
|
||||
Assignees: []*user.User{
|
||||
{
|
||||
ID: -100,
|
||||
Username: "dummy",
|
||||
Name: "dummy",
|
||||
Email: "dummy@vikunja",
|
||||
Created: time.Now(),
|
||||
Updated: time.Now(),
|
||||
},
|
||||
},
|
||||
Labels: []*Label{
|
||||
{
|
||||
ID: -110,
|
||||
Title: "dummylabel",
|
||||
Description: "Lorem Ipsum Dummy",
|
||||
HexColor: "000000",
|
||||
Created: time.Now(),
|
||||
Updated: time.Now(),
|
||||
},
|
||||
},
|
||||
Attachments: []*TaskAttachment{
|
||||
{
|
||||
ID: -120,
|
||||
TaskID: -100,
|
||||
Created: time.Now(),
|
||||
},
|
||||
},
|
||||
Comments: []*TaskComment{
|
||||
{
|
||||
ID: -220,
|
||||
Comment: "Lorem Ipsum Dummy",
|
||||
Created: time.Now(),
|
||||
Updated: time.Now(),
|
||||
Author: &user.User{
|
||||
ID: -100,
|
||||
Username: "dummy",
|
||||
Name: "dummy",
|
||||
Email: "dummy@vikunja",
|
||||
Created: time.Now(),
|
||||
Updated: time.Now(),
|
||||
},
|
||||
},
|
||||
},
|
||||
Positions: map[string]float64{
|
||||
"view_1": 10,
|
||||
"view_2": 30,
|
||||
"view_3": 5450,
|
||||
"view_4": 42,
|
||||
},
|
||||
Buckets: []int64{42},
|
||||
}
|
||||
|
||||
_, err = typesenseClient.Collection("tasks").
|
||||
Documents().
|
||||
Create(context.Background(), dummyTask)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
_, err = typesenseClient.Collection("tasks").
|
||||
Document(dummyTask.ID).
|
||||
Delete(context.Background())
|
||||
return
|
||||
}
|
||||
|
||||
type typesenseTask struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Done bool `json:"done"`
|
||||
DoneAt *int64 `json:"done_at"`
|
||||
DueDate *int64 `json:"due_date"`
|
||||
ProjectID int64 `json:"project_id"`
|
||||
RepeatAfter int64 `json:"repeat_after"`
|
||||
RepeatMode int `json:"repeat_mode"`
|
||||
Priority int64 `json:"priority"`
|
||||
StartDate *int64 `json:"start_date"`
|
||||
EndDate *int64 `json:"end_date"`
|
||||
HexColor string `json:"hex_color"`
|
||||
PercentDone float64 `json:"percent_done"`
|
||||
Identifier string `json:"identifier"`
|
||||
Index int64 `json:"index"`
|
||||
UID string `json:"uid"`
|
||||
CoverImageAttachmentID int64 `json:"cover_image_attachment_id"`
|
||||
Created int64 `json:"created"`
|
||||
Updated int64 `json:"updated"`
|
||||
CreatedByID int64 `json:"created_by_id"`
|
||||
Reminders interface{} `json:"reminders"`
|
||||
Assignees interface{} `json:"assignees"`
|
||||
Labels interface{} `json:"labels"`
|
||||
//RelatedTasks interface{} `json:"related_tasks"` // TODO
|
||||
Attachments interface{} `json:"attachments"`
|
||||
Comments interface{} `json:"comments"`
|
||||
Positions map[string]float64 `json:"positions"`
|
||||
Buckets []int64 `json:"buckets"`
|
||||
}
|
||||
|
||||
func convertTaskToTypesenseTask(task *Task, positions []*TaskPositionWithView, buckets []*TaskBucket) *typesenseTask {
|
||||
|
||||
tt := &typesenseTask{
|
||||
ID: fmt.Sprintf("%d", task.ID),
|
||||
Title: task.Title,
|
||||
Description: task.Description,
|
||||
Done: task.Done,
|
||||
DoneAt: pointer.Int64(task.DoneAt.UTC().Unix()),
|
||||
DueDate: pointer.Int64(task.DueDate.UTC().Unix()),
|
||||
ProjectID: task.ProjectID,
|
||||
RepeatAfter: task.RepeatAfter,
|
||||
RepeatMode: int(task.RepeatMode),
|
||||
Priority: task.Priority,
|
||||
StartDate: pointer.Int64(task.StartDate.UTC().Unix()),
|
||||
EndDate: pointer.Int64(task.EndDate.UTC().Unix()),
|
||||
HexColor: task.HexColor,
|
||||
PercentDone: task.PercentDone,
|
||||
Identifier: task.Identifier,
|
||||
Index: task.Index,
|
||||
UID: task.UID,
|
||||
CoverImageAttachmentID: task.CoverImageAttachmentID,
|
||||
Created: task.Created.UTC().Unix(),
|
||||
Updated: task.Updated.UTC().Unix(),
|
||||
CreatedByID: task.CreatedByID,
|
||||
Reminders: task.Reminders,
|
||||
Assignees: task.Assignees,
|
||||
Labels: task.Labels,
|
||||
//RelatedTasks: task.RelatedTasks,
|
||||
Attachments: task.Attachments,
|
||||
Positions: make(map[string]float64, len(positions)),
|
||||
Buckets: make([]int64, 0, len(buckets)),
|
||||
}
|
||||
|
||||
if task.DoneAt.IsZero() {
|
||||
tt.DoneAt = nil
|
||||
}
|
||||
if task.DueDate.IsZero() {
|
||||
tt.DueDate = nil
|
||||
}
|
||||
if task.StartDate.IsZero() {
|
||||
tt.StartDate = nil
|
||||
}
|
||||
if task.EndDate.IsZero() {
|
||||
tt.EndDate = nil
|
||||
}
|
||||
|
||||
for _, position := range positions {
|
||||
pos := position.TaskPosition.Position
|
||||
if pos == 0 {
|
||||
pos = float64(task.ID)
|
||||
}
|
||||
tt.Positions["view_"+strconv.FormatInt(position.ID, 10)] = pos
|
||||
}
|
||||
|
||||
for _, bucket := range buckets {
|
||||
tt.Buckets = append(tt.Buckets, bucket.BucketID)
|
||||
}
|
||||
|
||||
return tt
|
||||
}
|
||||
|
||||
// This function is only used to catch up with the Typesense Sync when it didn't index for some reason
|
||||
|
||||
func SyncUpdatedTasksIntoTypesense() (err error) {
|
||||
tasks := make(map[int64]*Task)
|
||||
|
||||
s := db.NewSession()
|
||||
defer s.Close()
|
||||
|
||||
lastSync := &TypesenseSync{}
|
||||
has, err := s.Where("collection = ?", "tasks").
|
||||
Get(lastSync)
|
||||
if err != nil {
|
||||
_ = s.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
if !has {
|
||||
log.Errorf("[Typesense Sync] No typesense sync stats yet, please run a full index via the CLI first")
|
||||
_ = s.Rollback()
|
||||
return
|
||||
}
|
||||
|
||||
currentSync := &TypesenseSync{SyncStartedAt: time.Now()}
|
||||
_, err = s.Where("collection = ?", "tasks").
|
||||
Cols("sync_started_at", "sync_finished_at").
|
||||
Update(currentSync)
|
||||
if err != nil {
|
||||
_ = s.Rollback()
|
||||
return
|
||||
}
|
||||
|
||||
err = s.
|
||||
Where("updated >= ?", lastSync.SyncStartedAt).
|
||||
And("updated != created"). // new tasks are already indexed via the event handler
|
||||
Find(tasks)
|
||||
if err != nil {
|
||||
_ = s.Rollback()
|
||||
return
|
||||
}
|
||||
|
||||
if len(tasks) > 0 {
|
||||
log.Debugf("[Typesense Sync] Updating %d tasks", len(tasks))
|
||||
|
||||
err = reindexTasksInTypesense(s, tasks)
|
||||
if err != nil {
|
||||
_ = s.Rollback()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if len(tasks) == 0 {
|
||||
log.Debugf("[Typesense Sync] No tasks changed since the last sync, not syncing")
|
||||
}
|
||||
|
||||
currentSync.SyncFinishedAt = time.Now()
|
||||
_, err = s.Where("collection = ?", "tasks").
|
||||
Cols("sync_finished_at").
|
||||
Update(currentSync)
|
||||
if err != nil {
|
||||
_ = s.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return s.Commit()
|
||||
}
|
||||
Loading…
Reference in New Issue