From 57c6f2cd10ff47b8b29526e15221e5618debd00e Mon Sep 17 00:00:00 2001
From: kolaente <k@knt.li>
Date: Tue, 29 Oct 2024 12:03:16 +0100
Subject: [PATCH] feat(filters): add "not in" operator for filters

Resolves https://community.vikunja.io/t/feature-requests-regarding-view-ordering-project-reference-in-kanban-and-a-notin-operator/2728
---
 .../project/partials/FilterInput.vue          |  2 +-
 .../project/partials/FilterInputDocs.vue      |  1 +
 frontend/src/helpers/filters.test.ts          | 20 ++++++++++
 frontend/src/helpers/filters.ts               | 13 +++---
 frontend/src/i18n/lang/en.json                |  3 +-
 pkg/models/task_collection_filter.go          | 13 ++++--
 pkg/models/task_collection_filter_test.go     | 12 ++++++
 pkg/models/task_collection_test.go            | 40 +++++++++++++++++++
 pkg/models/task_search.go                     |  4 +-
 pkg/models/tasks.go                           |  2 +
 10 files changed, 98 insertions(+), 12 deletions(-)

diff --git a/frontend/src/components/project/partials/FilterInput.vue b/frontend/src/components/project/partials/FilterInput.vue
index 61a0281f4..8718a2227 100644
--- a/frontend/src/components/project/partials/FilterInput.vue
+++ b/frontend/src/components/project/partials/FilterInput.vue
@@ -133,7 +133,7 @@ const highlightedFilterQuery = computed(() => {
 				}
 
 				let labelTitles = [value.trim()]
-				if (operator === 'in' || operator === '?=') {
+				if (operator === 'in' || operator === '?=' || operator === 'not in' || operator === '?!=') {
 					labelTitles = value.split(',').map(v => v.trim())
 				}
 
diff --git a/frontend/src/components/project/partials/FilterInputDocs.vue b/frontend/src/components/project/partials/FilterInputDocs.vue
index 01d09e212..d53c75ad8 100644
--- a/frontend/src/components/project/partials/FilterInputDocs.vue
+++ b/frontend/src/components/project/partials/FilterInputDocs.vue
@@ -45,6 +45,7 @@ const showDocs = ref(false)
 			<li><code>&lt;=</code>: {{ $t('filters.query.help.operators.lessThanOrEqual') }}</li>
 			<li><code>like</code>: {{ $t('filters.query.help.operators.like') }}</li>
 			<li><code>in</code>: {{ $t('filters.query.help.operators.in') }}</li>
+			<li><code>not in</code>: {{ $t('filters.query.help.operators.notIn') }}</li>
 		</ul>
 		<p>{{ $t('filters.query.help.logicalOperators.intro') }}</p>
 		<ul>
diff --git a/frontend/src/helpers/filters.test.ts b/frontend/src/helpers/filters.test.ts
index 4fd3bfbb8..a63e611b5 100644
--- a/frontend/src/helpers/filters.test.ts
+++ b/frontend/src/helpers/filters.test.ts
@@ -67,6 +67,16 @@ describe('Filter Transformation', () => {
 
 			expect(transformed).toBe('labels in 1, 2 && due_date = now')
 		})
+		
+		it('should correctly resolve multiple labels with a not in clause', () => {
+			const transformed = transformFilterStringForApi(
+				'labels not in lorem, ipsum && dueDate = now',
+				multipleDummyResolver,
+				nullTitleToIdResolver,
+			)
+
+			expect(transformed).toBe('labels not in 1, 2 && due_date = now')
+		})
 
 		it('should correctly resolve projects', () => {
 			const transformed = transformFilterStringForApi(
@@ -218,7 +228,17 @@ describe('Filter Transformation', () => {
 
 			expect(transformed).toBe('labels in lorem, ipsum')
 		})
+		
+		it('should correctly resolve multiple labels not in', () => {
+			const transformed = transformFilterStringFromApi(
+				'labels not in 1, 2',
+				multipleIdToTitleResolver,
+				nullIdToTitleResolver,
+			)
 
+			expect(transformed).toBe('labels not in lorem, ipsum')
+		})
+		
 		it('should not touch the label value when it is undefined', () => {
 			const transformed = transformFilterStringFromApi(
 				'labels = one',
diff --git a/frontend/src/helpers/filters.ts b/frontend/src/helpers/filters.ts
index 6d7d14abc..25733a288 100644
--- a/frontend/src/helpers/filters.ts
+++ b/frontend/src/helpers/filters.ts
@@ -46,6 +46,7 @@ export const FILTER_OPERATORS = [
 	'<',
 	'<=',
 	'like',
+	'not in',
 	'in',
 	'?=',
 ]
@@ -57,7 +58,7 @@ export const FILTER_JOIN_OPERATOR = [
 	')',
 ]
 
-export const FILTER_OPERATORS_REGEX = '(&lt;|&gt;|&lt;=|&gt;=|=|!=|in)'
+export const FILTER_OPERATORS_REGEX = '(&lt;|&gt;|&lt;=|&gt;=|=|!=|not in|in)'
 
 export function getFilterFieldRegexPattern(field: string): RegExp {
 	return new RegExp('(' + field + '\\s*' + FILTER_OPERATORS_REGEX + '\\s*)([\'"]?)([^\'"&|()<]+\\1?)?', 'ig')
@@ -68,13 +69,13 @@ export function transformFilterStringForApi(
 	labelResolver: (title: string) => number | null,
 	projectResolver: (title: string) => number | null,
 ): string {
-	
+
 	filter = filter.trim()
 
 	if (filter === '') {
 		return ''
 	}
-	
+
 	AVAILABLE_FILTER_FIELDS.forEach(f => {
 		filter = filter.replace(new RegExp(f, 'ig'), f)
 	})
@@ -97,10 +98,10 @@ export function transformFilterStringForApi(
 				}
 
 				let keywords = [keyword.trim()]
-				if (operator === 'in' || operator === '?=') {
+				if (operator === 'in' || operator === '?=' || operator === 'not in' || operator === '?!=') {
 					keywords = keyword.trim().split(',').map(k => k.trim())
 				}
-				
+
 				let replaced = keyword
 
 				keywords.forEach(k => {
@@ -162,7 +163,7 @@ export function transformFilterStringFromApi(
 				const [matched, prefix, operator, space, keyword] = match
 				if (keyword) {
 					let keywords = [keyword.trim()]
-					if (operator === 'in' || operator === '?=') {
+					if (operator === 'in' || operator === '?=' || operator === 'not in' || operator === '?!=') {
 						keywords = keyword.trim().split(',').map(k => k.trim())
 					}
 
diff --git a/frontend/src/i18n/lang/en.json b/frontend/src/i18n/lang/en.json
index f50430680..7d132a360 100644
--- a/frontend/src/i18n/lang/en.json
+++ b/frontend/src/i18n/lang/en.json
@@ -472,7 +472,8 @@
           "lessThan": "Less than",
           "lessThanOrEqual": "Less than or equal to",
           "like": "Matches a pattern (using wildcard %)",
-          "in": "Matches any value in a comma-seperated list of values"
+          "in": "Matches any value in a comma-seperated list of values",
+          "notIn": "Matches any value not present in a comma-seperated list of values"
         },
         "logicalOperators": {
           "intro": "To combine multiple conditions, you can use the following logical operators:",
diff --git a/pkg/models/task_collection_filter.go b/pkg/models/task_collection_filter.go
index 3448a459c..9cbc59f83 100644
--- a/pkg/models/task_collection_filter.go
+++ b/pkg/models/task_collection_filter.go
@@ -47,6 +47,7 @@ const (
 	taskFilterComparatorNotEquals    taskFilterComparator = "!="
 	taskFilterComparatorLike         taskFilterComparator = "like"
 	taskFilterComparatorIn           taskFilterComparator = "in"
+	taskFilterComparatorNotIn        taskFilterComparator = "not in"
 )
 
 // Guess what you get back if you ask Safari for a rfc 3339 formatted date?
@@ -153,11 +154,12 @@ func getTaskFiltersFromFilterString(filter string, filterTimezone string) (filte
 		return
 	}
 
+	filter = strings.ReplaceAll(filter, " not in ", " "+string(fexpr.SignAnyNeq)+" ")
 	filter = strings.ReplaceAll(filter, " in ", " ?= ")
 	filter = strings.ReplaceAll(filter, " like ", " ~ ")
 
 	// Regex pattern to match filter expressions
-	re := regexp.MustCompile(`(\w+)\s*(>=|<=|!=|~|\?=|=|>|<)\s*([^&|()]+)`)
+	re := regexp.MustCompile(`(\w+)\s*(>=|<=|!=|~|\?=|\?!=|=|>|<)\s*([^&|()]+)`)
 
 	filter = re.ReplaceAllStringFunc(filter, func(match string) string {
 		parts := re.FindStringSubmatch(match)
@@ -221,7 +223,8 @@ func validateTaskFieldComparator(comparator taskFilterComparator) error {
 		taskFilterComparatorLessEquals,
 		taskFilterComparatorNotEquals,
 		taskFilterComparatorLike,
-		taskFilterComparatorIn:
+		taskFilterComparatorIn,
+		taskFilterComparatorNotIn:
 		return nil
 	case taskFilterComparatorInvalid:
 		fallthrough
@@ -250,6 +253,10 @@ func getFilterComparatorFromOp(op fexpr.SignOp) (taskFilterComparator, error) {
 		fallthrough
 	case "in":
 		return taskFilterComparatorIn, nil
+	case fexpr.SignAnyNeq:
+		fallthrough
+	case "not in":
+		return taskFilterComparatorNotIn, nil
 	default:
 		return taskFilterComparatorInvalid, ErrInvalidTaskFilterComparator{Comparator: taskFilterComparator(op)}
 	}
@@ -337,7 +344,7 @@ func getNativeValueForTaskField(fieldName string, comparator taskFilterComparato
 		}
 	}
 
-	if comparator == taskFilterComparatorIn {
+	if comparator == taskFilterComparatorIn || comparator == taskFilterComparatorNotIn {
 		vals := strings.Split(value, ",")
 		valueSlice := []interface{}{}
 		for _, val := range vals {
diff --git a/pkg/models/task_collection_filter_test.go b/pkg/models/task_collection_filter_test.go
index ca9ea6cf9..4d2e5eb6b 100644
--- a/pkg/models/task_collection_filter_test.go
+++ b/pkg/models/task_collection_filter_test.go
@@ -74,6 +74,18 @@ func TestParseFilter(t *testing.T) {
 		assert.Equal(t, int64(2), result[0].value.([]interface{})[1])
 		assert.Equal(t, int64(3), result[0].value.([]interface{})[2])
 	})
+	t.Run("not in", func(t *testing.T) {
+		result, err := getTaskFiltersFromFilterString("project_id not in 1,2,3", "UTC")
+
+		require.NoError(t, err)
+		require.Len(t, result, 1)
+		assert.Equal(t, "project_id", result[0].field)
+		assert.Equal(t, taskFilterComparatorNotIn, result[0].comparator)
+		require.Len(t, result[0].value, 3)
+		assert.Equal(t, int64(1), result[0].value.([]interface{})[0])
+		assert.Equal(t, int64(2), result[0].value.([]interface{})[1])
+		assert.Equal(t, int64(3), result[0].value.([]interface{})[2])
+	})
 	t.Run("use project for project_id", func(t *testing.T) {
 		result, err := getTaskFiltersFromFilterString("project in 1,2,3", "UTC")
 
diff --git a/pkg/models/task_collection_test.go b/pkg/models/task_collection_test.go
index a6e5f42ad..05a15ccdb 100644
--- a/pkg/models/task_collection_test.go
+++ b/pkg/models/task_collection_test.go
@@ -26,6 +26,7 @@ import (
 	"code.vikunja.io/api/pkg/files"
 	"code.vikunja.io/api/pkg/user"
 	"code.vikunja.io/api/pkg/web"
+
 	"gopkg.in/d4l3k/messagediff.v1"
 )
 
@@ -1037,6 +1038,45 @@ func TestTaskCollection_ReadAll(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "filter not in",
+			fields: fields{
+				Filter: "id not in '1,2,3,4'",
+			},
+			args: defaultArgs,
+			want: []*Task{
+				task5,
+				task6,
+				task7,
+				task8,
+				task9,
+				task10,
+				task11,
+				task12,
+				task15,
+				task16,
+				task17,
+				task18,
+				task19,
+				task20,
+				task21,
+				task22,
+				task23,
+				task24,
+				task25,
+				task26,
+				task27,
+				task28,
+				task29,
+				task30,
+				task31,
+				task32,
+				task33,
+				task35,
+				task39,
+			},
+			wantErr: false,
+		},
 		{
 			name: "filter assignees by username",
 			fields: fields{
diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go
index a996bed19..7c6118a14 100644
--- a/pkg/models/task_search.go
+++ b/pkg/models/task_search.go
@@ -488,6 +488,8 @@ func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string,
 			filter += ":"
 		case taskFilterComparatorIn:
 			filter += ":["
+		case taskFilterComparatorNotIn:
+			filter += ":!["
 		case taskFilterComparatorInvalid:
 		// Nothing to do
 		default:
@@ -496,7 +498,7 @@ func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string,
 
 		filter += convertFilterValues(f.value)
 
-		if f.comparator == taskFilterComparatorIn {
+		if f.comparator == taskFilterComparatorIn || f.comparator == taskFilterComparatorNotIn {
 			filter += "]"
 		}
 
diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go
index 63df9856b..d75c969f0 100644
--- a/pkg/models/tasks.go
+++ b/pkg/models/tasks.go
@@ -235,6 +235,8 @@ func getFilterCond(f *taskFilter, includeNulls bool) (cond builder.Cond, err err
 		cond = &builder.Like{field, "%" + val + "%"}
 	case taskFilterComparatorIn:
 		cond = builder.In(field, f.value)
+	case taskFilterComparatorNotIn:
+		cond = builder.NotIn(field, f.value)
 	case taskFilterComparatorInvalid:
 		// Nothing to do
 	}