diff --git a/go.sum b/go.sum index 5518f1558..fe31e4880 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,6 @@ github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEX github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-oidc/v3 v3.16.0 h1:qRQUCFstKpXwmEjDQTIbyY/5jF00+asXzSkmkoa/mow= -github.com/coreos/go-oidc/v3 v3.16.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -398,8 +396,6 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= -github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4= -github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM= github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -521,8 +517,6 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/pkg/modules/migration/ticktick/main_test.go b/pkg/modules/migration/ticktick/main_test.go new file mode 100644 index 000000000..dfa8c6a12 --- /dev/null +++ b/pkg/modules/migration/ticktick/main_test.go @@ -0,0 +1,32 @@ +// Vikunja is a to-do list application to facilitate your life. +// Copyright 2018-present Vikunja and contributors. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ticktick + +import ( + "os" + "testing" + + "code.vikunja.io/api/pkg/log" +) + +// TestMain is the main test function used to bootstrap the test env +func TestMain(m *testing.M) { + // Initialize logger for tests + log.InitLogger() + + os.Exit(m.Run()) +} diff --git a/pkg/modules/migration/ticktick/testdata_ticktick_export.csv b/pkg/modules/migration/ticktick/testdata_ticktick_export.csv new file mode 100644 index 000000000..ed508d05e --- /dev/null +++ b/pkg/modules/migration/ticktick/testdata_ticktick_export.csv @@ -0,0 +1,18 @@ +"Date: 2025-11-25+0000" +"Version: 7.1" +"Status: +0 Normal +1 Completed +2 Archived" +"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId" +"Work","Project Alpha","Task with repeating schedule","TEXT","urgent, work","This task repeats weekly","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1","" +"Work","Project Alpha","Task with reminder and dates","TEXT","work, reminder","Task description with reminder","N","2018-12-11T23:00:00+0000","2018-12-11T23:00:00+0000","","","0","0","2018-12-11T20:10:46+0000","2018-12-11T20:12:14+0000","0","Europe/Berlin","true","false",,,"list","2","" +"Work","Project Alpha","Subtask example","TEXT","","This is a subtask","N","","","","","0","0","2022-10-09T15:20:55+0000","","1099511627776","Europe/Berlin","true","false",,,"list","3","2" +"Work","Project Alpha","Another subtask","TEXT","","Another subtask example","N","","","","","0","0","2022-10-09T15:20:59+0000","","2199023255552","Europe/Berlin","true","false",,,"list","4","2" +"Personal","Shopping","Buy groceries","TEXT","shopping, personal","Weekly grocery shopping","N","","","","","0","0","2018-12-29T21:48:09+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","5","" +"Personal","Shopping","Buy household items","TEXT","shopping","Cleaning supplies and toiletries","N","","","","","0","0","2018-12-29T21:48:00+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","6","" +"","Inbox","Long task description example","TEXT","","This is an example of a task with a very long description that might contain special characters and formatting.","N","","","","","0","2","2022-01-21T11:33:40+0000","2025-11-25T10:39:31+0000","-2748779069440","Europe/Berlin","false","false",,,"list","7","" +"","Inbox","Completed task example","TEXT","","This task was completed and shows the completed timestamp","N","","","","","0","2","2022-01-21T11:33:34+0000","2025-11-25T10:39:31+0000","-2473901162496","Europe/Berlin","false","false",,,"list","8","" +"","Inbox","Task with due date","TEXT","important, deadline","Task with specific due date and tags","N","2023-03-28T22:00:00+0000","2023-03-28T22:00:00+0000","","","0","0","2018-12-29T21:14:45+0000","","-2199023255552","Europe/Berlin","true","false",,,"list","9","" +"","Inbox","Welcome task","TEXT","","Welcome to the task management system. This task demonstrates basic functionality.","N","2023-06-13T22:00:00+0000","2023-06-13T22:00:00+0000","","","0","0","2018-12-11T20:09:58+0000","","-1099511627776","","true","false",,,"list","10","" +"","Inbox","Checklist example","CHECKLIST","","This is a checklist task with multiple items","Y","","","","","0","0","2018-12-11T20:09:58+0000","","2199023255552","",,"false",,,"list","11","" diff --git a/pkg/modules/migration/ticktick/testdata_ticktick_multiline.csv b/pkg/modules/migration/ticktick/testdata_ticktick_multiline.csv new file mode 100644 index 000000000..0f27e21a2 --- /dev/null +++ b/pkg/modules/migration/ticktick/testdata_ticktick_multiline.csv @@ -0,0 +1,14 @@ +"Date: 2025-11-25+0000" +"Version: 7.1" +"Status: +0 Normal +1 Completed +2 Archived" +"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId" +"Work","Project Alpha","Task with multiline +description","TEXT","urgent, work","This is a task description +that spans multiple lines. + +It has paragraphs and everything! +Including special characters: #, *, @","N","","","","","0","0","2022-10-09T15:09:48+0000","","-1099511627776","Europe/Berlin",,"false",,,"list","1","" +"Work","Project Alpha","Regular task","TEXT","","Simple description","N","","","","","0","0","2022-10-09T15:10:00+0000","","-1099511627775","Europe/Berlin",,"false",,,"list","2","" diff --git a/pkg/modules/migration/ticktick/ticktick.go b/pkg/modules/migration/ticktick/ticktick.go index f38ceb098..6984ed5d7 100644 --- a/pkg/modules/migration/ticktick/ticktick.go +++ b/pkg/modules/migration/ticktick/ticktick.go @@ -18,6 +18,7 @@ package ticktick import ( "bufio" + "bytes" "encoding/csv" "errors" "io" @@ -25,7 +26,6 @@ import ( "strings" "time" - "code.vikunja.io/api/pkg/log" "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" @@ -101,9 +101,13 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi labels := make([]*models.Label, 0, len(t.Tags)) for _, tag := range t.Tags { - labels = append(labels, &models.Label{ - Title: tag, - }) + // Only create labels for non-empty tags after trimming whitespace + trimmedTag := strings.TrimSpace(tag) + if trimmedTag != "" { + labels = append(labels, &models.Label{ + Title: trimmedTag, + }) + } } task := &models.TaskWithComments{ @@ -163,24 +167,79 @@ func (m *Migrator) Name() string { return "ticktick" } -func newLineSkipDecoder(r io.Reader, linesToSkip int) gocsv.SimpleDecoder { - reader := csv.NewReader(r) - for i := 0; i < linesToSkip; i++ { - _, err := reader.Read() - if err != nil { - if errors.Is(err, io.EOF) { +// stripBOM removes the UTF-8 BOM from the beginning of a reader +func stripBOM(r io.Reader) io.Reader { + // Read the first few bytes to check for BOM + buf := make([]byte, 3) + n, err := r.Read(buf) + if err != nil && err != io.EOF { + // If we read some bytes before the error, preserve them + if n > 0 { + return io.MultiReader(bytes.NewReader(buf[:n]), r) + } + return r + } + + // Check if it starts with UTF-8 BOM (0xEF, 0xBB, 0xBF) + // We need exactly 3 bytes and they must match the BOM sequence + if n == 3 && len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF { + // BOM found, return reader without BOM + return io.MultiReader(bytes.NewReader(buf[3:n]), r) + } + + // No BOM found, return reader with the bytes we read back + return io.MultiReader(bytes.NewReader(buf[:n]), r) +} + +func newLineSkipDecoder(r io.Reader, linesToSkip int) (gocsv.SimpleDecoder, error) { + // Strip BOM if present - this must be done consistently with linesToSkipBeforeHeader + r = stripBOM(r) + + // Read all content into memory so we can work with it + // This is acceptable since CSV imports are typically not huge files + allBytes, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + // Skip the metadata lines before the CSV header by finding newlines + // linesToSkipBeforeHeader counts raw text lines (newlines), not CSV records, + // because even metadata can have multiline quoted fields. + // We manually search for newlines (no buffer size limits like bufio.Scanner) + bytesSkipped := 0 + linesFound := 0 + for i := 0; i < len(allBytes) && linesFound < linesToSkip; i++ { + if allBytes[i] == '\n' { + linesFound++ + if linesFound == linesToSkip { + // Position is right after the Nth newline + bytesSkipped = i + 1 break } - log.Debugf("[TickTick Migration] CSV parse error: %s", err) } } - reader.FieldsPerRecord = 0 - return gocsv.NewSimpleDecoderFromCSVReader(reader) + + if linesFound < linesToSkip { + return nil, io.ErrUnexpectedEOF + } + + // Now create a CSV reader starting from after the skipped lines + // The CSV reader will properly handle any multiline quoted fields in the actual data + remainingContent := allBytes[bytesSkipped:] + reader := csv.NewReader(bytes.NewReader(remainingContent)) + + // Allow variable field counts and be lenient with parsing + reader.FieldsPerRecord = -1 + reader.LazyQuotes = true + reader.TrimLeadingSpace = true + return gocsv.NewSimpleDecoderFromCSVReader(reader), nil } func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) { sr := io.NewSectionReader(file, 0, size) - scanner := bufio.NewScanner(sr) + // Strip BOM before scanning for header + r := stripBOM(sr) + scanner := bufio.NewScanner(r) lines := 0 for scanner.Scan() { line := scanner.Text() @@ -243,7 +302,10 @@ func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error if err != nil { return err } - decode := newLineSkipDecoder(fr, skip) + decode, err := newLineSkipDecoder(fr, skip) + if err != nil { + return err + } err = gocsv.UnmarshalDecoder(decode, &allTasks) if err != nil { return err diff --git a/pkg/modules/migration/ticktick/ticktick_test.go b/pkg/modules/migration/ticktick/ticktick_test.go index c5826fbf1..aa8c3015e 100644 --- a/pkg/modules/migration/ticktick/ticktick_test.go +++ b/pkg/modules/migration/ticktick/ticktick_test.go @@ -17,7 +17,12 @@ package ticktick import ( + "bufio" "bytes" + "encoding/csv" + "io" + "os" + "strings" "testing" "time" @@ -155,10 +160,417 @@ func TestLinesToSkipBeforeHeader(t *testing.T) { assert.Equal(t, 2, lines) r2 := bytes.NewReader([]byte(csvContent)) - dec := newLineSkipDecoder(r2, lines) + dec, err := newLineSkipDecoder(r2, lines) + require.NoError(t, err) tasks := []*tickTickTask{} err = gocsv.UnmarshalDecoder(dec, &tasks) require.NoError(t, err) require.Len(t, tasks, 1) assert.Equal(t, "task1", tasks[0].Title) } + +func TestLinesToSkipBeforeHeaderWithRealCSV(t *testing.T) { + // This is the actual format from a real TickTick export with BOM and multi-line status + csvContent := "\uFEFF\"Date: 2025-11-25+0000\"\n" + + "\"Version: 7.1\"\n" + + "\"Status: \n" + + "0 Normal\n" + + "1 Completed\n" + + "2 Archived\"\n" + + "\"Folder Name\",\"List Name\",\"Title\",\"Kind\",\"Tags\",\"Content\",\"Is Check list\",\"Start Date\",\"Due Date\",\"Reminder\",\"Repeat\",\"Priority\",\"Status\",\"Created Time\",\"Completed Time\",\"Order\",\"Timezone\",\"Is All Day\",\"Is Floating\",\"Column Name\",\"Column Order\",\"View Mode\",\"taskId\",\"parentId\"\n" + + "\"dsx\",\"x\",\"this task repeats\",\"TEXT\",\"\",\"\",\"N\",\"\",\"\",\"\",\"\",\"0\",\"0\",\"2022-10-09T15:09:48+0000\",\"\",\"-1099511627776\",\"Europe/Berlin\",,\"false\",,,\"list\",\"2\",\"\"\n" + + t.Logf("CSV content length: %d", len(csvContent)) + t.Logf("CSV content first 100 chars: %q", csvContent[:100]) + + r := bytes.NewReader([]byte(csvContent)) + lines, err := linesToSkipBeforeHeader(r, int64(len(csvContent))) + require.NoError(t, err) + t.Logf("Lines to skip: %d", lines) + assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header + + r2 := bytes.NewReader([]byte(csvContent)) + dec, err := newLineSkipDecoder(r2, lines) + require.NoError(t, err) + tasks := []*tickTickTask{} + err = gocsv.UnmarshalDecoder(dec, &tasks) + require.NoError(t, err) + require.Len(t, tasks, 1) + assert.Equal(t, "this task repeats", tasks[0].Title) + assert.Equal(t, "dsx", tasks[0].FolderName) + assert.Equal(t, "x", tasks[0].ProjectName) +} + +func TestLinesToSkipBeforeHeaderWithCleanTestFile(t *testing.T) { + // Test with the cleaned-up test CSV file + file, err := os.Open("testdata_ticktick_export.csv") + require.NoError(t, err) + defer file.Close() + + stat, err := file.Stat() + require.NoError(t, err) + + lines, err := linesToSkipBeforeHeader(file, stat.Size()) + require.NoError(t, err) + t.Logf("Lines to skip in test file: %d", lines) + assert.Equal(t, 6, lines) // Should skip 6 lines to get to the header + + // Reset file position + _, err = file.Seek(0, io.SeekStart) + require.NoError(t, err) + + // Let's manually check what the header line looks like after skipping + r := stripBOM(file) + scanner := bufio.NewScanner(r) + for i := 0; i <= lines; i++ { + if !scanner.Scan() { + break + } + if i == lines { + t.Logf("Header line after skipping %d lines: %q", lines, scanner.Text()) + } + } + + // Reset file position again + _, err = file.Seek(0, io.SeekStart) + require.NoError(t, err) + + dec, err := newLineSkipDecoder(file, lines) + require.NoError(t, err) + tasks := []*tickTickTask{} + err = gocsv.UnmarshalDecoder(dec, &tasks) + require.NoError(t, err) + require.Greater(t, len(tasks), 0) + + // Verify that the first task has actual data + assert.Equal(t, "Work", tasks[0].FolderName) + assert.Equal(t, "Project Alpha", tasks[0].ProjectName) + assert.Equal(t, "Task with repeating schedule", tasks[0].Title) +} + +func TestBOMStripping(t *testing.T) { + // Test BOM stripping specifically + csvWithBOM := "\uFEFF\"Folder Name\",\"List Name\",\"Title\"\n\"test\",\"list\",\"task\"\n" + + r := stripBOM(bytes.NewReader([]byte(csvWithBOM))) + scanner := bufio.NewScanner(r) + + // Read first line (header) + require.True(t, scanner.Scan()) + header := scanner.Text() + t.Logf("Header after BOM stripping: %q", header) + + // Read second line (data) + require.True(t, scanner.Scan()) + data := scanner.Text() + t.Logf("Data line: %q", data) + + // Test CSV parsing + r2 := stripBOM(bytes.NewReader([]byte(csvWithBOM))) + reader := csv.NewReader(r2) + records, err := reader.ReadAll() + require.NoError(t, err) + require.Len(t, records, 2) + t.Logf("CSV records: %+v", records) +} + +func TestEmptyLabelHandling(t *testing.T) { + t.Run("Normal tags", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: "work, personal, urgent", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{"work", "personal", "urgent"} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Tags with extra spaces", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: "work, personal , urgent", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{"work", "personal", "urgent"} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Empty tags mixed with valid ones", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: "work, , urgent, ", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{"work", "urgent"} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Only whitespace tags", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: " , , ", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Empty string", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: "", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Single valid tag", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: "important", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{"important"} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Single empty tag", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: " ", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) + + t.Run("Tags with leading/trailing spaces", func(t *testing.T) { + task := &tickTickTask{ + Title: "Test Task", + ProjectName: "Test Project", + TagsList: " work , personal, urgent ", + } + task.Tags = strings.Split(task.TagsList, ", ") + + vikunjaTasks := convertTickTickToVikunja([]*tickTickTask{task}) + projectWithTasks := findProjectWithTasks(t, vikunjaTasks) + vikunjaTask := projectWithTasks.Tasks[0] + + expectedTags := []string{"work", "personal", "urgent"} + assertLabelsMatch(t, vikunjaTask, expectedTags) + }) +} + +// Helper function to find the project that contains tasks +func findProjectWithTasks(t *testing.T, vikunjaTasks []*models.ProjectWithTasksAndBuckets) *models.ProjectWithTasksAndBuckets { + t.Helper() + + // The function creates a parent project and child projects + // We expect 2 projects: parent "Migrated from TickTick" and child "Test Project" + require.Len(t, vikunjaTasks, 2) + + // Find the project with tasks (should be the child project) + for _, project := range vikunjaTasks { + if len(project.Tasks) > 0 { + require.Len(t, project.Tasks, 1) + return project + } + } + + t.Fatal("Should find a project with tasks") + return nil +} + +// Helper function to assert that labels match expected tags +func assertLabelsMatch(t *testing.T, vikunjaTask *models.TaskWithComments, expectedTags []string) { + t.Helper() + + // Check that only non-empty labels were created + assert.Len(t, vikunjaTask.Labels, len(expectedTags), "Number of labels should match expected") + + // Check that the label titles match expected tags + actualTags := make([]string, len(vikunjaTask.Labels)) + for i, label := range vikunjaTask.Labels { + actualTags[i] = label.Title + } + + assert.ElementsMatch(t, expectedTags, actualTags, "Label titles should match expected tags") + + // Ensure no empty labels were created + for _, label := range vikunjaTask.Labels { + assert.NotEmpty(t, strings.TrimSpace(label.Title), "No label should be empty or whitespace-only") + } +} + +func TestMultilineDescriptions(t *testing.T) { + // Test with a CSV fixture that contains actual multiline content in quoted fields + file, err := os.Open("testdata_ticktick_multiline.csv") + require.NoError(t, err, "Failed to open test fixture") + defer file.Close() + + stat, err := file.Stat() + require.NoError(t, err) + + lines, err := linesToSkipBeforeHeader(file, stat.Size()) + require.NoError(t, err) + t.Logf("Lines to skip: %d", lines) + assert.Equal(t, 6, lines, "Should skip 6 metadata lines") + + // Reset file position + _, err = file.Seek(0, io.SeekStart) + require.NoError(t, err) + + dec, err := newLineSkipDecoder(file, lines) + require.NoError(t, err) + tasks := []*tickTickTask{} + err = gocsv.UnmarshalDecoder(dec, &tasks) + require.NoError(t, err, "Failed to parse CSV with multiline descriptions") + + // We expect 2 tasks in this fixture + require.Len(t, tasks, 2, "Should parse exactly 2 tasks") + + // First task has multiline content in both Title and Content fields + task1 := tasks[0] + assert.Equal(t, "Work", task1.FolderName) + assert.Equal(t, "Project Alpha", task1.ProjectName) + + // The title contains a newline + assert.Contains(t, task1.Title, "Task with multiline") + assert.Contains(t, task1.Title, "description") + assert.Contains(t, task1.Title, "\n", "Title should contain actual newline character") + + // The content contains multiple newlines and paragraphs + assert.Contains(t, task1.Content, "This is a task description") + assert.Contains(t, task1.Content, "that spans multiple lines") + assert.Contains(t, task1.Content, "It has paragraphs and everything!") + assert.Contains(t, task1.Content, "Including special characters: #, *, @") + + // Count newlines in content - should have at least 3 (between the 4 lines) + newlineCount := strings.Count(task1.Content, "\n") + assert.GreaterOrEqual(t, newlineCount, 3, "Content should have multiple newlines") + + // Second task is a regular task without multiline content + task2 := tasks[1] + assert.Equal(t, "Regular task", task2.Title) + assert.Equal(t, "Simple description", task2.Content) + assert.NotContains(t, task2.Title, "\n", "Regular task title should not have newlines") + + t.Logf("Successfully parsed tasks with multiline content:") + t.Logf(" Task 1 title: %q", task1.Title) + t.Logf(" Task 1 content: %q", task1.Content) + t.Logf(" Task 2 title: %q", task2.Title) +} + +func TestEmptyLabelHandlingWithRealCSV(t *testing.T) { + t.Run("Parse CSV file", func(t *testing.T) { + file, err := os.Open("testdata_ticktick_export.csv") + require.NoError(t, err) + defer file.Close() + + stat, err := file.Stat() + require.NoError(t, err) + + lines, err := linesToSkipBeforeHeader(file, stat.Size()) + require.NoError(t, err) + + // Reset file position + _, err = file.Seek(0, io.SeekStart) + require.NoError(t, err) + + dec, err := newLineSkipDecoder(file, lines) + require.NoError(t, err) + tasks := []*tickTickTask{} + err = gocsv.UnmarshalDecoder(dec, &tasks) + require.NoError(t, err) + require.Greater(t, len(tasks), 0) + + t.Logf("Successfully parsed %d tasks from CSV file", len(tasks)) + }) + + t.Run("Process tags and check for empty labels", func(t *testing.T) { + file, err := os.Open("testdata_ticktick_export.csv") + require.NoError(t, err) + defer file.Close() + + stat, err := file.Stat() + require.NoError(t, err) + + lines, err := linesToSkipBeforeHeader(file, stat.Size()) + require.NoError(t, err) + + // Reset file position + _, err = file.Seek(0, io.SeekStart) + require.NoError(t, err) + + dec, err := newLineSkipDecoder(file, lines) + require.NoError(t, err) + tasks := []*tickTickTask{} + err = gocsv.UnmarshalDecoder(dec, &tasks) + require.NoError(t, err) + + // Process tags as the migration code does + for _, task := range tasks { + task.Tags = strings.Split(task.TagsList, ", ") + } + + // Convert to Vikunja format + vikunjaTasks := convertTickTickToVikunja(tasks) + + // Check all tasks for empty labels + totalLabels := 0 + for _, project := range vikunjaTasks { + for _, task := range project.Tasks { + totalLabels += len(task.Labels) + for _, label := range task.Labels { + assert.NotEmpty(t, strings.TrimSpace(label.Title), + "No label should be empty or whitespace-only. Found empty label in task: %s", task.Title) + } + } + } + + t.Logf("Successfully processed %d tasks with %d total labels, no empty labels created", len(tasks), totalLabels) + }) +}