fix(migration): tolerate non-numeric values in TickTick CSV exports
TickTick exports could contain non-numeric values in columns Vikunja parses as integers (Priority, taskId, parentId). gocsv's strconv.ParseInt then failed, aborting the entire import and surfacing as an internal server error reported to Sentry (e.g. parsing "p1": invalid syntax). Numeric ID columns now fall back to 0 for unparseable values instead of failing the import. The Priority column, which was previously parsed but never carried over to the imported task, is now mapped onto the task and accepts both the plain numeric form (0, 1, 3, 5) and the "pN" form (p1, p2, p3). Closes #2822
This commit is contained in:
Tink bot
kolaente
parent
e1c9ab5939
commit
ebb89ba4f3
|
|
@ -0,0 +1,9 @@
|
|||
"Date: 2026-02-27+0000"
|
||||
"Version: 7.1"
|
||||
"Status:
|
||||
0 Normal
|
||||
1 Completed
|
||||
2 Archived"
|
||||
"Folder Name","List Name","Title","Kind","Tags","Content","Is Check list","Start Date","Due Date","Reminder","Repeat","Priority","Status","Created Time","Completed Time","Order","Timezone","Is All Day","Is Floating","Column Name","Column Order","View Mode","taskId","parentId"
|
||||
"Work","Project Beta","Task with non-numeric priority","TEXT","","A task whose priority column is p1","N","","","","","p1","0","2026-02-15 09:30:00","","-1099511627776","Europe/Berlin",,"false",,,"list","1",""
|
||||
"Work","Project Beta","Task with non-numeric ids","TEXT","","A task with garbage in the id columns","N","","","","","0","0","2026-02-10 08:00:00","","0","Europe/Berlin","false","false",,,"list","abc","xyz"
|
||||
|
Can't render this file because it has a wrong number of fields in line 7.
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -44,32 +45,84 @@ type Migrator struct {
|
|||
}
|
||||
|
||||
type tickTickTask struct {
|
||||
FolderName string `csv:"Folder Name"`
|
||||
ProjectName string `csv:"List Name"`
|
||||
Title string `csv:"Title"`
|
||||
TagsList string `csv:"Tags"`
|
||||
Tags []string `csv:"-"`
|
||||
Content string `csv:"Content"`
|
||||
IsChecklistString string `csv:"Is Check list"`
|
||||
IsChecklist bool `csv:"-"`
|
||||
StartDate tickTickTime `csv:"Start Date"`
|
||||
DueDate tickTickTime `csv:"Due Date"`
|
||||
ReminderDuration string `csv:"Reminder"`
|
||||
Reminder time.Duration `csv:"-"`
|
||||
Repeat string `csv:"Repeat"`
|
||||
Priority int `csv:"Priority"`
|
||||
Status string `csv:"Status"`
|
||||
CreatedTime tickTickTime `csv:"Created Time"`
|
||||
CompletedTime tickTickTime `csv:"Completed Time"`
|
||||
Order float64 `csv:"Order"`
|
||||
TaskID int64 `csv:"taskId"`
|
||||
ParentID int64 `csv:"parentId"`
|
||||
FolderName string `csv:"Folder Name"`
|
||||
ProjectName string `csv:"List Name"`
|
||||
Title string `csv:"Title"`
|
||||
TagsList string `csv:"Tags"`
|
||||
Tags []string `csv:"-"`
|
||||
Content string `csv:"Content"`
|
||||
IsChecklistString string `csv:"Is Check list"`
|
||||
IsChecklist bool `csv:"-"`
|
||||
StartDate tickTickTime `csv:"Start Date"`
|
||||
DueDate tickTickTime `csv:"Due Date"`
|
||||
ReminderDuration string `csv:"Reminder"`
|
||||
Reminder time.Duration `csv:"-"`
|
||||
Repeat string `csv:"Repeat"`
|
||||
Priority tickTickPriority `csv:"Priority"`
|
||||
Status string `csv:"Status"`
|
||||
CreatedTime tickTickTime `csv:"Created Time"`
|
||||
CompletedTime tickTickTime `csv:"Completed Time"`
|
||||
Order float64 `csv:"Order"`
|
||||
TaskID tickTickNumber `csv:"taskId"`
|
||||
ParentID tickTickNumber `csv:"parentId"`
|
||||
}
|
||||
|
||||
type tickTickTime struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
// tickTickNumber is an int64 that tolerates non-numeric or malformed values in
|
||||
// the numeric ID columns (taskId, parentId) of TickTick exports. Such values can
|
||||
// occur through column misalignment caused by unescaped delimiters, which would
|
||||
// otherwise make gocsv fail the entire import with an internal server error
|
||||
// (go-vikunja/vikunja#2822). Rather than aborting the import, we fall back to 0
|
||||
// for any value we cannot parse.
|
||||
type tickTickNumber int64
|
||||
|
||||
func (n *tickTickNumber) UnmarshalCSV(csv string) error {
|
||||
csv = strings.TrimSpace(csv)
|
||||
if csv == "" {
|
||||
*n = 0
|
||||
return nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(csv, 10, 64)
|
||||
if err != nil {
|
||||
// Deliberately ignore the parse error and fall back to 0 so a single
|
||||
// malformed value does not abort the whole import.
|
||||
*n = 0
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
*n = tickTickNumber(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// tickTickPriority parses the TickTick "Priority" column. TickTick exports the
|
||||
// priority either as a plain number (0, 1, 3, 5) or, in some exports, prefixed
|
||||
// with "p" (p1, p2, p3). We accept both forms and fall back to 0 (no priority)
|
||||
// for anything we cannot parse, so a stray value never fails the whole import
|
||||
// (go-vikunja/vikunja#2822). Vikunja's task priority is a free-form sortable
|
||||
// integer, so the parsed value is carried over as-is.
|
||||
type tickTickPriority int64
|
||||
|
||||
func (p *tickTickPriority) UnmarshalCSV(csv string) error {
|
||||
csv = strings.TrimSpace(csv)
|
||||
csv = strings.TrimPrefix(csv, "p")
|
||||
csv = strings.TrimPrefix(csv, "P")
|
||||
if csv == "" {
|
||||
*p = 0
|
||||
return nil
|
||||
}
|
||||
parsed, err := strconv.ParseInt(csv, 10, 64)
|
||||
if err != nil {
|
||||
// Deliberately ignore the parse error and fall back to 0 so a single
|
||||
// malformed value does not abort the whole import.
|
||||
*p = 0
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
*p = tickTickPriority(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
|
||||
date.Time = time.Time{}
|
||||
if csv == "" {
|
||||
|
|
@ -88,12 +141,12 @@ func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
|
|||
// appears before any of its children. Tasks without a parent come first.
|
||||
// The relative order of siblings / unrelated tasks is preserved.
|
||||
func sortParentsBeforeChildren(tasks []*tickTickTask) []*tickTickTask {
|
||||
tasksByID := make(map[int64]*tickTickTask, len(tasks))
|
||||
tasksByID := make(map[tickTickNumber]*tickTickTask, len(tasks))
|
||||
for _, t := range tasks {
|
||||
tasksByID[t.TaskID] = t
|
||||
}
|
||||
|
||||
placed := make(map[int64]bool, len(tasks))
|
||||
placed := make(map[tickTickNumber]bool, len(tasks))
|
||||
result := make([]*tickTickTask, 0, len(tasks))
|
||||
|
||||
var place func(t *tickTickTask)
|
||||
|
|
@ -161,7 +214,7 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi
|
|||
|
||||
task := &models.TaskWithComments{
|
||||
Task: models.Task{
|
||||
ID: t.TaskID,
|
||||
ID: int64(t.TaskID),
|
||||
Title: t.Title,
|
||||
Description: t.Content,
|
||||
StartDate: t.StartDate.Time,
|
||||
|
|
@ -170,6 +223,7 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi
|
|||
Done: t.Status == "1" || t.Status == "2",
|
||||
DoneAt: t.CompletedTime.Time,
|
||||
Position: t.Order,
|
||||
Priority: int64(t.Priority),
|
||||
Labels: labels,
|
||||
},
|
||||
}
|
||||
|
|
@ -185,7 +239,7 @@ func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWi
|
|||
|
||||
if t.ParentID != 0 {
|
||||
task.RelatedTasks = map[models.RelationKind][]*models.Task{
|
||||
models.RelationKindParenttask: {{ID: t.ParentID}},
|
||||
models.RelationKindParenttask: {{ID: int64(t.ParentID)}},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ func TestConvertTicktickTasksToVikunja(t *testing.T) {
|
|||
assert.Equal(t, models.RelatedTaskMap{
|
||||
models.RelationKindParenttask: []*models.Task{
|
||||
{
|
||||
ID: tickTickTasks[1].ParentID,
|
||||
ID: int64(tickTickTasks[1].ParentID),
|
||||
},
|
||||
},
|
||||
}, vikunjaTasks[1].Tasks[1].RelatedTasks)
|
||||
|
|
@ -770,3 +770,79 @@ func TestEmptyLabelHandlingWithRealCSV(t *testing.T) {
|
|||
t.Logf("Successfully processed %d tasks with %d total labels, no empty labels created", len(tasks), totalLabels)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTickTickPriorityParsing(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
expected int64
|
||||
}{
|
||||
{"", 0},
|
||||
{"0", 0},
|
||||
{"1", 1},
|
||||
{"3", 3},
|
||||
{"5", 5},
|
||||
{"p1", 1},
|
||||
{"P2", 2},
|
||||
{"p3", 3},
|
||||
{" p5 ", 5},
|
||||
{"garbage", 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.input, func(t *testing.T) {
|
||||
var p tickTickPriority
|
||||
require.NoError(t, p.UnmarshalCSV(c.input))
|
||||
assert.Equal(t, tickTickPriority(c.expected), p)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNonNumericNumberColumns ensures that exports containing non-numeric values
|
||||
// in columns Vikunja parses as integers (Priority, taskId, parentId) do not fail
|
||||
// the whole import. See go-vikunja/vikunja#2822.
|
||||
func TestNonNumericNumberColumns(t *testing.T) {
|
||||
file, err := os.Open("testdata_ticktick_invalid_numbers.csv")
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
lines, err := linesToSkipBeforeHeader(file, stat.Size())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = file.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
dec, err := newLineSkipDecoder(file, lines)
|
||||
require.NoError(t, err)
|
||||
tasks := []*tickTickTask{}
|
||||
err = gocsv.UnmarshalDecoder(dec, &tasks)
|
||||
require.NoError(t, err, "non-numeric values in numeric columns should not fail the import")
|
||||
require.Len(t, tasks, 2)
|
||||
|
||||
// "p1" in the Priority column is parsed as priority 1 instead of crashing.
|
||||
assert.Equal(t, tickTickPriority(1), tasks[0].Priority)
|
||||
assert.Equal(t, tickTickNumber(1), tasks[0].TaskID)
|
||||
|
||||
// Non-numeric taskId/parentId fall back to 0 as well.
|
||||
assert.Equal(t, tickTickNumber(0), tasks[1].TaskID)
|
||||
assert.Equal(t, tickTickNumber(0), tasks[1].ParentID)
|
||||
|
||||
// And the tasks still convert to the Vikunja structure, carrying the parsed
|
||||
// priority over to the resulting task.
|
||||
for _, task := range tasks {
|
||||
task.Tags = strings.Split(task.TagsList, ", ")
|
||||
}
|
||||
vikunjaTasks := convertTickTickToVikunja(tasks)
|
||||
require.Greater(t, len(vikunjaTasks), 0)
|
||||
|
||||
var priority int64
|
||||
for _, project := range vikunjaTasks {
|
||||
for _, task := range project.Tasks {
|
||||
if task.Title == "Task with non-numeric priority" {
|
||||
priority = task.Priority
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, int64(1), priority)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue