// Vikunja is a to-do list application to facilitate your life. // Copyright 2018-present Vikunja and contributors. All rights reserved. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package ticktick import ( "bufio" "bytes" "encoding/csv" "errors" "io" "sort" "strings" "time" "code.vikunja.io/api/pkg/models" "code.vikunja.io/api/pkg/modules/migration" "code.vikunja.io/api/pkg/user" "code.vikunja.io/api/pkg/utils" "github.com/gocarina/gocsv" ) var timeFormats = []string{ "2006-01-02T15:04:05-0700", "2006-01-02 15:04:05", "2006-01-02T15:04:05Z", } type Migrator struct { } type tickTickTask struct { FolderName string `csv:"Folder Name"` ProjectName string `csv:"List Name"` Title string `csv:"Title"` TagsList string `csv:"Tags"` Tags []string `csv:"-"` Content string `csv:"Content"` IsChecklistString string `csv:"Is Check list"` IsChecklist bool `csv:"-"` StartDate tickTickTime `csv:"Start Date"` DueDate tickTickTime `csv:"Due Date"` ReminderDuration string `csv:"Reminder"` Reminder time.Duration `csv:"-"` Repeat string `csv:"Repeat"` Priority int `csv:"Priority"` Status string `csv:"Status"` CreatedTime tickTickTime `csv:"Created Time"` CompletedTime tickTickTime `csv:"Completed Time"` Order float64 `csv:"Order"` TaskID int64 `csv:"taskId"` ParentID int64 `csv:"parentId"` } type tickTickTime struct { time.Time } func (date *tickTickTime) UnmarshalCSV(csv string) (err error) { date.Time = time.Time{} if csv == "" { return nil } for _, format := range timeFormats { date.Time, err = time.Parse(format, csv) if err == nil { return nil } } return err } // sortParentsBeforeChildren reorders tasks so that every parent task // appears before any of its children. Tasks without a parent come first. // The relative order of siblings / unrelated tasks is preserved. func sortParentsBeforeChildren(tasks []*tickTickTask) []*tickTickTask { tasksByID := make(map[int64]*tickTickTask, len(tasks)) for _, t := range tasks { tasksByID[t.TaskID] = t } placed := make(map[int64]bool, len(tasks)) result := make([]*tickTickTask, 0, len(tasks)) var place func(t *tickTickTask) place = func(t *tickTickTask) { if placed[t.TaskID] { return } // If this task has a parent that we know about, place the parent first. if t.ParentID != 0 { if parent, ok := tasksByID[t.ParentID]; ok { place(parent) } } placed[t.TaskID] = true result = append(result, t) } for _, t := range tasks { place(t) } return result } func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWithTasksAndBuckets) { // Sort tasks so that parent tasks always come before their children. // Without this, create_from_structure.go would try to create a // placeholder for a not-yet-seen parent, which fails because the // placeholder has no title. (go-vikunja/vikunja#2487) tasks = sortParentsBeforeChildren(tasks) var pseudoParentID int64 = 1 result = []*models.ProjectWithTasksAndBuckets{ { Project: models.Project{ ID: pseudoParentID, Title: "Migrated from TickTick", }, }, } projects := make(map[string]*models.ProjectWithTasksAndBuckets) for index, t := range tasks { _, has := projects[t.ProjectName] if !has { projects[t.ProjectName] = &models.ProjectWithTasksAndBuckets{ Project: models.Project{ ID: int64(index+1) + pseudoParentID, ParentProjectID: pseudoParentID, Title: t.ProjectName, }, } } labels := make([]*models.Label, 0, len(t.Tags)) for _, tag := range t.Tags { // Only create labels for non-empty tags after trimming whitespace trimmedTag := strings.TrimSpace(tag) if trimmedTag != "" { labels = append(labels, &models.Label{ Title: trimmedTag, }) } } task := &models.TaskWithComments{ Task: models.Task{ ID: t.TaskID, Title: t.Title, Description: t.Content, StartDate: t.StartDate.Time, EndDate: t.DueDate.Time, DueDate: t.DueDate.Time, Done: t.Status == "1" || t.Status == "2", DoneAt: t.CompletedTime.Time, Position: t.Order, Labels: labels, }, } if !t.DueDate.IsZero() && t.Reminder > 0 { task.Reminders = []*models.TaskReminder{ { RelativeTo: models.ReminderRelationDueDate, RelativePeriod: int64((t.Reminder * -1).Seconds()), }, } } if t.ParentID != 0 { task.RelatedTasks = map[models.RelationKind][]*models.Task{ models.RelationKindParenttask: {{ID: t.ParentID}}, } } projects[t.ProjectName].Tasks = append(projects[t.ProjectName].Tasks, task) } for _, l := range projects { result = append(result, l) } sort.Slice(result, func(i, j int) bool { return result[i].Title < result[j].Title }) return } // Name is used to get the name of the ticktick migration - we're using the docs here to annotate the status route. // @Summary Get migration status // @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again. // @tags migration // @Produce json // @Security JWTKeyAuth // @Success 200 {object} migration.Status "The migration status" // @Failure 500 {object} models.Message "Internal server error" // @Router /migration/ticktick/status [get] func (m *Migrator) Name() string { return "ticktick" } // stripBOM removes the UTF-8 BOM from the beginning of a reader func stripBOM(r io.Reader) io.Reader { // Read the first few bytes to check for BOM buf := make([]byte, 3) n, err := r.Read(buf) if err != nil && err != io.EOF { // If we read some bytes before the error, preserve them if n > 0 { return io.MultiReader(bytes.NewReader(buf[:n]), r) } return r } // Check if it starts with UTF-8 BOM (0xEF, 0xBB, 0xBF) // We need exactly 3 bytes and they must match the BOM sequence if n == 3 && len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF { // BOM found, return reader without BOM return io.MultiReader(bytes.NewReader(buf[3:n]), r) } // No BOM found, return reader with the bytes we read back return io.MultiReader(bytes.NewReader(buf[:n]), r) } func newLineSkipDecoder(r io.Reader, linesToSkip int) (gocsv.SimpleDecoder, error) { // Strip BOM if present - this must be done consistently with linesToSkipBeforeHeader r = stripBOM(r) // Read all content into memory so we can work with it // This is acceptable since CSV imports are typically not huge files allBytes, err := io.ReadAll(r) if err != nil { return nil, err } // Skip the metadata lines before the CSV header by finding newlines // linesToSkipBeforeHeader counts raw text lines (newlines), not CSV records, // because even metadata can have multiline quoted fields. // We manually search for newlines (no buffer size limits like bufio.Scanner) bytesSkipped := 0 linesFound := 0 for i := 0; i < len(allBytes) && linesFound < linesToSkip; i++ { if allBytes[i] == '\n' { linesFound++ if linesFound == linesToSkip { // Position is right after the Nth newline bytesSkipped = i + 1 break } } } if linesFound < linesToSkip { return nil, io.ErrUnexpectedEOF } // Now create a CSV reader starting from after the skipped lines // The CSV reader will properly handle any multiline quoted fields in the actual data remainingContent := allBytes[bytesSkipped:] reader := csv.NewReader(bytes.NewReader(remainingContent)) // Allow variable field counts and be lenient with parsing reader.FieldsPerRecord = -1 reader.LazyQuotes = true reader.TrimLeadingSpace = true return gocsv.NewSimpleDecoderFromCSVReader(reader), nil } func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) { sr := io.NewSectionReader(file, 0, size) // Strip BOM before scanning for header r := stripBOM(sr) scanner := bufio.NewScanner(r) lines := 0 for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "Folder Name") && strings.Contains(line, "List Name") && strings.Contains(line, "Title") { break } lines++ } if err := scanner.Err(); err != nil { return 0, err } return lines, nil } // Migrate takes a ticktick export, parses it and imports everything in it into Vikunja. // @Summary Import all projects, tasks etc. from a TickTick backup export // @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja. // @tags migration // @Accept x-www-form-urlencoded // @Produce json // @Security JWTKeyAuth // @Param import formData string true "The TickTick backup csv file." // @Success 200 {object} models.Message "A message telling you everything was migrated successfully." // @Failure 500 {object} models.Message "Internal server error" // @Router /migration/ticktick/migrate [post] func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error { // Check if file is empty if size == 0 { return &migration.ErrFileIsEmpty{} } fr := io.NewSectionReader(file, 0, size) // Check if the file is a valid CSV buf := make([]byte, 1024) n, err := fr.Read(buf) if errors.Is(err, io.EOF) || n == 0 { return &migration.ErrFileIsEmpty{} } if err != nil { return err } // Reset the reader position to start _, err = fr.Seek(0, io.SeekStart) if err != nil { return err } // Check if the content looks like a CSV file content := string(buf[:n]) if !isValidCSV(content) { return &migration.ErrNotACSVFile{} } allTasks := []*tickTickTask{} skip, err := linesToSkipBeforeHeader(file, size) if err != nil { return err } decode, err := newLineSkipDecoder(fr, skip) if err != nil { return err } err = gocsv.UnmarshalDecoder(decode, &allTasks) if err != nil { return err } // Also check if no tasks were found after decoding if len(allTasks) == 0 { return &migration.ErrFileIsEmpty{} } for _, task := range allTasks { if task.IsChecklistString == "Y" { task.IsChecklist = true } reminder := utils.ParseISO8601Duration(task.ReminderDuration) if reminder > 0 { task.Reminder = reminder } task.Tags = strings.Split(task.TagsList, ", ") } vikunjaTasks := convertTickTickToVikunja(allTasks) return migration.InsertFromStructure(vikunjaTasks, user) } // isValidCSV performs a basic check to determine if the content looks like a CSV file func isValidCSV(content string) bool { // Check for common CSV headers from TickTick export if !strings.Contains(content, "Folder Name") || !strings.Contains(content, "List Name") || !strings.Contains(content, "Title") { return false } // Check if the file has commas as separators and multiple lines hasCommas := strings.Contains(content, ",") hasNewlines := strings.Contains(content, "\n") return hasCommas && hasNewlines }