401 lines
12 KiB
Go
401 lines
12 KiB
Go
// Vikunja is a to-do list application to facilitate your life.
|
|
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
package ticktick
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/csv"
|
|
"errors"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.vikunja.io/api/pkg/models"
|
|
"code.vikunja.io/api/pkg/modules/migration"
|
|
"code.vikunja.io/api/pkg/user"
|
|
"code.vikunja.io/api/pkg/utils"
|
|
|
|
"github.com/gocarina/gocsv"
|
|
)
|
|
|
|
var timeFormats = []string{
|
|
"2006-01-02T15:04:05-0700",
|
|
"2006-01-02 15:04:05",
|
|
"2006-01-02T15:04:05Z",
|
|
}
|
|
|
|
type Migrator struct {
|
|
}
|
|
|
|
type tickTickTask struct {
|
|
FolderName string `csv:"Folder Name"`
|
|
ProjectName string `csv:"List Name"`
|
|
Title string `csv:"Title"`
|
|
TagsList string `csv:"Tags"`
|
|
Tags []string `csv:"-"`
|
|
Content string `csv:"Content"`
|
|
IsChecklistString string `csv:"Is Check list"`
|
|
IsChecklist bool `csv:"-"`
|
|
StartDate tickTickTime `csv:"Start Date"`
|
|
DueDate tickTickTime `csv:"Due Date"`
|
|
ReminderDuration string `csv:"Reminder"`
|
|
Reminder time.Duration `csv:"-"`
|
|
Repeat string `csv:"Repeat"`
|
|
Priority int `csv:"Priority"`
|
|
Status string `csv:"Status"`
|
|
CreatedTime tickTickTime `csv:"Created Time"`
|
|
CompletedTime tickTickTime `csv:"Completed Time"`
|
|
Order float64 `csv:"Order"`
|
|
TaskID int64 `csv:"taskId"`
|
|
ParentID int64 `csv:"parentId"`
|
|
}
|
|
|
|
type tickTickTime struct {
|
|
time.Time
|
|
}
|
|
|
|
func (date *tickTickTime) UnmarshalCSV(csv string) (err error) {
|
|
date.Time = time.Time{}
|
|
if csv == "" {
|
|
return nil
|
|
}
|
|
for _, format := range timeFormats {
|
|
date.Time, err = time.Parse(format, csv)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// sortParentsBeforeChildren reorders tasks so that every parent task
|
|
// appears before any of its children. Tasks without a parent come first.
|
|
// The relative order of siblings / unrelated tasks is preserved.
|
|
func sortParentsBeforeChildren(tasks []*tickTickTask) []*tickTickTask {
|
|
tasksByID := make(map[int64]*tickTickTask, len(tasks))
|
|
for _, t := range tasks {
|
|
tasksByID[t.TaskID] = t
|
|
}
|
|
|
|
placed := make(map[int64]bool, len(tasks))
|
|
result := make([]*tickTickTask, 0, len(tasks))
|
|
|
|
var place func(t *tickTickTask)
|
|
place = func(t *tickTickTask) {
|
|
if placed[t.TaskID] {
|
|
return
|
|
}
|
|
// If this task has a parent that we know about, place the parent first.
|
|
if t.ParentID != 0 {
|
|
if parent, ok := tasksByID[t.ParentID]; ok {
|
|
place(parent)
|
|
}
|
|
}
|
|
placed[t.TaskID] = true
|
|
result = append(result, t)
|
|
}
|
|
|
|
for _, t := range tasks {
|
|
place(t)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func convertTickTickToVikunja(tasks []*tickTickTask) (result []*models.ProjectWithTasksAndBuckets) {
|
|
// Sort tasks so that parent tasks always come before their children.
|
|
// Without this, create_from_structure.go would try to create a
|
|
// placeholder for a not-yet-seen parent, which fails because the
|
|
// placeholder has no title. (go-vikunja/vikunja#2487)
|
|
tasks = sortParentsBeforeChildren(tasks)
|
|
|
|
var pseudoParentID int64 = 1
|
|
result = []*models.ProjectWithTasksAndBuckets{
|
|
{
|
|
Project: models.Project{
|
|
ID: pseudoParentID,
|
|
Title: "Migrated from TickTick",
|
|
},
|
|
},
|
|
}
|
|
|
|
projects := make(map[string]*models.ProjectWithTasksAndBuckets)
|
|
for index, t := range tasks {
|
|
_, has := projects[t.ProjectName]
|
|
if !has {
|
|
projects[t.ProjectName] = &models.ProjectWithTasksAndBuckets{
|
|
Project: models.Project{
|
|
ID: int64(index+1) + pseudoParentID,
|
|
ParentProjectID: pseudoParentID,
|
|
Title: t.ProjectName,
|
|
},
|
|
}
|
|
}
|
|
|
|
labels := make([]*models.Label, 0, len(t.Tags))
|
|
for _, tag := range t.Tags {
|
|
// Only create labels for non-empty tags after trimming whitespace
|
|
trimmedTag := strings.TrimSpace(tag)
|
|
if trimmedTag != "" {
|
|
labels = append(labels, &models.Label{
|
|
Title: trimmedTag,
|
|
})
|
|
}
|
|
}
|
|
|
|
task := &models.TaskWithComments{
|
|
Task: models.Task{
|
|
ID: t.TaskID,
|
|
Title: t.Title,
|
|
Description: t.Content,
|
|
StartDate: t.StartDate.Time,
|
|
EndDate: t.DueDate.Time,
|
|
DueDate: t.DueDate.Time,
|
|
Done: t.Status == "1" || t.Status == "2",
|
|
DoneAt: t.CompletedTime.Time,
|
|
Position: t.Order,
|
|
Labels: labels,
|
|
},
|
|
}
|
|
|
|
if !t.DueDate.IsZero() && t.Reminder > 0 {
|
|
task.Reminders = []*models.TaskReminder{
|
|
{
|
|
RelativeTo: models.ReminderRelationDueDate,
|
|
RelativePeriod: int64((t.Reminder * -1).Seconds()),
|
|
},
|
|
}
|
|
}
|
|
|
|
if t.ParentID != 0 {
|
|
task.RelatedTasks = map[models.RelationKind][]*models.Task{
|
|
models.RelationKindParenttask: {{ID: t.ParentID}},
|
|
}
|
|
}
|
|
|
|
projects[t.ProjectName].Tasks = append(projects[t.ProjectName].Tasks, task)
|
|
}
|
|
|
|
for _, l := range projects {
|
|
result = append(result, l)
|
|
}
|
|
|
|
sort.Slice(result, func(i, j int) bool {
|
|
return result[i].Title < result[j].Title
|
|
})
|
|
|
|
return
|
|
}
|
|
|
|
// Name is used to get the name of the ticktick migration - we're using the docs here to annotate the status route.
|
|
// @Summary Get migration status
|
|
// @Description Returns if the current user already did the migation or not. This is useful to show a confirmation message in the frontend if the user is trying to do the same migration again.
|
|
// @tags migration
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Success 200 {object} migration.Status "The migration status"
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/ticktick/status [get]
|
|
func (m *Migrator) Name() string {
|
|
return "ticktick"
|
|
}
|
|
|
|
// stripBOM removes the UTF-8 BOM from the beginning of a reader
|
|
func stripBOM(r io.Reader) io.Reader {
|
|
// Read the first few bytes to check for BOM
|
|
buf := make([]byte, 3)
|
|
n, err := r.Read(buf)
|
|
if err != nil && err != io.EOF {
|
|
// If we read some bytes before the error, preserve them
|
|
if n > 0 {
|
|
return io.MultiReader(bytes.NewReader(buf[:n]), r)
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Check if it starts with UTF-8 BOM (0xEF, 0xBB, 0xBF)
|
|
// We need exactly 3 bytes and they must match the BOM sequence
|
|
if n == 3 && len(buf) >= 3 && buf[0] == 0xEF && buf[1] == 0xBB && buf[2] == 0xBF {
|
|
// BOM found, return reader without BOM
|
|
return io.MultiReader(bytes.NewReader(buf[3:n]), r)
|
|
}
|
|
|
|
// No BOM found, return reader with the bytes we read back
|
|
return io.MultiReader(bytes.NewReader(buf[:n]), r)
|
|
}
|
|
|
|
func newLineSkipDecoder(r io.Reader, linesToSkip int) (gocsv.SimpleDecoder, error) {
|
|
// Strip BOM if present - this must be done consistently with linesToSkipBeforeHeader
|
|
r = stripBOM(r)
|
|
|
|
// Read all content into memory so we can work with it
|
|
// This is acceptable since CSV imports are typically not huge files
|
|
allBytes, err := io.ReadAll(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Skip the metadata lines before the CSV header by finding newlines
|
|
// linesToSkipBeforeHeader counts raw text lines (newlines), not CSV records,
|
|
// because even metadata can have multiline quoted fields.
|
|
// We manually search for newlines (no buffer size limits like bufio.Scanner)
|
|
bytesSkipped := 0
|
|
linesFound := 0
|
|
for i := 0; i < len(allBytes) && linesFound < linesToSkip; i++ {
|
|
if allBytes[i] == '\n' {
|
|
linesFound++
|
|
if linesFound == linesToSkip {
|
|
// Position is right after the Nth newline
|
|
bytesSkipped = i + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if linesFound < linesToSkip {
|
|
return nil, io.ErrUnexpectedEOF
|
|
}
|
|
|
|
// Now create a CSV reader starting from after the skipped lines
|
|
// The CSV reader will properly handle any multiline quoted fields in the actual data
|
|
remainingContent := allBytes[bytesSkipped:]
|
|
reader := csv.NewReader(bytes.NewReader(remainingContent))
|
|
|
|
// Allow variable field counts and be lenient with parsing
|
|
reader.FieldsPerRecord = -1
|
|
reader.LazyQuotes = true
|
|
reader.TrimLeadingSpace = true
|
|
return gocsv.NewSimpleDecoderFromCSVReader(reader), nil
|
|
}
|
|
|
|
func linesToSkipBeforeHeader(file io.ReaderAt, size int64) (int, error) {
|
|
sr := io.NewSectionReader(file, 0, size)
|
|
// Strip BOM before scanning for header
|
|
r := stripBOM(sr)
|
|
scanner := bufio.NewScanner(r)
|
|
lines := 0
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if strings.Contains(line, "Folder Name") &&
|
|
strings.Contains(line, "List Name") &&
|
|
strings.Contains(line, "Title") {
|
|
break
|
|
}
|
|
lines++
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
return lines, nil
|
|
}
|
|
|
|
// Migrate takes a ticktick export, parses it and imports everything in it into Vikunja.
|
|
// @Summary Import all projects, tasks etc. from a TickTick backup export
|
|
// @Description Imports all projects, tasks, notes, reminders, subtasks and files from a TickTick backup export into Vikunja.
|
|
// @tags migration
|
|
// @Accept x-www-form-urlencoded
|
|
// @Produce json
|
|
// @Security JWTKeyAuth
|
|
// @Param import formData string true "The TickTick backup csv file."
|
|
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
|
// @Failure 500 {object} models.Message "Internal server error"
|
|
// @Router /migration/ticktick/migrate [put]
|
|
func (m *Migrator) Migrate(user *user.User, file io.ReaderAt, size int64) error {
|
|
// Check if file is empty
|
|
if size == 0 {
|
|
return &migration.ErrFileIsEmpty{}
|
|
}
|
|
|
|
fr := io.NewSectionReader(file, 0, size)
|
|
|
|
// Check if the file is a valid CSV
|
|
buf := make([]byte, 1024)
|
|
n, err := fr.Read(buf)
|
|
if errors.Is(err, io.EOF) || n == 0 {
|
|
return &migration.ErrFileIsEmpty{}
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Reset the reader position to start
|
|
_, err = fr.Seek(0, io.SeekStart)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Check if the content looks like a CSV file
|
|
content := string(buf[:n])
|
|
if !isValidCSV(content) {
|
|
return &migration.ErrNotACSVFile{}
|
|
}
|
|
|
|
allTasks := []*tickTickTask{}
|
|
skip, err := linesToSkipBeforeHeader(file, size)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
decode, err := newLineSkipDecoder(fr, skip)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = gocsv.UnmarshalDecoder(decode, &allTasks)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Also check if no tasks were found after decoding
|
|
if len(allTasks) == 0 {
|
|
return &migration.ErrFileIsEmpty{}
|
|
}
|
|
|
|
for _, task := range allTasks {
|
|
if task.IsChecklistString == "Y" {
|
|
task.IsChecklist = true
|
|
}
|
|
|
|
reminder := utils.ParseISO8601Duration(task.ReminderDuration)
|
|
if reminder > 0 {
|
|
task.Reminder = reminder
|
|
}
|
|
|
|
task.Tags = strings.Split(task.TagsList, ", ")
|
|
}
|
|
|
|
vikunjaTasks := convertTickTickToVikunja(allTasks)
|
|
|
|
return migration.InsertFromStructure(vikunjaTasks, user)
|
|
}
|
|
|
|
// isValidCSV performs a basic check to determine if the content looks like a CSV file
|
|
func isValidCSV(content string) bool {
|
|
// Check for common CSV headers from TickTick export
|
|
if !strings.Contains(content, "Folder Name") ||
|
|
!strings.Contains(content, "List Name") ||
|
|
!strings.Contains(content, "Title") {
|
|
return false
|
|
}
|
|
|
|
// Check if the file has commas as separators and multiple lines
|
|
hasCommas := strings.Contains(content, ",")
|
|
hasNewlines := strings.Contains(content, "\n")
|
|
|
|
return hasCommas && hasNewlines
|
|
}
|