From e085fcaef2738b96ea54342451f7627c8a27decd Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 9 Jan 2026 22:50:27 +0100 Subject: [PATCH] feat(migration/todoist): migrate from Sync API v9 to API v1 (#2072) Migrates the Todoist migration module from the deprecated Sync API v9 to the new unified Todoist API v1. --- pkg/modules/migration/helpers.go | 107 ++++++++++++++--------- pkg/modules/migration/todoist/todoist.go | 101 ++++++++++++++------- 2 files changed, 134 insertions(+), 74 deletions(-) diff --git a/pkg/modules/migration/helpers.go b/pkg/modules/migration/helpers.go index 835d14b60..ba180bf52 100644 --- a/pkg/modules/migration/helpers.go +++ b/pkg/modules/migration/helpers.go @@ -19,15 +19,14 @@ package migration import ( "bytes" "context" - "crypto/rand" "fmt" "io" - "math" - "math/big" "net/http" "net/url" "strings" - "time" + + "code.vikunja.io/api/pkg/log" + "code.vikunja.io/api/pkg/utils" ) // DownloadFile downloads a file and returns its contents @@ -65,17 +64,56 @@ func DoPost(url string, form url.Values) (resp *http.Response, err error) { return DoPostWithHeaders(url, form, map[string]string{}) } -// DoPostWithHeaders does an api request and allows to pass in arbitrary headers -func DoPostWithHeaders(url string, form url.Values, headers map[string]string) (resp *http.Response, err error) { - const maxRetries = 3 - const baseDelay = 100 * time.Millisecond - +// DoGetWithHeaders makes an HTTP GET request with custom headers +func DoGetWithHeaders(urlStr string, headers map[string]string) (resp *http.Response, err error) { hc := http.Client{} - for attempt := 0; attempt < maxRetries; attempt++ { - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, strings.NewReader(form.Encode())) + err = utils.RetryWithBackoff("HTTP GET "+urlStr, func() error { + req, reqErr := http.NewRequestWithContext(context.Background(), http.MethodGet, urlStr, nil) + if reqErr != nil { + return reqErr + } + + for key, value := range headers { + req.Header.Add(key, value) + } + + resp, err = hc.Do(req) //nolint:bodyclose // Caller is responsible for closing on success if err != nil { - return nil, err + return err + } + + // Log 4xx errors for debugging + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + bodyBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + // Re-create the body so the caller can still read it + resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + log.Debugf("[Migration] HTTP GET %s returned %d: %s", urlStr, resp.StatusCode, string(bodyBytes)) + return nil // Don't retry on 4xx + } + + // Retry on 5xx status codes, include response body in error + if resp.StatusCode >= 500 { + bodyBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return fmt.Errorf("server returned status %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil + }) + + return resp, err +} + +// DoPostWithHeaders does an api request and allows to pass in arbitrary headers +func DoPostWithHeaders(urlStr string, form url.Values, headers map[string]string) (resp *http.Response, err error) { + hc := http.Client{} + + err = utils.RetryWithBackoff("HTTP POST "+urlStr, func() error { + req, reqErr := http.NewRequestWithContext(context.Background(), http.MethodPost, urlStr, strings.NewReader(form.Encode())) + if reqErr != nil { + return reqErr } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") @@ -84,41 +122,30 @@ func DoPostWithHeaders(url string, form url.Values, headers map[string]string) ( req.Header.Add(key, value) } - resp, err = hc.Do(req) + resp, err = hc.Do(req) //nolint:bodyclose // Caller is responsible for closing on success if err != nil { - return nil, err + return err } - // Don't retry on non-5xx status codes - if resp.StatusCode < 500 { - return resp, nil - } - - // Return error on last attempt if still getting 5xx - if attempt == maxRetries-1 { - bodyBytes, readErr := io.ReadAll(resp.Body) + // Log 4xx errors for debugging + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + bodyBytes, _ := io.ReadAll(resp.Body) resp.Body.Close() - - // Re-create the body so the caller can still read it if needed + // Re-create the body so the caller can still read it resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - - if readErr != nil { - return resp, fmt.Errorf("request failed after %d attempts with status code %d (could not read response body: %w)", maxRetries, resp.StatusCode, readErr) - } - - return resp, fmt.Errorf("request failed after %d attempts with status code %d: %s", maxRetries, resp.StatusCode, string(bodyBytes)) + log.Debugf("[Migration] HTTP POST %s returned %d: %s", urlStr, resp.StatusCode, string(bodyBytes)) + return nil // Don't retry on 4xx } - // Close the body before retrying - resp.Body.Close() + // Retry on 5xx status codes, include response body in error + if resp.StatusCode >= 500 { + bodyBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return fmt.Errorf("server returned status %d: %s", resp.StatusCode, string(bodyBytes)) + } - // Exponential backoff with jitter - delay := baseDelay * time.Duration(math.Pow(2, float64(attempt))) - maxJitter := int64(delay / 2) - jitterBig, _ := rand.Int(rand.Reader, big.NewInt(maxJitter)) - jitter := time.Duration(jitterBig.Int64()) - time.Sleep(delay + jitter) - } + return nil + }) - return nil, fmt.Errorf("request failed after %d attempts", maxRetries) + return resp, err } diff --git a/pkg/modules/migration/todoist/todoist.go b/pkg/modules/migration/todoist/todoist.go index 63564446d..e7a9098b5 100644 --- a/pkg/modules/migration/todoist/todoist.go +++ b/pkg/modules/migration/todoist/todoist.go @@ -97,10 +97,6 @@ type item struct { DateCompleted time.Time `json:"completed_at"` } -type itemWrapper struct { - Item *item `json:"item"` -} - type doneItem struct { CompletedDate time.Time `json:"completed_at"` Content string `json:"content"` @@ -109,9 +105,17 @@ type doneItem struct { TaskID string `json:"task_id"` } -type doneItemSync struct { - Items []*doneItem `json:"items"` - Projects map[string]*project `json:"projects"` +// paginatedCompletedTasks is the response structure for the v1 API completed tasks endpoint +type paginatedCompletedTasks struct { + Items []*doneItem `json:"items"` + Projects map[string]*project `json:"projects"` + NextCursor string `json:"next_cursor"` +} + +// paginatedProjects is the response structure for the v1 API paginated projects endpoints +type paginatedProjects struct { + Results []*project `json:"results"` + NextCursor string `json:"next_cursor"` } type fileAttachment struct { @@ -546,7 +550,7 @@ func (m *Migration) Migrate(u *user.User) (err error) { "Authorization": "Bearer " + token, } - resp, err := migration.DoPostWithHeaders("https://api.todoist.com/sync/v9/sync", form, bearerHeader) + resp, err := migration.DoPostWithHeaders("https://api.todoist.com/api/v1/sync", form, bearerHeader) if err != nil { return } @@ -560,19 +564,25 @@ func (m *Migration) Migrate(u *user.User) (err error) { log.Debugf("[Todoist Migration] Getting done items for user %d", u.ID) - // Get all done tasks and projects - offset := 0 + // Get all done tasks and projects using cursor-based pagination + var cursor string doneItems := make(map[string]*doneItem) + iteration := 0 for { - resp, err = migration.DoPostWithHeaders("https://api.todoist.com/sync/v9/completed/get_all?limit="+strconv.Itoa(paginationLimit)+"&offset="+strconv.Itoa(offset*paginationLimit), form, bearerHeader) + completedURL := "https://api.todoist.com/api/v1/tasks/completed?limit=" + strconv.Itoa(paginationLimit) + if cursor != "" { + completedURL += "&cursor=" + url.QueryEscape(cursor) + } + + resp, err = migration.DoGetWithHeaders(completedURL, bearerHeader) if err != nil { return } - defer resp.Body.Close() - completedSyncResponse := &doneItemSync{} + completedSyncResponse := &paginatedCompletedTasks{} err = json.NewDecoder(resp.Body).Decode(completedSyncResponse) + resp.Body.Close() if err != nil { return } @@ -596,55 +606,78 @@ func (m *Migration) Migrate(u *user.User) (err error) { } doneItems[i.TaskID] = i - // need to get done item data - resp, err = migration.DoPostWithHeaders("https://api.todoist.com/sync/v9/items/get", url.Values{ - "item_id": []string{i.TaskID}, - }, bearerHeader) + // need to get done item data using v1 API + resp, err = migration.DoGetWithHeaders("https://api.todoist.com/api/v1/tasks/"+i.TaskID, bearerHeader) if err != nil { return } - defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { // Done items of deleted projects may show up here but since the project is already deleted // we can't show them individually and the api returns a 404. buf := bytes.Buffer{} _, _ = buf.ReadFrom(resp.Body) + resp.Body.Close() log.Debugf("[Todoist Migration] Could not retrieve task details for task %s: %s", i.TaskID, buf.String()) continue } - doneI := &itemWrapper{} + // The v1 API returns the task directly, not wrapped + doneI := &item{} err = json.NewDecoder(resp.Body).Decode(doneI) + resp.Body.Close() if err != nil { return } log.Debugf("[Todoist Migration] Retrieved full task data for done task %s", i.TaskID) - syncResponse.Items = append(syncResponse.Items, doneI.Item) + syncResponse.Items = append(syncResponse.Items, doneI) } - if len(completedSyncResponse.Items) < paginationLimit { + // Check if there are more pages + if completedSyncResponse.NextCursor == "" { break } - offset++ - log.Debugf("[Todoist Migration] User %d has more than 200 done tasks or projects, looping to get more; iteration %d", u.ID, offset) + cursor = completedSyncResponse.NextCursor + iteration++ + log.Debugf("[Todoist Migration] User %d has more than %d done tasks or projects, looping to get more; iteration %d", u.ID, paginationLimit, iteration) } log.Debugf("[Todoist Migration] Got %d done items for user %d", len(doneItems), u.ID) log.Debugf("[Todoist Migration] Getting archived projects for user %d", u.ID) - // Get all archived projects - resp, err = migration.DoPostWithHeaders("https://api.todoist.com/sync/v9/projects/get_archived", form, bearerHeader) - if err != nil { - return - } - defer resp.Body.Close() - + // Get all archived projects using cursor-based pagination archivedProjects := []*project{} - err = json.NewDecoder(resp.Body).Decode(&archivedProjects) - if err != nil { - return + cursor = "" // reuse cursor variable + iteration = 0 + + for { + archivedURL := "https://api.todoist.com/api/v1/projects/archived" + if cursor != "" { + archivedURL += "?cursor=" + url.QueryEscape(cursor) + } + + resp, err = migration.DoGetWithHeaders(archivedURL, bearerHeader) + if err != nil { + return + } + + archivedResponse := &paginatedProjects{} + err = json.NewDecoder(resp.Body).Decode(archivedResponse) + resp.Body.Close() + if err != nil { + return + } + + archivedProjects = append(archivedProjects, archivedResponse.Results...) + + if archivedResponse.NextCursor == "" { + break + } + cursor = archivedResponse.NextCursor + iteration++ + log.Debugf("[Todoist Migration] User %d has more archived projects, fetching more; iteration %d", u.ID, iteration) } + syncResponse.Projects = append(syncResponse.Projects, archivedProjects...) log.Debugf("[Todoist Migration] Got %d archived projects for user %d", len(archivedProjects), u.ID) @@ -652,7 +685,7 @@ func (m *Migration) Migrate(u *user.User) (err error) { // Project data is not included in the regular sync for archived projects, so we need to get all of those by hand for _, p := range archivedProjects { - resp, err = migration.DoPostWithHeaders("https://api.todoist.com/sync/v9/projects/get_data?project_id="+p.ID, form, bearerHeader) + resp, err = migration.DoGetWithHeaders("https://api.todoist.com/api/v1/projects/"+p.ID+"/full", bearerHeader) if err != nil { return }