feat(migration): add generic CSV import with column mapping
Add a new CSV migration module that allows users to import tasks from any CSV file with custom column mapping and parsing options. Backend changes: - New CSV migrator module with detection, preview, and import endpoints - Auto-detection of delimiter, quote character, and date format - Suggested column mappings based on column name patterns - Transactional import using InsertFromStructure Frontend changes: - New CSV migration UI with two-step flow (upload -> mapping -> import) - Column mapping selectors for all task attributes - Live preview showing first 5 tasks with current mapping - Parsing option controls for delimiter and date format The CSV migrator creates a parent "Imported from CSV" project with child projects based on the project column if provided, or a default "Tasks" project for tasks without a specified project.
This commit is contained in:
Claude
kolaente
parent
b20df2ef63
commit
07c68e04ef
|
|
@ -641,7 +641,42 @@
|
|||
"importUpload": "To import data from {name} into Vikunja, click the button below to select a file.",
|
||||
"upload": "Upload file",
|
||||
"migrationStartedWillReciveEmail": "Vikunja will now import your lists/projects, tasks, notes, reminders and files from {service}. As this will take a while, we will send you an email once done. You can close this window now.",
|
||||
"migrationInProgress": "A migration is currently in progress. Please wait until it is done."
|
||||
"migrationInProgress": "A migration is currently in progress. Please wait until it is done.",
|
||||
"csv": {
|
||||
"description": "Import tasks from a CSV file with custom column mapping.",
|
||||
"uploadDescription": "Select a CSV file to import. The file should contain task data with headers in the first row.",
|
||||
"selectFile": "Select CSV file",
|
||||
"columnMapping": "Column Mapping",
|
||||
"columnMappingDescription": "Map each column in your CSV file to a task attribute. Vikunja has auto-detected the most likely mappings.",
|
||||
"parsingOptions": "Parsing Options",
|
||||
"delimiter": "Delimiter",
|
||||
"dateFormat": "Date Format",
|
||||
"mapColumns": "Map Columns",
|
||||
"example": "e.g.",
|
||||
"preview": "Preview",
|
||||
"previewDescription": "Showing first 5 of {count} tasks that will be imported.",
|
||||
"previewErrors": "{count} rows had parsing errors and will be skipped.",
|
||||
"import": "Import Tasks",
|
||||
"untitled": "Untitled Task",
|
||||
"completed": "Completed",
|
||||
"dueDate": "Due",
|
||||
"priority": "Priority",
|
||||
"project": "Project",
|
||||
"labels": "Labels",
|
||||
"attributes": {
|
||||
"title": "Title",
|
||||
"description": "Description",
|
||||
"due_date": "Due Date",
|
||||
"start_date": "Start Date",
|
||||
"end_date": "End Date",
|
||||
"done": "Done/Completed",
|
||||
"priority": "Priority",
|
||||
"labels": "Labels/Tags",
|
||||
"project": "Project",
|
||||
"reminder": "Reminder",
|
||||
"ignore": "Ignore"
|
||||
}
|
||||
}
|
||||
},
|
||||
"label": {
|
||||
"title": "Labels",
|
||||
|
|
|
|||
|
|
@ -154,6 +154,11 @@ const router = createRouter({
|
|||
name: 'migrate.start',
|
||||
component: () => import('@/views/migrate/Migration.vue'),
|
||||
},
|
||||
{
|
||||
path: '/migrate/csv',
|
||||
name: 'migrate.csv',
|
||||
component: () => import('@/views/migrate/MigrationCSV.vue'),
|
||||
},
|
||||
{
|
||||
path: '/migrate/:service',
|
||||
name: 'migrate.service',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
import AbstractService from '../abstractService'
|
||||
|
||||
export interface ColumnMapping {
|
||||
column_index: number
|
||||
column_name: string
|
||||
attribute: TaskAttribute
|
||||
}
|
||||
|
||||
export type TaskAttribute =
|
||||
| 'title'
|
||||
| 'description'
|
||||
| 'due_date'
|
||||
| 'start_date'
|
||||
| 'end_date'
|
||||
| 'done'
|
||||
| 'priority'
|
||||
| 'labels'
|
||||
| 'project'
|
||||
| 'reminder'
|
||||
| 'ignore'
|
||||
|
||||
export const TASK_ATTRIBUTES: { value: TaskAttribute; label: string }[] = [
|
||||
{ value: 'title', label: 'Title' },
|
||||
{ value: 'description', label: 'Description' },
|
||||
{ value: 'due_date', label: 'Due Date' },
|
||||
{ value: 'start_date', label: 'Start Date' },
|
||||
{ value: 'end_date', label: 'End Date' },
|
||||
{ value: 'done', label: 'Done/Completed' },
|
||||
{ value: 'priority', label: 'Priority' },
|
||||
{ value: 'labels', label: 'Labels/Tags' },
|
||||
{ value: 'project', label: 'Project' },
|
||||
{ value: 'reminder', label: 'Reminder' },
|
||||
{ value: 'ignore', label: 'Ignore' },
|
||||
]
|
||||
|
||||
export interface DetectionResult {
|
||||
columns: string[]
|
||||
delimiter: string
|
||||
quote_char: string
|
||||
date_format: string
|
||||
suggested_mapping: ColumnMapping[]
|
||||
preview_rows: string[][]
|
||||
}
|
||||
|
||||
export interface ImportConfig {
|
||||
delimiter: string
|
||||
quote_char: string
|
||||
date_format: string
|
||||
mapping: ColumnMapping[]
|
||||
}
|
||||
|
||||
export interface PreviewTask {
|
||||
title: string
|
||||
description: string
|
||||
due_date?: string
|
||||
start_date?: string
|
||||
end_date?: string
|
||||
done: boolean
|
||||
priority: number
|
||||
labels?: string[]
|
||||
project?: string
|
||||
}
|
||||
|
||||
export interface PreviewResult {
|
||||
tasks: PreviewTask[]
|
||||
total_rows: number
|
||||
error_count: number
|
||||
errors?: string[]
|
||||
}
|
||||
|
||||
export interface MigrationStatus {
|
||||
started_at: string | null
|
||||
finished_at: string | null
|
||||
}
|
||||
|
||||
export const SUPPORTED_DELIMITERS = [
|
||||
{ value: ',', label: 'Comma (,)' },
|
||||
{ value: ';', label: 'Semicolon (;)' },
|
||||
{ value: '\t', label: 'Tab' },
|
||||
{ value: '|', label: 'Pipe (|)' },
|
||||
]
|
||||
|
||||
export const SUPPORTED_DATE_FORMATS = [
|
||||
{ value: '2006-01-02', label: 'YYYY-MM-DD (2024-01-15)' },
|
||||
{ value: '2006-01-02T15:04:05', label: 'ISO DateTime (2024-01-15T10:30:00)' },
|
||||
{ value: '02/01/2006', label: 'DD/MM/YYYY (15/01/2024)' },
|
||||
{ value: '01/02/2006', label: 'MM/DD/YYYY (01/15/2024)' },
|
||||
{ value: '02-01-2006', label: 'DD-MM-YYYY (15-01-2024)' },
|
||||
{ value: '01-02-2006', label: 'MM-DD-YYYY (01-15-2024)' },
|
||||
{ value: '02.01.2006', label: 'DD.MM.YYYY (15.01.2024)' },
|
||||
{ value: '2006/01/02', label: 'YYYY/MM/DD (2024/01/15)' },
|
||||
{ value: '2006-01-02 15:04:05', label: 'DateTime (2024-01-15 10:30:00)' },
|
||||
]
|
||||
|
||||
export default class CSVMigrationService extends AbstractService {
|
||||
constructor() {
|
||||
super({})
|
||||
}
|
||||
|
||||
getStatus(): Promise<MigrationStatus> {
|
||||
return this.getM('/migration/csv/status')
|
||||
}
|
||||
|
||||
useCreateInterceptor() {
|
||||
return false
|
||||
}
|
||||
|
||||
async detect(file: File): Promise<DetectionResult> {
|
||||
return this.uploadFile(
|
||||
'/migration/csv/detect',
|
||||
file,
|
||||
'import',
|
||||
)
|
||||
}
|
||||
|
||||
async preview(file: File, config: ImportConfig): Promise<PreviewResult> {
|
||||
const data = new FormData()
|
||||
data.append('import', file)
|
||||
data.append('config', JSON.stringify(config))
|
||||
return this.uploadFormData('/migration/csv/preview', data)
|
||||
}
|
||||
|
||||
async migrate(file: File, config: ImportConfig): Promise<{ message: string }> {
|
||||
const data = new FormData()
|
||||
data.append('import', file)
|
||||
data.append('config', JSON.stringify(config))
|
||||
return this.uploadFormData('/migration/csv/migrate', data)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,489 @@
|
|||
<template>
|
||||
<div class="content csv-migration">
|
||||
<h1>{{ $t('migrate.titleService', {name: 'CSV'}) }}</h1>
|
||||
<p>{{ $t('migrate.csv.description') }}</p>
|
||||
|
||||
<!-- Step 1: File Upload -->
|
||||
<div
|
||||
v-if="step === 'upload'"
|
||||
class="upload-step"
|
||||
>
|
||||
<Message
|
||||
v-if="error"
|
||||
variant="danger"
|
||||
class="mbe-4"
|
||||
>
|
||||
{{ error }}
|
||||
</Message>
|
||||
<p>{{ $t('migrate.csv.uploadDescription') }}</p>
|
||||
<input
|
||||
ref="uploadInput"
|
||||
class="is-hidden"
|
||||
type="file"
|
||||
accept=".csv,.txt"
|
||||
@change="handleFileUpload"
|
||||
>
|
||||
<XButton
|
||||
:loading="isLoading"
|
||||
:disabled="isLoading || undefined"
|
||||
@click="uploadInput?.click()"
|
||||
>
|
||||
{{ $t('migrate.csv.selectFile') }}
|
||||
</XButton>
|
||||
</div>
|
||||
|
||||
<!-- Step 2: Column Mapping -->
|
||||
<div
|
||||
v-else-if="step === 'mapping'"
|
||||
class="mapping-step"
|
||||
>
|
||||
<div class="mapping-header">
|
||||
<h2>{{ $t('migrate.csv.columnMapping') }}</h2>
|
||||
<p>{{ $t('migrate.csv.columnMappingDescription') }}</p>
|
||||
</div>
|
||||
|
||||
<!-- Parsing Options -->
|
||||
<div class="parsing-options card">
|
||||
<h3>{{ $t('migrate.csv.parsingOptions') }}</h3>
|
||||
<div class="options-grid">
|
||||
<div class="option-group">
|
||||
<label for="delimiter">{{ $t('migrate.csv.delimiter') }}</label>
|
||||
<select
|
||||
id="delimiter"
|
||||
v-model="config.delimiter"
|
||||
@change="updatePreview"
|
||||
>
|
||||
<option
|
||||
v-for="delim in SUPPORTED_DELIMITERS"
|
||||
:key="delim.value"
|
||||
:value="delim.value"
|
||||
>
|
||||
{{ delim.label }}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="option-group">
|
||||
<label for="dateFormat">{{ $t('migrate.csv.dateFormat') }}</label>
|
||||
<select
|
||||
id="dateFormat"
|
||||
v-model="config.date_format"
|
||||
@change="updatePreview"
|
||||
>
|
||||
<option
|
||||
v-for="format in SUPPORTED_DATE_FORMATS"
|
||||
:key="format.value"
|
||||
:value="format.value"
|
||||
>
|
||||
{{ format.label }}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Column Mappings -->
|
||||
<div class="column-mappings card">
|
||||
<h3>{{ $t('migrate.csv.mapColumns') }}</h3>
|
||||
<div class="mappings-grid">
|
||||
<div
|
||||
v-for="(mapping, index) in config.mapping"
|
||||
:key="index"
|
||||
class="mapping-row"
|
||||
>
|
||||
<div class="column-name">
|
||||
<strong>{{ mapping.column_name }}</strong>
|
||||
<span
|
||||
v-if="detectionResult && detectionResult.preview_rows[0]"
|
||||
class="preview-value"
|
||||
>
|
||||
{{ $t('migrate.csv.example') }}: {{ detectionResult.preview_rows[0][index] || '-' }}
|
||||
</span>
|
||||
</div>
|
||||
<select
|
||||
v-model="mapping.attribute"
|
||||
@change="updatePreview"
|
||||
>
|
||||
<option
|
||||
v-for="attr in TASK_ATTRIBUTES"
|
||||
:key="attr.value"
|
||||
:value="attr.value"
|
||||
>
|
||||
{{ $t('migrate.csv.attributes.' + attr.value) }}
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Preview -->
|
||||
<div
|
||||
v-if="previewResult"
|
||||
class="preview-section card"
|
||||
>
|
||||
<h3>{{ $t('migrate.csv.preview') }}</h3>
|
||||
<p>{{ $t('migrate.csv.previewDescription', {count: previewResult.total_rows}) }}</p>
|
||||
|
||||
<div
|
||||
v-if="previewResult.errors && previewResult.errors.length > 0"
|
||||
class="preview-errors"
|
||||
>
|
||||
<Message variant="warning">
|
||||
{{ $t('migrate.csv.previewErrors', {count: previewResult.error_count}) }}
|
||||
</Message>
|
||||
</div>
|
||||
|
||||
<div class="preview-tasks">
|
||||
<div
|
||||
v-for="(task, index) in previewResult.tasks"
|
||||
:key="index"
|
||||
class="preview-task card"
|
||||
>
|
||||
<div class="task-title">
|
||||
<strong>{{ task.title || $t('migrate.csv.untitled') }}</strong>
|
||||
<span
|
||||
v-if="task.done"
|
||||
class="done-badge"
|
||||
>{{ $t('migrate.csv.completed') }}</span>
|
||||
</div>
|
||||
<div
|
||||
v-if="task.description"
|
||||
class="task-description"
|
||||
>
|
||||
{{ truncate(task.description, 100) }}
|
||||
</div>
|
||||
<div class="task-meta">
|
||||
<span v-if="task.due_date">
|
||||
{{ $t('migrate.csv.dueDate') }}: {{ task.due_date }}
|
||||
</span>
|
||||
<span v-if="task.priority > 0">
|
||||
{{ $t('migrate.csv.priority') }}: {{ task.priority }}
|
||||
</span>
|
||||
<span v-if="task.project">
|
||||
{{ $t('migrate.csv.project') }}: {{ task.project }}
|
||||
</span>
|
||||
<span v-if="task.labels && task.labels.length > 0">
|
||||
{{ $t('migrate.csv.labels') }}: {{ task.labels.join(', ') }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="actions">
|
||||
<XButton
|
||||
variant="tertiary"
|
||||
@click="resetToUpload"
|
||||
>
|
||||
{{ $t('misc.cancel') }}
|
||||
</XButton>
|
||||
<XButton
|
||||
:loading="isLoading"
|
||||
:disabled="!hasValidMapping || isLoading"
|
||||
@click="performImport"
|
||||
>
|
||||
{{ $t('migrate.csv.import') }}
|
||||
</XButton>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Step 3: Success -->
|
||||
<div
|
||||
v-else-if="step === 'success'"
|
||||
class="success-step"
|
||||
>
|
||||
<Message class="mbe-4">
|
||||
{{ successMessage }}
|
||||
</Message>
|
||||
<XButton :to="{name: 'home'}">
|
||||
{{ $t('home.goToOverview') }}
|
||||
</XButton>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import {computed, ref, shallowReactive} from 'vue'
|
||||
import {useI18n} from 'vue-i18n'
|
||||
|
||||
import Message from '@/components/misc/Message.vue'
|
||||
|
||||
import CSVMigrationService, {
|
||||
type DetectionResult,
|
||||
type ImportConfig,
|
||||
type PreviewResult,
|
||||
TASK_ATTRIBUTES,
|
||||
SUPPORTED_DELIMITERS,
|
||||
SUPPORTED_DATE_FORMATS,
|
||||
} from '@/services/migrator/csvMigration'
|
||||
|
||||
import {useTitle} from '@/composables/useTitle'
|
||||
import {useProjectStore} from '@/stores/projects'
|
||||
import {getErrorText} from '@/message'
|
||||
|
||||
type Step = 'upload' | 'mapping' | 'success'
|
||||
|
||||
const {t} = useI18n({useScope: 'global'})
|
||||
|
||||
useTitle(() => t('migrate.titleService', {name: 'CSV'}))
|
||||
|
||||
const csvService = shallowReactive(new CSVMigrationService())
|
||||
|
||||
const step = ref<Step>('upload')
|
||||
const error = ref('')
|
||||
const successMessage = ref('')
|
||||
const isLoading = ref(false)
|
||||
const uploadInput = ref<HTMLInputElement | null>(null)
|
||||
const selectedFile = ref<File | null>(null)
|
||||
const detectionResult = ref<DetectionResult | null>(null)
|
||||
const previewResult = ref<PreviewResult | null>(null)
|
||||
|
||||
const config = ref<ImportConfig>({
|
||||
delimiter: ',',
|
||||
quote_char: '"',
|
||||
date_format: '2006-01-02',
|
||||
mapping: [],
|
||||
})
|
||||
|
||||
const hasValidMapping = computed(() => {
|
||||
if (!config.value.mapping.length) return false
|
||||
// At least one column should be mapped to title
|
||||
return config.value.mapping.some(m => m.attribute === 'title')
|
||||
})
|
||||
|
||||
function truncate(text: string, length: number): string {
|
||||
if (text.length <= length) return text
|
||||
return text.substring(0, length) + '...'
|
||||
}
|
||||
|
||||
async function handleFileUpload() {
|
||||
const files = uploadInput.value?.files
|
||||
if (!files || files.length === 0) return
|
||||
|
||||
selectedFile.value = files[0]
|
||||
error.value = ''
|
||||
isLoading.value = true
|
||||
|
||||
try {
|
||||
const result = await csvService.detect(selectedFile.value)
|
||||
detectionResult.value = result
|
||||
|
||||
// Apply detected values
|
||||
config.value = {
|
||||
delimiter: result.delimiter,
|
||||
quote_char: result.quote_char,
|
||||
date_format: result.date_format,
|
||||
mapping: result.suggested_mapping,
|
||||
}
|
||||
|
||||
// Get initial preview
|
||||
await updatePreview()
|
||||
|
||||
step.value = 'mapping'
|
||||
} catch (e) {
|
||||
error.value = getErrorText(e)
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function updatePreview() {
|
||||
if (!selectedFile.value) return
|
||||
|
||||
isLoading.value = true
|
||||
try {
|
||||
previewResult.value = await csvService.preview(selectedFile.value, config.value)
|
||||
} catch (e) {
|
||||
error.value = getErrorText(e)
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function performImport() {
|
||||
if (!selectedFile.value || !hasValidMapping.value) return
|
||||
|
||||
isLoading.value = true
|
||||
error.value = ''
|
||||
|
||||
try {
|
||||
const result = await csvService.migrate(selectedFile.value, config.value)
|
||||
successMessage.value = result.message
|
||||
|
||||
// Reload projects
|
||||
const projectStore = useProjectStore()
|
||||
await projectStore.loadAllProjects()
|
||||
|
||||
step.value = 'success'
|
||||
} catch (e) {
|
||||
error.value = getErrorText(e)
|
||||
} finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function resetToUpload() {
|
||||
step.value = 'upload'
|
||||
selectedFile.value = null
|
||||
detectionResult.value = null
|
||||
previewResult.value = null
|
||||
error.value = ''
|
||||
config.value = {
|
||||
delimiter: ',',
|
||||
quote_char: '"',
|
||||
date_format: '2006-01-02',
|
||||
mapping: [],
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style lang="scss" scoped>
|
||||
.csv-migration {
|
||||
max-inline-size: 900px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.card {
|
||||
background: var(--white);
|
||||
border-radius: var(--border-radius);
|
||||
padding: 1.5rem;
|
||||
margin-block-end: 1.5rem;
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.mapping-header {
|
||||
margin-block-end: 1.5rem;
|
||||
|
||||
h2 {
|
||||
margin-block-end: 0.5rem;
|
||||
}
|
||||
}
|
||||
|
||||
.parsing-options {
|
||||
h3 {
|
||||
margin-block-end: 1rem;
|
||||
}
|
||||
}
|
||||
|
||||
.options-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.option-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
|
||||
label {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
select {
|
||||
padding: 0.5rem;
|
||||
border: 1px solid var(--grey-300);
|
||||
border-radius: var(--border-radius);
|
||||
background: var(--white);
|
||||
}
|
||||
}
|
||||
|
||||
.column-mappings {
|
||||
h3 {
|
||||
margin-block-end: 1rem;
|
||||
}
|
||||
}
|
||||
|
||||
.mappings-grid {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.mapping-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
padding: 0.75rem;
|
||||
background: var(--grey-100);
|
||||
border-radius: var(--border-radius);
|
||||
|
||||
@media (width <= 600px) {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
.column-name {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.25rem;
|
||||
|
||||
.preview-value {
|
||||
font-size: 0.85rem;
|
||||
color: var(--grey-500);
|
||||
}
|
||||
}
|
||||
|
||||
.preview-section {
|
||||
h3 {
|
||||
margin-block-end: 0.5rem;
|
||||
}
|
||||
}
|
||||
|
||||
.preview-errors {
|
||||
margin-block: 1rem;
|
||||
}
|
||||
|
||||
.preview-tasks {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
margin-block-start: 1rem;
|
||||
}
|
||||
|
||||
.preview-task {
|
||||
padding: 1rem;
|
||||
background: var(--grey-50);
|
||||
}
|
||||
|
||||
.task-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-block-end: 0.5rem;
|
||||
}
|
||||
|
||||
.done-badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 0.125rem 0.5rem;
|
||||
background: var(--success);
|
||||
color: var(--white);
|
||||
border-radius: var(--border-radius);
|
||||
}
|
||||
|
||||
.task-description {
|
||||
color: var(--grey-600);
|
||||
margin-block-end: 0.5rem;
|
||||
}
|
||||
|
||||
.task-meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 1rem;
|
||||
font-size: 0.85rem;
|
||||
color: var(--grey-500);
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
justify-content: flex-end;
|
||||
margin-block-start: 1.5rem;
|
||||
}
|
||||
|
||||
.success-step {
|
||||
text-align: center;
|
||||
padding: 2rem;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<svg viewBox="0 0 88 88" xmlns="http://www.w3.org/2000/svg">
|
||||
<g fill="none" fill-rule="evenodd">
|
||||
<!-- Document background -->
|
||||
<rect x="14" y="4" width="60" height="80" rx="4" fill="#4772FA"/>
|
||||
<!-- Folded corner -->
|
||||
<path d="M54 4 L74 24 L54 24 Z" fill="#2a5bd7"/>
|
||||
<!-- CSV text lines representing data rows -->
|
||||
<rect x="22" y="34" width="44" height="6" rx="2" fill="#ffffff"/>
|
||||
<rect x="22" y="46" width="44" height="6" rx="2" fill="#ffffff" opacity="0.8"/>
|
||||
<rect x="22" y="58" width="44" height="6" rx="2" fill="#ffffff" opacity="0.6"/>
|
||||
<rect x="22" y="70" width="30" height="6" rx="2" fill="#ffffff" opacity="0.4"/>
|
||||
<!-- Vertical separators (comma separators) -->
|
||||
<rect x="36" y="34" width="2" height="42" fill="#4772FA" opacity="0.3"/>
|
||||
<rect x="52" y="34" width="2" height="42" fill="#4772FA" opacity="0.3"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 875 B |
|
|
@ -5,11 +5,13 @@ import microsoftTodoIcon from './icons/microsoft-todo.svg?url'
|
|||
import vikunjaFileIcon from './icons/vikunja-file.png?url'
|
||||
import tickTickIcon from './icons/ticktick.svg?url'
|
||||
import wekanIcon from './icons/wekan.png?url'
|
||||
import csvIcon from './icons/csv.svg?url'
|
||||
|
||||
export interface Migrator {
|
||||
id: string
|
||||
name: string
|
||||
isFileMigrator?: boolean
|
||||
isCSVMigrator?: boolean
|
||||
icon: string
|
||||
}
|
||||
|
||||
|
|
@ -56,4 +58,11 @@ export const MIGRATORS = {
|
|||
icon: wekanIcon,
|
||||
isFileMigrator: true,
|
||||
},
|
||||
csv: {
|
||||
id: 'csv',
|
||||
name: 'CSV',
|
||||
icon: csvIcon as string,
|
||||
isFileMigrator: true,
|
||||
isCSVMigrator: true,
|
||||
},
|
||||
} as const satisfies IMigratorRecord
|
||||
|
|
|
|||
|
|
@ -0,0 +1,683 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
"code.vikunja.io/api/pkg/user"
|
||||
)
|
||||
|
||||
// Migrator is the CSV migrator
|
||||
type Migrator struct{}
|
||||
|
||||
// Name returns the name of this migrator
|
||||
func (m *Migrator) Name() string {
|
||||
return "csv"
|
||||
}
|
||||
|
||||
// SupportedDelimiters contains all supported CSV delimiters
|
||||
var SupportedDelimiters = []string{",", ";", "\t", "|"}
|
||||
|
||||
// SupportedQuoteChars contains all supported quote characters
|
||||
var SupportedQuoteChars = []string{"\"", "'"}
|
||||
|
||||
// SupportedDateFormats contains common date formats for parsing
|
||||
var SupportedDateFormats = []string{
|
||||
"2006-01-02", // ISO date
|
||||
"2006-01-02T15:04:05", // ISO datetime
|
||||
"2006-01-02T15:04:05Z07:00", // RFC3339
|
||||
"2006-01-02T15:04:05-0700", // ISO with timezone
|
||||
"02/01/2006", // DD/MM/YYYY
|
||||
"01/02/2006", // MM/DD/YYYY
|
||||
"02-01-2006", // DD-MM-YYYY
|
||||
"01-02-2006", // MM-DD-YYYY
|
||||
"Jan 2, 2006", // Month D, YYYY
|
||||
"2 Jan 2006", // D Month YYYY
|
||||
"02/01/2006 15:04", // DD/MM/YYYY HH:MM
|
||||
"01/02/2006 15:04", // MM/DD/YYYY HH:MM
|
||||
"2006-01-02 15:04:05", // MySQL datetime
|
||||
"2006/01/02", // YYYY/MM/DD
|
||||
"02.01.2006", // DD.MM.YYYY (European)
|
||||
"02.01.2006 15:04", // DD.MM.YYYY HH:MM (European)
|
||||
time.RFC1123, // RFC1123
|
||||
time.RFC1123Z, // RFC1123 with numeric zone
|
||||
time.RFC822, // RFC822
|
||||
time.RFC822Z, // RFC822 with numeric zone
|
||||
time.RFC850, // RFC850
|
||||
time.ANSIC, // ANSIC
|
||||
time.UnixDate, // Unix date
|
||||
}
|
||||
|
||||
// TaskAttribute represents a task attribute that can be mapped from CSV
|
||||
type TaskAttribute string
|
||||
|
||||
const (
|
||||
AttrTitle TaskAttribute = "title"
|
||||
AttrDescription TaskAttribute = "description"
|
||||
AttrDueDate TaskAttribute = "due_date"
|
||||
AttrStartDate TaskAttribute = "start_date"
|
||||
AttrEndDate TaskAttribute = "end_date"
|
||||
AttrDone TaskAttribute = "done"
|
||||
AttrPriority TaskAttribute = "priority"
|
||||
AttrLabels TaskAttribute = "labels"
|
||||
AttrProject TaskAttribute = "project"
|
||||
AttrReminder TaskAttribute = "reminder"
|
||||
AttrIgnore TaskAttribute = "ignore"
|
||||
)
|
||||
|
||||
// AllTaskAttributes returns all available task attributes for mapping
|
||||
var AllTaskAttributes = []TaskAttribute{
|
||||
AttrTitle,
|
||||
AttrDescription,
|
||||
AttrDueDate,
|
||||
AttrStartDate,
|
||||
AttrEndDate,
|
||||
AttrDone,
|
||||
AttrPriority,
|
||||
AttrLabels,
|
||||
AttrProject,
|
||||
AttrReminder,
|
||||
AttrIgnore,
|
||||
}
|
||||
|
||||
// ColumnMapping represents a mapping from a CSV column to a task attribute
|
||||
type ColumnMapping struct {
|
||||
ColumnIndex int `json:"column_index"`
|
||||
ColumnName string `json:"column_name"`
|
||||
Attribute TaskAttribute `json:"attribute"`
|
||||
}
|
||||
|
||||
// DetectionResult contains the auto-detected CSV structure
|
||||
type DetectionResult struct {
|
||||
Columns []string `json:"columns"`
|
||||
Delimiter string `json:"delimiter"`
|
||||
QuoteChar string `json:"quote_char"`
|
||||
DateFormat string `json:"date_format"`
|
||||
SuggestedMapping []ColumnMapping `json:"suggested_mapping"`
|
||||
PreviewRows [][]string `json:"preview_rows"`
|
||||
}
|
||||
|
||||
// ImportConfig contains the configuration for CSV import
|
||||
type ImportConfig struct {
|
||||
Delimiter string `json:"delimiter"`
|
||||
QuoteChar string `json:"quote_char"`
|
||||
DateFormat string `json:"date_format"`
|
||||
Mapping []ColumnMapping `json:"mapping"`
|
||||
}
|
||||
|
||||
// PreviewTask represents a task preview before import
|
||||
type PreviewTask struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
DueDate string `json:"due_date,omitempty"`
|
||||
StartDate string `json:"start_date,omitempty"`
|
||||
EndDate string `json:"end_date,omitempty"`
|
||||
Done bool `json:"done"`
|
||||
Priority int `json:"priority"`
|
||||
Labels []string `json:"labels,omitempty"`
|
||||
Project string `json:"project,omitempty"`
|
||||
}
|
||||
|
||||
// PreviewResult contains preview data before import
|
||||
type PreviewResult struct {
|
||||
Tasks []PreviewTask `json:"tasks"`
|
||||
TotalRows int `json:"total_rows"`
|
||||
ErrorCount int `json:"error_count"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// stripBOM removes the UTF-8 BOM from the beginning of a reader
|
||||
func stripBOM(data []byte) []byte {
|
||||
if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF {
|
||||
return data[3:]
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// detectDelimiter attempts to auto-detect the CSV delimiter
|
||||
func detectDelimiter(data []byte) string {
|
||||
content := string(data)
|
||||
|
||||
// Count occurrences of each delimiter in the first few lines
|
||||
lines := strings.SplitN(content, "\n", 5)
|
||||
if len(lines) < 2 {
|
||||
return "," // Default to comma
|
||||
}
|
||||
|
||||
delimiterCounts := make(map[string]int)
|
||||
for _, delim := range SupportedDelimiters {
|
||||
count := 0
|
||||
for _, line := range lines[:min(3, len(lines))] {
|
||||
count += strings.Count(line, delim)
|
||||
}
|
||||
delimiterCounts[delim] = count
|
||||
}
|
||||
|
||||
// Find the delimiter with the most consistent count across lines
|
||||
bestDelimiter := ","
|
||||
maxCount := 0
|
||||
for delim, count := range delimiterCounts {
|
||||
if count > maxCount {
|
||||
maxCount = count
|
||||
bestDelimiter = delim
|
||||
}
|
||||
}
|
||||
|
||||
return bestDelimiter
|
||||
}
|
||||
|
||||
// detectQuoteChar attempts to auto-detect the quote character
|
||||
func detectQuoteChar(data []byte) string {
|
||||
content := string(data)
|
||||
|
||||
doubleQuotes := strings.Count(content, "\"")
|
||||
singleQuotes := strings.Count(content, "'")
|
||||
|
||||
if singleQuotes > doubleQuotes {
|
||||
return "'"
|
||||
}
|
||||
return "\""
|
||||
}
|
||||
|
||||
// detectDateFormat attempts to detect the date format from sample data
|
||||
func detectDateFormat(sampleDates []string) string {
|
||||
if len(sampleDates) == 0 {
|
||||
return SupportedDateFormats[0] // Default to ISO
|
||||
}
|
||||
|
||||
for _, format := range SupportedDateFormats {
|
||||
matches := 0
|
||||
for _, dateStr := range sampleDates {
|
||||
dateStr = strings.TrimSpace(dateStr)
|
||||
if dateStr == "" {
|
||||
continue
|
||||
}
|
||||
_, err := time.Parse(format, dateStr)
|
||||
if err == nil {
|
||||
matches++
|
||||
}
|
||||
}
|
||||
// If most dates match this format, use it
|
||||
if matches > 0 && matches >= len(sampleDates)/2 {
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
return SupportedDateFormats[0]
|
||||
}
|
||||
|
||||
// suggestMapping suggests column mappings based on column names
|
||||
func suggestMapping(columns []string) []ColumnMapping {
|
||||
mappings := make([]ColumnMapping, len(columns))
|
||||
|
||||
// Common column name patterns for each attribute
|
||||
patterns := map[TaskAttribute][]string{
|
||||
AttrTitle: {"title", "name", "task", "subject", "summary"},
|
||||
AttrDescription: {"description", "content", "notes", "details", "body", "text"},
|
||||
AttrDueDate: {"due", "due_date", "duedate", "deadline", "due date"},
|
||||
AttrStartDate: {"start", "start_date", "startdate", "begin", "start date"},
|
||||
AttrEndDate: {"end", "end_date", "enddate", "finish", "end date"},
|
||||
AttrDone: {"done", "completed", "complete", "finished", "status", "is_done"},
|
||||
AttrPriority: {"priority", "importance", "urgent", "prio"},
|
||||
AttrLabels: {"labels", "tags", "categories", "category", "label", "tag"},
|
||||
AttrProject: {"project", "list", "folder", "group", "project_name", "list_name"},
|
||||
AttrReminder: {"reminder", "remind", "alert", "notification"},
|
||||
}
|
||||
|
||||
usedAttributes := make(map[TaskAttribute]bool)
|
||||
|
||||
for i, col := range columns {
|
||||
colLower := strings.ToLower(strings.TrimSpace(col))
|
||||
mappings[i] = ColumnMapping{
|
||||
ColumnIndex: i,
|
||||
ColumnName: col,
|
||||
Attribute: AttrIgnore,
|
||||
}
|
||||
|
||||
for attr, keywords := range patterns {
|
||||
if usedAttributes[attr] && attr != AttrLabels {
|
||||
continue // Don't map the same attribute twice (except labels)
|
||||
}
|
||||
for _, keyword := range keywords {
|
||||
if strings.Contains(colLower, keyword) || colLower == keyword {
|
||||
mappings[i].Attribute = attr
|
||||
usedAttributes[attr] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if mappings[i].Attribute != AttrIgnore {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return mappings
|
||||
}
|
||||
|
||||
// parseCSV parses CSV data with the given configuration
|
||||
func parseCSV(data []byte, delimiter, quoteChar string) ([]string, [][]string, error) {
|
||||
data = stripBOM(data)
|
||||
reader := csv.NewReader(bytes.NewReader(data))
|
||||
|
||||
if len(delimiter) > 0 {
|
||||
reader.Comma = rune(delimiter[0])
|
||||
}
|
||||
reader.FieldsPerRecord = -1 // Allow variable field counts
|
||||
reader.LazyQuotes = true
|
||||
reader.TrimLeadingSpace = true
|
||||
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil, nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
headers := records[0]
|
||||
var dataRows [][]string
|
||||
if len(records) > 1 {
|
||||
dataRows = records[1:]
|
||||
}
|
||||
|
||||
return headers, dataRows, nil
|
||||
}
|
||||
|
||||
// DetectCSVStructure analyzes a CSV file and returns detection results
|
||||
func DetectCSVStructure(file io.ReaderAt, size int64) (*DetectionResult, error) {
|
||||
if size == 0 {
|
||||
return nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
// Read the entire file
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Detect delimiter and quote character
|
||||
delimiter := detectDelimiter(data)
|
||||
quoteChar := detectQuoteChar(data)
|
||||
|
||||
// Parse CSV
|
||||
headers, rows, err := parseCSV(data, delimiter, quoteChar)
|
||||
if err != nil {
|
||||
return nil, &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
// Suggest column mappings
|
||||
suggestedMapping := suggestMapping(headers)
|
||||
|
||||
// Collect sample dates for format detection
|
||||
var sampleDates []string
|
||||
for _, mapping := range suggestedMapping {
|
||||
if mapping.Attribute == AttrDueDate || mapping.Attribute == AttrStartDate || mapping.Attribute == AttrEndDate {
|
||||
for _, row := range rows {
|
||||
if mapping.ColumnIndex < len(row) && row[mapping.ColumnIndex] != "" {
|
||||
sampleDates = append(sampleDates, row[mapping.ColumnIndex])
|
||||
if len(sampleDates) >= 10 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dateFormat := detectDateFormat(sampleDates)
|
||||
|
||||
// Get preview rows (first 5)
|
||||
previewRows := rows
|
||||
if len(previewRows) > 5 {
|
||||
previewRows = previewRows[:5]
|
||||
}
|
||||
|
||||
return &DetectionResult{
|
||||
Columns: headers,
|
||||
Delimiter: delimiter,
|
||||
QuoteChar: quoteChar,
|
||||
DateFormat: dateFormat,
|
||||
SuggestedMapping: suggestedMapping,
|
||||
PreviewRows: previewRows,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PreviewImport generates a preview of the import based on current mapping
|
||||
func PreviewImport(file io.ReaderAt, size int64, config ImportConfig) (*PreviewResult, error) {
|
||||
if size == 0 {
|
||||
return nil, &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||
if err != nil {
|
||||
return nil, &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
result := &PreviewResult{
|
||||
Tasks: make([]PreviewTask, 0, min(5, len(rows))),
|
||||
TotalRows: len(rows),
|
||||
}
|
||||
|
||||
previewCount := min(5, len(rows))
|
||||
for i := 0; i < previewCount; i++ {
|
||||
task, err := rowToPreviewTask(rows[i], config)
|
||||
if err != nil {
|
||||
result.ErrorCount++
|
||||
result.Errors = append(result.Errors, err.Error())
|
||||
continue
|
||||
}
|
||||
result.Tasks = append(result.Tasks, task)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// rowToPreviewTask converts a CSV row to a preview task
|
||||
func rowToPreviewTask(row []string, config ImportConfig) (PreviewTask, error) {
|
||||
task := PreviewTask{}
|
||||
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.ColumnIndex >= len(row) {
|
||||
continue
|
||||
}
|
||||
|
||||
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch mapping.Attribute {
|
||||
case AttrTitle:
|
||||
task.Title = value
|
||||
case AttrDescription:
|
||||
task.Description = value
|
||||
case AttrDueDate:
|
||||
task.DueDate = value
|
||||
case AttrStartDate:
|
||||
task.StartDate = value
|
||||
case AttrEndDate:
|
||||
task.EndDate = value
|
||||
case AttrDone:
|
||||
task.Done = parseBool(value)
|
||||
case AttrPriority:
|
||||
task.Priority = parsePriority(value)
|
||||
case AttrLabels:
|
||||
task.Labels = parseLabels(value)
|
||||
case AttrProject:
|
||||
task.Project = value
|
||||
}
|
||||
}
|
||||
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// parseBool parses various boolean representations
|
||||
func parseBool(value string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
return lower == "true" || lower == "yes" || lower == "1" || lower == "done" || lower == "completed"
|
||||
}
|
||||
|
||||
// parsePriority parses priority value
|
||||
func parsePriority(value string) int {
|
||||
// Try to parse as number
|
||||
if p, err := strconv.Atoi(strings.TrimSpace(value)); err == nil {
|
||||
// Vikunja uses 0-5 priority (0=unset, 1=low, 5=urgent)
|
||||
if p < 0 {
|
||||
return 0
|
||||
}
|
||||
if p > 5 {
|
||||
return 5
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Try to parse text priority
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
switch {
|
||||
case strings.Contains(lower, "urgent") || strings.Contains(lower, "highest"):
|
||||
return 5
|
||||
case strings.Contains(lower, "high"):
|
||||
return 4
|
||||
case strings.Contains(lower, "medium") || strings.Contains(lower, "normal"):
|
||||
return 3
|
||||
case strings.Contains(lower, "low"):
|
||||
return 2
|
||||
case strings.Contains(lower, "lowest"):
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseLabels parses comma-separated labels
|
||||
func parseLabels(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
labels := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
label := strings.TrimSpace(part)
|
||||
if label != "" {
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
// parseDate parses a date string with the given format
|
||||
func parseDate(value, format string) time.Time {
|
||||
if value == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Try the specified format first
|
||||
if t, err := time.Parse(format, strings.TrimSpace(value)); err == nil {
|
||||
return t
|
||||
}
|
||||
|
||||
// Try all supported formats as fallback
|
||||
for _, f := range SupportedDateFormats {
|
||||
if t, err := time.Parse(f, strings.TrimSpace(value)); err == nil {
|
||||
return t
|
||||
}
|
||||
}
|
||||
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// Migrate imports CSV data into Vikunja
|
||||
// @Summary Import all tasks from a CSV file
|
||||
// @Description Imports tasks from a CSV file into Vikunja. Requires a mapping configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to import"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/migrate [put]
|
||||
func (m *Migrator) Migrate(u *user.User, file io.ReaderAt, size int64) error {
|
||||
// This will be called with the standard file migrator handler
|
||||
// The actual configuration will come through the handler
|
||||
return &migration.ErrNotACSVFile{} // Need config, use MigrateWithConfig instead
|
||||
}
|
||||
|
||||
// MigrateWithConfig imports CSV data into Vikunja with the provided configuration
|
||||
func MigrateWithConfig(u *user.User, file io.ReaderAt, size int64, config ImportConfig) error {
|
||||
if size == 0 {
|
||||
return &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
data := make([]byte, size)
|
||||
_, err := file.ReadAt(data, 0)
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
|
||||
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||
if err != nil {
|
||||
return &migration.ErrNotACSVFile{}
|
||||
}
|
||||
|
||||
if len(rows) == 0 {
|
||||
return &migration.ErrFileIsEmpty{}
|
||||
}
|
||||
|
||||
// Convert rows to Vikunja structure
|
||||
vikunjaTasks := convertToVikunja(rows, config)
|
||||
|
||||
return migration.InsertFromStructure(vikunjaTasks, u)
|
||||
}
|
||||
|
||||
// convertToVikunja converts CSV rows to Vikunja project/task structure
|
||||
func convertToVikunja(rows [][]string, config ImportConfig) []*models.ProjectWithTasksAndBuckets {
|
||||
var pseudoParentID int64 = 1
|
||||
result := []*models.ProjectWithTasksAndBuckets{
|
||||
{
|
||||
Project: models.Project{
|
||||
ID: pseudoParentID,
|
||||
Title: "Imported from CSV",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
projects := make(map[string]*models.ProjectWithTasksAndBuckets)
|
||||
defaultProjectName := "Tasks"
|
||||
|
||||
for i, row := range rows {
|
||||
task := rowToTask(row, config, int64(i+1))
|
||||
|
||||
// Determine project name
|
||||
projectName := defaultProjectName
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.Attribute == AttrProject && mapping.ColumnIndex < len(row) {
|
||||
if pn := strings.TrimSpace(row[mapping.ColumnIndex]); pn != "" {
|
||||
projectName = pn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get or create project
|
||||
if _, exists := projects[projectName]; !exists {
|
||||
projects[projectName] = &models.ProjectWithTasksAndBuckets{
|
||||
Project: models.Project{
|
||||
ID: int64(len(projects)+2) + pseudoParentID,
|
||||
ParentProjectID: pseudoParentID,
|
||||
Title: projectName,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Add task to project
|
||||
projects[projectName].Tasks = append(projects[projectName].Tasks, &models.TaskWithComments{Task: task})
|
||||
}
|
||||
|
||||
// Collect all projects
|
||||
for _, p := range projects {
|
||||
result = append(result, p)
|
||||
}
|
||||
|
||||
// Sort projects by title for consistent ordering
|
||||
sort.Slice(result[1:], func(i, j int) bool {
|
||||
return result[i+1].Title < result[j+1].Title
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// rowToTask converts a CSV row to a Vikunja task
|
||||
func rowToTask(row []string, config ImportConfig, taskID int64) models.Task {
|
||||
task := models.Task{
|
||||
ID: taskID,
|
||||
}
|
||||
|
||||
for _, mapping := range config.Mapping {
|
||||
if mapping.ColumnIndex >= len(row) {
|
||||
continue
|
||||
}
|
||||
|
||||
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch mapping.Attribute {
|
||||
case AttrTitle:
|
||||
task.Title = value
|
||||
case AttrDescription:
|
||||
task.Description = value
|
||||
case AttrDueDate:
|
||||
task.DueDate = parseDate(value, config.DateFormat)
|
||||
case AttrStartDate:
|
||||
task.StartDate = parseDate(value, config.DateFormat)
|
||||
case AttrEndDate:
|
||||
task.EndDate = parseDate(value, config.DateFormat)
|
||||
case AttrDone:
|
||||
task.Done = parseBool(value)
|
||||
if task.Done {
|
||||
task.DoneAt = time.Now()
|
||||
}
|
||||
case AttrPriority:
|
||||
task.Priority = int64(parsePriority(value))
|
||||
case AttrLabels:
|
||||
labels := parseLabels(value)
|
||||
for _, labelTitle := range labels {
|
||||
task.Labels = append(task.Labels, &models.Label{Title: labelTitle})
|
||||
}
|
||||
case AttrReminder:
|
||||
// Parse reminder as duration or date
|
||||
reminderDate := parseDate(value, config.DateFormat)
|
||||
if !reminderDate.IsZero() {
|
||||
task.Reminders = []*models.TaskReminder{
|
||||
{
|
||||
Reminder: reminderDate,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure task has a title
|
||||
if task.Title == "" {
|
||||
task.Title = "Untitled Task"
|
||||
}
|
||||
|
||||
return task
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
|
@ -0,0 +1,581 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStripBOM(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expected []byte
|
||||
}{
|
||||
{
|
||||
name: "with BOM",
|
||||
input: []byte{0xEF, 0xBB, 0xBF, 'H', 'e', 'l', 'l', 'o'},
|
||||
expected: []byte("Hello"),
|
||||
},
|
||||
{
|
||||
name: "without BOM",
|
||||
input: []byte("Hello"),
|
||||
expected: []byte("Hello"),
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
input: []byte{},
|
||||
expected: []byte{},
|
||||
},
|
||||
{
|
||||
name: "only BOM",
|
||||
input: []byte{0xEF, 0xBB, 0xBF},
|
||||
expected: []byte{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := stripBOM(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDelimiter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "comma separated",
|
||||
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||
expected: ",",
|
||||
},
|
||||
{
|
||||
name: "semicolon separated",
|
||||
input: "name;email;phone\nJohn;john@test.com;123\nJane;jane@test.com;456",
|
||||
expected: ";",
|
||||
},
|
||||
{
|
||||
name: "tab separated",
|
||||
input: "name\temail\tphone\nJohn\tjohn@test.com\t123\nJane\tjane@test.com\t456",
|
||||
expected: "\t",
|
||||
},
|
||||
{
|
||||
name: "pipe separated",
|
||||
input: "name|email|phone\nJohn|john@test.com|123\nJane|jane@test.com|456",
|
||||
expected: "|",
|
||||
},
|
||||
{
|
||||
name: "single line defaults to comma",
|
||||
input: "just a single line",
|
||||
expected: ",",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectDelimiter([]byte(tc.input))
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectQuoteChar(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "double quotes",
|
||||
input: `"name","email"\n"John","john@test.com"`,
|
||||
expected: "\"",
|
||||
},
|
||||
{
|
||||
name: "single quotes",
|
||||
input: `'name','email'\n'John','john@test.com'`,
|
||||
expected: "'",
|
||||
},
|
||||
{
|
||||
name: "no quotes defaults to double",
|
||||
input: "name,email\nJohn,john@test.com",
|
||||
expected: "\"",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectQuoteChar([]byte(tc.input))
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectDateFormat(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sampleDates []string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "ISO date",
|
||||
sampleDates: []string{"2024-01-15", "2024-02-20", "2024-03-25"},
|
||||
expected: "2006-01-02",
|
||||
},
|
||||
{
|
||||
name: "ISO datetime",
|
||||
sampleDates: []string{"2024-01-15T10:30:00", "2024-02-20T14:45:00"},
|
||||
expected: "2006-01-02T15:04:05",
|
||||
},
|
||||
{
|
||||
name: "European format",
|
||||
sampleDates: []string{"15.01.2024", "20.02.2024", "25.03.2024"},
|
||||
expected: "02.01.2006",
|
||||
},
|
||||
{
|
||||
name: "empty defaults to ISO",
|
||||
sampleDates: []string{},
|
||||
expected: "2006-01-02",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := detectDateFormat(tc.sampleDates)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
columns []string
|
||||
expected map[int]TaskAttribute
|
||||
}{
|
||||
{
|
||||
name: "standard column names",
|
||||
columns: []string{"Title", "Description", "Due Date", "Priority", "Labels"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrTitle,
|
||||
1: AttrDescription,
|
||||
2: AttrDueDate,
|
||||
3: AttrPriority,
|
||||
4: AttrLabels,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "alternative column names",
|
||||
columns: []string{"Task Name", "Notes", "Deadline", "Tags", "Project"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrTitle,
|
||||
1: AttrDescription,
|
||||
2: AttrDueDate,
|
||||
3: AttrLabels,
|
||||
4: AttrProject,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown columns",
|
||||
columns: []string{"ID", "Random Column", "Unknown"},
|
||||
expected: map[int]TaskAttribute{
|
||||
0: AttrIgnore,
|
||||
1: AttrIgnore,
|
||||
2: AttrIgnore,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mappings := suggestMapping(tc.columns)
|
||||
require.Len(t, mappings, len(tc.columns))
|
||||
|
||||
for idx, expectedAttr := range tc.expected {
|
||||
assert.Equal(t, expectedAttr, mappings[idx].Attribute, "Column %d (%s)", idx, tc.columns[idx])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCSV(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
delimiter string
|
||||
quoteChar string
|
||||
expectedCols []string
|
||||
expectedRows int
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "simple comma CSV",
|
||||
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email", "phone"},
|
||||
expectedRows: 2,
|
||||
},
|
||||
{
|
||||
name: "semicolon CSV",
|
||||
input: "name;email;phone\nJohn;john@test.com;123",
|
||||
delimiter: ";",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email", "phone"},
|
||||
expectedRows: 1,
|
||||
},
|
||||
{
|
||||
name: "quoted fields",
|
||||
input: "name,description\n\"John Doe\",\"A long, complicated description\"\nJane,Simple",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "description"},
|
||||
expectedRows: 2,
|
||||
},
|
||||
{
|
||||
name: "with BOM",
|
||||
input: "\xEF\xBB\xBFname,email\nJohn,john@test.com",
|
||||
delimiter: ",",
|
||||
quoteChar: "\"",
|
||||
expectedCols: []string{"name", "email"},
|
||||
expectedRows: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
headers, rows, err := parseCSV([]byte(tc.input), tc.delimiter, tc.quoteChar)
|
||||
|
||||
if tc.expectedError {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCols, headers)
|
||||
assert.Len(t, rows, tc.expectedRows)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBool(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{"true", true},
|
||||
{"True", true},
|
||||
{"TRUE", true},
|
||||
{"yes", true},
|
||||
{"Yes", true},
|
||||
{"1", true},
|
||||
{"done", true},
|
||||
{"completed", true},
|
||||
{"false", false},
|
||||
{"no", false},
|
||||
{"0", false},
|
||||
{"", false},
|
||||
{"random", false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parseBool(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePriority(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected int
|
||||
}{
|
||||
{"0", 0},
|
||||
{"1", 1},
|
||||
{"3", 3},
|
||||
{"5", 5},
|
||||
{"10", 5}, // capped at 5
|
||||
{"-1", 0}, // minimum 0
|
||||
{"low", 2},
|
||||
{"medium", 3},
|
||||
{"high", 4},
|
||||
{"urgent", 5},
|
||||
{"highest", 5},
|
||||
{"lowest", 1},
|
||||
{"normal", 3},
|
||||
{"random", 0},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parsePriority(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLabels(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{"work, personal, urgent", []string{"work", "personal", "urgent"}},
|
||||
{"single", []string{"single"}},
|
||||
{" spaced , labels ", []string{"spaced", "labels"}},
|
||||
{"", []string{}},
|
||||
{",,,", []string{}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := parseLabels(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectCSVStructure(t *testing.T) {
|
||||
csvContent := `Title,Description,Due Date,Priority,Labels
|
||||
Task 1,Description 1,2024-01-15,high,work
|
||||
Task 2,Description 2,2024-01-20,low,"personal, urgent"
|
||||
Task 3,Description 3,2024-01-25,medium,home`
|
||||
|
||||
reader := bytes.NewReader([]byte(csvContent))
|
||||
|
||||
result, err := DetectCSVStructure(reader, int64(len(csvContent)))
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []string{"Title", "Description", "Due Date", "Priority", "Labels"}, result.Columns)
|
||||
assert.Equal(t, ",", result.Delimiter)
|
||||
assert.Len(t, result.SuggestedMapping, 5)
|
||||
assert.Len(t, result.PreviewRows, 3)
|
||||
|
||||
// Check suggested mappings
|
||||
titleMapping := result.SuggestedMapping[0]
|
||||
assert.Equal(t, AttrTitle, titleMapping.Attribute)
|
||||
assert.Equal(t, "Title", titleMapping.ColumnName)
|
||||
|
||||
descMapping := result.SuggestedMapping[1]
|
||||
assert.Equal(t, AttrDescription, descMapping.Attribute)
|
||||
|
||||
dueDateMapping := result.SuggestedMapping[2]
|
||||
assert.Equal(t, AttrDueDate, dueDateMapping.Attribute)
|
||||
}
|
||||
|
||||
func TestPreviewImport(t *testing.T) {
|
||||
csvContent := `Title,Description,Done,Priority
|
||||
Task 1,Description 1,true,high
|
||||
Task 2,Description 2,false,low
|
||||
Task 3,Description 3,yes,medium
|
||||
Task 4,Description 4,no,urgent
|
||||
Task 5,Description 5,1,normal
|
||||
Task 6,Description 6,0,low`
|
||||
|
||||
config := ImportConfig{
|
||||
Delimiter: ",",
|
||||
QuoteChar: "\"",
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, ColumnName: "Title", Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, ColumnName: "Description", Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, ColumnName: "Done", Attribute: AttrDone},
|
||||
{ColumnIndex: 3, ColumnName: "Priority", Attribute: AttrPriority},
|
||||
},
|
||||
}
|
||||
|
||||
reader := bytes.NewReader([]byte(csvContent))
|
||||
|
||||
result, err := PreviewImport(reader, int64(len(csvContent)), config)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 6, result.TotalRows)
|
||||
assert.Len(t, result.Tasks, 5) // Preview limited to 5
|
||||
|
||||
// Check first task
|
||||
assert.Equal(t, "Task 1", result.Tasks[0].Title)
|
||||
assert.Equal(t, "Description 1", result.Tasks[0].Description)
|
||||
assert.True(t, result.Tasks[0].Done)
|
||||
assert.Equal(t, 4, result.Tasks[0].Priority) // "high" -> 4
|
||||
|
||||
// Check second task
|
||||
assert.Equal(t, "Task 2", result.Tasks[1].Title)
|
||||
assert.False(t, result.Tasks[1].Done)
|
||||
assert.Equal(t, 2, result.Tasks[1].Priority) // "low" -> 2
|
||||
}
|
||||
|
||||
func TestConvertToVikunja(t *testing.T) {
|
||||
rows := [][]string{
|
||||
{"Task 1", "Description 1", "Project A"},
|
||||
{"Task 2", "Description 2", "Project A"},
|
||||
{"Task 3", "Description 3", "Project B"},
|
||||
{"Task 4", "Description 4", ""}, // No project -> default
|
||||
}
|
||||
|
||||
config := ImportConfig{
|
||||
Delimiter: ",",
|
||||
QuoteChar: "\"",
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, Attribute: AttrProject},
|
||||
},
|
||||
}
|
||||
|
||||
result := convertToVikunja(rows, config)
|
||||
|
||||
// Should have parent project + child projects
|
||||
require.GreaterOrEqual(t, len(result), 2)
|
||||
|
||||
// First project should be the parent "Imported from CSV"
|
||||
assert.Equal(t, "Imported from CSV", result[0].Title)
|
||||
|
||||
// Find Project A
|
||||
var projectA, projectB, tasksProject *struct {
|
||||
title string
|
||||
numTasks int
|
||||
}
|
||||
for _, p := range result[1:] {
|
||||
switch p.Title {
|
||||
case "Project A":
|
||||
projectA = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
case "Project B":
|
||||
projectB = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
case "Tasks":
|
||||
tasksProject = &struct {
|
||||
title string
|
||||
numTasks int
|
||||
}{p.Title, len(p.Tasks)}
|
||||
}
|
||||
}
|
||||
|
||||
assert.NotNil(t, projectA, "Project A should exist")
|
||||
assert.Equal(t, 2, projectA.numTasks, "Project A should have 2 tasks")
|
||||
|
||||
assert.NotNil(t, projectB, "Project B should exist")
|
||||
assert.Equal(t, 1, projectB.numTasks, "Project B should have 1 task")
|
||||
|
||||
assert.NotNil(t, tasksProject, "Tasks project should exist for tasks without project")
|
||||
assert.Equal(t, 1, tasksProject.numTasks, "Tasks project should have 1 task")
|
||||
}
|
||||
|
||||
func TestRowToTask(t *testing.T) {
|
||||
row := []string{"My Task", "Task description", "2024-01-15", "high", "work, urgent"}
|
||||
|
||||
config := ImportConfig{
|
||||
DateFormat: "2006-01-02",
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
{ColumnIndex: 2, Attribute: AttrDueDate},
|
||||
{ColumnIndex: 3, Attribute: AttrPriority},
|
||||
{ColumnIndex: 4, Attribute: AttrLabels},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
assert.Equal(t, "My Task", task.Title)
|
||||
assert.Equal(t, "Task description", task.Description)
|
||||
assert.Equal(t, 2024, task.DueDate.Year())
|
||||
assert.Equal(t, 1, int(task.DueDate.Month()))
|
||||
assert.Equal(t, 15, task.DueDate.Day())
|
||||
assert.Equal(t, int64(4), task.Priority) // "high" -> 4
|
||||
require.Len(t, task.Labels, 2)
|
||||
assert.Equal(t, "work", task.Labels[0].Title)
|
||||
assert.Equal(t, "urgent", task.Labels[1].Title)
|
||||
}
|
||||
|
||||
func TestMigratorName(t *testing.T) {
|
||||
m := &Migrator{}
|
||||
assert.Equal(t, "csv", m.Name())
|
||||
}
|
||||
|
||||
func TestEmptyFile(t *testing.T) {
|
||||
reader := bytes.NewReader([]byte{})
|
||||
|
||||
_, err := DetectCSVStructure(reader, 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRowToTaskWithMissingColumns(t *testing.T) {
|
||||
// Row with fewer columns than expected
|
||||
row := []string{"My Task"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription}, // Index 1 doesn't exist
|
||||
{ColumnIndex: 2, Attribute: AttrDueDate}, // Index 2 doesn't exist
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
// Should still work with available columns
|
||||
assert.Equal(t, "My Task", task.Title)
|
||||
assert.Empty(t, task.Description)
|
||||
assert.True(t, task.DueDate.IsZero())
|
||||
}
|
||||
|
||||
func TestRowToTaskWithEmptyTitle(t *testing.T) {
|
||||
row := []string{"", "Some description"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
// Should have default title
|
||||
assert.Equal(t, "Untitled Task", task.Title)
|
||||
assert.Equal(t, "Some description", task.Description)
|
||||
}
|
||||
|
||||
func TestDoneTask(t *testing.T) {
|
||||
row := []string{"Done Task", "completed"}
|
||||
|
||||
config := ImportConfig{
|
||||
Mapping: []ColumnMapping{
|
||||
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||
{ColumnIndex: 1, Attribute: AttrDone},
|
||||
},
|
||||
}
|
||||
|
||||
task := rowToTask(row, config, 1)
|
||||
|
||||
assert.Equal(t, "Done Task", task.Title)
|
||||
assert.True(t, task.Done)
|
||||
assert.False(t, task.DoneAt.IsZero()) // DoneAt should be set
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
// Vikunja is a to-do list application to facilitate your life.
|
||||
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package csv
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"code.vikunja.io/api/pkg/models"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
user2 "code.vikunja.io/api/pkg/user"
|
||||
"code.vikunja.io/api/pkg/web/handler"
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// CSVMigratorWeb handles CSV migration HTTP routes
|
||||
type CSVMigratorWeb struct{}
|
||||
|
||||
// RegisterRoutes registers all CSV migration routes
|
||||
func (c *CSVMigratorWeb) RegisterRoutes(g *echo.Group) {
|
||||
g.GET("/csv/status", c.Status)
|
||||
g.PUT("/csv/detect", c.Detect)
|
||||
g.PUT("/csv/preview", c.Preview)
|
||||
g.PUT("/csv/migrate", c.Migrate)
|
||||
}
|
||||
|
||||
// Status returns the migration status
|
||||
// @Summary Get CSV migration status
|
||||
// @Description Returns if the current user already did the CSV migration or not.
|
||||
// @tags migration
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Success 200 {object} migration.Status "The migration status"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/status [get]
|
||||
func (c *CSVMigratorWeb) Status(ctx echo.Context) error {
|
||||
u, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
m := &Migrator{}
|
||||
s, err := migration.GetMigrationStatus(m, u)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, s)
|
||||
}
|
||||
|
||||
// Detect analyzes a CSV file and returns detection results
|
||||
// @Summary Detect CSV structure
|
||||
// @Description Analyzes a CSV file and returns auto-detected columns, delimiter, quote character, and date format with suggested column mappings.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to analyze"
|
||||
// @Success 200 {object} DetectionResult "Detection results with suggested mappings"
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/detect [put]
|
||||
func (c *CSVMigratorWeb) Detect(ctx echo.Context) error {
|
||||
_, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
result, err := DetectCSVStructure(src, file.Size)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// Preview generates a preview of the import
|
||||
// @Summary Preview CSV import
|
||||
// @Description Generates a preview of the first 5 tasks that would be imported with the given configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to preview"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} PreviewResult "Preview of tasks to import"
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/preview [put]
|
||||
func (c *CSVMigratorWeb) Preview(ctx echo.Context) error {
|
||||
_, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
configStr := ctx.FormValue("config")
|
||||
if configStr == "" {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||
}
|
||||
|
||||
var config ImportConfig
|
||||
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
result, err := PreviewImport(src, file.Size, config)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
// Migrate imports the CSV file
|
||||
// @Summary Import CSV file
|
||||
// @Description Imports tasks from a CSV file into Vikunja with the provided configuration.
|
||||
// @tags migration
|
||||
// @Accept multipart/form-data
|
||||
// @Produce json
|
||||
// @Security JWTKeyAuth
|
||||
// @Param import formData file true "The CSV file to import"
|
||||
// @Param config formData string true "The import configuration JSON"
|
||||
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||
// @Failure 500 {object} models.Message "Internal server error"
|
||||
// @Router /migration/csv/migrate [put]
|
||||
func (c *CSVMigratorWeb) Migrate(ctx echo.Context) error {
|
||||
u, err := user2.GetCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
file, err := ctx.FormFile("import")
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||
}
|
||||
|
||||
configStr := ctx.FormValue("config")
|
||||
if configStr == "" {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||
}
|
||||
|
||||
var config ImportConfig
|
||||
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
m := &Migrator{}
|
||||
status, err := migration.StartMigration(m, u)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
err = MigrateWithConfig(u, src, file.Size, config)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
err = migration.FinishMigration(status)
|
||||
if err != nil {
|
||||
return handler.HandleHTTPError(err)
|
||||
}
|
||||
|
||||
return ctx.JSON(http.StatusOK, models.Message{Message: "Everything was migrated successfully."})
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"code.vikunja.io/api/pkg/config"
|
||||
"code.vikunja.io/api/pkg/log"
|
||||
"code.vikunja.io/api/pkg/modules/auth/openid"
|
||||
csvmigrator "code.vikunja.io/api/pkg/modules/migration/csv"
|
||||
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
||||
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
||||
"code.vikunja.io/api/pkg/modules/migration/todoist"
|
||||
|
|
@ -108,6 +109,7 @@ func Info(c *echo.Context) error {
|
|||
(&vikunja_file.FileMigrator{}).Name(),
|
||||
(&ticktick.Migrator{}).Name(),
|
||||
(&wekan.Migrator{}).Name(),
|
||||
(&csvmigrator.Migrator{}).Name(),
|
||||
},
|
||||
Legal: legalInfo{
|
||||
ImprintURL: config.LegalImprintURL.GetString(),
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ import (
|
|||
"code.vikunja.io/api/pkg/modules/background/unsplash"
|
||||
"code.vikunja.io/api/pkg/modules/background/upload"
|
||||
"code.vikunja.io/api/pkg/modules/migration"
|
||||
csvmigrator "code.vikunja.io/api/pkg/modules/migration/csv"
|
||||
migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler"
|
||||
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
||||
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
||||
|
|
@ -861,6 +862,10 @@ func registerMigrations(m *echo.Group) {
|
|||
},
|
||||
}
|
||||
wekanFileMigrator.RegisterRoutes(m)
|
||||
|
||||
// CSV File Migrator (always enabled - generic import)
|
||||
csvFileMigrator := &csvmigrator.CSVMigratorWeb{}
|
||||
csvFileMigrator.RegisterRoutes(m)
|
||||
}
|
||||
|
||||
func registerCalDavRoutes(c *echo.Group) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue