Compare commits
14 Commits
main
...
csv-import
| Author | SHA1 | Date |
|---|---|---|
|
|
c0b90ba009 | |
|
|
56c4d97b50 | |
|
|
e031a8d428 | |
|
|
c213f8eed8 | |
|
|
c6cdd551a1 | |
|
|
ada0dc1ffa | |
|
|
0dd62b6a4f | |
|
|
2b076cb7f7 | |
|
|
5317a87d90 | |
|
|
46438fc74b | |
|
|
61fa94e672 | |
|
|
ee349cc548 | |
|
|
12475aa497 | |
|
|
07c68e04ef |
|
|
@ -641,7 +641,33 @@
|
||||||
"importUpload": "To import data from {name} into Vikunja, click the button below to select a file.",
|
"importUpload": "To import data from {name} into Vikunja, click the button below to select a file.",
|
||||||
"upload": "Upload file",
|
"upload": "Upload file",
|
||||||
"migrationStartedWillReciveEmail": "Vikunja will now import your lists/projects, tasks, notes, reminders and files from {service}. As this will take a while, we will send you an email once done. You can close this window now.",
|
"migrationStartedWillReciveEmail": "Vikunja will now import your lists/projects, tasks, notes, reminders and files from {service}. As this will take a while, we will send you an email once done. You can close this window now.",
|
||||||
"migrationInProgress": "A migration is currently in progress. Please wait until it is done."
|
"migrationInProgress": "A migration is currently in progress. Please wait until it is done.",
|
||||||
|
"csv": {
|
||||||
|
"description": "Import tasks from a CSV file with custom column mapping.",
|
||||||
|
"uploadDescription": "Select a CSV file to import. The file should contain task data with headers in the first row.",
|
||||||
|
"selectFile": "Select CSV file",
|
||||||
|
"columnMapping": "Column Mapping",
|
||||||
|
"columnMappingDescription": "Map each column in your CSV file to a task attribute. Vikunja has auto-detected the most likely mappings.",
|
||||||
|
"parsingOptions": "Parsing Options",
|
||||||
|
"delimiter": "Delimiter",
|
||||||
|
"dateFormat": "Date Format",
|
||||||
|
"skipRows": "Skip Rows",
|
||||||
|
"mapColumns": "Map Columns",
|
||||||
|
"example": "e.g.",
|
||||||
|
"preview": "Preview",
|
||||||
|
"previewDescription": "Showing first 5 of {count} tasks that will be imported.",
|
||||||
|
"previewErrors": "{count} rows had parsing errors and will be skipped.",
|
||||||
|
"import": "Import Tasks",
|
||||||
|
"untitled": "Untitled Task",
|
||||||
|
"completed": "Completed",
|
||||||
|
"ignore": "Ignore",
|
||||||
|
"delimiters": {
|
||||||
|
"comma": "Comma (,)",
|
||||||
|
"semicolon": "Semicolon (;)",
|
||||||
|
"tab": "Tab",
|
||||||
|
"pipe": "Pipe (|)"
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"label": {
|
"label": {
|
||||||
"title": "Labels",
|
"title": "Labels",
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,11 @@ const router = createRouter({
|
||||||
name: 'migrate.start',
|
name: 'migrate.start',
|
||||||
component: () => import('@/views/migrate/Migration.vue'),
|
component: () => import('@/views/migrate/Migration.vue'),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
path: '/migrate/csv',
|
||||||
|
name: 'migrate.csv',
|
||||||
|
component: () => import('@/views/migrate/MigrationCSV.vue'),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
path: '/migrate/:service',
|
path: '/migrate/:service',
|
||||||
name: 'migrate.service',
|
name: 'migrate.service',
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,123 @@
|
||||||
|
import AbstractService from '../abstractService'
|
||||||
|
|
||||||
|
export interface ColumnMapping {
|
||||||
|
column_index: number
|
||||||
|
column_name: string
|
||||||
|
attribute: TaskAttribute
|
||||||
|
}
|
||||||
|
|
||||||
|
export type TaskAttribute =
|
||||||
|
| 'title'
|
||||||
|
| 'description'
|
||||||
|
| 'due_date'
|
||||||
|
| 'start_date'
|
||||||
|
| 'end_date'
|
||||||
|
| 'done'
|
||||||
|
| 'priority'
|
||||||
|
| 'labels'
|
||||||
|
| 'project'
|
||||||
|
| 'reminder'
|
||||||
|
| 'ignore'
|
||||||
|
|
||||||
|
export const TASK_ATTRIBUTES: TaskAttribute[] = [
|
||||||
|
'title',
|
||||||
|
'description',
|
||||||
|
'due_date',
|
||||||
|
'start_date',
|
||||||
|
'end_date',
|
||||||
|
'done',
|
||||||
|
'priority',
|
||||||
|
'labels',
|
||||||
|
'project',
|
||||||
|
'reminder',
|
||||||
|
'ignore',
|
||||||
|
]
|
||||||
|
|
||||||
|
export interface DetectionResult {
|
||||||
|
columns: string[]
|
||||||
|
delimiter: string
|
||||||
|
quote_char: string
|
||||||
|
date_format: string
|
||||||
|
suggested_mapping: ColumnMapping[]
|
||||||
|
preview_rows: string[][]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportConfig {
|
||||||
|
delimiter: string
|
||||||
|
quote_char: string
|
||||||
|
date_format: string
|
||||||
|
skip_rows: number
|
||||||
|
mapping: ColumnMapping[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PreviewTask {
|
||||||
|
title: string
|
||||||
|
description: string
|
||||||
|
due_date?: string
|
||||||
|
start_date?: string
|
||||||
|
end_date?: string
|
||||||
|
done: boolean
|
||||||
|
priority: number
|
||||||
|
labels?: string[]
|
||||||
|
project?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PreviewResult {
|
||||||
|
tasks: PreviewTask[]
|
||||||
|
total_rows: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MigrationStatus {
|
||||||
|
started_at: string | null
|
||||||
|
finished_at: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SUPPORTED_DELIMITERS = [',', ';', '\t', '|'] as const
|
||||||
|
|
||||||
|
export const SUPPORTED_DATE_FORMATS = [
|
||||||
|
'2006-01-02',
|
||||||
|
'2006-01-02T15:04:05',
|
||||||
|
'02/01/2006',
|
||||||
|
'01/02/2006',
|
||||||
|
'02-01-2006',
|
||||||
|
'01-02-2006',
|
||||||
|
'02.01.2006',
|
||||||
|
'2006/01/02',
|
||||||
|
'2006-01-02 15:04:05',
|
||||||
|
] as const
|
||||||
|
|
||||||
|
export default class CSVMigrationService extends AbstractService {
|
||||||
|
constructor() {
|
||||||
|
super({})
|
||||||
|
}
|
||||||
|
|
||||||
|
getStatus(): Promise<MigrationStatus> {
|
||||||
|
return this.getM('/migration/csv/status')
|
||||||
|
}
|
||||||
|
|
||||||
|
useCreateInterceptor() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
async detect(file: File): Promise<DetectionResult> {
|
||||||
|
return this.uploadFile(
|
||||||
|
'/migration/csv/detect',
|
||||||
|
file,
|
||||||
|
'import',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async preview(file: File, config: ImportConfig): Promise<PreviewResult> {
|
||||||
|
const data = new FormData()
|
||||||
|
data.append('import', file)
|
||||||
|
data.append('config', JSON.stringify(config))
|
||||||
|
return this.uploadFormData('/migration/csv/preview', data)
|
||||||
|
}
|
||||||
|
|
||||||
|
async migrate(file: File, config: ImportConfig): Promise<{ message: string }> {
|
||||||
|
const data = new FormData()
|
||||||
|
data.append('import', file)
|
||||||
|
data.append('config', JSON.stringify(config))
|
||||||
|
return this.uploadFormData('/migration/csv/migrate', data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -4,10 +4,10 @@
|
||||||
<p>{{ $t('migrate.description') }}</p>
|
<p>{{ $t('migrate.description') }}</p>
|
||||||
<div class="migration-services">
|
<div class="migration-services">
|
||||||
<RouterLink
|
<RouterLink
|
||||||
v-for="{name, id, icon} in availableMigrators"
|
v-for="{name, id, icon, isCSVMigrator} in availableMigrators"
|
||||||
:key="id"
|
:key="id"
|
||||||
class="migration-service-link"
|
class="migration-service-link"
|
||||||
:to="{name: 'migrate.service', params: {service: id}}"
|
:to="isCSVMigrator ? {name: 'migrate.csv'} : {name: 'migrate.service', params: {service: id}}"
|
||||||
>
|
>
|
||||||
<img
|
<img
|
||||||
class="migration-service-image"
|
class="migration-service-image"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,536 @@
|
||||||
|
<template>
|
||||||
|
<div class="content csv-migration">
|
||||||
|
<h1>{{ $t('migrate.titleService', {name: 'CSV'}) }}</h1>
|
||||||
|
<p>{{ $t('migrate.csv.description') }}</p>
|
||||||
|
|
||||||
|
<Message
|
||||||
|
v-if="error"
|
||||||
|
variant="danger"
|
||||||
|
class="mbe-4"
|
||||||
|
>
|
||||||
|
{{ error }}
|
||||||
|
</Message>
|
||||||
|
|
||||||
|
<!-- Step 1: File Upload -->
|
||||||
|
<div
|
||||||
|
v-if="step === 'upload'"
|
||||||
|
class="upload-step"
|
||||||
|
>
|
||||||
|
<p>{{ $t('migrate.csv.uploadDescription') }}</p>
|
||||||
|
<input
|
||||||
|
ref="uploadInput"
|
||||||
|
class="is-hidden"
|
||||||
|
type="file"
|
||||||
|
accept=".csv,.txt"
|
||||||
|
@change="handleFileUpload"
|
||||||
|
>
|
||||||
|
<XButton
|
||||||
|
:loading="isLoading"
|
||||||
|
:disabled="isLoading || undefined"
|
||||||
|
@click="uploadInput?.click()"
|
||||||
|
>
|
||||||
|
{{ $t('migrate.csv.selectFile') }}
|
||||||
|
</XButton>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Step 2: Column Mapping -->
|
||||||
|
<div
|
||||||
|
v-else-if="step === 'mapping'"
|
||||||
|
class="mapping-step"
|
||||||
|
>
|
||||||
|
<div class="mapping-header">
|
||||||
|
<h2>{{ $t('migrate.csv.columnMapping') }}</h2>
|
||||||
|
<p>{{ $t('migrate.csv.columnMappingDescription') }}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Parsing Options -->
|
||||||
|
<div class="parsing-options card">
|
||||||
|
<h3>{{ $t('migrate.csv.parsingOptions') }}</h3>
|
||||||
|
<div class="options-grid">
|
||||||
|
<div class="option-group">
|
||||||
|
<label for="delimiter">{{ $t('migrate.csv.delimiter') }}</label>
|
||||||
|
<div class="select is-fullwidth">
|
||||||
|
<select
|
||||||
|
id="delimiter"
|
||||||
|
v-model="config.delimiter"
|
||||||
|
@change="updatePreview"
|
||||||
|
>
|
||||||
|
<option
|
||||||
|
v-for="delim in SUPPORTED_DELIMITERS"
|
||||||
|
:key="delim"
|
||||||
|
:value="delim"
|
||||||
|
>
|
||||||
|
{{ getDelimiterLabel(delim) }}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="option-group">
|
||||||
|
<label for="dateFormat">{{ $t('migrate.csv.dateFormat') }}</label>
|
||||||
|
<div class="select is-fullwidth">
|
||||||
|
<select
|
||||||
|
id="dateFormat"
|
||||||
|
v-model="config.date_format"
|
||||||
|
@change="updatePreview"
|
||||||
|
>
|
||||||
|
<option
|
||||||
|
v-for="format in SUPPORTED_DATE_FORMATS"
|
||||||
|
:key="format"
|
||||||
|
:value="format"
|
||||||
|
>
|
||||||
|
{{ getDateFormatLabel(format) }}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="option-group">
|
||||||
|
<label for="skipRows">{{ $t('migrate.csv.skipRows') }}</label>
|
||||||
|
<input
|
||||||
|
id="skipRows"
|
||||||
|
v-model.number="config.skip_rows"
|
||||||
|
type="number"
|
||||||
|
class="input"
|
||||||
|
min="0"
|
||||||
|
@change="updatePreview"
|
||||||
|
>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Column Mappings -->
|
||||||
|
<div class="column-mappings card">
|
||||||
|
<h3>{{ $t('migrate.csv.mapColumns') }}</h3>
|
||||||
|
<div class="mappings-grid">
|
||||||
|
<div
|
||||||
|
v-for="(mapping, index) in config.mapping"
|
||||||
|
:key="index"
|
||||||
|
class="mapping-row"
|
||||||
|
>
|
||||||
|
<div class="column-name">
|
||||||
|
<strong>{{ mapping.column_name }}</strong>
|
||||||
|
<span
|
||||||
|
v-if="detectionResult && detectionResult.preview_rows[0]"
|
||||||
|
class="preview-value"
|
||||||
|
>
|
||||||
|
{{ $t('migrate.csv.example') }}: {{ detectionResult.preview_rows[0][index] || '-' }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="select is-fullwidth">
|
||||||
|
<select
|
||||||
|
v-model="mapping.attribute"
|
||||||
|
@change="updatePreview"
|
||||||
|
>
|
||||||
|
<option
|
||||||
|
v-for="attr in TASK_ATTRIBUTES"
|
||||||
|
:key="attr"
|
||||||
|
:value="attr"
|
||||||
|
>
|
||||||
|
{{ getAttributeLabel(attr) }}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Preview -->
|
||||||
|
<div
|
||||||
|
v-if="previewResult"
|
||||||
|
class="preview-section card"
|
||||||
|
>
|
||||||
|
<h3>{{ $t('migrate.csv.preview') }}</h3>
|
||||||
|
<p>{{ $t('migrate.csv.previewDescription', {count: previewResult.total_rows}) }}</p>
|
||||||
|
|
||||||
|
<div class="preview-tasks">
|
||||||
|
<div
|
||||||
|
v-for="(task, index) in previewResult.tasks"
|
||||||
|
:key="index"
|
||||||
|
class="preview-task card"
|
||||||
|
>
|
||||||
|
<div class="task-title">
|
||||||
|
<strong>{{ task.title || $t('migrate.csv.untitled') }}</strong>
|
||||||
|
<span
|
||||||
|
v-if="task.done"
|
||||||
|
class="done-badge"
|
||||||
|
>{{ $t('migrate.csv.completed') }}</span>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
v-if="task.description"
|
||||||
|
class="task-description"
|
||||||
|
>
|
||||||
|
{{ truncate(task.description, 100) }}
|
||||||
|
</div>
|
||||||
|
<div class="task-meta">
|
||||||
|
<span v-if="task.due_date">
|
||||||
|
{{ $t('task.attributes.dueDate') }}: {{ task.due_date }}
|
||||||
|
</span>
|
||||||
|
<span v-if="task.priority > 0">
|
||||||
|
{{ $t('task.attributes.priority') }}: {{ task.priority }}
|
||||||
|
</span>
|
||||||
|
<span v-if="task.project">
|
||||||
|
{{ $t('project.title') }}: {{ task.project }}
|
||||||
|
</span>
|
||||||
|
<span v-if="task.labels && task.labels.length > 0">
|
||||||
|
{{ $t('task.attributes.labels') }}: {{ task.labels.join(', ') }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Actions -->
|
||||||
|
<div class="actions">
|
||||||
|
<XButton
|
||||||
|
variant="tertiary"
|
||||||
|
@click="resetToUpload"
|
||||||
|
>
|
||||||
|
{{ $t('misc.cancel') }}
|
||||||
|
</XButton>
|
||||||
|
<XButton
|
||||||
|
:loading="isLoading"
|
||||||
|
:disabled="!hasValidMapping || isLoading"
|
||||||
|
@click="performImport"
|
||||||
|
>
|
||||||
|
{{ $t('migrate.csv.import') }}
|
||||||
|
</XButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Step 3: Success -->
|
||||||
|
<div
|
||||||
|
v-else-if="step === 'success'"
|
||||||
|
class="success-step"
|
||||||
|
>
|
||||||
|
<Message class="mbe-4">
|
||||||
|
{{ successMessage }}
|
||||||
|
</Message>
|
||||||
|
<XButton :to="{name: 'home'}">
|
||||||
|
{{ $t('home.goToOverview') }}
|
||||||
|
</XButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup lang="ts">
|
||||||
|
import {computed, ref, shallowReactive} from 'vue'
|
||||||
|
import {useI18n} from 'vue-i18n'
|
||||||
|
|
||||||
|
import Message from '@/components/misc/Message.vue'
|
||||||
|
|
||||||
|
import CSVMigrationService, {
|
||||||
|
type DetectionResult,
|
||||||
|
type ImportConfig,
|
||||||
|
type PreviewResult,
|
||||||
|
TASK_ATTRIBUTES,
|
||||||
|
SUPPORTED_DELIMITERS,
|
||||||
|
SUPPORTED_DATE_FORMATS,
|
||||||
|
} from '@/services/migrator/csvMigration'
|
||||||
|
|
||||||
|
import {useTitle} from '@/composables/useTitle'
|
||||||
|
import {useProjectStore} from '@/stores/projects'
|
||||||
|
import {getErrorText} from '@/message'
|
||||||
|
|
||||||
|
type Step = 'upload' | 'mapping' | 'success'
|
||||||
|
|
||||||
|
const {t} = useI18n({useScope: 'global'})
|
||||||
|
|
||||||
|
useTitle(() => t('migrate.titleService', {name: 'CSV'}))
|
||||||
|
|
||||||
|
const csvService = shallowReactive(new CSVMigrationService())
|
||||||
|
|
||||||
|
const step = ref<Step>('upload')
|
||||||
|
const error = ref('')
|
||||||
|
const successMessage = ref('')
|
||||||
|
const isLoading = ref(false)
|
||||||
|
const uploadInput = ref<HTMLInputElement | null>(null)
|
||||||
|
const selectedFile = ref<File | null>(null)
|
||||||
|
const detectionResult = ref<DetectionResult | null>(null)
|
||||||
|
const previewResult = ref<PreviewResult | null>(null)
|
||||||
|
|
||||||
|
const config = ref<ImportConfig>({
|
||||||
|
delimiter: ',',
|
||||||
|
quote_char: '"',
|
||||||
|
date_format: '2006-01-02',
|
||||||
|
skip_rows: 0,
|
||||||
|
mapping: [],
|
||||||
|
})
|
||||||
|
|
||||||
|
const hasValidMapping = computed(() => {
|
||||||
|
if (!config.value.mapping.length) return false
|
||||||
|
// At least one column should be mapped to title
|
||||||
|
return config.value.mapping.some(m => m.attribute === 'title')
|
||||||
|
})
|
||||||
|
|
||||||
|
// Map snake_case attribute names to translation keys
|
||||||
|
function getAttributeLabel(attribute: string): string {
|
||||||
|
const attributeMap: Record<string, string> = {
|
||||||
|
title: 'task.attributes.title',
|
||||||
|
description: 'task.attributes.description',
|
||||||
|
due_date: 'task.attributes.dueDate',
|
||||||
|
start_date: 'task.attributes.startDate',
|
||||||
|
end_date: 'task.attributes.endDate',
|
||||||
|
done: 'task.attributes.done',
|
||||||
|
priority: 'task.attributes.priority',
|
||||||
|
labels: 'task.attributes.labels',
|
||||||
|
reminder: 'task.attributes.reminders',
|
||||||
|
project: 'project.title',
|
||||||
|
ignore: 'migrate.csv.ignore',
|
||||||
|
}
|
||||||
|
return t(attributeMap[attribute] || attribute)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDelimiterLabel(delimiter: string): string {
|
||||||
|
const labels: Record<string, string> = {
|
||||||
|
',': t('migrate.csv.delimiters.comma'),
|
||||||
|
';': t('migrate.csv.delimiters.semicolon'),
|
||||||
|
'\t': t('migrate.csv.delimiters.tab'),
|
||||||
|
'|': t('migrate.csv.delimiters.pipe'),
|
||||||
|
}
|
||||||
|
return labels[delimiter] || delimiter
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDateFormatLabel(format: string): string {
|
||||||
|
const labels: Record<string, string> = {
|
||||||
|
'2006-01-02': 'YYYY-MM-DD (2024-01-15)',
|
||||||
|
'2006-01-02T15:04:05': 'ISO DateTime (2024-01-15T10:30:00)',
|
||||||
|
'02/01/2006': 'DD/MM/YYYY (15/01/2024)',
|
||||||
|
'01/02/2006': 'MM/DD/YYYY (01/15/2024)',
|
||||||
|
'02-01-2006': 'DD-MM-YYYY (15-01-2024)',
|
||||||
|
'01-02-2006': 'MM-DD-YYYY (01-15-2024)',
|
||||||
|
'02.01.2006': 'DD.MM.YYYY (15.01.2024)',
|
||||||
|
'2006/01/02': 'YYYY/MM/DD (2024/01/15)',
|
||||||
|
'2006-01-02 15:04:05': 'DateTime (2024-01-15 10:30:00)',
|
||||||
|
}
|
||||||
|
return labels[format] || format
|
||||||
|
}
|
||||||
|
|
||||||
|
function truncate(text: string, length: number): string {
|
||||||
|
if (text.length <= length) return text
|
||||||
|
return text.substring(0, length) + '...'
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleFileUpload() {
|
||||||
|
const files = uploadInput.value?.files
|
||||||
|
if (!files || files.length === 0) return
|
||||||
|
|
||||||
|
selectedFile.value = files[0]
|
||||||
|
error.value = ''
|
||||||
|
isLoading.value = true
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await csvService.detect(selectedFile.value)
|
||||||
|
detectionResult.value = result
|
||||||
|
|
||||||
|
// Apply detected values
|
||||||
|
config.value = {
|
||||||
|
delimiter: result.delimiter,
|
||||||
|
quote_char: result.quote_char,
|
||||||
|
date_format: result.date_format,
|
||||||
|
skip_rows: 0,
|
||||||
|
mapping: result.suggested_mapping,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get initial preview
|
||||||
|
await updatePreview()
|
||||||
|
|
||||||
|
step.value = 'mapping'
|
||||||
|
} catch (e) {
|
||||||
|
error.value = getErrorText(e)
|
||||||
|
} finally {
|
||||||
|
isLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updatePreview() {
|
||||||
|
if (!selectedFile.value) return
|
||||||
|
|
||||||
|
isLoading.value = true
|
||||||
|
try {
|
||||||
|
previewResult.value = await csvService.preview(selectedFile.value, config.value)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = getErrorText(e)
|
||||||
|
} finally {
|
||||||
|
isLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function performImport() {
|
||||||
|
if (!selectedFile.value || !hasValidMapping.value) return
|
||||||
|
|
||||||
|
isLoading.value = true
|
||||||
|
error.value = ''
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await csvService.migrate(selectedFile.value, config.value)
|
||||||
|
successMessage.value = result.message
|
||||||
|
|
||||||
|
// Reload projects
|
||||||
|
const projectStore = useProjectStore()
|
||||||
|
await projectStore.loadAllProjects()
|
||||||
|
|
||||||
|
step.value = 'success'
|
||||||
|
} catch (e) {
|
||||||
|
error.value = getErrorText(e)
|
||||||
|
} finally {
|
||||||
|
isLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetToUpload() {
|
||||||
|
step.value = 'upload'
|
||||||
|
selectedFile.value = null
|
||||||
|
detectionResult.value = null
|
||||||
|
previewResult.value = null
|
||||||
|
error.value = ''
|
||||||
|
if (uploadInput.value) {
|
||||||
|
uploadInput.value.value = ''
|
||||||
|
}
|
||||||
|
config.value = {
|
||||||
|
delimiter: ',',
|
||||||
|
quote_char: '"',
|
||||||
|
date_format: '2006-01-02',
|
||||||
|
skip_rows: 0,
|
||||||
|
mapping: [],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style lang="scss" scoped>
|
||||||
|
.csv-migration {
|
||||||
|
max-inline-size: 900px;
|
||||||
|
margin: 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.card {
|
||||||
|
background: var(--white);
|
||||||
|
border-radius: var(--border-radius);
|
||||||
|
padding: 1.5rem;
|
||||||
|
margin-block-end: 1.5rem;
|
||||||
|
box-shadow: var(--shadow-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
.mapping-header {
|
||||||
|
margin-block-end: 1.5rem;
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
margin-block-end: 0.5rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.parsing-options {
|
||||||
|
h3 {
|
||||||
|
margin-block-end: 1rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.options-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||||
|
gap: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.option-group {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.5rem;
|
||||||
|
|
||||||
|
label {
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.column-mappings {
|
||||||
|
h3 {
|
||||||
|
margin-block-end: 1rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.mappings-grid {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.mapping-row {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 1rem;
|
||||||
|
align-items: center;
|
||||||
|
padding: 0.75rem;
|
||||||
|
background: var(--grey-100);
|
||||||
|
border-radius: var(--border-radius);
|
||||||
|
|
||||||
|
@media (width <= 600px) {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.column-name {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.25rem;
|
||||||
|
|
||||||
|
.preview-value {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--grey-500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.preview-section {
|
||||||
|
h3 {
|
||||||
|
margin-block-end: 0.5rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.preview-tasks {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 1rem;
|
||||||
|
margin-block-start: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.preview-task {
|
||||||
|
padding: 1rem;
|
||||||
|
background: var(--grey-50);
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-title {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin-block-end: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.done-badge {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
padding: 0.125rem 0.5rem;
|
||||||
|
background: var(--success);
|
||||||
|
color: var(--white);
|
||||||
|
border-radius: var(--border-radius);
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-description {
|
||||||
|
color: var(--grey-600);
|
||||||
|
margin-block-end: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-meta {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 1rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--grey-500);
|
||||||
|
}
|
||||||
|
|
||||||
|
.actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 1rem;
|
||||||
|
justify-content: flex-end;
|
||||||
|
margin-block-start: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.success-step {
|
||||||
|
text-align: center;
|
||||||
|
padding: 2rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
<svg viewBox="0 0 88 88" xmlns="http://www.w3.org/2000/svg">
|
||||||
|
<g fill="none" fill-rule="evenodd">
|
||||||
|
<!-- Document background -->
|
||||||
|
<rect x="14" y="4" width="60" height="80" rx="4" fill="#4772FA"/>
|
||||||
|
<!-- Folded corner -->
|
||||||
|
<path d="M54 4 L74 24 L54 24 Z" fill="#2a5bd7"/>
|
||||||
|
<!-- CSV text lines representing data rows -->
|
||||||
|
<rect x="22" y="34" width="44" height="6" rx="2" fill="#ffffff"/>
|
||||||
|
<rect x="22" y="46" width="44" height="6" rx="2" fill="#ffffff" opacity="0.8"/>
|
||||||
|
<rect x="22" y="58" width="44" height="6" rx="2" fill="#ffffff" opacity="0.6"/>
|
||||||
|
<rect x="22" y="70" width="30" height="6" rx="2" fill="#ffffff" opacity="0.4"/>
|
||||||
|
<!-- Vertical separators (comma separators) -->
|
||||||
|
<rect x="36" y="34" width="2" height="42" fill="#4772FA" opacity="0.3"/>
|
||||||
|
<rect x="52" y="34" width="2" height="42" fill="#4772FA" opacity="0.3"/>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 875 B |
|
|
@ -5,11 +5,13 @@ import microsoftTodoIcon from './icons/microsoft-todo.svg?url'
|
||||||
import vikunjaFileIcon from './icons/vikunja-file.png?url'
|
import vikunjaFileIcon from './icons/vikunja-file.png?url'
|
||||||
import tickTickIcon from './icons/ticktick.svg?url'
|
import tickTickIcon from './icons/ticktick.svg?url'
|
||||||
import wekanIcon from './icons/wekan.png?url'
|
import wekanIcon from './icons/wekan.png?url'
|
||||||
|
import csvIcon from './icons/csv.svg?url'
|
||||||
|
|
||||||
export interface Migrator {
|
export interface Migrator {
|
||||||
id: string
|
id: string
|
||||||
name: string
|
name: string
|
||||||
isFileMigrator?: boolean
|
isFileMigrator?: boolean
|
||||||
|
isCSVMigrator?: boolean
|
||||||
icon: string
|
icon: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -56,4 +58,11 @@ export const MIGRATORS = {
|
||||||
icon: wekanIcon,
|
icon: wekanIcon,
|
||||||
isFileMigrator: true,
|
isFileMigrator: true,
|
||||||
},
|
},
|
||||||
|
csv: {
|
||||||
|
id: 'csv',
|
||||||
|
name: 'CSV',
|
||||||
|
icon: csvIcon as string,
|
||||||
|
isFileMigrator: true,
|
||||||
|
isCSVMigrator: true,
|
||||||
|
},
|
||||||
} as const satisfies IMigratorRecord
|
} as const satisfies IMigratorRecord
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,714 @@
|
||||||
|
// Vikunja is a to-do list application to facilitate your life.
|
||||||
|
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package csv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/csv"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.vikunja.io/api/pkg/models"
|
||||||
|
"code.vikunja.io/api/pkg/modules/migration"
|
||||||
|
"code.vikunja.io/api/pkg/user"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Migrator is the CSV migrator
|
||||||
|
type Migrator struct{}
|
||||||
|
|
||||||
|
// Name returns the name of this migrator
|
||||||
|
func (m *Migrator) Name() string {
|
||||||
|
return "csv"
|
||||||
|
}
|
||||||
|
|
||||||
|
// SupportedDelimiters contains all supported CSV delimiters
|
||||||
|
var SupportedDelimiters = []string{",", ";", "\t", "|"}
|
||||||
|
|
||||||
|
// SupportedQuoteChars contains all supported quote characters
|
||||||
|
var SupportedQuoteChars = []string{"\"", "'"}
|
||||||
|
|
||||||
|
// SupportedDateFormats contains common date formats for parsing
|
||||||
|
var SupportedDateFormats = []string{
|
||||||
|
"2006-01-02", // ISO date
|
||||||
|
"2006-01-02T15:04:05", // ISO datetime
|
||||||
|
"2006-01-02T15:04:05Z07:00", // RFC3339
|
||||||
|
"2006-01-02T15:04:05-0700", // ISO with timezone
|
||||||
|
"02/01/2006", // DD/MM/YYYY
|
||||||
|
"01/02/2006", // MM/DD/YYYY
|
||||||
|
"02-01-2006", // DD-MM-YYYY
|
||||||
|
"01-02-2006", // MM-DD-YYYY
|
||||||
|
"Jan 2, 2006", // Month D, YYYY
|
||||||
|
"2 Jan 2006", // D Month YYYY
|
||||||
|
"02/01/2006 15:04", // DD/MM/YYYY HH:MM
|
||||||
|
"01/02/2006 15:04", // MM/DD/YYYY HH:MM
|
||||||
|
"2006-01-02 15:04:05", // MySQL datetime
|
||||||
|
"2006/01/02", // YYYY/MM/DD
|
||||||
|
"02.01.2006", // DD.MM.YYYY (European)
|
||||||
|
"02.01.2006 15:04", // DD.MM.YYYY HH:MM (European)
|
||||||
|
time.RFC1123, // RFC1123
|
||||||
|
time.RFC1123Z, // RFC1123 with numeric zone
|
||||||
|
time.RFC822, // RFC822
|
||||||
|
time.RFC822Z, // RFC822 with numeric zone
|
||||||
|
time.RFC850, // RFC850
|
||||||
|
time.ANSIC, // ANSIC
|
||||||
|
time.UnixDate, // Unix date
|
||||||
|
}
|
||||||
|
|
||||||
|
// TaskAttribute represents a task attribute that can be mapped from CSV
|
||||||
|
type TaskAttribute string
|
||||||
|
|
||||||
|
const (
|
||||||
|
AttrTitle TaskAttribute = "title"
|
||||||
|
AttrDescription TaskAttribute = "description"
|
||||||
|
AttrDueDate TaskAttribute = "due_date"
|
||||||
|
AttrStartDate TaskAttribute = "start_date"
|
||||||
|
AttrEndDate TaskAttribute = "end_date"
|
||||||
|
AttrDone TaskAttribute = "done"
|
||||||
|
AttrPriority TaskAttribute = "priority"
|
||||||
|
AttrLabels TaskAttribute = "labels"
|
||||||
|
AttrProject TaskAttribute = "project"
|
||||||
|
AttrReminder TaskAttribute = "reminder"
|
||||||
|
AttrIgnore TaskAttribute = "ignore"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AllTaskAttributes returns all available task attributes for mapping
|
||||||
|
var AllTaskAttributes = []TaskAttribute{
|
||||||
|
AttrTitle,
|
||||||
|
AttrDescription,
|
||||||
|
AttrDueDate,
|
||||||
|
AttrStartDate,
|
||||||
|
AttrEndDate,
|
||||||
|
AttrDone,
|
||||||
|
AttrPriority,
|
||||||
|
AttrLabels,
|
||||||
|
AttrProject,
|
||||||
|
AttrReminder,
|
||||||
|
AttrIgnore,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ColumnMapping represents a mapping from a CSV column to a task attribute
|
||||||
|
type ColumnMapping struct {
|
||||||
|
ColumnIndex int `json:"column_index"`
|
||||||
|
ColumnName string `json:"column_name"`
|
||||||
|
Attribute TaskAttribute `json:"attribute"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectionResult contains the auto-detected CSV structure
|
||||||
|
type DetectionResult struct {
|
||||||
|
Columns []string `json:"columns"`
|
||||||
|
Delimiter string `json:"delimiter"`
|
||||||
|
QuoteChar string `json:"quote_char"`
|
||||||
|
DateFormat string `json:"date_format"`
|
||||||
|
SuggestedMapping []ColumnMapping `json:"suggested_mapping"`
|
||||||
|
PreviewRows [][]string `json:"preview_rows"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImportConfig contains the configuration for CSV import
|
||||||
|
type ImportConfig struct {
|
||||||
|
Delimiter string `json:"delimiter"`
|
||||||
|
QuoteChar string `json:"quote_char"`
|
||||||
|
DateFormat string `json:"date_format"`
|
||||||
|
SkipRows int `json:"skip_rows"`
|
||||||
|
Mapping []ColumnMapping `json:"mapping"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreviewTask represents a task preview before import
|
||||||
|
type PreviewTask struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
DueDate string `json:"due_date,omitempty"`
|
||||||
|
StartDate string `json:"start_date,omitempty"`
|
||||||
|
EndDate string `json:"end_date,omitempty"`
|
||||||
|
Done bool `json:"done"`
|
||||||
|
Priority int `json:"priority"`
|
||||||
|
Labels []string `json:"labels,omitempty"`
|
||||||
|
Project string `json:"project,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreviewResult contains preview data before import
|
||||||
|
type PreviewResult struct {
|
||||||
|
Tasks []PreviewTask `json:"tasks"`
|
||||||
|
TotalRows int `json:"total_rows"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripBOM removes the UTF-8 BOM from the beginning of a reader
|
||||||
|
func stripBOM(data []byte) []byte {
|
||||||
|
if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF {
|
||||||
|
return data[3:]
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectDelimiter attempts to auto-detect the CSV delimiter
|
||||||
|
func detectDelimiter(data []byte) string {
|
||||||
|
content := string(data)
|
||||||
|
|
||||||
|
// Count occurrences of each delimiter in the first few lines
|
||||||
|
lines := strings.SplitN(content, "\n", 5)
|
||||||
|
if len(lines) < 2 {
|
||||||
|
return "," // Default to comma
|
||||||
|
}
|
||||||
|
|
||||||
|
delimiterCounts := make(map[string]int)
|
||||||
|
for _, delim := range SupportedDelimiters {
|
||||||
|
count := 0
|
||||||
|
for _, line := range lines[:minInt(3, len(lines))] {
|
||||||
|
count += strings.Count(line, delim)
|
||||||
|
}
|
||||||
|
delimiterCounts[delim] = count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the delimiter with the most consistent count across lines
|
||||||
|
bestDelimiter := ","
|
||||||
|
maxCount := 0
|
||||||
|
for delim, count := range delimiterCounts {
|
||||||
|
if count > maxCount {
|
||||||
|
maxCount = count
|
||||||
|
bestDelimiter = delim
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return bestDelimiter
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectQuoteChar attempts to auto-detect the quote character
|
||||||
|
func detectQuoteChar(data []byte) string {
|
||||||
|
content := string(data)
|
||||||
|
|
||||||
|
doubleQuotes := strings.Count(content, "\"")
|
||||||
|
singleQuotes := strings.Count(content, "'")
|
||||||
|
|
||||||
|
if singleQuotes > doubleQuotes {
|
||||||
|
return "'"
|
||||||
|
}
|
||||||
|
return "\""
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectDateFormat attempts to detect the date format from sample data
|
||||||
|
func detectDateFormat(sampleDates []string) string {
|
||||||
|
if len(sampleDates) == 0 {
|
||||||
|
return SupportedDateFormats[0] // Default to ISO
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, format := range SupportedDateFormats {
|
||||||
|
matches := 0
|
||||||
|
for _, dateStr := range sampleDates {
|
||||||
|
dateStr = strings.TrimSpace(dateStr)
|
||||||
|
if dateStr == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, err := time.Parse(format, dateStr)
|
||||||
|
if err == nil {
|
||||||
|
matches++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If most dates match this format, use it
|
||||||
|
if matches > 0 && matches >= len(sampleDates)/2 {
|
||||||
|
return format
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return SupportedDateFormats[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// suggestMapping suggests column mappings based on column names
|
||||||
|
func suggestMapping(columns []string) []ColumnMapping {
|
||||||
|
mappings := make([]ColumnMapping, len(columns))
|
||||||
|
|
||||||
|
// Common column name patterns for each attribute
|
||||||
|
patterns := map[TaskAttribute][]string{
|
||||||
|
AttrTitle: {"title", "name", "task", "subject", "summary"},
|
||||||
|
AttrDescription: {"description", "content", "notes", "details", "body", "text"},
|
||||||
|
AttrDueDate: {"due", "due_date", "duedate", "deadline", "due date"},
|
||||||
|
AttrStartDate: {"start", "start_date", "startdate", "begin", "start date"},
|
||||||
|
AttrEndDate: {"end", "end_date", "enddate", "finish", "end date"},
|
||||||
|
AttrDone: {"done", "completed", "complete", "finished", "status", "is_done"},
|
||||||
|
AttrPriority: {"priority", "importance", "urgent", "prio"},
|
||||||
|
AttrLabels: {"labels", "tags", "categories", "category", "label", "tag"},
|
||||||
|
AttrProject: {"project", "list", "folder", "group", "project_name", "list_name"},
|
||||||
|
AttrReminder: {"reminder", "remind", "alert", "notification"},
|
||||||
|
}
|
||||||
|
|
||||||
|
usedAttributes := make(map[TaskAttribute]bool)
|
||||||
|
|
||||||
|
for i, col := range columns {
|
||||||
|
colLower := strings.ToLower(strings.TrimSpace(col))
|
||||||
|
mappings[i] = ColumnMapping{
|
||||||
|
ColumnIndex: i,
|
||||||
|
ColumnName: col,
|
||||||
|
Attribute: AttrIgnore,
|
||||||
|
}
|
||||||
|
|
||||||
|
for attr, keywords := range patterns {
|
||||||
|
if usedAttributes[attr] && attr != AttrLabels {
|
||||||
|
continue // Don't map the same attribute twice (except labels)
|
||||||
|
}
|
||||||
|
for _, keyword := range keywords {
|
||||||
|
if strings.Contains(colLower, keyword) || colLower == keyword {
|
||||||
|
mappings[i].Attribute = attr
|
||||||
|
usedAttributes[attr] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mappings[i].Attribute != AttrIgnore {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mappings
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCSV parses CSV data with the given configuration
|
||||||
|
func parseCSV(data []byte, delimiter, quoteChar string) ([]string, [][]string, error) {
|
||||||
|
data = stripBOM(data)
|
||||||
|
|
||||||
|
// Go's csv.Reader only supports double-quote as the quote character.
|
||||||
|
// If a different quote character is specified, replace it with double-quote
|
||||||
|
// before parsing so that quoted fields are handled correctly.
|
||||||
|
if quoteChar != "" && quoteChar != "\"" {
|
||||||
|
data = bytes.ReplaceAll(data, []byte(quoteChar), []byte("\""))
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := csv.NewReader(bytes.NewReader(data))
|
||||||
|
|
||||||
|
if len(delimiter) > 0 {
|
||||||
|
reader.Comma = rune(delimiter[0])
|
||||||
|
}
|
||||||
|
reader.FieldsPerRecord = -1 // Allow variable field counts
|
||||||
|
reader.LazyQuotes = true
|
||||||
|
reader.TrimLeadingSpace = true
|
||||||
|
|
||||||
|
records, err := reader.ReadAll()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(records) == 0 {
|
||||||
|
return nil, nil, &migration.ErrFileIsEmpty{}
|
||||||
|
}
|
||||||
|
|
||||||
|
headers := records[0]
|
||||||
|
var dataRows [][]string
|
||||||
|
if len(records) > 1 {
|
||||||
|
dataRows = records[1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
return headers, dataRows, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectCSVStructure analyzes a CSV file and returns detection results
|
||||||
|
func DetectCSVStructure(file io.ReaderAt, size int64) (*DetectionResult, error) {
|
||||||
|
if size == 0 {
|
||||||
|
return nil, &migration.ErrFileIsEmpty{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the entire file
|
||||||
|
data := make([]byte, size)
|
||||||
|
_, err := file.ReadAt(data, 0)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect delimiter and quote character
|
||||||
|
delimiter := detectDelimiter(data)
|
||||||
|
quoteChar := detectQuoteChar(data)
|
||||||
|
|
||||||
|
// Parse CSV
|
||||||
|
headers, rows, err := parseCSV(data, delimiter, quoteChar)
|
||||||
|
if err != nil {
|
||||||
|
var emptyErr *migration.ErrFileIsEmpty
|
||||||
|
if errors.As(err, &emptyErr) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, &migration.ErrNotACSVFile{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Suggest column mappings
|
||||||
|
suggestedMapping := suggestMapping(headers)
|
||||||
|
|
||||||
|
// Collect sample dates for format detection
|
||||||
|
var sampleDates []string
|
||||||
|
for _, mapping := range suggestedMapping {
|
||||||
|
if mapping.Attribute == AttrDueDate || mapping.Attribute == AttrStartDate || mapping.Attribute == AttrEndDate {
|
||||||
|
for _, row := range rows {
|
||||||
|
if mapping.ColumnIndex < len(row) && row[mapping.ColumnIndex] != "" {
|
||||||
|
sampleDates = append(sampleDates, row[mapping.ColumnIndex])
|
||||||
|
if len(sampleDates) >= 10 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dateFormat := detectDateFormat(sampleDates)
|
||||||
|
|
||||||
|
// Get preview rows (first 5)
|
||||||
|
previewRows := rows
|
||||||
|
if len(previewRows) > 5 {
|
||||||
|
previewRows = previewRows[:5]
|
||||||
|
}
|
||||||
|
|
||||||
|
return &DetectionResult{
|
||||||
|
Columns: headers,
|
||||||
|
Delimiter: delimiter,
|
||||||
|
QuoteChar: quoteChar,
|
||||||
|
DateFormat: dateFormat,
|
||||||
|
SuggestedMapping: suggestedMapping,
|
||||||
|
PreviewRows: previewRows,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PreviewImport generates a preview of the import based on current mapping
|
||||||
|
func PreviewImport(file io.ReaderAt, size int64, config ImportConfig) (*PreviewResult, error) {
|
||||||
|
if size == 0 {
|
||||||
|
return nil, &migration.ErrFileIsEmpty{}
|
||||||
|
}
|
||||||
|
|
||||||
|
data := make([]byte, size)
|
||||||
|
_, err := file.ReadAt(data, 0)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||||
|
if err != nil {
|
||||||
|
var emptyErr *migration.ErrFileIsEmpty
|
||||||
|
if errors.As(err, &emptyErr) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, &migration.ErrNotACSVFile{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip rows if configured
|
||||||
|
if config.SkipRows > 0 && config.SkipRows < len(rows) {
|
||||||
|
rows = rows[config.SkipRows:]
|
||||||
|
}
|
||||||
|
|
||||||
|
result := &PreviewResult{
|
||||||
|
Tasks: make([]PreviewTask, 0, minInt(5, len(rows))),
|
||||||
|
TotalRows: len(rows),
|
||||||
|
}
|
||||||
|
|
||||||
|
previewCount := minInt(5, len(rows))
|
||||||
|
for i := 0; i < previewCount; i++ {
|
||||||
|
task := rowToPreviewTask(rows[i], config)
|
||||||
|
result.Tasks = append(result.Tasks, task)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rowToPreviewTask converts a CSV row to a preview task
|
||||||
|
func rowToPreviewTask(row []string, config ImportConfig) PreviewTask {
|
||||||
|
task := PreviewTask{}
|
||||||
|
|
||||||
|
for _, mapping := range config.Mapping {
|
||||||
|
if mapping.ColumnIndex >= len(row) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch mapping.Attribute {
|
||||||
|
case AttrTitle:
|
||||||
|
task.Title = value
|
||||||
|
case AttrDescription:
|
||||||
|
task.Description = value
|
||||||
|
case AttrDueDate:
|
||||||
|
task.DueDate = value
|
||||||
|
case AttrStartDate:
|
||||||
|
task.StartDate = value
|
||||||
|
case AttrEndDate:
|
||||||
|
task.EndDate = value
|
||||||
|
case AttrDone:
|
||||||
|
task.Done = parseBool(value)
|
||||||
|
case AttrPriority:
|
||||||
|
task.Priority = parsePriority(value)
|
||||||
|
case AttrLabels:
|
||||||
|
task.Labels = parseLabels(value)
|
||||||
|
case AttrProject:
|
||||||
|
task.Project = value
|
||||||
|
case AttrReminder:
|
||||||
|
// Reminders are not supported in preview tasks
|
||||||
|
case AttrIgnore:
|
||||||
|
// Ignored attributes are not processed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseBool parses various boolean representations
|
||||||
|
func parseBool(value string) bool {
|
||||||
|
lower := strings.ToLower(strings.TrimSpace(value))
|
||||||
|
return lower == "true" || lower == "yes" || lower == "1" || lower == "done" || lower == "completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsePriority parses priority value
|
||||||
|
func parsePriority(value string) int {
|
||||||
|
// Try to parse as number
|
||||||
|
if p, err := strconv.Atoi(strings.TrimSpace(value)); err == nil {
|
||||||
|
// Vikunja uses 0-5 priority (0=unset, 1=low, 5=urgent)
|
||||||
|
if p < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if p > 5 {
|
||||||
|
return 5
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to parse text priority
|
||||||
|
lower := strings.ToLower(strings.TrimSpace(value))
|
||||||
|
switch {
|
||||||
|
case strings.Contains(lower, "urgent") || strings.Contains(lower, "highest"):
|
||||||
|
return 5
|
||||||
|
case strings.Contains(lower, "high"):
|
||||||
|
return 4
|
||||||
|
case strings.Contains(lower, "medium") || strings.Contains(lower, "normal"):
|
||||||
|
return 3
|
||||||
|
case strings.Contains(lower, "lowest"):
|
||||||
|
return 1
|
||||||
|
case strings.Contains(lower, "low"):
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseLabels parses comma-separated labels
|
||||||
|
func parseLabels(value string) []string {
|
||||||
|
parts := strings.Split(value, ",")
|
||||||
|
labels := make([]string, 0, len(parts))
|
||||||
|
for _, part := range parts {
|
||||||
|
label := strings.TrimSpace(part)
|
||||||
|
if label != "" {
|
||||||
|
labels = append(labels, label)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return labels
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDate parses a date string with the given format
|
||||||
|
func parseDate(value, format string) time.Time {
|
||||||
|
if value == "" {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try the specified format first
|
||||||
|
if t, err := time.Parse(format, strings.TrimSpace(value)); err == nil {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try all supported formats as fallback
|
||||||
|
for _, f := range SupportedDateFormats {
|
||||||
|
if t, err := time.Parse(f, strings.TrimSpace(value)); err == nil {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Migrate imports CSV data into Vikunja
|
||||||
|
// @Summary Import all tasks from a CSV file
|
||||||
|
// @Description Imports tasks from a CSV file into Vikunja. Requires a mapping configuration.
|
||||||
|
// @tags migration
|
||||||
|
// @Accept multipart/form-data
|
||||||
|
// @Produce json
|
||||||
|
// @Security JWTKeyAuth
|
||||||
|
// @Param import formData file true "The CSV file to import"
|
||||||
|
// @Param config formData string true "The import configuration JSON"
|
||||||
|
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||||
|
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||||
|
// @Failure 500 {object} models.Message "Internal server error"
|
||||||
|
// @Router /migration/csv/migrate [put]
|
||||||
|
func (m *Migrator) Migrate(_ *user.User, _ io.ReaderAt, _ int64) error {
|
||||||
|
return &migration.ErrCSVConfigRequired{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MigrateWithConfig imports CSV data into Vikunja with the provided configuration
|
||||||
|
func MigrateWithConfig(u *user.User, file io.ReaderAt, size int64, config ImportConfig) error {
|
||||||
|
if size == 0 {
|
||||||
|
return &migration.ErrFileIsEmpty{}
|
||||||
|
}
|
||||||
|
|
||||||
|
data := make([]byte, size)
|
||||||
|
_, err := file.ReadAt(data, 0)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rows, err := parseCSV(data, config.Delimiter, config.QuoteChar)
|
||||||
|
if err != nil {
|
||||||
|
var emptyErr *migration.ErrFileIsEmpty
|
||||||
|
if errors.As(err, &emptyErr) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return &migration.ErrNotACSVFile{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip rows if configured
|
||||||
|
if config.SkipRows > 0 && config.SkipRows < len(rows) {
|
||||||
|
rows = rows[config.SkipRows:]
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return &migration.ErrFileIsEmpty{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert rows to Vikunja structure
|
||||||
|
vikunjaTasks := convertToVikunja(rows, config)
|
||||||
|
|
||||||
|
return migration.InsertFromStructure(vikunjaTasks, u)
|
||||||
|
}
|
||||||
|
|
||||||
|
// convertToVikunja converts CSV rows to Vikunja project/task structure
|
||||||
|
func convertToVikunja(rows [][]string, config ImportConfig) []*models.ProjectWithTasksAndBuckets {
|
||||||
|
var pseudoParentID int64 = 1
|
||||||
|
result := []*models.ProjectWithTasksAndBuckets{
|
||||||
|
{
|
||||||
|
Project: models.Project{
|
||||||
|
ID: pseudoParentID,
|
||||||
|
Title: "Imported from CSV",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
projects := make(map[string]*models.ProjectWithTasksAndBuckets)
|
||||||
|
defaultProjectName := "Tasks"
|
||||||
|
|
||||||
|
for i, row := range rows {
|
||||||
|
task := rowToTask(row, config, int64(i+1))
|
||||||
|
|
||||||
|
// Determine project name
|
||||||
|
projectName := defaultProjectName
|
||||||
|
for _, mapping := range config.Mapping {
|
||||||
|
if mapping.Attribute == AttrProject && mapping.ColumnIndex < len(row) {
|
||||||
|
if pn := strings.TrimSpace(row[mapping.ColumnIndex]); pn != "" {
|
||||||
|
projectName = pn
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get or create project
|
||||||
|
if _, exists := projects[projectName]; !exists {
|
||||||
|
projects[projectName] = &models.ProjectWithTasksAndBuckets{
|
||||||
|
Project: models.Project{
|
||||||
|
ID: int64(len(projects)+2) + pseudoParentID,
|
||||||
|
ParentProjectID: pseudoParentID,
|
||||||
|
Title: projectName,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add task to project
|
||||||
|
projects[projectName].Tasks = append(projects[projectName].Tasks, &models.TaskWithComments{Task: task})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect all projects
|
||||||
|
for _, p := range projects {
|
||||||
|
result = append(result, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort projects by title for consistent ordering
|
||||||
|
sort.Slice(result[1:], func(i, j int) bool {
|
||||||
|
return result[i+1].Title < result[j+1].Title
|
||||||
|
})
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// rowToTask converts a CSV row to a Vikunja task
|
||||||
|
func rowToTask(row []string, config ImportConfig, taskID int64) models.Task {
|
||||||
|
task := models.Task{
|
||||||
|
ID: taskID,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, mapping := range config.Mapping {
|
||||||
|
if mapping.ColumnIndex >= len(row) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
value := strings.TrimSpace(row[mapping.ColumnIndex])
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch mapping.Attribute {
|
||||||
|
case AttrTitle:
|
||||||
|
task.Title = value
|
||||||
|
case AttrDescription:
|
||||||
|
task.Description = value
|
||||||
|
case AttrDueDate:
|
||||||
|
task.DueDate = parseDate(value, config.DateFormat)
|
||||||
|
case AttrStartDate:
|
||||||
|
task.StartDate = parseDate(value, config.DateFormat)
|
||||||
|
case AttrEndDate:
|
||||||
|
task.EndDate = parseDate(value, config.DateFormat)
|
||||||
|
case AttrDone:
|
||||||
|
task.Done = parseBool(value)
|
||||||
|
if task.Done {
|
||||||
|
task.DoneAt = time.Now()
|
||||||
|
}
|
||||||
|
case AttrPriority:
|
||||||
|
task.Priority = int64(parsePriority(value))
|
||||||
|
case AttrLabels:
|
||||||
|
labels := parseLabels(value)
|
||||||
|
for _, labelTitle := range labels {
|
||||||
|
task.Labels = append(task.Labels, &models.Label{Title: labelTitle})
|
||||||
|
}
|
||||||
|
case AttrReminder:
|
||||||
|
// Parse reminder as duration or date
|
||||||
|
reminderDate := parseDate(value, config.DateFormat)
|
||||||
|
if !reminderDate.IsZero() {
|
||||||
|
task.Reminders = []*models.TaskReminder{
|
||||||
|
{
|
||||||
|
Reminder: reminderDate,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case AttrProject:
|
||||||
|
// Project attribute is handled separately for task creation
|
||||||
|
case AttrIgnore:
|
||||||
|
// Ignored attributes are not processed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure task has a title
|
||||||
|
if task.Title == "" {
|
||||||
|
task.Title = "Untitled Task"
|
||||||
|
}
|
||||||
|
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
|
||||||
|
func minInt(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,581 @@
|
||||||
|
// Vikunja is a to-do list application to facilitate your life.
|
||||||
|
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package csv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStripBOM(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input []byte
|
||||||
|
expected []byte
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "with BOM",
|
||||||
|
input: []byte{0xEF, 0xBB, 0xBF, 'H', 'e', 'l', 'l', 'o'},
|
||||||
|
expected: []byte("Hello"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "without BOM",
|
||||||
|
input: []byte("Hello"),
|
||||||
|
expected: []byte("Hello"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty",
|
||||||
|
input: []byte{},
|
||||||
|
expected: []byte{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "only BOM",
|
||||||
|
input: []byte{0xEF, 0xBB, 0xBF},
|
||||||
|
expected: []byte{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result := stripBOM(tc.input)
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectDelimiter(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "comma separated",
|
||||||
|
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||||
|
expected: ",",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "semicolon separated",
|
||||||
|
input: "name;email;phone\nJohn;john@test.com;123\nJane;jane@test.com;456",
|
||||||
|
expected: ";",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "tab separated",
|
||||||
|
input: "name\temail\tphone\nJohn\tjohn@test.com\t123\nJane\tjane@test.com\t456",
|
||||||
|
expected: "\t",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pipe separated",
|
||||||
|
input: "name|email|phone\nJohn|john@test.com|123\nJane|jane@test.com|456",
|
||||||
|
expected: "|",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single line defaults to comma",
|
||||||
|
input: "just a single line",
|
||||||
|
expected: ",",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result := detectDelimiter([]byte(tc.input))
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectQuoteChar(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "double quotes",
|
||||||
|
input: `"name","email"\n"John","john@test.com"`,
|
||||||
|
expected: "\"",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single quotes",
|
||||||
|
input: `'name','email'\n'John','john@test.com'`,
|
||||||
|
expected: "'",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no quotes defaults to double",
|
||||||
|
input: "name,email\nJohn,john@test.com",
|
||||||
|
expected: "\"",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result := detectQuoteChar([]byte(tc.input))
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectDateFormat(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
sampleDates []string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "ISO date",
|
||||||
|
sampleDates: []string{"2024-01-15", "2024-02-20", "2024-03-25"},
|
||||||
|
expected: "2006-01-02",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ISO datetime",
|
||||||
|
sampleDates: []string{"2024-01-15T10:30:00", "2024-02-20T14:45:00"},
|
||||||
|
expected: "2006-01-02T15:04:05",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "European format",
|
||||||
|
sampleDates: []string{"15.01.2024", "20.02.2024", "25.03.2024"},
|
||||||
|
expected: "02.01.2006",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty defaults to ISO",
|
||||||
|
sampleDates: []string{},
|
||||||
|
expected: "2006-01-02",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result := detectDateFormat(tc.sampleDates)
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSuggestMapping(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
columns []string
|
||||||
|
expected map[int]TaskAttribute
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "standard column names",
|
||||||
|
columns: []string{"Title", "Description", "Due Date", "Priority", "Labels"},
|
||||||
|
expected: map[int]TaskAttribute{
|
||||||
|
0: AttrTitle,
|
||||||
|
1: AttrDescription,
|
||||||
|
2: AttrDueDate,
|
||||||
|
3: AttrPriority,
|
||||||
|
4: AttrLabels,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "alternative column names",
|
||||||
|
columns: []string{"Task Name", "Notes", "Deadline", "Tags", "Project"},
|
||||||
|
expected: map[int]TaskAttribute{
|
||||||
|
0: AttrTitle,
|
||||||
|
1: AttrDescription,
|
||||||
|
2: AttrDueDate,
|
||||||
|
3: AttrLabels,
|
||||||
|
4: AttrProject,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown columns",
|
||||||
|
columns: []string{"ID", "Random Column", "Unknown"},
|
||||||
|
expected: map[int]TaskAttribute{
|
||||||
|
0: AttrIgnore,
|
||||||
|
1: AttrIgnore,
|
||||||
|
2: AttrIgnore,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
mappings := suggestMapping(tc.columns)
|
||||||
|
require.Len(t, mappings, len(tc.columns))
|
||||||
|
|
||||||
|
for idx, expectedAttr := range tc.expected {
|
||||||
|
assert.Equal(t, expectedAttr, mappings[idx].Attribute, "Column %d (%s)", idx, tc.columns[idx])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCSV(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
delimiter string
|
||||||
|
quoteChar string
|
||||||
|
expectedCols []string
|
||||||
|
expectedRows int
|
||||||
|
expectedError bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "simple comma CSV",
|
||||||
|
input: "name,email,phone\nJohn,john@test.com,123\nJane,jane@test.com,456",
|
||||||
|
delimiter: ",",
|
||||||
|
quoteChar: "\"",
|
||||||
|
expectedCols: []string{"name", "email", "phone"},
|
||||||
|
expectedRows: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "semicolon CSV",
|
||||||
|
input: "name;email;phone\nJohn;john@test.com;123",
|
||||||
|
delimiter: ";",
|
||||||
|
quoteChar: "\"",
|
||||||
|
expectedCols: []string{"name", "email", "phone"},
|
||||||
|
expectedRows: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "quoted fields",
|
||||||
|
input: "name,description\n\"John Doe\",\"A long, complicated description\"\nJane,Simple",
|
||||||
|
delimiter: ",",
|
||||||
|
quoteChar: "\"",
|
||||||
|
expectedCols: []string{"name", "description"},
|
||||||
|
expectedRows: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with BOM",
|
||||||
|
input: "\xEF\xBB\xBFname,email\nJohn,john@test.com",
|
||||||
|
delimiter: ",",
|
||||||
|
quoteChar: "\"",
|
||||||
|
expectedCols: []string{"name", "email"},
|
||||||
|
expectedRows: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
headers, rows, err := parseCSV([]byte(tc.input), tc.delimiter, tc.quoteChar)
|
||||||
|
|
||||||
|
if tc.expectedError {
|
||||||
|
require.Error(t, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.expectedCols, headers)
|
||||||
|
assert.Len(t, rows, tc.expectedRows)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseBool(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
input string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{"true", true},
|
||||||
|
{"True", true},
|
||||||
|
{"TRUE", true},
|
||||||
|
{"yes", true},
|
||||||
|
{"Yes", true},
|
||||||
|
{"1", true},
|
||||||
|
{"done", true},
|
||||||
|
{"completed", true},
|
||||||
|
{"false", false},
|
||||||
|
{"no", false},
|
||||||
|
{"0", false},
|
||||||
|
{"", false},
|
||||||
|
{"random", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
|
result := parseBool(tc.input)
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePriority(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
input string
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
|
{"0", 0},
|
||||||
|
{"1", 1},
|
||||||
|
{"3", 3},
|
||||||
|
{"5", 5},
|
||||||
|
{"10", 5}, // capped at 5
|
||||||
|
{"-1", 0}, // minimum 0
|
||||||
|
{"low", 2},
|
||||||
|
{"medium", 3},
|
||||||
|
{"high", 4},
|
||||||
|
{"urgent", 5},
|
||||||
|
{"highest", 5},
|
||||||
|
{"lowest", 1},
|
||||||
|
{"normal", 3},
|
||||||
|
{"random", 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
|
result := parsePriority(tc.input)
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLabels(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
input string
|
||||||
|
expected []string
|
||||||
|
}{
|
||||||
|
{"work, personal, urgent", []string{"work", "personal", "urgent"}},
|
||||||
|
{"single", []string{"single"}},
|
||||||
|
{" spaced , labels ", []string{"spaced", "labels"}},
|
||||||
|
{"", []string{}},
|
||||||
|
{",,,", []string{}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.input, func(t *testing.T) {
|
||||||
|
result := parseLabels(tc.input)
|
||||||
|
assert.Equal(t, tc.expected, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectCSVStructure(t *testing.T) {
|
||||||
|
csvContent := `Title,Description,Due Date,Priority,Labels
|
||||||
|
Task 1,Description 1,2024-01-15,high,work
|
||||||
|
Task 2,Description 2,2024-01-20,low,"personal, urgent"
|
||||||
|
Task 3,Description 3,2024-01-25,medium,home`
|
||||||
|
|
||||||
|
reader := bytes.NewReader([]byte(csvContent))
|
||||||
|
|
||||||
|
result, err := DetectCSVStructure(reader, int64(len(csvContent)))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, []string{"Title", "Description", "Due Date", "Priority", "Labels"}, result.Columns)
|
||||||
|
assert.Equal(t, ",", result.Delimiter)
|
||||||
|
assert.Len(t, result.SuggestedMapping, 5)
|
||||||
|
assert.Len(t, result.PreviewRows, 3)
|
||||||
|
|
||||||
|
// Check suggested mappings
|
||||||
|
titleMapping := result.SuggestedMapping[0]
|
||||||
|
assert.Equal(t, AttrTitle, titleMapping.Attribute)
|
||||||
|
assert.Equal(t, "Title", titleMapping.ColumnName)
|
||||||
|
|
||||||
|
descMapping := result.SuggestedMapping[1]
|
||||||
|
assert.Equal(t, AttrDescription, descMapping.Attribute)
|
||||||
|
|
||||||
|
dueDateMapping := result.SuggestedMapping[2]
|
||||||
|
assert.Equal(t, AttrDueDate, dueDateMapping.Attribute)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreviewImport(t *testing.T) {
|
||||||
|
csvContent := `Title,Description,Done,Priority
|
||||||
|
Task 1,Description 1,true,high
|
||||||
|
Task 2,Description 2,false,low
|
||||||
|
Task 3,Description 3,yes,medium
|
||||||
|
Task 4,Description 4,no,urgent
|
||||||
|
Task 5,Description 5,1,normal
|
||||||
|
Task 6,Description 6,0,low`
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
Delimiter: ",",
|
||||||
|
QuoteChar: "\"",
|
||||||
|
DateFormat: "2006-01-02",
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, ColumnName: "Title", Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, ColumnName: "Description", Attribute: AttrDescription},
|
||||||
|
{ColumnIndex: 2, ColumnName: "Done", Attribute: AttrDone},
|
||||||
|
{ColumnIndex: 3, ColumnName: "Priority", Attribute: AttrPriority},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := bytes.NewReader([]byte(csvContent))
|
||||||
|
|
||||||
|
result, err := PreviewImport(reader, int64(len(csvContent)), config)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, 6, result.TotalRows)
|
||||||
|
assert.Len(t, result.Tasks, 5) // Preview limited to 5
|
||||||
|
|
||||||
|
// Check first task
|
||||||
|
assert.Equal(t, "Task 1", result.Tasks[0].Title)
|
||||||
|
assert.Equal(t, "Description 1", result.Tasks[0].Description)
|
||||||
|
assert.True(t, result.Tasks[0].Done)
|
||||||
|
assert.Equal(t, 4, result.Tasks[0].Priority) // "high" -> 4
|
||||||
|
|
||||||
|
// Check second task
|
||||||
|
assert.Equal(t, "Task 2", result.Tasks[1].Title)
|
||||||
|
assert.False(t, result.Tasks[1].Done)
|
||||||
|
assert.Equal(t, 2, result.Tasks[1].Priority) // "low" -> 2
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvertToVikunja(t *testing.T) {
|
||||||
|
rows := [][]string{
|
||||||
|
{"Task 1", "Description 1", "Project A"},
|
||||||
|
{"Task 2", "Description 2", "Project A"},
|
||||||
|
{"Task 3", "Description 3", "Project B"},
|
||||||
|
{"Task 4", "Description 4", ""}, // No project -> default
|
||||||
|
}
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
Delimiter: ",",
|
||||||
|
QuoteChar: "\"",
|
||||||
|
DateFormat: "2006-01-02",
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||||
|
{ColumnIndex: 2, Attribute: AttrProject},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := convertToVikunja(rows, config)
|
||||||
|
|
||||||
|
// Should have parent project + child projects
|
||||||
|
require.GreaterOrEqual(t, len(result), 2)
|
||||||
|
|
||||||
|
// First project should be the parent "Imported from CSV"
|
||||||
|
assert.Equal(t, "Imported from CSV", result[0].Title)
|
||||||
|
|
||||||
|
// Find Project A
|
||||||
|
var projectA, projectB, tasksProject *struct {
|
||||||
|
title string
|
||||||
|
numTasks int
|
||||||
|
}
|
||||||
|
for _, p := range result[1:] {
|
||||||
|
switch p.Title {
|
||||||
|
case "Project A":
|
||||||
|
projectA = &struct {
|
||||||
|
title string
|
||||||
|
numTasks int
|
||||||
|
}{p.Title, len(p.Tasks)}
|
||||||
|
case "Project B":
|
||||||
|
projectB = &struct {
|
||||||
|
title string
|
||||||
|
numTasks int
|
||||||
|
}{p.Title, len(p.Tasks)}
|
||||||
|
case "Tasks":
|
||||||
|
tasksProject = &struct {
|
||||||
|
title string
|
||||||
|
numTasks int
|
||||||
|
}{p.Title, len(p.Tasks)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NotNil(t, projectA, "Project A should exist")
|
||||||
|
assert.Equal(t, 2, projectA.numTasks, "Project A should have 2 tasks")
|
||||||
|
|
||||||
|
assert.NotNil(t, projectB, "Project B should exist")
|
||||||
|
assert.Equal(t, 1, projectB.numTasks, "Project B should have 1 task")
|
||||||
|
|
||||||
|
assert.NotNil(t, tasksProject, "Tasks project should exist for tasks without project")
|
||||||
|
assert.Equal(t, 1, tasksProject.numTasks, "Tasks project should have 1 task")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRowToTask(t *testing.T) {
|
||||||
|
row := []string{"My Task", "Task description", "2024-01-15", "high", "work, urgent"}
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
DateFormat: "2006-01-02",
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||||
|
{ColumnIndex: 2, Attribute: AttrDueDate},
|
||||||
|
{ColumnIndex: 3, Attribute: AttrPriority},
|
||||||
|
{ColumnIndex: 4, Attribute: AttrLabels},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := rowToTask(row, config, 1)
|
||||||
|
|
||||||
|
assert.Equal(t, "My Task", task.Title)
|
||||||
|
assert.Equal(t, "Task description", task.Description)
|
||||||
|
assert.Equal(t, 2024, task.DueDate.Year())
|
||||||
|
assert.Equal(t, 1, int(task.DueDate.Month()))
|
||||||
|
assert.Equal(t, 15, task.DueDate.Day())
|
||||||
|
assert.Equal(t, int64(4), task.Priority) // "high" -> 4
|
||||||
|
require.Len(t, task.Labels, 2)
|
||||||
|
assert.Equal(t, "work", task.Labels[0].Title)
|
||||||
|
assert.Equal(t, "urgent", task.Labels[1].Title)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigratorName(t *testing.T) {
|
||||||
|
m := &Migrator{}
|
||||||
|
assert.Equal(t, "csv", m.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyFile(t *testing.T) {
|
||||||
|
reader := bytes.NewReader([]byte{})
|
||||||
|
|
||||||
|
_, err := DetectCSVStructure(reader, 0)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRowToTaskWithMissingColumns(t *testing.T) {
|
||||||
|
// Row with fewer columns than expected
|
||||||
|
row := []string{"My Task"}
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, Attribute: AttrDescription}, // Index 1 doesn't exist
|
||||||
|
{ColumnIndex: 2, Attribute: AttrDueDate}, // Index 2 doesn't exist
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := rowToTask(row, config, 1)
|
||||||
|
|
||||||
|
// Should still work with available columns
|
||||||
|
assert.Equal(t, "My Task", task.Title)
|
||||||
|
assert.Empty(t, task.Description)
|
||||||
|
assert.True(t, task.DueDate.IsZero())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRowToTaskWithEmptyTitle(t *testing.T) {
|
||||||
|
row := []string{"", "Some description"}
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, Attribute: AttrDescription},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := rowToTask(row, config, 1)
|
||||||
|
|
||||||
|
// Should have default title
|
||||||
|
assert.Equal(t, "Untitled Task", task.Title)
|
||||||
|
assert.Equal(t, "Some description", task.Description)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoneTask(t *testing.T) {
|
||||||
|
row := []string{"Done Task", "completed"}
|
||||||
|
|
||||||
|
config := ImportConfig{
|
||||||
|
Mapping: []ColumnMapping{
|
||||||
|
{ColumnIndex: 0, Attribute: AttrTitle},
|
||||||
|
{ColumnIndex: 1, Attribute: AttrDone},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
task := rowToTask(row, config, 1)
|
||||||
|
|
||||||
|
assert.Equal(t, "Done Task", task.Title)
|
||||||
|
assert.True(t, task.Done)
|
||||||
|
assert.False(t, task.DoneAt.IsZero()) // DoneAt should be set
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,206 @@
|
||||||
|
// Vikunja is a to-do list application to facilitate your life.
|
||||||
|
// Copyright 2018-present Vikunja and contributors. All rights reserved.
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package csv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"code.vikunja.io/api/pkg/models"
|
||||||
|
"code.vikunja.io/api/pkg/modules/migration"
|
||||||
|
user2 "code.vikunja.io/api/pkg/user"
|
||||||
|
"github.com/labstack/echo/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MigratorWeb handles CSV migration HTTP routes
|
||||||
|
type MigratorWeb struct{}
|
||||||
|
|
||||||
|
// RegisterRoutes registers all CSV migration routes
|
||||||
|
func (c *MigratorWeb) RegisterRoutes(g *echo.Group) {
|
||||||
|
g.GET("/csv/status", c.Status)
|
||||||
|
g.PUT("/csv/detect", c.Detect)
|
||||||
|
g.PUT("/csv/preview", c.Preview)
|
||||||
|
g.PUT("/csv/migrate", c.Migrate)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the migration status
|
||||||
|
// @Summary Get CSV migration status
|
||||||
|
// @Description Returns if the current user already did the CSV migration or not.
|
||||||
|
// @tags migration
|
||||||
|
// @Produce json
|
||||||
|
// @Security JWTKeyAuth
|
||||||
|
// @Success 200 {object} migration.Status "The migration status"
|
||||||
|
// @Failure 500 {object} models.Message "Internal server error"
|
||||||
|
// @Router /migration/csv/status [get]
|
||||||
|
func (c *MigratorWeb) Status(ctx *echo.Context) error {
|
||||||
|
u, err := user2.GetCurrentUser(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
m := &Migrator{}
|
||||||
|
s, err := migration.GetMigrationStatus(m, u)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.JSON(http.StatusOK, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect analyzes a CSV file and returns detection results
|
||||||
|
// @Summary Detect CSV structure
|
||||||
|
// @Description Analyzes a CSV file and returns auto-detected columns, delimiter, quote character, and date format with suggested column mappings.
|
||||||
|
// @tags migration
|
||||||
|
// @Accept multipart/form-data
|
||||||
|
// @Produce json
|
||||||
|
// @Security JWTKeyAuth
|
||||||
|
// @Param import formData file true "The CSV file to analyze"
|
||||||
|
// @Success 200 {object} DetectionResult "Detection results with suggested mappings"
|
||||||
|
// @Failure 400 {object} models.Message "Invalid CSV file"
|
||||||
|
// @Failure 500 {object} models.Message "Internal server error"
|
||||||
|
// @Router /migration/csv/detect [put]
|
||||||
|
func (c *MigratorWeb) Detect(ctx *echo.Context) error {
|
||||||
|
_, err := user2.GetCurrentUser(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := ctx.FormFile("import")
|
||||||
|
if err != nil {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := file.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
result, err := DetectCSVStructure(src, file.Size)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.JSON(http.StatusOK, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preview generates a preview of the import
|
||||||
|
// @Summary Preview CSV import
|
||||||
|
// @Description Generates a preview of the first 5 tasks that would be imported with the given configuration.
|
||||||
|
// @tags migration
|
||||||
|
// @Accept multipart/form-data
|
||||||
|
// @Produce json
|
||||||
|
// @Security JWTKeyAuth
|
||||||
|
// @Param import formData file true "The CSV file to preview"
|
||||||
|
// @Param config formData string true "The import configuration JSON"
|
||||||
|
// @Success 200 {object} PreviewResult "Preview of tasks to import"
|
||||||
|
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||||
|
// @Failure 500 {object} models.Message "Internal server error"
|
||||||
|
// @Router /migration/csv/preview [put]
|
||||||
|
func (c *MigratorWeb) Preview(ctx *echo.Context) error {
|
||||||
|
_, err := user2.GetCurrentUser(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := ctx.FormFile("import")
|
||||||
|
if err != nil {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
configStr := ctx.FormValue("config")
|
||||||
|
if configStr == "" {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
var config ImportConfig
|
||||||
|
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := file.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
result, err := PreviewImport(src, file.Size, config)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.JSON(http.StatusOK, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Migrate imports the CSV file
|
||||||
|
// @Summary Import CSV file
|
||||||
|
// @Description Imports tasks from a CSV file into Vikunja with the provided configuration.
|
||||||
|
// @tags migration
|
||||||
|
// @Accept multipart/form-data
|
||||||
|
// @Produce json
|
||||||
|
// @Security JWTKeyAuth
|
||||||
|
// @Param import formData file true "The CSV file to import"
|
||||||
|
// @Param config formData string true "The import configuration JSON"
|
||||||
|
// @Success 200 {object} models.Message "A message telling you everything was migrated successfully."
|
||||||
|
// @Failure 400 {object} models.Message "Invalid CSV file or configuration"
|
||||||
|
// @Failure 500 {object} models.Message "Internal server error"
|
||||||
|
// @Router /migration/csv/migrate [put]
|
||||||
|
func (c *MigratorWeb) Migrate(ctx *echo.Context) error {
|
||||||
|
u, err := user2.GetCurrentUser(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := ctx.FormFile("import")
|
||||||
|
if err != nil {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "No file provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
configStr := ctx.FormValue("config")
|
||||||
|
if configStr == "" {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "No configuration provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
var config ImportConfig
|
||||||
|
if err := json.Unmarshal([]byte(configStr), &config); err != nil {
|
||||||
|
return echo.NewHTTPError(http.StatusBadRequest, "Invalid configuration: "+err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
src, err := file.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
m := &Migrator{}
|
||||||
|
status, err := migration.StartMigration(m, u)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = MigrateWithConfig(u, src, file.Size, config)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = migration.FinishMigration(status)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.JSON(http.StatusOK, models.Message{Message: "Everything was migrated successfully."})
|
||||||
|
}
|
||||||
|
|
@ -60,6 +60,28 @@ func (err *ErrFileIsEmpty) HTTPError() web.HTTPError {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ErrCSVConfigRequired represents an error when the CSV migration endpoint
|
||||||
|
// is called without the required configuration. The CSV migrator requires
|
||||||
|
// a mapping configuration and must be used via /migration/csv/migrate with
|
||||||
|
// a config form field.
|
||||||
|
type ErrCSVConfigRequired struct{}
|
||||||
|
|
||||||
|
func (err *ErrCSVConfigRequired) Error() string {
|
||||||
|
return "CSV import requires a configuration with column mappings. Use the /migration/csv/detect endpoint to get suggested mappings, then call /migration/csv/migrate with a config form field."
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrCodeCSVConfigRequired holds the unique world-error code of this error
|
||||||
|
const ErrCodeCSVConfigRequired = 14004
|
||||||
|
|
||||||
|
// HTTPError holds the http error description
|
||||||
|
func (err *ErrCSVConfigRequired) HTTPError() web.HTTPError {
|
||||||
|
return web.HTTPError{
|
||||||
|
HTTPCode: http.StatusBadRequest,
|
||||||
|
Code: ErrCodeCSVConfigRequired,
|
||||||
|
Message: "CSV import requires a configuration with column mappings. Use the /migration/csv/detect endpoint to get suggested mappings, then call /migration/csv/migrate with a config form field.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ErrNotACSVFile represents a "ErrNotACSVFile" kind of error.
|
// ErrNotACSVFile represents a "ErrNotACSVFile" kind of error.
|
||||||
type ErrNotACSVFile struct{}
|
type ErrNotACSVFile struct{}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ import (
|
||||||
"code.vikunja.io/api/pkg/config"
|
"code.vikunja.io/api/pkg/config"
|
||||||
"code.vikunja.io/api/pkg/log"
|
"code.vikunja.io/api/pkg/log"
|
||||||
"code.vikunja.io/api/pkg/modules/auth/openid"
|
"code.vikunja.io/api/pkg/modules/auth/openid"
|
||||||
|
csvmigrator "code.vikunja.io/api/pkg/modules/migration/csv"
|
||||||
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
||||||
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
||||||
"code.vikunja.io/api/pkg/modules/migration/todoist"
|
"code.vikunja.io/api/pkg/modules/migration/todoist"
|
||||||
|
|
@ -108,6 +109,7 @@ func Info(c *echo.Context) error {
|
||||||
(&vikunja_file.FileMigrator{}).Name(),
|
(&vikunja_file.FileMigrator{}).Name(),
|
||||||
(&ticktick.Migrator{}).Name(),
|
(&ticktick.Migrator{}).Name(),
|
||||||
(&wekan.Migrator{}).Name(),
|
(&wekan.Migrator{}).Name(),
|
||||||
|
(&csvmigrator.Migrator{}).Name(),
|
||||||
},
|
},
|
||||||
Legal: legalInfo{
|
Legal: legalInfo{
|
||||||
ImprintURL: config.LegalImprintURL.GetString(),
|
ImprintURL: config.LegalImprintURL.GetString(),
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,7 @@ import (
|
||||||
"code.vikunja.io/api/pkg/modules/background/unsplash"
|
"code.vikunja.io/api/pkg/modules/background/unsplash"
|
||||||
"code.vikunja.io/api/pkg/modules/background/upload"
|
"code.vikunja.io/api/pkg/modules/background/upload"
|
||||||
"code.vikunja.io/api/pkg/modules/migration"
|
"code.vikunja.io/api/pkg/modules/migration"
|
||||||
|
csvmigrator "code.vikunja.io/api/pkg/modules/migration/csv"
|
||||||
migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler"
|
migrationHandler "code.vikunja.io/api/pkg/modules/migration/handler"
|
||||||
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
microsofttodo "code.vikunja.io/api/pkg/modules/migration/microsoft-todo"
|
||||||
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
"code.vikunja.io/api/pkg/modules/migration/ticktick"
|
||||||
|
|
@ -861,6 +862,10 @@ func registerMigrations(m *echo.Group) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
wekanFileMigrator.RegisterRoutes(m)
|
wekanFileMigrator.RegisterRoutes(m)
|
||||||
|
|
||||||
|
// CSV File Migrator (always enabled - generic import)
|
||||||
|
csvFileMigrator := &csvmigrator.MigratorWeb{}
|
||||||
|
csvFileMigrator.RegisterRoutes(m)
|
||||||
}
|
}
|
||||||
|
|
||||||
func registerCalDavRoutes(c *echo.Group) {
|
func registerCalDavRoutes(c *echo.Group) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue