From ea45f6fcaf8e578d3badac1321d1541c7a6b69e8 Mon Sep 17 00:00:00 2001 From: "bert.hausmans" Date: Wed, 20 May 2026 21:06:44 +0200 Subject: [PATCH] feat(backend): excel import and export --- packages/backend/src/routes/cards.ts | 26 ++++ packages/backend/src/services/import.test.ts | 57 +++++++++ packages/backend/src/services/import.ts | 127 +++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 packages/backend/src/services/import.test.ts create mode 100644 packages/backend/src/services/import.ts diff --git a/packages/backend/src/routes/cards.ts b/packages/backend/src/routes/cards.ts index 887df32..425eb67 100644 --- a/packages/backend/src/routes/cards.ts +++ b/packages/backend/src/routes/cards.ts @@ -1,10 +1,14 @@ import { Router } from 'express'; +import multer from 'multer'; import { cardCreateSchema, cardUpdateSchema } from '@flashcard/shared'; import type { Db } from '../db/client.js'; +import { ApiError } from '../lib/errors.js'; import { createCard, deleteCard, listCardsByLesson, updateCard } from '../services/cards.js'; +import { exportCardsToBuffer, importCardsFromBuffer } from '../services/import.js'; export function cardsRouter(db: Db): Router { const r = Router({ mergeParams: true }); + const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 10 * 1024 * 1024 } }); r.get('/lessons/:lessonId/cards', async (req, res, next) => { try { res.json(await listCardsByLesson(db, Number(req.params.lessonId))); } catch (e) { next(e); } @@ -28,5 +32,27 @@ export function cardsRouter(db: Db): Router { try { await deleteCard(db, Number(req.params.id)); res.status(204).end(); } catch (e) { next(e); } }); + r.post('/lessons/:lessonId/cards/import', upload.single('file'), async (req, res, next) => { + try { + if (!req.file) throw ApiError.validation('file is required'); + const updateExisting = req.body.updateExisting !== 'false'; + const createMissingLessons = req.body.createMissingLessons === 'true'; + const result = await importCardsFromBuffer( + db, Number(req.params.lessonId), req.file.buffer, { updateExisting, createMissingLessons } + ); + res.json(result); + } catch (e) { next(e); } + }); + + r.get('/lessons/:lessonId/cards/export', async (req, res, next) => { + try { + const includeDescendants = req.query.include_descendants === 'true'; + const buf = await exportCardsToBuffer(db, Number(req.params.lessonId), includeDescendants); + res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); + res.setHeader('Content-Disposition', `attachment; filename="cards-lesson-${req.params.lessonId}.xlsx"`); + res.send(buf); + } catch (e) { next(e); } + }); + return r; } diff --git a/packages/backend/src/services/import.test.ts b/packages/backend/src/services/import.test.ts new file mode 100644 index 0000000..a5e3fee --- /dev/null +++ b/packages/backend/src/services/import.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import * as XLSX from 'xlsx'; +import { makeTestDb } from '../tests/dbHelper.js'; +import { createLesson } from './lessons.js'; +import { importCardsFromBuffer, exportCardsToBuffer } from './import.js'; +import { listCardsByLesson } from './cards.js'; + +let env: ReturnType; +beforeEach(() => { env = makeTestDb(); }); + +function buildXlsx(rows: Record[]): Buffer { + const ws = XLSX.utils.json_to_sheet(rows); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, ws, 'cards'); + return XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }) as Buffer; +} + +describe('excel import', () => { + it('imports cards into the target lesson when no lesson_path column present', async () => { + const l = await createLesson(env.db, { name: 'L' }); + const buf = buildXlsx([ + { question: 'q1', answer: 'a1' }, + { question: 'q2', answer: 'a2', hint: 'tip' }, + ]); + const result = await importCardsFromBuffer(env.db, l.id, buf, { updateExisting: true, createMissingLessons: false }); + expect(result.inserted).toBe(2); + expect(result.errors).toHaveLength(0); + expect(await listCardsByLesson(env.db, l.id)).toHaveLength(2); + }); + + it('creates intermediate lessons from lesson_path when allowed', async () => { + const root = await createLesson(env.db, { name: 'Spaans' }); + const buf = buildXlsx([ + { question: 'hola', answer: 'hello', lesson_path: 'Spaans/Begroetingen' }, + ]); + const result = await importCardsFromBuffer(env.db, root.id, buf, { updateExisting: true, createMissingLessons: true }); + expect(result.inserted).toBe(1); + }); + + it('updates an existing card on duplicate question in same lesson', async () => { + const l = await createLesson(env.db, { name: 'L' }); + await importCardsFromBuffer(env.db, l.id, buildXlsx([{ question: 'q', answer: 'a' }]), { updateExisting: true, createMissingLessons: false }); + const res = await importCardsFromBuffer(env.db, l.id, buildXlsx([{ question: 'q', answer: 'b' }]), { updateExisting: true, createMissingLessons: false }); + expect(res.updated).toBe(1); + const cards = await listCardsByLesson(env.db, l.id); + expect(cards[0]!.answer).toBe('b'); + }); + + it('exports a buffer that round-trips back via import', async () => { + const l = await createLesson(env.db, { name: 'L' }); + await importCardsFromBuffer(env.db, l.id, buildXlsx([{ question: 'q', answer: 'a' }]), { updateExisting: true, createMissingLessons: false }); + const buf = await exportCardsToBuffer(env.db, l.id, false); + const wb = XLSX.read(buf, { type: 'buffer' }); + const rows = XLSX.utils.sheet_to_json(wb.Sheets[wb.SheetNames[0]!]!); + expect(rows).toHaveLength(1); + }); +}); diff --git a/packages/backend/src/services/import.ts b/packages/backend/src/services/import.ts new file mode 100644 index 0000000..80b50fd --- /dev/null +++ b/packages/backend/src/services/import.ts @@ -0,0 +1,127 @@ +import { and, eq, inArray, isNull } from 'drizzle-orm'; +import * as XLSX from 'xlsx'; +import type { Db } from '../db/client.js'; +import { cardProgress, cards, lessons } from '../db/schema.js'; +import { getDescendantLessonIds } from './lessons.js'; + +export interface ImportRow { + question: string; + answer: string; + hint?: string; + lesson_path?: string; +} +export interface ImportOptions { + updateExisting: boolean; + createMissingLessons: boolean; +} +export interface ImportResult { + inserted: number; + updated: number; + skipped: number; + errors: { row: number; message: string }[]; +} + +function parseSheet(buf: Buffer): ImportRow[] { + const wb = XLSX.read(buf, { type: 'buffer' }); + const firstName = wb.SheetNames[0]; + if (!firstName) return []; + const sheet = wb.Sheets[firstName]; + if (!sheet) return []; + return XLSX.utils.sheet_to_json(sheet, { defval: '' }); +} + +function nowSec() { return Math.floor(Date.now() / 1000); } + +async function resolveLesson( + db: Db, + defaultLessonId: number, + lessonPath: string | undefined, + createMissing: boolean +): Promise { + if (!lessonPath || lessonPath.trim() === '') return defaultLessonId; + const parts = lessonPath.split('/').map((s) => s.trim()).filter(Boolean); + if (parts.length === 0) return defaultLessonId; + let parentId: number | null = null; + for (const name of parts) { + const found: typeof lessons.$inferSelect | undefined = parentId === null + ? db.select().from(lessons).where(and(eq(lessons.name, name), isNull(lessons.parentId))).get() + : db.select().from(lessons).where(and(eq(lessons.name, name), eq(lessons.parentId, parentId))).get(); + if (found) { parentId = found.id; continue; } + if (!createMissing) return null; + const [row] = db.insert(lessons).values({ name, parentId, position: 0 }).returning().all(); + parentId = row!.id; + } + return parentId; +} + +export async function importCardsFromBuffer( + db: Db, defaultLessonId: number, buffer: Buffer, opts: ImportOptions +): Promise { + const rows = parseSheet(buffer); + const result: ImportResult = { inserted: 0, updated: 0, skipped: 0, errors: [] }; + for (let i = 0; i < rows.length; i++) { + const row = rows[i]!; + const rowNum = i + 2; + const q = String(row.question ?? '').trim(); + const a = String(row.answer ?? '').trim(); + if (!q || !a) { + result.errors.push({ row: rowNum, message: 'question and answer are required' }); + continue; + } + const targetLessonId = await resolveLesson(db, defaultLessonId, row.lesson_path, opts.createMissingLessons); + if (targetLessonId === null) { + result.errors.push({ row: rowNum, message: `lesson_path not found: ${row.lesson_path}` }); + continue; + } + const existing = db.select().from(cards).where(and(eq(cards.lessonId, targetLessonId), eq(cards.question, q))).get(); + const hint = row.hint && String(row.hint).trim() !== '' ? String(row.hint) : null; + if (existing) { + if (!opts.updateExisting) { result.skipped += 1; continue; } + db.update(cards).set({ answer: a, hint, updatedAt: nowSec() }).where(eq(cards.id, existing.id)).run(); + result.updated += 1; + } else { + const positions = db.select({ pos: cards.position }).from(cards).where(eq(cards.lessonId, targetLessonId)).all(); + const position = positions.length === 0 ? 0 : Math.max(...positions.map((p) => p.pos)) + 1; + const [inserted] = db.insert(cards).values({ lessonId: targetLessonId, question: q, answer: a, hint, position }).returning().all(); + db.insert(cardProgress).values({ cardId: inserted!.id, direction: 'forward', box: 1, nextDueAt: 0 }).run(); + const lesson = db.select().from(lessons).where(eq(lessons.id, targetLessonId)).get(); + if (lesson?.bidirectional) { + db.insert(cardProgress).values({ cardId: inserted!.id, direction: 'backward', box: 1, nextDueAt: 0 }).run(); + } + result.inserted += 1; + } + } + return result; +} + +export async function exportCardsToBuffer(db: Db, lessonId: number, includeDescendants: boolean): Promise { + const ids = includeDescendants ? await getDescendantLessonIds(db, lessonId) : [lessonId]; + const lessonRows = db.select().from(lessons).where(inArray(lessons.id, ids)).all(); + const pathById = buildPathMap(lessonRows); + const cardRows = db.select().from(cards).where(inArray(cards.lessonId, ids)).all(); + const data = cardRows.map((c) => ({ + question: c.question, + answer: c.answer, + hint: c.hint ?? '', + lesson_path: pathById.get(c.lessonId) ?? '', + })); + const ws = XLSX.utils.json_to_sheet(data); + const wb = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(wb, ws, 'cards'); + return XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }) as Buffer; +} + +function buildPathMap(allLessons: { id: number; name: string; parentId: number | null }[]): Map { + const byId = new Map(allLessons.map((l) => [l.id, l])); + const cache = new Map(); + function path(id: number): string { + if (cache.has(id)) return cache.get(id)!; + const l = byId.get(id); + if (!l) return ''; + const p = l.parentId === null ? l.name : `${path(l.parentId)}/${l.name}`; + cache.set(id, p); + return p; + } + for (const l of allLessons) path(l.id); + return cache; +}