feat(forms): add data migration and verification commands
forms:migrate-legacy-data {--dry-run} {--verify-only}
Per-org transaction (outer loop); inside each org, one form_schema per
distinct event_id in registration_form_fields, one form_field per legacy
field (with lowercase→uppercase field_type mapping and PII heuristic),
one form_submission per distinct person_field_values author, one form_value
per legacy row. form_templates derive schema_snapshot in ARCH §4.6.1 shape.
Idempotent via existence checks; skips if registration_form_fields absent.
Wrapped in App\Support\ActivityLog::suppressed() so --dry-run and re-runs
don't storm the activity log.
forms:verify-data-integrity {--strict}
Nine coherence checks: schemas/fields/submissions/values/user_profiles
structure, data migration counts (skipped when legacy tables absent),
orphans, section/schema relation consistency, and strict reachability
(opt-in). Runs all checks to completion; exit 1 on any failure.
Validates binding JSON against config/form_binding.php registry and
field_type against FormFieldType::values() ∪ custom_field_types config.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
472
api/app/Console/Commands/MigrateLegacyFormsData.php
Normal file
472
api/app/Console/Commands/MigrateLegacyFormsData.php
Normal file
@@ -0,0 +1,472 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Enums\FormBuilder\FormFieldType;
|
||||
use App\Enums\FormBuilder\FormPurpose;
|
||||
use App\Enums\FormBuilder\FormSchemaSnapshotMode;
|
||||
use App\Enums\FormBuilder\FormSubmissionMode;
|
||||
use App\Enums\FormBuilder\FormSubmissionStatus;
|
||||
use App\Enums\FormBuilder\FormValueStorageHint;
|
||||
use App\Models\FormBuilder\FormField;
|
||||
use App\Models\FormBuilder\FormSchema;
|
||||
use App\Models\FormBuilder\FormSubmission;
|
||||
use App\Models\FormBuilder\FormTemplate;
|
||||
use App\Models\FormBuilder\FormValue;
|
||||
use App\Support\ActivityLog;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
final class MigrateLegacyFormsData extends Command
|
||||
{
|
||||
protected $signature = 'forms:migrate-legacy-data {--dry-run : Simulate; write nothing} {--verify-only : Skip migration, only run integrity checks}';
|
||||
|
||||
protected $description = 'Migrate legacy registration_form_fields / person_field_values / registration_field_templates to the new form_* tables';
|
||||
|
||||
/** Legacy field_type (lowercase) → new FormFieldType (uppercase). */
|
||||
private const FIELD_TYPE_MAP = [
|
||||
'text' => 'TEXT',
|
||||
'textarea' => 'TEXTAREA',
|
||||
'select' => 'SELECT',
|
||||
'multiselect' => 'MULTISELECT',
|
||||
'checkbox' => 'CHECKBOX_LIST',
|
||||
'radio' => 'RADIO',
|
||||
'boolean' => 'BOOLEAN',
|
||||
'number' => 'NUMBER',
|
||||
'tag_picker' => 'TAG_PICKER',
|
||||
'heading' => 'HEADING',
|
||||
];
|
||||
|
||||
private const PII_SLUG_PATTERNS = [
|
||||
'email', 'phone', 'telefoon', 'adres', 'address',
|
||||
'emergency_contact', 'noodcontact', 'noodnummer',
|
||||
'geboort', 'birthdate', 'birth_date', 'dob',
|
||||
'allergie', 'allergy', 'medisch', 'medical',
|
||||
'dieet', 'diet',
|
||||
'toegangs', 'access',
|
||||
'bsn', 'social_security',
|
||||
'iban', 'bank',
|
||||
];
|
||||
|
||||
private const PII_FIELD_TYPES = ['EMAIL', 'PHONE'];
|
||||
|
||||
public function handle(): int
|
||||
{
|
||||
$dryRun = (bool) $this->option('dry-run');
|
||||
$verifyOnly = (bool) $this->option('verify-only');
|
||||
|
||||
if (! Schema::hasTable('registration_form_fields')) {
|
||||
$this->info('No legacy data to migrate (registration_form_fields table absent). Skipping.');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
if ($verifyOnly) {
|
||||
return $this->verify();
|
||||
}
|
||||
|
||||
return ActivityLog::suppressed(function () use ($dryRun): int {
|
||||
$this->info($dryRun ? 'DRY RUN — no data will be written.' : 'Starting legacy data migration…');
|
||||
|
||||
$orgIds = DB::table('registration_form_fields')
|
||||
->join('events', 'registration_form_fields.event_id', '=', 'events.id')
|
||||
->distinct()
|
||||
->pluck('events.organisation_id');
|
||||
|
||||
$totals = ['schemas' => 0, 'fields' => 0, 'submissions' => 0, 'values' => 0, 'templates' => 0];
|
||||
|
||||
foreach ($orgIds as $orgId) {
|
||||
$orgName = DB::table('organisations')->where('id', $orgId)->value('name') ?? $orgId;
|
||||
$this->info("Migrating organisation {$orgName}…");
|
||||
|
||||
if ($dryRun) {
|
||||
$this->runForOrganisation($orgId, $totals, true);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
DB::transaction(function () use ($orgId, &$totals): void {
|
||||
$this->runForOrganisation($orgId, $totals, false);
|
||||
});
|
||||
}
|
||||
|
||||
// Templates are org-scoped but not per-event.
|
||||
foreach (DB::table('registration_field_templates')->select('organisation_id')->distinct()->pluck('organisation_id') as $orgId) {
|
||||
if ($dryRun) {
|
||||
$this->migrateTemplatesForOrg($orgId, $totals, true);
|
||||
|
||||
continue;
|
||||
}
|
||||
DB::transaction(function () use ($orgId, &$totals): void {
|
||||
$this->migrateTemplatesForOrg($orgId, $totals, false);
|
||||
});
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info('Summary:');
|
||||
$this->table(
|
||||
['What', 'Count'],
|
||||
[
|
||||
['form_schemas', $totals['schemas']],
|
||||
['form_fields', $totals['fields']],
|
||||
['form_submissions', $totals['submissions']],
|
||||
['form_values', $totals['values']],
|
||||
['form_templates', $totals['templates']],
|
||||
]
|
||||
);
|
||||
|
||||
if ($dryRun) {
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info('Running verification…');
|
||||
|
||||
return $this->verify();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $totals
|
||||
*/
|
||||
private function runForOrganisation(string $orgId, array &$totals, bool $dryRun): void
|
||||
{
|
||||
$eventIds = DB::table('registration_form_fields')
|
||||
->join('events', 'registration_form_fields.event_id', '=', 'events.id')
|
||||
->where('events.organisation_id', $orgId)
|
||||
->distinct()
|
||||
->pluck('registration_form_fields.event_id');
|
||||
|
||||
foreach ($eventIds as $eventId) {
|
||||
$event = DB::table('events')->where('id', $eventId)->first();
|
||||
if ($event === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$existingSchema = DB::table('form_schemas')
|
||||
->where('organisation_id', $orgId)
|
||||
->where('owner_type', 'event')
|
||||
->where('owner_id', $eventId)
|
||||
->where('purpose', FormPurpose::EVENT_REGISTRATION->value)
|
||||
->first();
|
||||
|
||||
if ($existingSchema !== null) {
|
||||
$this->line(" Event '{$event->name}': schema already exists, skipping (idempotent).");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$isPublished = in_array(
|
||||
$event->status,
|
||||
['registration_open', 'buildup', 'showday', 'teardown', 'closed'],
|
||||
true
|
||||
);
|
||||
|
||||
$createdByUserId = DB::table('event_user_roles')
|
||||
->where('event_id', $eventId)
|
||||
->where('role', 'admin')
|
||||
->value('user_id');
|
||||
|
||||
$schemaId = (string) Str::ulid();
|
||||
$schemaSlug = $this->dedupeSlug(
|
||||
$orgId,
|
||||
Str::slug(($event->slug ?? $event->name).'-registratie')
|
||||
);
|
||||
|
||||
if (! $dryRun) {
|
||||
DB::table('form_schemas')->insert([
|
||||
'id' => $schemaId,
|
||||
'organisation_id' => $orgId,
|
||||
'owner_type' => 'event',
|
||||
'owner_id' => $eventId,
|
||||
'name' => $event->name.' registratie',
|
||||
'slug' => $schemaSlug,
|
||||
'purpose' => FormPurpose::EVENT_REGISTRATION->value,
|
||||
'description' => null,
|
||||
'is_published' => $isPublished,
|
||||
'submission_mode' => FormSubmissionMode::DRAFT_SINGLE->value,
|
||||
'locale' => 'nl',
|
||||
'version' => 1,
|
||||
'snapshot_mode' => FormSchemaSnapshotMode::NEVER->value,
|
||||
'freeze_on_submit' => false,
|
||||
'section_level_submit' => false,
|
||||
'auto_save_enabled' => false,
|
||||
'created_by_user_id' => $createdByUserId,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]);
|
||||
}
|
||||
|
||||
$totals['schemas']++;
|
||||
|
||||
// Fields — preserve legacy id → new id mapping for form_values.
|
||||
$fieldMap = [];
|
||||
$fieldCount = 0;
|
||||
foreach (DB::table('registration_form_fields')->where('event_id', $eventId)->orderBy('sort_order')->get() as $rff) {
|
||||
$newFieldType = self::FIELD_TYPE_MAP[$rff->field_type] ?? 'TEXT';
|
||||
$hint = $this->hintFor($newFieldType);
|
||||
$isPii = $this->isPii($newFieldType, $rff->slug);
|
||||
|
||||
$newFieldId = (string) Str::ulid();
|
||||
$fieldMap[$rff->id] = ['id' => $newFieldId, 'type' => $newFieldType, 'hint' => $hint];
|
||||
|
||||
if (! $dryRun) {
|
||||
DB::table('form_fields')->insert([
|
||||
'id' => $newFieldId,
|
||||
'form_schema_id' => $schemaId,
|
||||
'field_type' => $newFieldType,
|
||||
'slug' => $rff->slug,
|
||||
'label' => $rff->label,
|
||||
'help_text' => $rff->help_text,
|
||||
'options' => $rff->options,
|
||||
'is_required' => (bool) $rff->is_required,
|
||||
'is_filterable' => (bool) $rff->is_filterable,
|
||||
'is_portal_visible' => (bool) $rff->is_portal_visible,
|
||||
'is_admin_only' => (bool) $rff->is_admin_only,
|
||||
'is_unique' => false,
|
||||
'is_pii' => $isPii,
|
||||
'display_width' => $rff->display_width ?: 'full',
|
||||
'value_storage_hint' => $hint,
|
||||
'review_required' => false,
|
||||
'sort_order' => (int) $rff->sort_order,
|
||||
'created_at' => $rff->created_at ?: now(),
|
||||
'updated_at' => $rff->updated_at ?: now(),
|
||||
]);
|
||||
}
|
||||
$fieldCount++;
|
||||
$totals['fields']++;
|
||||
}
|
||||
|
||||
// Submissions: one per distinct person who has field values for this event.
|
||||
$personIds = DB::table('person_field_values')
|
||||
->join('registration_form_fields', 'person_field_values.registration_form_field_id', '=', 'registration_form_fields.id')
|
||||
->where('registration_form_fields.event_id', $eventId)
|
||||
->distinct()
|
||||
->pluck('person_field_values.person_id');
|
||||
|
||||
$submissionCount = 0;
|
||||
foreach ($personIds as $personId) {
|
||||
$person = DB::table('persons')->where('id', $personId)->first();
|
||||
if ($person === null) {
|
||||
continue;
|
||||
}
|
||||
$submissionStatus = in_array(
|
||||
$person->status,
|
||||
['applied', 'approved', 'no_show'],
|
||||
true
|
||||
)
|
||||
? FormSubmissionStatus::SUBMITTED
|
||||
: FormSubmissionStatus::DRAFT;
|
||||
|
||||
$firstValueCreatedAt = DB::table('person_field_values')
|
||||
->join('registration_form_fields', 'person_field_values.registration_form_field_id', '=', 'registration_form_fields.id')
|
||||
->where('person_field_values.person_id', $personId)
|
||||
->where('registration_form_fields.event_id', $eventId)
|
||||
->min('person_field_values.id'); // id ordering matches creation on AI PKs
|
||||
|
||||
$submissionId = (string) Str::ulid();
|
||||
$submittedAt = $submissionStatus === FormSubmissionStatus::SUBMITTED
|
||||
? ($firstValueCreatedAt ? now() : now())
|
||||
: null;
|
||||
|
||||
if (! $dryRun) {
|
||||
DB::table('form_submissions')->insert([
|
||||
'id' => $submissionId,
|
||||
'form_schema_id' => $schemaId,
|
||||
'subject_type' => 'person',
|
||||
'subject_id' => $personId,
|
||||
'submitted_by_user_id' => $person->user_id ?? null,
|
||||
'status' => $submissionStatus->value,
|
||||
'is_test' => false,
|
||||
'submitted_in_locale' => 'nl',
|
||||
'submitted_at' => $submittedAt,
|
||||
'schema_version_at_submit' => $submissionStatus === FormSubmissionStatus::SUBMITTED ? 1 : null,
|
||||
'auto_save_count' => 0,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
]);
|
||||
}
|
||||
|
||||
// Values for this person.
|
||||
$pfvRows = DB::table('person_field_values')
|
||||
->join('registration_form_fields', 'person_field_values.registration_form_field_id', '=', 'registration_form_fields.id')
|
||||
->where('person_field_values.person_id', $personId)
|
||||
->where('registration_form_fields.event_id', $eventId)
|
||||
->select([
|
||||
'person_field_values.id',
|
||||
'person_field_values.registration_form_field_id',
|
||||
'person_field_values.value',
|
||||
'person_field_values.selected_options',
|
||||
])
|
||||
->get();
|
||||
|
||||
$searchParts = [];
|
||||
foreach ($pfvRows as $pfv) {
|
||||
$map = $fieldMap[$pfv->registration_form_field_id] ?? null;
|
||||
if ($map === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$valueJson = $this->wrapValue($map['type'], $pfv->value, $pfv->selected_options);
|
||||
|
||||
if (! $dryRun) {
|
||||
$fv = FormValue::create([
|
||||
'form_submission_id' => $submissionId,
|
||||
'form_field_id' => $map['id'],
|
||||
'value' => $valueJson,
|
||||
]);
|
||||
unset($fv);
|
||||
}
|
||||
|
||||
if (in_array($map['type'], ['TEXT', 'TEXTAREA', 'EMAIL', 'URL'], true) && is_string($pfv->value)) {
|
||||
$searchParts[] = $pfv->value;
|
||||
}
|
||||
|
||||
$totals['values']++;
|
||||
}
|
||||
|
||||
if (! $dryRun && $searchParts !== []) {
|
||||
DB::table('form_submissions')
|
||||
->where('id', $submissionId)
|
||||
->update(['search_index' => Str::limit(implode(' ', $searchParts), 10000, '')]);
|
||||
}
|
||||
|
||||
$submissionCount++;
|
||||
$totals['submissions']++;
|
||||
}
|
||||
|
||||
$this->line(" Event '{$event->name}': {$fieldCount} fields → 1 schema, {$submissionCount} submissions");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $totals
|
||||
*/
|
||||
private function migrateTemplatesForOrg(string $orgId, array &$totals, bool $dryRun): void
|
||||
{
|
||||
$templates = DB::table('registration_field_templates')->where('organisation_id', $orgId)->get();
|
||||
foreach ($templates as $tpl) {
|
||||
$existing = DB::table('form_templates')
|
||||
->where('organisation_id', $orgId)
|
||||
->where('slug', $tpl->slug)
|
||||
->exists();
|
||||
if ($existing) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$newType = self::FIELD_TYPE_MAP[$tpl->field_type] ?? 'TEXT';
|
||||
$snapshot = [
|
||||
'schema_version' => 1,
|
||||
'snapshot_created_at' => now()->toIso8601String(),
|
||||
'schema' => [
|
||||
'name' => $tpl->label.' (template)',
|
||||
'slug' => $tpl->slug,
|
||||
'purpose' => FormPurpose::EVENT_REGISTRATION->value,
|
||||
'description' => null,
|
||||
'locale' => 'nl',
|
||||
'freeze_on_submit' => false,
|
||||
'section_level_submit' => false,
|
||||
'settings' => [],
|
||||
],
|
||||
'sections' => [],
|
||||
'fields' => [[
|
||||
'id' => $tpl->id,
|
||||
'slug' => $tpl->slug,
|
||||
'field_type' => $newType,
|
||||
'label' => $tpl->label,
|
||||
'help_text' => $tpl->help_text,
|
||||
'section_slug' => null,
|
||||
'options' => $tpl->options ? json_decode($tpl->options, true) : null,
|
||||
'validation_rules' => null,
|
||||
'is_required' => (bool) $tpl->is_required,
|
||||
'is_filterable' => (bool) $tpl->is_filterable,
|
||||
'is_pii' => $this->isPii($newType, $tpl->slug),
|
||||
'binding' => null,
|
||||
'conditional_logic' => null,
|
||||
'translations' => null,
|
||||
'value_storage_hint' => $this->hintFor($newType),
|
||||
'sort_order' => (int) $tpl->sort_order,
|
||||
]],
|
||||
];
|
||||
|
||||
if (! $dryRun) {
|
||||
DB::table('form_templates')->insert([
|
||||
'id' => (string) Str::ulid(),
|
||||
'organisation_id' => $orgId,
|
||||
'name' => $tpl->label,
|
||||
'slug' => $tpl->slug,
|
||||
'purpose' => FormPurpose::EVENT_REGISTRATION->value,
|
||||
'description' => null,
|
||||
'schema_snapshot' => json_encode($snapshot, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'is_system' => (bool) $tpl->is_system,
|
||||
'is_active' => (bool) $tpl->is_active,
|
||||
'created_at' => $tpl->created_at ?: now(),
|
||||
'updated_at' => $tpl->updated_at ?: now(),
|
||||
]);
|
||||
}
|
||||
|
||||
$totals['templates']++;
|
||||
}
|
||||
}
|
||||
|
||||
private function hintFor(string $fieldTypeValue): string
|
||||
{
|
||||
$case = FormFieldType::tryFrom($fieldTypeValue);
|
||||
|
||||
return ($case?->recommendedValueStorageHint() ?? FormValueStorageHint::JSON)->value;
|
||||
}
|
||||
|
||||
private function isPii(string $fieldType, string $slug): bool
|
||||
{
|
||||
if (in_array($fieldType, self::PII_FIELD_TYPES, true)) {
|
||||
return true;
|
||||
}
|
||||
$lower = strtolower($slug);
|
||||
foreach (self::PII_SLUG_PATTERNS as $pattern) {
|
||||
if (str_contains($lower, $pattern)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the canonical JSON-wrapped payload for a form_values row.
|
||||
*/
|
||||
private function wrapValue(string $fieldType, ?string $rawValue, ?string $selectedOptionsJson): array
|
||||
{
|
||||
if (in_array($fieldType, ['MULTISELECT', 'CHECKBOX_LIST', 'TAG_PICKER'], true)) {
|
||||
if ($selectedOptionsJson !== null && $selectedOptionsJson !== '') {
|
||||
$decoded = json_decode($selectedOptionsJson, true);
|
||||
if (is_array($decoded)) {
|
||||
return $decoded;
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
return ['value' => $rawValue];
|
||||
}
|
||||
|
||||
private function dedupeSlug(string $orgId, string $base): string
|
||||
{
|
||||
$slug = $base;
|
||||
$i = 1;
|
||||
while (DB::table('form_schemas')->where('organisation_id', $orgId)->where('slug', $slug)->exists()) {
|
||||
$slug = $base.'-'.(++$i);
|
||||
}
|
||||
|
||||
return $slug;
|
||||
}
|
||||
|
||||
private function verify(): int
|
||||
{
|
||||
return $this->call('forms:verify-data-integrity');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user