bert.hausmans/crewli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(forms): add data migration and verification commands

Browse Source 
forms:migrate-legacy-data {--dry-run} {--verify-only}
  Per-org transaction (outer loop); inside each org, one form_schema per
  distinct event_id in registration_form_fields, one form_field per legacy
  field (with lowercase→uppercase field_type mapping and PII heuristic),
  one form_submission per distinct person_field_values author, one form_value
  per legacy row. form_templates derive schema_snapshot in ARCH §4.6.1 shape.
  Idempotent via existence checks; skips if registration_form_fields absent.
  Wrapped in App\Support\ActivityLog::suppressed() so --dry-run and re-runs
  don't storm the activity log.

forms:verify-data-integrity {--strict}
  Nine coherence checks: schemas/fields/submissions/values/user_profiles
  structure, data migration counts (skipped when legacy tables absent),
  orphans, section/schema relation consistency, and strict reachability
  (opt-in). Runs all checks to completion; exit 1 on any failure.
  Validates binding JSON against config/form_binding.php registry and
  field_type against FormFieldType::values() ∪ custom_field_types config.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
bert.hausmans
2026-04-17 13:18:42 +02:00
parent 85815ccb16
commit 72892d38f4
2 changed files with 879 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

407
api/app/Console/Commands/VerifyFormsDataIntegrity.php Normal file
View File

@@ -0,0 +1,407 @@
<?php
declare(strict_types=1);
namespace App\Console\Commands;
use App\Enums\FormBuilder\FormFieldType;
use App\Enums\FormBuilder\FormPurpose;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
/**
* Exit codes:
* 0 — all checks passed
* 1 — at least one check failed (strict or non-strict)
*
* Checks always run to completion; the final exit code reflects the worst
* result across all checks.
*/
final class VerifyFormsDataIntegrity extends Command
{
protected $signature = 'forms:verify-data-integrity {--strict : Also run strict reachability and activity-log spot-checks}';
protected $description = 'Verify integrity of form-builder data (schemas, fields, submissions, values, user_profiles)';
private bool $hasFailure = false;
public function handle(): int
{
$this->checkSchemaCoherence();
$this->checkFieldCoherence();
$this->checkSubmissionCoherence();
$this->checkValueCoherence();
$this->checkUserProfileCoherence();
$this->checkDataMigrationCounts();
$this->checkOrphans();
$this->checkRelationConsistency();
if ((bool) $this->option('strict')) {
$this->checkStrictReachability();
}
return $this->hasFailure ? self::FAILURE : self::SUCCESS;
}
private function recordPass(string $check, string $note = ''): void
{
$this->info("[PASS] {$check}".($note !== '' ? ": {$note}" : ''));
}
private function recordFailure(string $check, string $detail, string $fix = ''): void
{
$this->hasFailure = true;
$this->error("[FAIL] {$check}: {$detail}");
if ($fix !== '') {
$this->line(" → Suggested fix: {$fix}");
}
}
// ---- 1. schemas ----
private function checkSchemaCoherence(): void
{
$total = DB::table('form_schemas')->count();
$purposeValues = FormPurpose::values();
$invalidPurpose = DB::table('form_schemas')
->whereNotIn('purpose', $purposeValues)
->count();
$customMismatch = DB::table('form_schemas')
->where(function ($q): void {
$q->where(function ($q): void {
$q->where('purpose', FormPurpose::CUSTOM->value)
->whereNull('custom_purpose_slug');
})->orWhere(function ($q): void {
$q->where('purpose', '!=', FormPurpose::CUSTOM->value)
->whereNotNull('custom_purpose_slug');
});
})
->count();
$publicTokenDupes = DB::table('form_schemas')
->whereNotNull('public_token')
->select('public_token')
->groupBy('public_token')
->havingRaw('COUNT(*) > 1')
->count();
if ($invalidPurpose > 0 || $customMismatch > 0 || $publicTokenDupes > 0) {
$this->recordFailure('Schema coherence',
"{$invalidPurpose} invalid purpose, {$customMismatch} custom_purpose_slug mismatches, {$publicTokenDupes} duplicate public_tokens"
);
return;
}
$this->recordPass('Schema coherence', "{$total} schemas verified");
}
// ---- 2. fields ----
private function checkFieldCoherence(): void
{
$total = DB::table('form_fields')->count();
$orphanSchema = DB::table('form_fields')
->leftJoin('form_schemas', 'form_fields.form_schema_id', '=', 'form_schemas.id')
->whereNull('form_schemas.id')
->count();
$builtInTypes = FormFieldType::values();
$customTypes = array_keys((array) config('form_builder.custom_field_types', []));
$validTypes = array_merge($builtInTypes, $customTypes);
$invalidType = DB::table('form_fields')
->whereNotIn('field_type', $validTypes)
->count();
$nonFilterableMarkedFilterable = DB::table('form_fields')
->where('is_filterable', true)
->whereIn('field_type', [
FormFieldType::TEXTAREA->value,
FormFieldType::FILE_UPLOAD->value,
FormFieldType::IMAGE_UPLOAD->value,
FormFieldType::SIGNATURE->value,
FormFieldType::HEADING->value,
FormFieldType::PARAGRAPH->value,
FormFieldType::TABLE_ROWS->value,
FormFieldType::SECTION_PRIORITY->value,
FormFieldType::AVAILABILITY_PICKER->value,
])
->count();
// slug uniqueness within schema (non-deleted)
$dupSlugs = DB::table('form_fields')
->whereNull('deleted_at')
->select('form_schema_id', 'slug')
->groupBy('form_schema_id', 'slug')
->havingRaw('COUNT(*) > 1')
->count();
// Binding registry cross-check
$binding = (array) config('form_binding', []);
$badBindings = 0;
$invalidBindings = DB::table('form_fields')->whereNotNull('binding')->select('binding')->get();
foreach ($invalidBindings as $row) {
$b = is_string($row->binding) ? json_decode($row->binding, true) : null;
if (! is_array($b) || ! isset($b['mode'], $b['entity'], $b['column'])) {
$badBindings++;
continue;
}
if (! in_array($b['mode'], ['entity_owned', 'mirrored'], true)) {
$badBindings++;
continue;
}
if (! isset($binding[$b['entity']][$b['column']])) {
$badBindings++;
continue;
}
if (($binding[$b['entity']][$b['column']]['writable'] ?? false) !== true) {
$badBindings++;
}
}
if ($orphanSchema > 0 || $invalidType > 0 || $nonFilterableMarkedFilterable > 0 || $dupSlugs > 0 || $badBindings > 0) {
$this->recordFailure('Field coherence',
"{$orphanSchema} orphan schema, {$invalidType} invalid field_type, {$nonFilterableMarkedFilterable} unfilterable-but-marked, {$dupSlugs} duplicate slugs, {$badBindings} invalid bindings"
);
return;
}
$this->recordPass('Field coherence', "{$total} fields verified");
}
// ---- 3. submissions ----
private function checkSubmissionCoherence(): void
{
$total = DB::table('form_submissions')->count();
$subjectMismatch = DB::table('form_submissions')
->where(function ($q): void {
$q->whereNotNull('subject_type')->whereNull('subject_id');
})
->orWhere(function ($q): void {
$q->whereNull('subject_type')->whereNotNull('subject_id');
})
->count();
$subjectTypes = array_keys((array) config('form_subjects', []));
$invalidSubjectType = DB::table('form_submissions')
->whereNotNull('subject_type')
->whereNotIn('subject_type', $subjectTypes)
->count();
$submittedWithoutTs = DB::table('form_submissions')
->where('status', 'submitted')
->whereNull('submitted_at')
->count();
$draftWithTs = DB::table('form_submissions')
->where('status', 'draft')
->whereNotNull('submitted_at')
->count();
if ($subjectMismatch > 0 || $invalidSubjectType > 0 || $submittedWithoutTs > 0 || $draftWithTs > 0) {
$this->recordFailure('Submission coherence',
"{$subjectMismatch} subject_type/id mismatch, {$invalidSubjectType} invalid subject_type, {$submittedWithoutTs} submitted without submitted_at, {$draftWithTs} draft with submitted_at"
);
return;
}
$this->recordPass('Submission coherence', "{$total} submissions verified");
}
// ---- 4. values ----
private function checkValueCoherence(): void
{
$total = DB::table('form_values')->count();
$orphanSub = DB::table('form_values')
->leftJoin('form_submissions', 'form_values.form_submission_id', '=', 'form_submissions.id')
->whereNull('form_submissions.id')
->count();
$orphanField = DB::table('form_values')
->leftJoin('form_fields', 'form_values.form_field_id', '=', 'form_fields.id')
->whereNull('form_fields.id')
->count();
$dup = DB::table('form_values')
->select('form_submission_id', 'form_field_id')
->groupBy('form_submission_id', 'form_field_id')
->havingRaw('COUNT(*) > 1')
->count();
$longIndexed = DB::table('form_values')
->whereNotNull('value_indexed')
->whereRaw('CHAR_LENGTH(value_indexed) > 255')
->count();
$nonFilterableIndexed = DB::table('form_values')
->join('form_fields', 'form_values.form_field_id', '=', 'form_fields.id')
->whereNotNull('form_values.value_indexed')
->where('form_fields.is_filterable', false)
->count();
$multiValueIndexed = DB::table('form_values')
->join('form_fields', 'form_values.form_field_id', '=', 'form_fields.id')
->whereNotNull('form_values.value_indexed')
->whereIn('form_fields.field_type', [
FormFieldType::MULTISELECT->value,
FormFieldType::CHECKBOX_LIST->value,
FormFieldType::TAG_PICKER->value,
])
->count();
if ($orphanSub > 0 || $orphanField > 0 || $dup > 0 || $longIndexed > 0 || $multiValueIndexed > 0) {
$this->recordFailure('Value coherence',
"{$orphanSub} orphan submission, {$orphanField} orphan field, {$dup} duplicate pairs, {$longIndexed} over-length value_indexed, {$multiValueIndexed} multi-value rows with value_indexed set"
);
return;
}
// Warning only — doesn't fail the check.
if ($nonFilterableIndexed > 0) {
$this->warn(" [WARN] {$nonFilterableIndexed} form_values have value_indexed set for a non-filterable field");
}
$this->recordPass('Value coherence', "{$total} values verified");
}
// ---- 5. user profiles ----
private function checkUserProfileCoherence(): void
{
$userCount = DB::table('users')->whereNull('deleted_at')->count();
$profilesCount = DB::table('user_profiles')->count();
$missing = DB::table('users')
->whereNull('deleted_at')
->whereNotIn('id', DB::table('user_profiles')->select('user_id'))
->count();
$orphans = DB::table('user_profiles')
->whereNotIn('user_id', DB::table('users')->select('id'))
->count();
$outOfRange = DB::table('user_profiles')
->where(function ($q): void {
$q->where('reliability_score', '<', 0.00)
->orWhere('reliability_score', '>', 5.00);
})
->count();
if ($missing > 0 || $orphans > 0 || $outOfRange > 0) {
$this->recordFailure('User profile coherence',
"{$missing} users without profile, {$orphans} orphan profiles, {$outOfRange} score out of [0,5]"
);
return;
}
$this->recordPass('User profile coherence', "{$profilesCount} profiles (users: {$userCount})");
}
// ---- 6. data migration counts (only if legacy tables present) ----
private function checkDataMigrationCounts(): void
{
if (! Schema::hasTable('registration_form_fields')) {
$this->recordPass('Data migration counts', 'legacy tables already dropped, skipping');
return;
}
$legacyFields = DB::table('registration_form_fields')->count();
$legacyValues = DB::table('person_field_values')->count();
$legacyTemplates = DB::table('registration_field_templates')->count();
$newFields = DB::table('form_fields')->whereNull('deleted_at')->count();
$newValues = DB::table('form_values')->count();
$newTemplates = DB::table('form_templates')->count();
if ($legacyFields === 0 && $legacyValues === 0 && $legacyTemplates === 0) {
$this->recordPass('Data migration counts', 'legacy tables are empty, nothing to verify');
return;
}
$ok = $newFields >= $legacyFields && $newValues >= $legacyValues && $newTemplates >= $legacyTemplates;
if (! $ok) {
$this->recordFailure('Data migration counts',
"fields: legacy={$legacyFields} new={$newFields}; values: legacy={$legacyValues} new={$newValues}; templates: legacy={$legacyTemplates} new={$newTemplates}"
);
return;
}
$this->recordPass('Data migration counts', "fields {$legacyFields}→{$newFields}, values {$legacyValues}→{$newValues}, templates {$legacyTemplates}→{$newTemplates}");
}
// ---- 7. orphans ----
private function checkOrphans(): void
{
$orphanPivotValue = DB::table('form_value_options')
->leftJoin('form_values', 'form_value_options.form_value_id', '=', 'form_values.id')
->whereNull('form_values.id')
->count();
$orphanSectionStatus = DB::table('form_submission_section_statuses')
->leftJoin('form_submissions', 'form_submission_section_statuses.form_submission_id', '=', 'form_submissions.id')
->whereNull('form_submissions.id')
->count();
if ($orphanPivotValue > 0 || $orphanSectionStatus > 0) {
$this->recordFailure('Orphan records', "{$orphanPivotValue} orphan form_value_options, {$orphanSectionStatus} orphan section_statuses");
return;
}
$this->recordPass('Orphan records', 'none');
}
// ---- 8. relation consistency ----
private function checkRelationConsistency(): void
{
$mismatchedSection = DB::table('form_fields as ff')
->join('form_schema_sections as fss', 'ff.form_schema_section_id', '=', 'fss.id')
->whereColumn('ff.form_schema_id', '!=', 'fss.form_schema_id')
->count();
if ($mismatchedSection > 0) {
$this->recordFailure('Relation consistency',
"{$mismatchedSection} form_fields.form_schema_section_id points at a section in a different schema"
);
return;
}
$this->recordPass('Relation consistency', 'all section/schema relations aligned');
}
// ---- 9. strict ----
private function checkStrictReachability(): void
{
$unreachableValues = DB::table('form_values as fv')
->join('form_submissions as fs', 'fv.form_submission_id', '=', 'fs.id')
->join('form_fields as ff', 'fv.form_field_id', '=', 'ff.id')
->whereColumn('ff.form_schema_id', '!=', 'fs.form_schema_id')
->count();
if ($unreachableValues > 0) {
$this->recordFailure('Strict reachability',
"{$unreachableValues} form_values where field.schema != submission.schema"
);
return;
}
$this->recordPass('Strict reachability', 'all values reachable via submission.schema→field');
}
}