Fix schema mismatch: load files individually and strip metadata

Files have different metadata columns (e.g., correction_applied in E8 file).
Load each category file separately, remove metadata columns, then concatenate.

Strategy:
1. Load each of 6 category files individually
2. Keep only essential columns (task_id, category, description, options, etc.)
3. Remove varying metadata (correction_applied, source_date, etc.)
4. Concatenate into single dataset

This handles schema differences between corrected and uncorrected files.

app.py

@@ -77,12 +77,45 @@ def load_benchmark_dataset(subset="australian", num_samples=200):
     global dataset_cache
 
     if dataset_cache is None:
-        # Load data files directly as JSON to avoid schema mismatch issues
-        dataset_cache = load_dataset(
-            "json",
-            data_files=f"hf://datasets/Zen0/AusCyberBench/data/{subset}/*.jsonl",
-            split="train"
-        )
+        # Load data files individually to handle different schemas per file
+        from datasets import concatenate_datasets
+
+        # Get list of category files for the subset
+        import glob
+        from huggingface_hub import hf_hub_download
+
+        # Manually specify the categories to avoid globbing issues
+        categories = [
+            "knowledge_terminology",
+            "knowledge_threat_intelligence",
+            "regulatory_essential_eight",
+            "regulatory_ism_controls",
+            "regulatory_privacy_act",
+            "regulatory_soci_act"
+        ]
+
+        datasets_list = []
+        for category in categories:
+            try:
+                ds = load_dataset(
+                    "json",
+                    data_files=f"hf://datasets/Zen0/AusCyberBench/data/{subset}/{category}.jsonl",
+                    split="train"
+                )
+                # Remove metadata columns that may differ between files
+                cols_to_remove = [col for col in ds.column_names if col not in [
+                    'task_id', 'category', 'subcategory', 'title', 'description',
+                    'task_type', 'difficulty', 'answer', 'options', 'context',
+                    'australian_focus', 'regulatory_references'
+                ]]
+                if cols_to_remove:
+                    ds = ds.remove_columns(cols_to_remove)
+                datasets_list.append(ds)
+            except Exception as e:
+                print(f"Warning: Could not load {category}: {e}")
+
+        # Concatenate all datasets
+        dataset_cache = concatenate_datasets(datasets_list)
 
     # Proportional sampling
     import random