{ "schema_version": "verified-gameplay-icon-classifier-public-review.v1", "article_language": "zh-CN", "article_version": "first-public-review-draft", "experiment_status": "IMPLEMENTED_SMOKE_ONLY_FINAL_BENCHMARK_NOT_RUN", "headline_claim": "Experiment 4 now has an isolated, reproducible data and training pipeline; no scientific model winner is claimed because only a one-update smoke run was executed.", "scope": { "input": "one crop image", "outputs": [ "validity probability", "four-way visual type", "type-specific identity" ], "included_tasks": [ "crop-level icon classification", "crop validity rejection" ], "excluded_model_inputs": [ "OCR output", "scene-classification prior", "full screenshot", "video context", "detector proposals", "bounding-box regression", "Simulator state" ] }, "historical_audit": { "v1_production": { "description": "B02 catalog-only 354-class seed; its gameplay adaptation failed the gate, so production remained unchanged.", "validation": {"correct": 208, "total": 213, "accuracy": 0.9765}, "safety_validation": {"correct": 72, "total": 72, "accuracy": 1.0} }, "rejected_v5_full_head": { "description": "Used 1,055 deduplicated Hero/Skill gameplay crops and updated the full identity head; it regressed and was never deployed.", "validation": {"correct": 205, "total": 213, "accuracy": 0.9624}, "safety_validation": {"correct": 66, "total": 72, "accuracy": 0.9167}, "paired_changes_vs_v1": { "regressions": 10, "improvements": 1, "safety_regressions_from_one_identity_and_video_condition": 6 } }, "old_weights_safe_v5": { "description": "Old 354 rows matched V1; the failed appended class exposed a fail-closed publishing defect that has since been fixed.", "validation": {"correct": 208, "total": 213, "accuracy": 0.9765}, "safety_validation": {"correct": 72, "total": 72, "accuracy": 1.0} }, "confirmed_engineering_defects": [ "The historical balanced sampler used only 545 of 770 available real training crops and never consumed 225 crops.", "One synthetic validation crop dominated checkpoint selection.", "A failed appended-class gate still left an unvalidated output row enabled.", "Historical run configuration and sample provenance were too mutable or aggregate for exact retraining." ] }, "refactor": { "stable_core_retained": [ "annotation and review UI", "annotation database and human review records", "video and frame loading", "fixed-layout crop extraction", "battle class-agnostic detection", "catalog/reference reading", "OCR infrastructure", "scene-classification infrastructure", "generic dataset snapshot/export utilities" ], "experimental_code_moved_out_of_core": [ "icon model architecture", "icon training and losses", "icon-specific sampling", "icon checkpoint selection", "icon evaluation and reporting", "experimental confidence rejection" ], "preserved_history": [ "B01 and B02 experiment sources", "B03 detector results and configs", "V1/V5 audit reports", "historical checkpoints and local run artifacts" ], "production_side_effects": { "production_checkpoint_overwritten": false, "production_predictions_refreshed": false, "annotation_state_rebuilt": false, "raw_videos_modified": false } }, "frozen_snapshot": { "name": "verified-gameplay-crops-2026-08-31", "physically_copied": true, "fail_if_destination_exists": true, "catalog_visual_count": 390, "catalog_identity_count": 354, "gameplay_crop_count": 1746, "gameplay_identity_count": 112, "gameplay_valid_count": 1552, "gameplay_reject_count": 194, "total_sample_count": 2136, "verified_file_count": 2148, "video_count": 16, "excluded_unreviewed_count": 0, "eligibility": [ "A gameplay box must be USER_CONFIRMED.", "A VALID crop also needs a USER_CONFIRMED catalog identity matching the scene visual type.", "A REJECT crop needs a USER_CONFIRMED invalid-crop reason.", "Model suggestions, OCR suggestions, drafts, and unreviewed rows are not ground truth." ], "by_validity": {"VALID": 1552, "REJECT": 194}, "by_visual_type": {"hero": 921, "skill": 738, "artifact": 57, "creature": 30}, "by_reject_reason": { "OCR_TEXT_OCCLUDED": 84, "SCENE_MISMATCH": 63, "HERO_SELECTION_NON_HERO": 34, "TOOLTIP_OCCLUDED": 13 }, "by_partition": {"train": 1297, "validation": 348, "safety_validation": 101, "final_test": 0}, "by_video": { "QQ20260804-151259": 348, "QQ2026730-1273": 109, "QQ2026730-14939": 101, "QQ2026730-202130": 159, "QQ2026730-204448": 75, "QQ2026730-211354": 93, "QQ2026730-21210": 117, "QQ2026730-213736": 78, "QQ2026730-221734": 201, "QQ2026730-2322": 66, "QQ2026731-23122": 108, "QQ202683-22543": 225, "Recording-2026-08-30-155825": 24, "ScreenRecording-2026-08-22-214131": 3, "ScreenRecording-2026-08-26-214535": 12, "ScreenRecording-2026-08-30-151540": 27 }, "duplicate_audit": { "exact_groups": 21, "exact_groups_crossing_partitions": 0, "perceptual_near_duplicate_groups": 146, "perceptual_groups_crossing_partitions": 38, "policy": "Exact duplicates can be disabled on the lower-priority side; perceptual groups are audit evidence only and are never silently deleted because dHash can create false merges." }, "final_test": { "status": "BLOCKED_NO_UNTOUCHED_VIDEO", "reason": "All 16 reviewed videos were already inspected or used during V1-V5 development." } }, "model": { "scientific_backbone": "torchvision ConvNeXt-Tiny with ImageNet-1K V1 initialization", "input_size": 224, "type_order": ["hero", "skill", "creature", "artifact"], "heads": { "validity": "one binary logit", "type": "four logits", "identity": "one dynamically sized head per visual type" }, "global_identity_softmax": false, "reject_loss_mask": "REJECT records contribute only validity loss; type and identity targets are masked with -1.", "inference": "Predict validity, predict type, then route to the selected type-specific identity head." }, "controlled_stages": { "M0": { "purpose": "catalog-only classification baseline", "epochs": 12, "steps_per_epoch": 356, "batch_size": 32, "gameplay_fraction": 0.0, "backbone_lr": 0.000012, "head_lr": 0.0004, "weight_decay": 0.02, "loss_weights": {"validity": 0.0, "type": 0.35, "identity": 1.0} }, "M1": { "purpose": "same baseline plus verified VALID gameplay", "epochs": 12, "steps_per_epoch": 356, "batch_size": 32, "gameplay_fraction": 0.5, "backbone_lr": 0.000012, "head_lr": 0.0004, "weight_decay": 0.02, "loss_weights": {"validity": 0.0, "type": 0.35, "identity": 1.0} }, "M2": { "purpose": "freeze M1 and train only the validity head", "epochs": 8, "steps_per_epoch": 160, "batch_size": 32, "gameplay_fraction": 0.5, "backbone_lr": 0.0, "head_lr": 0.0004, "weight_decay": 0.02, "loss_weights": {"validity": 1.0, "type": 0.0, "identity": 0.0} }, "seeds": [20260831, 20260832, 20260833, 20260834, 20260835], "fixed_between_m0_and_m1": [ "initialization", "taxonomy", "optimizer-update count", "learning-rate schedule", "identity batch schedule", "evaluation partitions" ] }, "sampling": { "M0_M1": "Identity-first schedule. Catalog and gameplay pools rotate deterministically; M0 and M1 share the exact identity sequence.", "M2": "Balanced across VALID/REJECT, reject reason, and source video.", "records_written": [ "every batch sample ID", "identity IDs", "sample domains", "validity labels", "unique consumption", "unused gameplay samples", "replacement draw count" ] }, "evaluation": { "classification_metrics": [ "type accuracy", "within-type identity accuracy", "joint type-and-identity accuracy", "macro identity accuracy", "macro type accuracy", "macro video accuracy", "gameplay-covered versus catalog-only identity accuracy", "type and identity confusion" ], "validity_metrics": [ "AUROC", "AUPRC", "balanced accuracy", "valid accept recall", "reject recall", "false accept rate", "false reject rate", "per reject reason", "per video" ], "threshold_selection": "Select on validation by maximum balanced accuracy; ties prefer higher valid recall and then a higher threshold.", "end_to_end_metrics": [ "coverage", "valid coverage", "selective identity accuracy", "end-to-end accuracy" ], "paired_analysis": "M0 and M1 predictions are joined by sample ID to list every improvement and regression." }, "nested_small_large_input": { "status": "MANIFEST_BUILT_TRAINING_NOT_RUN", "small_count": 118, "large_count": 472, "identity_count": 44, "identity_video_cell_count": 59, "small_is_strict_subset_of_large": true, "required_controls": [ "same identities", "same source-video cells", "same optimizer updates", "same identity schedule" ] }, "smoke_run": { "status": "SMOKE_COMPLETE", "scientific_result": false, "backbone": "tiny test-only CNN", "input_size": 64, "seed": 20260831, "updates_per_stage": 1, "hardware": { "os": "Windows 11", "python": "3.12.13", "torch": "2.13.0 with CUDA 13.2", "torchvision": "0.28.0 with CUDA 13.2", "gpu": "NVIDIA GeForce RTX 5080", "amp": "bfloat16" }, "invariants_verified": { "m0_m1_same_initialization": true, "m0_catalog_path_exercised": true, "m1_gameplay_path_exercised": true, "m2_only_validity_head_trainable": true, "m2_trainable_tensor_count": 2, "checkpoint_optimizer_scheduler_records_written": true, "predictions_metrics_and_report_written": true }, "metrics_not_for_model_selection": { "M0": { "validation_joint_identity": 0.0, "validation_confidence_balanced_validity": 0.5907, "safety_validation_joint_identity": 0.0, "safety_confidence_balanced_validity": 0.7043 }, "M1": { "validation_joint_identity": 0.0, "validation_confidence_balanced_validity": 0.5644, "safety_validation_joint_identity": 0.0, "safety_confidence_balanced_validity": 0.7043 }, "M2": { "validation_joint_identity": 0.0, "validation_learned_balanced_validity": 0.5076, "safety_validation_joint_identity": 0.0, "safety_learned_balanced_validity": 0.5 } } }, "verification": { "core_perception_tests": {"passed": 149, "failed": 0}, "experiment_4_tests": {"passed": 13, "failed": 0}, "b03_compatibility_tests": {"passed": 11, "failed": 0}, "python_compile_check": "passed", "snapshot_independent_verification": "passed", "ui_test_baseline": {"passed": 56, "known_preexisting_failures": 19, "new_failures": 0} }, "known_limitations": [ "The formal five-seed ConvNeXt-Tiny run has not been executed.", "No untouched final-test video exists.", "Thirty-eight perceptual near-duplicate groups cross partitions and need a reviewed policy.", "Gameplay supervision covers only 112 of 354 catalog identities and is concentrated in Hero and Main Skill.", "Artifact and Creature gameplay counts are too small for confident per-type conclusions.", "The safety-validation set has already influenced earlier development decisions.", "The smoke metrics come from one optimizer update and must not be interpreted as model quality." ], "questions_for_external_review": [ "Does M0 versus M1 isolate the effect of verified gameplay positives strongly enough?", "Should the same fixed identity schedule be supplemented with a fixed augmentation schedule?", "Is a frozen-backbone validity head sufficient, or should M2 use a small adapter while preserving identity behavior through distillation?", "How should the 38 cross-partition perceptual groups be reviewed without allowing dHash false merges to delete valid diversity?", "Which clustered confidence interval or hierarchical bootstrap best matches correlated crops from videos?", "What minimum new-video final-test design is credible for per-type and rejection claims?", "Should Artifact and Creature be excluded from the first formal claim until more verified gameplay examples exist?" ] }