| { |
| "eval_type_id": "absorption_first_letter", |
| "eval_config": { |
| "model_name": "gemma-2-2b", |
| "random_seed": 42, |
| "f1_jump_threshold": 0.03, |
| "max_k_value": 10, |
| "prompt_template": "{word} has the first letter:", |
| "prompt_token_pos": -6, |
| "llm_batch_size": 32, |
| "llm_dtype": "bfloat16", |
| "k_sparse_probe_l1_decay": 0.01, |
| "k_sparse_probe_batch_size": 4096, |
| "k_sparse_probe_num_epochs": 50, |
| "min_GT_probe_f1": 0.6, |
| "min_feats_for_eval": 20 |
| }, |
| "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6", |
| "datetime_epoch_millis": 1745711399847, |
| "eval_result_metrics": { |
| "mean": { |
| "mean_absorption_fraction_score": 0.2136590900204307, |
| "mean_full_absorption_score": 0.13025611386034586, |
| "mean_num_split_features": 1.1153846153846154, |
| "std_dev_absorption_fraction_score": 0.1512903704740959, |
| "std_dev_full_absorption_score": 0.12280437217624496, |
| "std_dev_num_split_features": 0.3258125936084211 |
| } |
| }, |
| "eval_result_details": [ |
| { |
| "first_letter": "a", |
| "mean_absorption_fraction": 0.09981917710500524, |
| "full_absorption_rate": 0.04317656129529684, |
| "num_full_absorption": 112, |
| "num_probe_true_positives": 2594, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "b", |
| "mean_absorption_fraction": 0.0178804588069183, |
| "full_absorption_rate": 0.016738995660260384, |
| "num_full_absorption": 27, |
| "num_probe_true_positives": 1613, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "c", |
| "mean_absorption_fraction": 0.006105862765139088, |
| "full_absorption_rate": 0.002468265162200282, |
| "num_full_absorption": 7, |
| "num_probe_true_positives": 2836, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "d", |
| "mean_absorption_fraction": 0.42574197296226696, |
| "full_absorption_rate": 0.3020434227330779, |
| "num_full_absorption": 473, |
| "num_probe_true_positives": 1566, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "e", |
| "mean_absorption_fraction": 0.40849163556717905, |
| "full_absorption_rate": 0.3877292852624921, |
| "num_full_absorption": 613, |
| "num_probe_true_positives": 1581, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "f", |
| "mean_absorption_fraction": 0.11936547904313492, |
| "full_absorption_rate": 0.09583333333333334, |
| "num_full_absorption": 115, |
| "num_probe_true_positives": 1200, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "g", |
| "mean_absorption_fraction": 0.1335060595531976, |
| "full_absorption_rate": 0.08003597122302158, |
| "num_full_absorption": 89, |
| "num_probe_true_positives": 1112, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "h", |
| "mean_absorption_fraction": 0.0745683286568553, |
| "full_absorption_rate": 0.03464870067372473, |
| "num_full_absorption": 36, |
| "num_probe_true_positives": 1039, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "i", |
| "mean_absorption_fraction": 0.28645583256440704, |
| "full_absorption_rate": 0.21691599539700807, |
| "num_full_absorption": 377, |
| "num_probe_true_positives": 1738, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "j", |
| "mean_absorption_fraction": 0.30129102884895986, |
| "full_absorption_rate": 0.12672811059907835, |
| "num_full_absorption": 55, |
| "num_probe_true_positives": 434, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "k", |
| "mean_absorption_fraction": 0.23079556906538518, |
| "full_absorption_rate": 0.08695652173913043, |
| "num_full_absorption": 60, |
| "num_probe_true_positives": 690, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "l", |
| "mean_absorption_fraction": 0.44400501320725594, |
| "full_absorption_rate": 0.22727272727272727, |
| "num_full_absorption": 270, |
| "num_probe_true_positives": 1188, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "m", |
| "mean_absorption_fraction": 0.045952685160575914, |
| "full_absorption_rate": 0.018312985571587125, |
| "num_full_absorption": 33, |
| "num_probe_true_positives": 1802, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "n", |
| "mean_absorption_fraction": 0.15397806417162374, |
| "full_absorption_rate": 0.07268464243845252, |
| "num_full_absorption": 62, |
| "num_probe_true_positives": 853, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "o", |
| "mean_absorption_fraction": 0.19192364641148027, |
| "full_absorption_rate": 0.12465116279069767, |
| "num_full_absorption": 134, |
| "num_probe_true_positives": 1075, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "p", |
| "mean_absorption_fraction": 0.1722726027285375, |
| "full_absorption_rate": 0.17988064791133845, |
| "num_full_absorption": 422, |
| "num_probe_true_positives": 2346, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "q", |
| "mean_absorption_fraction": 0.06788049832866817, |
| "full_absorption_rate": 0.022099447513812154, |
| "num_full_absorption": 4, |
| "num_probe_true_positives": 181, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "r", |
| "mean_absorption_fraction": 0.14085158177840548, |
| "full_absorption_rate": 0.1404109589041096, |
| "num_full_absorption": 246, |
| "num_probe_true_positives": 1752, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "s", |
| "mean_absorption_fraction": 0.44617707284748986, |
| "full_absorption_rate": 0.48686030428769017, |
| "num_full_absorption": 1408, |
| "num_probe_true_positives": 2892, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "t", |
| "mean_absorption_fraction": 0.13247951304053526, |
| "full_absorption_rate": 0.11044776119402985, |
| "num_full_absorption": 185, |
| "num_probe_true_positives": 1675, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "u", |
| "mean_absorption_fraction": 0.5430063448969557, |
| "full_absorption_rate": 0.18294190358467244, |
| "num_full_absorption": 148, |
| "num_probe_true_positives": 809, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "v", |
| "mean_absorption_fraction": 0.15342048568015876, |
| "full_absorption_rate": 0.047505938242280284, |
| "num_full_absorption": 40, |
| "num_probe_true_positives": 842, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "w", |
| "mean_absorption_fraction": 0.3521976406799161, |
| "full_absorption_rate": 0.26277372262773724, |
| "num_full_absorption": 180, |
| "num_probe_true_positives": 685, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "x", |
| "mean_absorption_fraction": 0.09641425744376537, |
| "full_absorption_rate": 0.0, |
| "num_full_absorption": 0, |
| "num_probe_true_positives": 102, |
| "num_split_features": 2 |
| }, |
| { |
| "first_letter": "y", |
| "mean_absorption_fraction": 0.3655916846706601, |
| "full_absorption_rate": 0.07738095238095238, |
| "num_full_absorption": 13, |
| "num_probe_true_positives": 168, |
| "num_split_features": 1 |
| }, |
| { |
| "first_letter": "z", |
| "mean_absorption_fraction": 0.14496384454672162, |
| "full_absorption_rate": 0.040160642570281124, |
| "num_full_absorption": 10, |
| "num_probe_true_positives": 249, |
| "num_split_features": 1 |
| } |
| ], |
| "sae_bench_commit_hash": "Unknown", |
| "sae_lens_id": "blocks.0.hook_resid_post", |
| "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc", |
| "sae_lens_version": "5.9.1", |
| "sae_cfg_dict": { |
| "architecture": "jumprelu", |
| "d_in": 2304, |
| "d_sae": 32768, |
| "activation_fn_str": "relu", |
| "apply_b_dec_to_input": true, |
| "finetuning_scaling_factor": false, |
| "context_size": 1024, |
| "model_name": "gemma-2-2b", |
| "hook_name": "blocks.0.hook_resid_post", |
| "hook_layer": 0, |
| "hook_head_index": null, |
| "prepend_bos": true, |
| "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B", |
| "dataset_trust_remote_code": true, |
| "normalize_activations": "none", |
| "dtype": "torch.bfloat16", |
| "device": "cuda", |
| "sae_lens_training_version": "5.5.1", |
| "activation_fn_kwargs": { |
| "k": 40 |
| }, |
| "neuronpedia_id": null, |
| "model_from_pretrained_kwargs": { |
| "center_writing_weights": false |
| }, |
| "seqpos_slice": [ |
| null |
| ] |
| }, |
| "eval_result_unstructured": null |
| } |
