diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..08b5cf333843db18f8ae48587f93665b9ad039cf --- /dev/null +++ b/config.json @@ -0,0 +1 @@ +{"sae": {"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true}, "batch_size": 8, "grad_acc_steps": 4, "micro_acc_steps": 1, "optimizer": "signum", "lr": null, "lr_warmup_steps": 1000, "k_decay_steps": 0, "auxk_alpha": 0.0, "dead_feature_threshold": 10000000, "hookpoints": ["model.layers.8.mlp"], "init_seeds": [0], "layers": [], "layer_stride": 1, "distribute_modules": false, "save_every": 1000, "log_to_wandb": true, "run_name": "single_128x", "wandb_log_frequency": 1, "model": "HuggingFaceTB/SmolLM2-135M", "dataset": "EleutherAI/fineweb-edu-dedup-10b", "split": "train", "ctx_len": 2048, "hf_token": null, "revision": null, "load_in_8bit": false, "max_examples": 500000, "resume": false, "text_column": "text", "finetune": null, "shuffle_seed": 42, "data_preprocessing_num_proc": 48} \ No newline at end of file diff --git a/model.layers.0.mlp/cfg.json b/model.layers.0.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.0.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.0.mlp/sae.safetensors b/model.layers.0.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7019a567cdb49559c2ad4c5fda6e79e6ef2d23b --- /dev/null +++ b/model.layers.0.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56ce70caa3f7167f8aefc3a2ba6cc0b782795a29f15f5ac21f0caaf5b7fa2677 +size 341363360 diff --git a/model.layers.1.mlp/cfg.json b/model.layers.1.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.1.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.1.mlp/sae.safetensors b/model.layers.1.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9c22da0864044184b519b68e14845de447a78b2d --- /dev/null +++ b/model.layers.1.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a0bd89cde09b2b4f9b0e48726cdc7e4cb20337840d9281354946f3c11a4b3ad +size 341363360 diff --git a/model.layers.10.mlp/cfg.json b/model.layers.10.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.10.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.10.mlp/sae.safetensors b/model.layers.10.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..19225cb20e0de9c7d7b71bb97ff5c272ba2937ef --- /dev/null +++ b/model.layers.10.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0b4b58019665f4358c54f451b11a41c34cd48c2fc78b4e0eff730455fc57d7 +size 341363360 diff --git a/model.layers.11.mlp/cfg.json b/model.layers.11.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.11.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.11.mlp/sae.safetensors b/model.layers.11.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64ca7a0cba4e62a1e427ef3ee371e3278f23bb23 --- /dev/null +++ b/model.layers.11.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:616287eec940898fc3cb72ef2bb2d09b6a5cebd6256217b197bcb5058cae92f9 +size 341363360 diff --git a/model.layers.12.mlp/cfg.json b/model.layers.12.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.12.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.12.mlp/sae.safetensors b/model.layers.12.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbe925301f9fad740a4c4e81580ae39a0a39ea61 --- /dev/null +++ b/model.layers.12.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d6c03f54af342471bfdb8a46b7707c2839a320d30a93d1ba6506e16ee9a4e43 +size 341363360 diff --git a/model.layers.13.mlp/cfg.json b/model.layers.13.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.13.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.13.mlp/sae.safetensors b/model.layers.13.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b101015746c8ba7eb4c99f7e58d75c81257d7960 --- /dev/null +++ b/model.layers.13.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c02de3b5882e3053cf5d75b703a63ed8b0ddcc8843bb3305adf8ab2252a17672 +size 341363360 diff --git a/model.layers.14.mlp/cfg.json b/model.layers.14.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.14.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.14.mlp/sae.safetensors b/model.layers.14.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6ba0967e7024d965a3427a4f24530e9f5903a7d5 --- /dev/null +++ b/model.layers.14.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55a455c3c2350aec1824ad9e8041305c7a6011196730f3ef0e104647eb59d9a3 +size 341363360 diff --git a/model.layers.15.mlp/cfg.json b/model.layers.15.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.15.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.15.mlp/sae.safetensors b/model.layers.15.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e22da527075f8f786173790851c44685070ff70f --- /dev/null +++ b/model.layers.15.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93b1262ece6ba839c2825649f84ef7a1264038f5c561472e1d8c7665a3cf42a1 +size 341363360 diff --git a/model.layers.16.mlp/cfg.json b/model.layers.16.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.16.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.16.mlp/sae.safetensors b/model.layers.16.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..063b41ffca9a971737996104e31f50e194248da5 --- /dev/null +++ b/model.layers.16.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b4207e91baeacb6094a314b995b29886fcbf82fbf90dfbe5effc90fe5bf3cfb +size 341363360 diff --git a/model.layers.17.mlp/cfg.json b/model.layers.17.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.17.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.17.mlp/sae.safetensors b/model.layers.17.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2867b525c994e35de3415937ab85e56787eb05a7 --- /dev/null +++ b/model.layers.17.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:837099fd7dc4f9b36da8b403fe410fb85bc03f4343160057770fe7a220101eab +size 341363360 diff --git a/model.layers.18.mlp/cfg.json b/model.layers.18.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.18.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.18.mlp/sae.safetensors b/model.layers.18.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f75e64ababccc0fac8d26cdff6bfc87498d9186 --- /dev/null +++ b/model.layers.18.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad15c681e948baafe0bb48ddc66e14910f17967825dfcff070fb5a6132e6a91 +size 341363360 diff --git a/model.layers.19.mlp/cfg.json b/model.layers.19.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.19.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.19.mlp/sae.safetensors b/model.layers.19.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d41333e0e900dac30bce2842c4df85f9ab5a829 --- /dev/null +++ b/model.layers.19.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d720fd8ebfaf1d66abf36d82b7249e89dbe3c9bfc2c01228885576babd9ee7f0 +size 341363360 diff --git a/model.layers.2.mlp/cfg.json b/model.layers.2.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.2.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.2.mlp/sae.safetensors b/model.layers.2.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a973314147c6a5f64c7d3584862401c00e669612 --- /dev/null +++ b/model.layers.2.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f981a5e57d2e40cf9601057809d72909fa8fed78fcae3775f4f8f72c423fc11 +size 341363360 diff --git a/model.layers.20.mlp/cfg.json b/model.layers.20.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcbb010e5653dc79cb70afd8da41db56e8c115 --- /dev/null +++ b/model.layers.20.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.20.mlp/sae.safetensors b/model.layers.20.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9cf5e80157e2284f825596fee9e9f55d81ca9757 --- /dev/null +++ b/model.layers.20.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bbf6f1dc5377a17525fb999c4b10897c86ced205579c37121c8d6dc735b06ac +size 341363360 diff --git a/model.layers.21.mlp/cfg.json b/model.layers.21.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcbb010e5653dc79cb70afd8da41db56e8c115 --- /dev/null +++ b/model.layers.21.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.21.mlp/sae.safetensors b/model.layers.21.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..baad6d2560ae2ce86bb1768a4d3fa5caee4eb0fe --- /dev/null +++ b/model.layers.21.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a89aa886e34339841f1d18aca73a4d07800d3628680f894ad5c322d924be86b0 +size 341363360 diff --git a/model.layers.22.mlp/cfg.json b/model.layers.22.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcbb010e5653dc79cb70afd8da41db56e8c115 --- /dev/null +++ b/model.layers.22.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.22.mlp/sae.safetensors b/model.layers.22.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c68aff46f9b2a2335c4bffba11f072e256d8754c --- /dev/null +++ b/model.layers.22.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d176fb7a5a2688cad50a9c22cfcebb8cd1e0972639dc79f9e77a2fa49aed2750 +size 341363360 diff --git a/model.layers.23.mlp/cfg.json b/model.layers.23.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.23.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.23.mlp/sae.safetensors b/model.layers.23.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..383797d817ba0555e08d6f5a16d2f81a56dd2ca9 --- /dev/null +++ b/model.layers.23.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69df70a0918bb431f9e4d3b2bbdc80d48af11c94eb8f367520142d94c551fd42 +size 341363360 diff --git a/model.layers.24.mlp/cfg.json b/model.layers.24.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcbb010e5653dc79cb70afd8da41db56e8c115 --- /dev/null +++ b/model.layers.24.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.24.mlp/sae.safetensors b/model.layers.24.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d46067e2df62cb10ada72557559e6036c6f033f --- /dev/null +++ b/model.layers.24.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e39c2c700f3f31a99e4ac0a2751176f5f2472c1c48e28bdcf0279314605c92 +size 341363360 diff --git a/model.layers.26.mlp/cfg.json b/model.layers.26.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..acaab9e38dd921b9acd7d395481ed3ff5eef6dec --- /dev/null +++ b/model.layers.26.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true,"d_in": 576} \ No newline at end of file diff --git a/model.layers.26.mlp/sae.safetensors b/model.layers.26.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2652fe4349c09a1117a3ef3a222d385af672392e --- /dev/null +++ b/model.layers.26.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:430a72cf5fa68b97b887b93b289ee007353db785d04971fe64059edc02b703ab +size 341363360 diff --git a/model.layers.27.mlp/cfg.json b/model.layers.27.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.27.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.27.mlp/sae.safetensors b/model.layers.27.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..655aea11237f47d6a60356a0f8242273744ade41 --- /dev/null +++ b/model.layers.27.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8317d1b216779c8667832b55664fabfe08fab6c3444856ce096c980d1b3bddb1 +size 341363360 diff --git a/model.layers.28.mlp/cfg.json b/model.layers.28.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcbb010e5653dc79cb70afd8da41db56e8c115 --- /dev/null +++ b/model.layers.28.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.28.mlp/sae.safetensors b/model.layers.28.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a4f2d00ffbe8dc7ecc26317e9d9be73c41d7870f --- /dev/null +++ b/model.layers.28.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb59358c254fbebfba816bdb06a6fc508b83c189d4a3d3eb7569b7d3f9f19cea +size 341363360 diff --git a/model.layers.29.mlp/cfg.json b/model.layers.29.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.29.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.29.mlp/sae.safetensors b/model.layers.29.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e8206e5bf65cddae48208bb1aac5d73c5fe4c33 --- /dev/null +++ b/model.layers.29.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb725eee13d13ffcc495640d082ff12699e056d3af2dcad279597288bfd52670 +size 341363360 diff --git a/model.layers.3.mlp/cfg.json b/model.layers.3.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.3.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.3.mlp/sae.safetensors b/model.layers.3.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..17913aad76e3819835dd1627e34dc67b6c77ce69 --- /dev/null +++ b/model.layers.3.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842613d636debd426d83fc6f8f0e2eaa4a788d28fdd6305a996e862fb6d67e0b +size 341363360 diff --git a/model.layers.4.mlp/cfg.json b/model.layers.4.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.4.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.4.mlp/sae.safetensors b/model.layers.4.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f909b1340dc6d114cdc4e2bb3b6c1a5ba689b70 --- /dev/null +++ b/model.layers.4.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3d7788cac85144c347d4f49f5706b1be856756e92c58f32791777afbb2bd1a7 +size 341363360 diff --git a/model.layers.5.mlp/cfg.json b/model.layers.5.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.5.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.5.mlp/sae.safetensors b/model.layers.5.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2f19d46f55af14072d479fe6517f60fd00d242af --- /dev/null +++ b/model.layers.5.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a73ef4943c221e109f82dc8775e70747a9294fb25d278e1cf2bf1d3feb24ed80 +size 341363360 diff --git a/model.layers.6.mlp/cfg.json b/model.layers.6.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.6.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.6.mlp/sae.safetensors b/model.layers.6.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d00e099e443c45acba0158598e9f653d702e916e --- /dev/null +++ b/model.layers.6.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ad8823ff70dcca1e08d5050ca9465a3f61d95a75b6a614d2d588285ce6823e +size 341363360 diff --git a/model.layers.7.mlp/cfg.json b/model.layers.7.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.7.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.7.mlp/sae.safetensors b/model.layers.7.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e1511b8cea7060977e4f6189e40b8f69bd2af5f7 --- /dev/null +++ b/model.layers.7.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203181e0397b5c2a58ef0f042176178c07d496e0b0a88be3d0e2ba3c095aa9a9 +size 341363360 diff --git a/model.layers.8.mlp/cfg.json b/model.layers.8.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.8.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.8.mlp/sae.safetensors b/model.layers.8.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bdffa51f8485cf02c3051f7b61ca28d948fc4b29 --- /dev/null +++ b/model.layers.8.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f811a32b7c0deb5cb15ad3f1e8529bb38265768eafba0713b3c208b4bc02e5 +size 341363360 diff --git a/model.layers.9.mlp/cfg.json b/model.layers.9.mlp/cfg.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad40bc4d1716b507aa3b6f9fb99b1b715c3b25e --- /dev/null +++ b/model.layers.9.mlp/cfg.json @@ -0,0 +1 @@ +{"activation": "topk", "expansion_factor": 128, "normalize_decoder": true, "num_latents": 0, "k": 128, "multi_topk": false, "skip_connection": true, "transcode": true, "d_in": 576} \ No newline at end of file diff --git a/model.layers.9.mlp/sae.safetensors b/model.layers.9.mlp/sae.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbc09c77f857eb1f461d25e425fafa419130305d --- /dev/null +++ b/model.layers.9.mlp/sae.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99f5feb458c152aad8ef64f28fc73a42a2c29cc95b2650154892ffe61e7d08fd +size 341363360 diff --git a/optimizer_0.pt b/optimizer_0.pt new file mode 100644 index 0000000000000000000000000000000000000000..67335c1b7dfd1e5c7cc88d2685b47ecb41d6a9a4 --- /dev/null +++ b/optimizer_0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1e1ddb390cfe9689d4bec282f673db6f91ffaa022a35c72d64ae235a5e591b5 +size 341365424 diff --git a/rank_0_state.pt b/rank_0_state.pt new file mode 100644 index 0000000000000000000000000000000000000000..88e256e1aa5f9a8d33512db2a2ac81285bc16e21 --- /dev/null +++ b/rank_0_state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcf5a61e652eea1c84e8142e9a6335d965e8fff0160055b4b9194ae7a43cce0a +size 591093 diff --git a/state.pt b/state.pt new file mode 100644 index 0000000000000000000000000000000000000000..4eff9ae8a896157c1a0553106682e0d7d6128d55 --- /dev/null +++ b/state.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d738715009486e24de8c546db27158058643accac7eb9d7303ad52a9515bf5 +size 856