@misc{song2026phasediagramtraininginference,
      title={A phase diagram for training and inference in language models}, 
      author={Zhuo-Yang Song and Zeyu Li and Qing-Hong Cao and Ming-xing Luo and Hua Xing Zhu},
      year={2026},
      eprint={2611.19848},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2611.19848}, 
}

@article{e2017proposal,
  title = {A Proposal on Machine Learning via Dynamical Systems},
  volume = {5},
  issn = {2194-671X},
  url = {http://dx.doi.org/10.1007/s40304-017-0103-z},
  doi = {10.1007/s40304-017-0103-z},
  number = {1},
  journal = {Communications in Mathematics and Statistics},
  publisher = {Springer Science and Business Media LLC},
  author = {E, Weinan},
  year = {2017},
  month = {Mar},
  pages = {1--11}
}

@article{haber2017stable,
  title = {Stable architectures for deep neural networks},
  volume = {34},
  issn = {1361-6420},
  url = {http://dx.doi.org/10.1088/1361-6420/aa9a90},
  doi = {10.1088/1361-6420/aa9a90},
  number = {1},
  journal = {Inverse Problems},
  publisher = {IOP Publishing},
  author = {Haber, Eldad and Ruthotto, Lars},
  year = {2017},
  month = {Dec},
  pages = {014004}
}

@inproceedings{chen2018neural,
 author = {Chen, Ricky T. Q. and Rubanova, Yulia and Bettencourt, Jesse and Duvenaud, David K},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Neural Ordinary Differential Equations},
 url = {https://proceedings.neurips.cc/paper_files/paper/2018/file/69386f6bb1dfed68692a24c8686939b9-Paper.pdf},
 volume = {31},
 year = {2018}
}

@misc{lu2019understanding,
  title = {Understanding and Improving Transformer From a Multi-Particle Dynamic System Point of View},
  author = {Yiping Lu and Zhuohan Li and Di He and Zhiqing Sun and Bin Dong and Tao Qin and Liwei Wang and Tie-Yan Liu},
  year = {2019},
  eprint = {1906.02762},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url = {https://arxiv.org/abs/1906.02762}
}

@inproceedings{roy2007effective,
  title={The effective rank: A measure of effective dimensionality},
  author={Roy, Olivier and Vetterli, Martin},
  booktitle={2007 15th European signal processing conference},
  pages={606--610},
  year={2007},
  organization={IEEE},
  url       = {https://eurasip.org/Proceedings/Eusipco/Eusipco2007/Papers/a5p-h05.pdf}
}

@article{facco2017estimating, title={Estimating the intrinsic dimension of datasets by a minimal neighborhood information}, volume={7}, ISSN={2045-2322}, url={http://dx.doi.org/10.1038/s41598-017-11873-y}, DOI={10.1038/s41598-017-11873-y}, number={1}, journal={Scientific Reports}, publisher={Springer Science and Business Media LLC}, author={Facco, Elena and d’Errico, Maria and Rodriguez, Alex and Laio, Alessandro}, year={2017}, month=sep }

@inproceedings{ansuini2019intrinsic,
 author = {Ansuini, Alessio and Laio, Alessandro and Macke, Jakob H and Zoccolan, Davide},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Intrinsic dimension of data representations in deep neural networks},
 url = {https://proceedings.neurips.cc/paper_files/paper/2019/file/cfcce0621b49c983991ead4c3d4d3b6b-Paper.pdf},
 volume = {32},
 year = {2019}
}

@article{papyan2020prevalence,
  title = {Prevalence of neural collapse during the terminal phase of deep learning training},
  volume = {117},
  issn = {1091-6490},
  url = {http://dx.doi.org/10.1073/pnas.2015509117},
  doi = {10.1073/pnas.2015509117},
  number = {40},
  journal = {Proceedings of the National Academy of Sciences},
  publisher = {National Academy of Sciences},
  author = {Papyan, Vardan and Han, X. Y. and Donoho, David L.},
  year = {2020},
  month = {Sept},
  pages = {24652--24663}
}

@misc{alain2017understanding,
title={Understanding intermediate layers using linear classifier probes},
author={Guillaume Alain and Yoshua Bengio},
year={2017},
url={https://openreview.net/forum?id=ryF7rTqgl}
}

@misc{belinkov2022probing,
  title = {Probing Classifiers: Promises, Shortcomings, and Advances},
  author = {Yonatan Belinkov},
  year = {2021},
  eprint = {2102.12452},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  url = {https://arxiv.org/abs/2102.12452}
}

@inproceedings{park2024linear,
  title = 	 {The Linear Representation Hypothesis and the Geometry of Large Language Models},
  author =       {Park, Kiho and Choe, Yo Joong and Veitch, Victor},
  booktitle = 	 {Proceedings of the 41st International Conference on Machine Learning},
  pages = 	 {39643--39666},
  year = 	 {2024},
  editor = 	 {Salakhutdinov, Ruslan and Kolter, Zico and Heller, Katherine and Weller, Adrian and Oliver, Nuria and Scarlett, Jonathan and Berkenkamp, Felix},
  volume = 	 {235},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {21--27 Jul},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v235/main/assets/park24c/park24c.pdf},
  url = 	 {https://proceedings.mlr.press/v235/park24c.html},
  abstract = 	 {Informally, the "linear representation hypothesis" is the idea that high-level concepts are represented linearly as directions in some representation space. In this paper, we address two closely related questions: What does "linear representation" actually mean? And, how do we make sense of geometric notions (e.g., cosine similarity and projection) in the representation space? To answer these, we use the language of counterfactuals to give two formalizations of linear representation, one in the output (word) representation space, and one in the input (context) space. We then prove that these connect to linear probing and model steering, respectively. To make sense of geometric notions, we use the formalization to identify a particular (non-Euclidean) inner product that respects language structure in a sense we make precise. Using this <em>causal inner product</em>, we show how to unify all notions of linear representation. In particular, this allows the construction of probes and steering vectors using counterfactual pairs. Experiments with LLaMA-2 demonstrate the existence of linear representations of concepts, the connection to interpretation and control, and the fundamental role of the choice of inner product.}
}

@inproceedings{marks2024geometry,
title={The Geometry of Truth: Emergent Linear Structure in Large Language Model Representations of True/False Datasets},
author={Samuel Marks and Max Tegmark},
booktitle={First Conference on Language Modeling},
year={2024},
url={https://openreview.net/forum?id=aajyHYjjsk}
}

@inproceedings{ethayarajh2019contextual,
    title = "How Contextual are Contextualized Word Representations? {C}omparing the Geometry of {BERT}, {ELM}o, and {GPT}-2 Embeddings",
    author = "Ethayarajh, Kawin",
    editor = "Inui, Kentaro  and
      Jiang, Jing  and
      Ng, Vincent  and
      Wan, Xiaojun",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing ({EMNLP}-{IJCNLP})",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-1006/",
    doi = "10.18653/v1/D19-1006",
    pages = "55--65",
    abstract = "Replacing static word embeddings with contextualized word representations has yielded significant improvements on many NLP tasks. However, just how contextual are the contextualized representations produced by models such as ELMo and BERT? Are there infinitely many context-specific representations for each word, or are words essentially assigned one of a finite number of word-sense representations? For one, we find that the contextualized representations of all words are not isotropic in any layer of the contextualizing model. While representations of the same word in different contexts still have a greater cosine similarity than those of two different words, this self-similarity is much lower in upper layers. This suggests that upper layers of contextualizing models produce more context-specific representations, much like how upper layers of LSTMs produce more task-specific representations. In all layers of ELMo, BERT, and GPT-2, on average, less than 5{\%} of the variance in a word{'}s contextualized representations can be explained by a static embedding for that word, providing some justification for the success of contextualized representations."
}

@inproceedings{gao2019representation,
title={Representation Degeneration Problem in Training Natural Language Generation Models},
author={Jun Gao and Di He and Xu Tan and Tao Qin and Liwei Wang and Tie-Yan Liu},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=SkEYojRqtm},
}

@inproceedings{
mu2018all,
title={All-but-the-Top: Simple and Effective Postprocessing for Word Representations},
author={Jiaqi Mu and Pramod Viswanath},
booktitle={International Conference on Learning Representations},
year={2018},
url={https://openreview.net/forum?id=HkuGJ3kCb},
}

@inproceedings{valeriani2023geometry,
 author = {Valeriani, Lucrezia and Doimo, Diego and Cuturello, Francesca and Laio, Alessandro and Ansuini, Alessio and Cazzaniga, Alberto},
 booktitle = {Advances in Neural Information Processing Systems},
 doi = {10.52202/075280-2230},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {51234--51252},
 publisher = {Curran Associates, Inc.},
 title = {The geometry of hidden representations of large transformer models},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/a0e66093d7168b40246af1cddc025daa-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}

@inproceedings{cheng2025emergence,
 author = {Cheng, Emily and Doimo, Diego and Kervadec, Corentin and Macocco, Iuri and Yu, Lei and Laio, Alessandro and Baroni, Marco},
 booktitle = {International Conference on Learning Representations},
 editor = {Y. Yue and A. Garg and N. Peng and F. Sha and R. Yu},
 pages = {35389--35414},
 title = {Emergence of a High-Dimensional Abstraction Phase in Language Transformers},
 url = {https://proceedings.iclr.cc/paper_files/paper/2025/file/57568e093cbe0a222de0334b36e83cf5-Paper-Conference.pdf},
 year = {2025}
}

@inproceedings{veit2016residual,
 author = {Veit, Andreas and Wilber, Michael J and Belongie, Serge},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Lee and M. Sugiyama and U. Luxburg and I. Guyon and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Residual Networks Behave Like Ensembles of Relatively Shallow Networks},
 url = {https://proceedings.neurips.cc/paper_files/paper/2016/file/37bc2f75bf1bcfe8450a1a41c200364c-Paper.pdf},
 volume = {29},
 year = {2016}
}

@inproceedings{gromov2025unreasonable,
 author = {Gromov, Andrey and Tirumala, Kushal and Shapourian, Hassan and Glorioso, Paolo and Roberts, Daniel A.},
 booktitle = {International Conference on Learning Representations},
 editor = {Y. Yue and A. Garg and N. Peng and F. Sha and R. Yu},
 pages = {81906--81920},
 title = {The Unreasonable Ineffectiveness of the Deeper Layers},
 url = {https://proceedings.iclr.cc/paper_files/paper/2025/file/cbabc2f70de2dd09f491a8715ec3e80f-Paper-Conference.pdf},
 year = {2025}
}

@inproceedings{men2025shortgpt,
  title = {{ShortGPT}: Layers in Large Language Models are More Redundant Than You Expect},
  url = {http://dx.doi.org/10.18653/v1/2025.findings-acl.1035},
  doi = {10.18653/v1/2025.findings-acl.1035},
  booktitle = {Findings of the Association for Computational Linguistics: ACL 2025},
  publisher = {Association for Computational Linguistics},
  author = {Men, Xin and Xu, Mingyu and Zhang, Qingyu and Yuan, Qianhao and Wang, Bingning and Lin, Hongyu and Lu, Yaojie and Han, Xianpei and Chen, Weipeng},
  year = {2025},
  pages = {20192--20204}
}

@misc{lad2024remarkable,
  title = {The Remarkable Robustness of {LLMs}: Stages of Inference?},
  author = {Vedang Lad and Jin Hwa Lee and Wes Gurnee and Max Tegmark},
  year = {2024},
  eprint = {2406.19384},
  archivePrefix = {arXiv},
  primaryClass = {cs.LG},
  url = {https://arxiv.org/abs/2406.19384}
}

@inproceedings{dong2021attention,
  title = 	 {Attention is not all you need: pure attention loses rank doubly exponentially with depth},
  author =       {Dong, Yihe and Cordonnier, Jean-Baptiste and Loukas, Andreas},
  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},
  pages = 	 {2793--2803},
  year = 	 {2021},
  editor = 	 {Meila, Marina and Zhang, Tong},
  volume = 	 {139},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {18--24 Jul},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v139/dong21a/dong21a.pdf},
  url = 	 {https://proceedings.mlr.press/v139/dong21a.html},
  abstract = 	 {Attention-based architectures have become ubiquitous in machine learning. Yet, our understanding of the reasons for their effectiveness remains limited. This work proposes a new way to understand self-attention networks: we show that their output can be decomposed into a sum of smaller terms—or paths—each involving the operation of a sequence of attention heads across layers. Using this path decomposition, we prove that self-attention possesses a strong inductive bias towards "token uniformity". Specifically, without skip connections or multi-layer perceptrons (MLPs), the output converges doubly exponentially to a rank-1 matrix. On the other hand, skip connections and MLPs stop the output from degeneration. Our experiments verify the convergence results on standard transformer architectures.}
}

@inproceedings{noci2022signal,
 author = {Noci, Lorenzo and Anagnostidis, Sotiris and Biggio, Luca and Orvieto, Antonio and Singh, Sidak Pal and Lucchi, Aurelien},
 booktitle = {Advances in Neural Information Processing Systems},
 doi = {10.52202/068431-1972},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {27198--27211},
 publisher = {Curran Associates, Inc.},
 title = {Signal Propagation in Transformers: Theoretical Perspectives and the Role of Rank Collapse},
 url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/ae0cba715b60c4052359b3d52a2cff7f-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}

@inproceedings{giorlandino2026two,
title={Two failure modes of deep transformers and how to avoid them: a unified theory of signal propagation at initialisation},
author={Alessio Giorlandino and Sebastian Goldt},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=utSqpxQHXq}
}

@inproceedings{poole2016exponential,
 author = {Poole, Ben and Lahiri, Subhaneil and Raghu, Maithra and Sohl-Dickstein, Jascha and Ganguli, Surya},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Lee and M. Sugiyama and U. Luxburg and I. Guyon and R. Garnett},
 publisher = {Curran Associates, Inc.},
 title = {Exponential expressivity in deep neural networks through transient chaos},
 url = {https://proceedings.neurips.cc/paper_files/paper/2016/file/148510031349642de5ca0c544f31b2ef-Paper.pdf},
 volume = {29},
 year = {2016}
}

@inproceedings{schoenholz2017deep,
title={Deep Information Propagation},
author={Samuel S. Schoenholz and Justin Gilmer and Surya Ganguli and Jascha Sohl-Dickstein},
booktitle={International Conference on Learning Representations},
year={2017},
url={https://openreview.net/forum?id=H1W1UN9gg}
}

@article{bahri2020statistical,
  title = {Statistical Mechanics of Deep Learning},
  volume = {11},
  issn = {1947-5462},
  url = {http://dx.doi.org/10.1146/annurev-conmatphys-031119-050745},
  doi = {10.1146/annurev-conmatphys-031119-050745},
  number = {1},
  journal = {Annual Review of Condensed Matter Physics},
  publisher = {Annual Reviews},
  author = {Bahri, Yasaman and Kadmon, Jonathan and Pennington, Jeffrey and Schoenholz, Sam S. and Sohl-Dickstein, Jascha and Ganguli, Surya},
  year = {2020},
  month = {Mar},
  pages = {501--528}
}

@inproceedings{geshkovski2023emergence,
 author = {Geshkovski, Borjan and Letrouit, Cyril and Polyanskiy, Yury and Rigollet, Philippe},
 booktitle = {Advances in Neural Information Processing Systems},
 doi = {10.52202/075280-2493},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {57026--57037},
 publisher = {Curran Associates, Inc.},
 title = {The emergence of clusters in self-attention dynamics},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/b2b3e1d9840eba17ad9bbf073e009afe-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}

@article{geshkovski2023mathematical,
    title = {A mathematical perspective on transformers},
  volume = {62},
  ISSN = {1088-9485},
  url = {http://dx.doi.org/10.1090/bull/1863},
  DOI = {10.1090/bull/1863},
  number = {3},
  journal = {Bulletin of the American Mathematical Society},
  publisher = {American Mathematical Society (AMS)},
  author = {Geshkovski,  Borjan and Letrouit,  Cyril and Polyanskiy,  Yury and Rigollet,  Philippe},
  year = {2025},
  month = Apr,
  pages = {427–479}
}

@inproceedings{li2025tracing,
 author = {Li, Melody and Agrawal, Kumar Krishna and Ghosh, Arna and Teru, Komal and Santoro, Adam and Lajoie, Guillaume and Richards, Blake},
 booktitle = {Advances in Neural Information Processing Systems},
 doi = {10.52202/085713-1829},
 editor = {D. Belgrave and C. Zhang and H. Lin and R. Pascanu and P. Koniusz and M. Ghassemi and N. Chen},
 pages = {54691--54724},
 publisher = {Curran Associates, Inc.},
 title = {Tracing the Representation Geometry of Language Models from Pretraining to Post-training},
 url = {https://proceedings.neurips.cc/paper_files/paper/2025/file/4eac651e6c06d6b374332cd14ab50f55-Paper-Conference.pdf},
 volume = {38},
 year = {2025}
}

@inproceedings{gopalani2025what,
 author = {Gopalani, Pulkit and Hu, Wei},
 booktitle = {Advances in Neural Information Processing Systems},
 doi = {10.52202/085713-1838},
 editor = {D. Belgrave and C. Zhang and H. Lin and R. Pascanu and P. Koniusz and M. Ghassemi and N. Chen},
 pages = {54928--54960},
 publisher = {Curran Associates, Inc.},
 title = {What Happens During the Loss Plateau? {U}nderstanding Abrupt Learning in Transformers},
 url = {https://proceedings.neurips.cc/paper_files/paper/2025/file/4f06c73c45d2625f0e687f7e6a206332-Paper-Conference.pdf},
 volume = {38},
 year = {2025}
}

@inproceedings{liu2021probing,
    title = "Probing Across Time: What Does {R}o{BERT}a Know and When?",
    author = "Liu, Leo Z.  and
      Wang, Yizhong  and
      Kasai, Jungo  and
      Hajishirzi, Hannaneh  and
      Smith, Noah A.",
    editor = "Moens, Marie-Francine  and
      Huang, Xuanjing  and
      Specia, Lucia  and
      Yih, Scott Wen-tau",
    booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EMNLP} 2021",
    month = nov,
    year = "2021",
    address = "Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.findings-emnlp.71/",
    doi = "10.18653/v1/2021.findings-emnlp.71",
    pages = "820--842",
    abstract = "Models of language trained on very large corpora have been demonstrated useful for natural language processing. As fixed artifacts, they have become the object of intense study, with many researchers ``probing'' the extent to which they acquire and readily demonstrate linguistic abstractions, factual and commonsense knowledge, and reasoning abilities. Recent work applied several probes to intermediate training stages to observe the developmental process of a large-scale model (Chiang et al., 2020). Following this effort, we systematically answer a question: for various types of knowledge a language model learns, when during (pre)training are they acquired? Using RoBERTa as a case study, we find: linguistic knowledge is acquired fast, stably, and robustly across domains. Facts and commonsense are slower and more domain-sensitive. Reasoning abilities are, in general, not stably acquired. As new datasets, pretraining protocols, and probes emerge, we believe that probing-across-time analyses can help researchers understand the complex, intermingled learning that these models undergo and guide us toward more efficient approaches that accomplish necessary learning faster."
}
