@misc{Rumelhart1986A001,
  title = {Backpropagation (Learning representations by back-propagating errors)},
  author = {David Rumelhart and Geoffrey Hinton and Ronald Williams},
  year = {1986},
  url = {https://www.nature.com/articles/323533a0}
}

@misc{Bengio2003A002,
  title = {A Neural Probabilistic Language Model},
  author = {Yoshua Bengio and Réjean Ducharme and Pascal Vincent and Christian Jauvin},
  year = {2003},
  url = {https://www.jmlr.org/papers/v3/bengio03a.html}
}

@article{Mikolov2013A004,
  title = {Efficient Estimation of Word Representations (Word2Vec)},
  author = {Tomas Mikolov and Kai Chen and Greg Corrado and Jeffrey Dean},
  year = {2013},
  url = {https://arxiv.org/abs/1301.3781},
  note = {arXiv:1301.3781}
}

@misc{Pennington2014A005,
  title = {GloVe: Global Vectors for Word Representation},
  author = {Jeffrey Pennington and Richard Socher and Christopher Manning},
  year = {2014},
  url = {https://aclanthology.org/D14-1162/}
}

@article{Sutskever2014A006,
  title = {Sequence to Sequence Learning with Neural Networks},
  author = {Ilya Sutskever and Oriol Vinyals and Quoc Le},
  year = {2014},
  url = {https://arxiv.org/abs/1409.3215},
  note = {arXiv:1409.3215}
}

@article{Bahdanau2014A007,
  title = {Neural Machine Translation by Jointly Learning to Align and Translate (Bahdanau attention)},
  author = {Dzmitry Bahdanau and Kyunghyun Cho and Yoshua Bengio},
  year = {2014},
  url = {https://arxiv.org/abs/1409.0473},
  note = {arXiv:1409.0473}
}

@article{Luong2015A008,
  title = {Effective Approaches to Attention-based NMT (Luong attention)},
  author = {Minh-Thang Luong and Hieu Pham and Christopher Manning},
  year = {2015},
  url = {https://arxiv.org/abs/1508.04025},
  note = {arXiv:1508.04025}
}

@article{He2015A009,
  title = {Deep Residual Learning for Image Recognition (ResNet)},
  author = {Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  year = {2015},
  url = {https://arxiv.org/abs/1512.03385},
  note = {arXiv:1512.03385}
}

@article{Ba2016A010,
  title = {Layer Normalization},
  author = {Jimmy Lei Ba and Jamie Ryan Kiros and Geoffrey Hinton},
  year = {2016},
  url = {https://arxiv.org/abs/1607.06450},
  note = {arXiv:1607.06450}
}

@article{Kingma2014A011,
  title = {Adam: A Method for Stochastic Optimization},
  author = {Diederik Kingma and Jimmy Ba},
  year = {2014},
  url = {https://arxiv.org/abs/1412.6980},
  note = {arXiv:1412.6980}
}

@misc{Srivastava2014A012,
  title = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
  author = {Nitish Srivastava and Geoffrey Hinton and Alex Krizhevsky and Ilya Sutskever and Ruslan Salakhutdinov},
  year = {2014},
  url = {https://jmlr.org/papers/v15/srivastava14a.html}
}

@article{Sennrich2015A013,
  title = {Neural Machine Translation of Rare Words with Subword Units (BPE)},
  author = {Rico Sennrich and Barry Haddow and Alexandra Birch},
  year = {2015},
  url = {https://arxiv.org/abs/1508.07909},
  note = {arXiv:1508.07909}
}

@misc{Cybenko1989A014,
  title = {Universal Approximation Theorem},
  author = {George Cybenko and Kurt Hornik},
  year = {1989},
  url = {https://link.springer.com/article/10.1007/BF02551274}
}

@misc{Glorot2010A015,
  title = {Understanding the Difficulty of Training Deep Nets (Xavier/He init)},
  author = {Xavier Glorot and Yoshua Bengio and Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
  year = {2010},
  url = {https://proceedings.mlr.press/v9/glorot10a.html}
}

@misc{Nair2010A016,
  title = {Rectified Linear Units (ReLU)},
  author = {Vinod Nair and Geoffrey Hinton and Xavier Glorot and Antoine Bordes and Yoshua Bengio},
  year = {2010},
  url = {https://www.cs.toronto.edu/~hinton/absps/reluICML.pdf}
}

@article{Ioffe2015A017,
  title = {Batch Normalization},
  author = {Sergey Ioffe and Christian Szegedy},
  year = {2015},
  url = {https://arxiv.org/abs/1502.03167},
  note = {arXiv:1502.03167}
}

@misc{LeCun1998A018,
  title = {LeNet: Gradient-Based Learning Applied to Document Recognition},
  author = {Yann LeCun and Léon Bottou and Yoshua Bengio and Patrick Haffner},
  year = {1998},
  url = {http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf}
}

@misc{Krizhevsky2012A019,
  title = {AlexNet: ImageNet Classification with Deep CNNs},
  author = {Alex Krizhevsky and Ilya Sutskever and Geoffrey Hinton},
  year = {2012},
  url = {https://proceedings.neurips.cc/paper/2012/hash/c399862d3b9d6b76c8436e924a68c45b-Abstract.html}
}

@misc{Elman1990A020,
  title = {Finding Structure in Time (Elman RNN)},
  author = {Jeffrey Elman},
  year = {1990},
  url = {https://onlinelibrary.wiley.com/doi/10.1207/s15516709cog1402_1}
}

@misc{Hochreiter1994A021,
  title = {Learning Long-Term Dependencies is Difficult (vanishing gradients)},
  author = {Sepp Hochreiter and Yoshua Bengio and Patrice Simard and Paolo Frasconi},
  year = {1994},
  url = {https://ieeexplore.ieee.org/document/279181}
}

@misc{Hochreiter1997A022,
  title = {Long Short-Term Memory (LSTM)},
  author = {Sepp Hochreiter and Jürgen Schmidhuber and Felix Gers},
  year = {1997},
  url = {https://www.bioinf.jku.at/publications/older/2604.pdf}
}

@misc{Schuster1997A023,
  title = {Bidirectional Recurrent Neural Networks},
  author = {Mike Schuster and Kuldip Paliwal},
  year = {1997},
  url = {https://ieeexplore.ieee.org/document/650093}
}

@article{Cho2014A024,
  title = {Gated Recurrent Unit (GRU) / RNN Encoder-Decoder},
  author = {Kyunghyun Cho and Bart van Merriënboer and Dzmitry Bahdanau and Yoshua Bengio},
  year = {2014},
  url = {https://arxiv.org/abs/1406.1078},
  note = {arXiv:1406.1078}
}

@misc{Graves2006A025,
  title = {Connectionist Temporal Classification (CTC)},
  author = {Alex Graves and Santiago Fernández and Faustino Gomez and Jürgen Schmidhuber},
  year = {2006},
  url = {https://www.cs.toronto.edu/~graves/icml_2006.pdf}
}

@misc{Hinton2006A026,
  title = {Deep Belief Nets / Reducing Dimensionality},
  author = {Geoffrey Hinton and Simon Osindero and Yee-Whye Teh and Ruslan Salakhutdinov},
  year = {2006},
  url = {https://www.science.org/doi/10.1126/science.1127647}
}

@article{Collobert2011A027,
  title = {Natural Language Processing (almost) from Scratch},
  author = {Ronan Collobert and Jason Weston and Léon Bottou and et al.},
  year = {2011},
  url = {https://arxiv.org/abs/1103.0398},
  note = {arXiv:1103.0398}
}

@misc{Brown1993A028,
  title = {The Mathematics of Statistical MT (IBM alignment models)},
  author = {Peter F. Brown and Vincent J. Della Pietra and Stephen A. Della Pietra and Robert L. Mercer},
  year = {1993},
  url = {https://aclanthology.org/J93-2003/}
}

@misc{Sutton1988A029,
  title = {Learning to Predict by the Methods of Temporal Differences},
  author = {Richard S. Sutton},
  year = {1988},
  url = {https://link.springer.com/article/10.1007/BF00115009}
}

@misc{Watkins1992A030,
  title = {Q-learning},
  author = {Christopher J. C. H. Watkins and Peter Dayan},
  year = {1992},
  url = {https://link.springer.com/article/10.1007/BF00992698}
}

@misc{Williams1992A031,
  title = {Simple Statistical Gradient-Following Algorithms (REINFORCE)},
  author = {Ronald J. Williams},
  year = {1992},
  url = {https://link.springer.com/article/10.1007/BF00992696}
}

@misc{Tesauro1995A032,
  title = {TD-Gammon},
  author = {Gerald Tesauro},
  year = {1995},
  url = {https://dl.acm.org/doi/10.1145/203330.203343}
}

@misc{Mnih2015A033,
  title = {Human-level Control through Deep RL (DQN)},
  author = {Volodymyr Mnih and Koray Kavukcuoglu and David Silver and et al.},
  year = {2015},
  url = {https://www.nature.com/articles/nature14236}
}

@misc{Silver2016A034,
  title = {Mastering the Game of Go (AlphaGo/AlphaZero)},
  author = {David Silver and Aja Huang and Chris J. Maddison and Julian Schrittwieser and Thomas Hubert and Demis Hassabis},
  year = {2016},
  url = {https://www.nature.com/articles/nature16961}
}

@article{Kingma2013A035,
  title = {Auto-Encoding Variational Bayes (VAE)},
  author = {Diederik P. Kingma and Max Welling},
  year = {2013},
  url = {https://arxiv.org/abs/1312.6114},
  note = {arXiv:1312.6114}
}

@article{Goodfellow2014A036,
  title = {Generative Adversarial Networks (GAN)},
  author = {Ian J. Goodfellow and Jean Pouget-Abadie and Mehdi Mirza and et al.},
  year = {2014},
  url = {https://arxiv.org/abs/1406.2661},
  note = {arXiv:1406.2661}
}

@article{Graves2014A037,
  title = {Neural Turing Machines / Memory Networks},
  author = {Alex Graves and Greg Wayne and Ivo Danihelka and Jason Weston and Sumit Chopra and Antoine Bordes},
  year = {2014},
  url = {https://arxiv.org/abs/1410.5401},
  note = {arXiv:1410.5401}
}

@misc{Hinton2012A038,
  title = {Deep Neural Networks for Acoustic Modeling in Speech},
  author = {Geoffrey Hinton and Li Deng and Dong Yu and et al.},
  year = {2012},
  url = {https://ieeexplore.ieee.org/document/6296526}
}

@misc{Deng2009A040,
  title = {ImageNet: A Large-Scale Hierarchical Image Database},
  author = {Jia Deng and Wei Dong and Richard Socher and Li-Jia Li and Kai Li and Li Fei-Fei},
  year = {2009},
  url = {https://ieeexplore.ieee.org/document/5206848}
}

@misc{Boole1854O001,
  title = {Boolean Algebra (The Laws of Thought)},
  author = {George Boole},
  year = {1854},
  url = {https://en.wikipedia.org/wiki/The_Laws_of_Thought}
}

@misc{Babbage1843O002,
  title = {Analytical Engine & the First Algorithm},
  author = {Charles Babbage and Ada Lovelace},
  year = {1843},
  url = {https://en.wikipedia.org/wiki/Analytical_Engine}
}

@misc{Turing1936O003,
  title = {On Computable Numbers (Turing Machine)},
  author = {Alan Turing},
  year = {1936},
  url = {https://en.wikipedia.org/wiki/Turing_machine}
}

@misc{Shannon1937O004,
  title = {A Symbolic Analysis of Relay and Switching Circuits},
  author = {Claude Shannon},
  year = {1937},
  url = {https://en.wikipedia.org/wiki/A_Symbolic_Analysis_of_Relay_and_Switching_Circuits}
}

@misc{McCulloch1943O005,
  title = {A Logical Calculus of Ideas Immanent in Nervous Activity (artificial neuron)},
  author = {Warren McCulloch and Walter Pitts},
  year = {1943},
  url = {https://en.wikipedia.org/wiki/Artificial_neuron}
}

@misc{Neumann1945O006,
  title = {First Draft of a Report on the EDVAC (stored-program architecture)},
  author = {von Neumann},
  year = {1945},
  url = {https://en.wikipedia.org/wiki/Von_Neumann_architecture}
}

@misc{Bush1945O007,
  title = {As We May Think (Memex) + Science the Endless Frontier},
  author = {Vannevar Bush},
  year = {1945},
  url = {https://en.wikipedia.org/wiki/As_We_May_Think}
}

@misc{Shannon1948O008,
  title = {A Mathematical Theory of Communication (information theory)},
  author = {Claude Shannon},
  year = {1948},
  url = {https://en.wikipedia.org/wiki/A_Mathematical_Theory_of_Communication}
}

@book{Wiener1948O009,
  title = {Cybernetics: Control and Communication in the Animal and the Machine},
  author = {Norbert Wiener},
  year = {1948},
  url = {https://en.wikipedia.org/wiki/Cybernetics}
}

@book{Hebb1949O010,
  title = {The Organization of Behavior (Hebbian learning)},
  author = {Donald Hebb},
  year = {1949},
  url = {https://en.wikipedia.org/wiki/Hebbian_theory}
}

@misc{Turing1950O011,
  title = {Computing Machinery and Intelligence (Turing Test)},
  author = {Alan Turing},
  year = {1950},
  url = {https://en.wikipedia.org/wiki/Computing_Machinery_and_Intelligence}
}

@misc{Oppenheimer1945O012,
  title = {The Manhattan Project / Los Alamos (big-science template)},
  author = {J. Robert Oppenheimer and Leslie Groves and John von Neumann and et al.},
  year = {1945},
  url = {https://en.wikipedia.org/wiki/Manhattan_Project}
}

@misc{Ulam1949O013,
  title = {The Monte Carlo Method / Metropolis Algorithm},
  author = {Stanislaw Ulam and John von Neumann and Nicholas Metropolis},
  year = {1949},
  url = {https://en.wikipedia.org/wiki/Monte_Carlo_method}
}

@misc{Turing1943O014,
  title = {Enigma / Bletchley Park codebreaking (Bombe / Colossus)},
  author = {Alan Turing and Gordon Welchman and Tommy Flowers and Bill Tutte},
  year = {1943},
  url = {https://en.wikipedia.org/wiki/Bletchley_Park}
}

@book{Neumann1944O015,
  title = {Theory of Games and Economic Behavior (game theory)},
  author = {John von Neumann and Oskar Morgenstern},
  year = {1944},
  url = {https://en.wikipedia.org/wiki/Theory_of_Games_and_Economic_Behavior}
}

@misc{McCarthy1956O016,
  title = {The Dartmouth Workshop (birth of Artificial Intelligence)},
  author = {John McCarthy and Marvin Minsky and Claude Shannon and Nathaniel Rochester},
  year = {1956},
  url = {https://en.wikipedia.org/wiki/Dartmouth_workshop}
}

@misc{Rosenblatt1958O017,
  title = {The Perceptron},
  author = {Frank Rosenblatt},
  year = {1958},
  url = {https://en.wikipedia.org/wiki/Perceptron}
}

@misc{Minsky1969O018,
  title = {Perceptrons (Minsky-Papert critique)},
  author = {Marvin Minsky and Seymour Papert},
  year = {1969},
  url = {https://en.wikipedia.org/wiki/Perceptrons_(book)}
}

@misc{Solomonoff1964O019,
  title = {Algorithmic Information Theory (Solomonoff / Kolmogorov)},
  author = {Ray Solomonoff and Andrey Kolmogorov and Gregory Chaitin},
  year = {1964},
  url = {https://en.wikipedia.org/wiki/Kolmogorov_complexity}
}

@article{Vaswani2017P001,
  title = {Attention Is All You Need},
  author = {Ashish Vaswani and Noam Shazeer and Niki Parmar and Jakob Uszkoreit and Llion Jones and Aidan N. Gomez and Lukasz Kaiser and Illia Polosukhin},
  year = {2017},
  url = {https://arxiv.org/abs/1706.03762},
  note = {arXiv:1706.03762}
}

@article{Su2021P002,
  title = {RoFormer: Rotary Position Embedding (RoPE)},
  author = {Jianlin Su and Yu Lu and Shengfeng Pan and Ahmed Murtadha and Bo Wen and Yunfeng Liu},
  year = {2021},
  url = {https://arxiv.org/abs/2104.09864},
  note = {arXiv:2104.09864}
}

@article{Zhang2019P003,
  title = {Root Mean Square Layer Normalization (RMSNorm)},
  author = {Biao Zhang and Rico Sennrich},
  year = {2019},
  url = {https://arxiv.org/abs/1910.07467},
  note = {arXiv:1910.07467}
}

@article{Shazeer2020P004,
  title = {GLU Variants Improve Transformer (SwiGLU)},
  author = {Noam Shazeer},
  year = {2020},
  url = {https://arxiv.org/abs/2002.05202},
  note = {arXiv:2002.05202}
}

@article{Press2021P005,
  title = {Train Short Test Long: Attention with Linear Biases (ALiBi)},
  author = {Ofir Press and Noah A. Smith and Mike Lewis},
  year = {2021},
  url = {https://arxiv.org/abs/2108.12409},
  note = {arXiv:2108.12409}
}

@article{Shaw2018P006,
  title = {Self-Attention with Relative Position Representations / Transformer-XL},
  author = {Peter Shaw and Jakob Uszkoreit and Ashish Vaswani and Zihang Dai and Zhilin Yang and Quoc V. Le and Ruslan Salakhutdinov},
  year = {2018},
  url = {https://arxiv.org/abs/1901.02860},
  note = {arXiv:1901.02860}
}

@article{Xiong2020P007,
  title = {On Layer Normalization in the Transformer Architecture (pre-norm)},
  author = {Ruibin Xiong and Yunchang Yang and Di He and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2002.04745},
  note = {arXiv:2002.04745}
}

@article{Shazeer2019P008,
  title = {Fast Transformer Decoding: One Write-Head is All You Need (MQA)},
  author = {Noam Shazeer},
  year = {2019},
  url = {https://arxiv.org/abs/1911.02150},
  note = {arXiv:1911.02150}
}

@article{Ainslie2023P009,
  title = {GQA: Training Generalized Multi-Query Transformer Models},
  author = {Joshua Ainslie and James Lee-Thorp and Michiel de Jong and Yury Zemlyanskiy and Federico Lebron and Sumit Sanghai},
  year = {2023},
  url = {https://arxiv.org/abs/2305.13245},
  note = {arXiv:2305.13245}
}

@misc{Radford2018P010,
  title = {Improving Language Understanding by Generative Pre-Training (GPT)},
  author = {Alec Radford and Karthik Narasimhan and Tim Salimans and Ilya Sutskever},
  year = {2018},
  url = {https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf}
}

@misc{Radford2019P011,
  title = {Language Models are Unsupervised Multitask Learners (GPT-2)},
  author = {Alec Radford and Jeffrey Wu and Rewon Child and David Luan and Dario Amodei and Ilya Sutskever},
  year = {2019},
  url = {https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf}
}

@article{Brown2020P012,
  title = {Language Models are Few-Shot Learners (GPT-3)},
  author = {Tom B. Brown and Benjamin Mann and Nick Ryder and Melanie Subbiah and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2005.14165},
  note = {arXiv:2005.14165}
}

@article{Devlin2018P013,
  title = {BERT: Pre-training of Deep Bidirectional Transformers},
  author = {Jacob Devlin and Ming-Wei Chang and Kenton Lee and Kristina Toutanova},
  year = {2018},
  url = {https://arxiv.org/abs/1810.04805},
  note = {arXiv:1810.04805}
}

@article{Liu2019P014,
  title = {RoBERTa: A Robustly Optimized BERT Pretraining Approach},
  author = {Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and et al.},
  year = {2019},
  url = {https://arxiv.org/abs/1907.11692},
  note = {arXiv:1907.11692}
}

@article{Lan2019P015,
  title = {ALBERT: A Lite BERT},
  author = {Zhenzhong Lan and Mingda Chen and Sebastian Goodman and Kevin Gimpel and Piyush Sharma and Radu Soricut},
  year = {2019},
  url = {https://arxiv.org/abs/1909.11942},
  note = {arXiv:1909.11942}
}

@article{Clark2020P016,
  title = {ELECTRA: Pre-training Text Encoders as Discriminators},
  author = {Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning},
  year = {2020},
  url = {https://arxiv.org/abs/2003.10555},
  note = {arXiv:2003.10555}
}

@article{Raffel2020P020,
  title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (T5)},
  author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
  year = {2020},
  url = {https://arxiv.org/abs/1910.10683},
  note = {arXiv:1910.10683}
}

@article{Lewis2020P021,
  title = {BART: Denoising Seq2Seq Pre-training},
  author = {Mike Lewis and Yinhan Liu and Naman Goyal and Marjan Ghazvininejad and Abdelrahman Mohamed and Omer Levy and Ves Stoyanov and Luke Zettlemoyer},
  year = {2020},
  url = {https://arxiv.org/abs/1910.13461},
  note = {arXiv:1910.13461}
}

@article{Tay2023P022,
  title = {UL2: Unifying Language Learning Paradigms},
  author = {Yi Tay and Mostafa Dehghani and Vinh Q. Tran and Xavier Garcia and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2205.05131},
  note = {arXiv:2205.05131}
}

@article{He2021P024,
  title = {DeBERTa: Decoding-enhanced BERT with Disentangled Attention},
  author = {Pengcheng He and Xiaodong Liu and Jianfeng Gao and Weizhu Chen},
  year = {2021},
  url = {https://arxiv.org/abs/2006.03654},
  note = {arXiv:2006.03654}
}

@article{Kaplan2020P100,
  title = {Scaling Laws for Neural Language Models},
  author = {Jared Kaplan and Sam McCandlish and Tom Henighan and Tom B. Brown and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2001.08361},
  note = {arXiv:2001.08361}
}

@article{Hoffmann2022P101,
  title = {Training Compute-Optimal Large Language Models (Chinchilla)},
  author = {Jordan Hoffmann and Sebastian Borgeaud and Arthur Mensch and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2203.15556},
  note = {arXiv:2203.15556}
}

@article{Wei2022P102,
  title = {Emergent Abilities of Large Language Models},
  author = {Jason Wei and Yi Tay and Rishi Bommasani and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2206.07682},
  note = {arXiv:2206.07682}
}

@article{Schaeffer2023P103,
  title = {Are Emergent Abilities of LLMs a Mirage?},
  author = {Rylan Schaeffer and Brando Miranda and Sanmi Koyejo},
  year = {2023},
  url = {https://arxiv.org/abs/2304.15004},
  note = {arXiv:2304.15004}
}

@article{Muennighoff2023P104,
  title = {Scaling Data-Constrained Language Models},
  author = {Niklas Muennighoff and Alexander M. Rush and Boaz Barak and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2305.16264},
  note = {arXiv:2305.16264}
}

@article{Shazeer2017P110,
  title = {Outrageously Large Neural Networks: Sparsely-Gated MoE},
  author = {Noam Shazeer and Azalia Mirhoseini and Krzysztof Maziarz and Andy Davis and Quoc Le and Geoffrey Hinton and Jeff Dean},
  year = {2017},
  url = {https://arxiv.org/abs/1701.06538},
  note = {arXiv:1701.06538}
}

@article{Lepikhin2021P111,
  title = {GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding},
  author = {Dmitry Lepikhin and HyoukJoong Lee and Yuanzhong Xu and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2006.16668},
  note = {arXiv:2006.16668}
}

@article{Fedus2022P112,
  title = {Switch Transformer},
  author = {William Fedus and Barret Zoph and Noam Shazeer},
  year = {2022},
  url = {https://arxiv.org/abs/2101.03961},
  note = {arXiv:2101.03961}
}

@article{Du2022P113,
  title = {GLaM: Efficient Scaling with Mixture-of-Experts},
  author = {Nan Du and Yanping Huang and Andrew M. Dai and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2112.06905},
  note = {arXiv:2112.06905}
}

@article{Zhou2022P114,
  title = {Mixture-of-Experts with Expert Choice Routing},
  author = {Yanqi Zhou and Tao Lei and Hanxiao Liu and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2202.09368},
  note = {arXiv:2202.09368}
}

@article{Zoph2022P115,
  title = {ST-MoE: Designing Stable and Transferable Sparse Expert Models},
  author = {Barret Zoph and Irwan Bello and Sameer Kumar and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2202.08906},
  note = {arXiv:2202.08906}
}

@article{Jiang2024P116,
  title = {Mixtral of Experts},
  author = {Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and et al.},
  year = {2024},
  url = {https://arxiv.org/abs/2401.04088},
  note = {arXiv:2401.04088}
}

@article{Dai2024P117,
  title = {DeepSeekMoE: Towards Ultimate Expert Specialization},
  author = {Damai Dai and Chengqi Deng and Chenggang Zhao and et al.},
  year = {2024},
  url = {https://arxiv.org/abs/2401.06066},
  note = {arXiv:2401.06066}
}

@article{Kudo2018P120,
  title = {SentencePiece},
  author = {Taku Kudo and John Richardson},
  year = {2018},
  url = {https://arxiv.org/abs/1808.06226},
  note = {arXiv:1808.06226}
}

@article{Kudo2018P121,
  title = {Subword Regularization / Unigram LM Tokenization},
  author = {Taku Kudo},
  year = {2018},
  url = {https://arxiv.org/abs/1804.10959},
  note = {arXiv:1804.10959}
}

@article{Lee2022P122,
  title = {Deduplicating Training Data Makes Language Models Better},
  author = {Katherine Lee and Daphne Ippolito and Andrew Nystrom and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2107.06499},
  note = {arXiv:2107.06499}
}

@article{Gao2021P123,
  title = {The Pile},
  author = {Leo Gao and Stella Biderman and Sid Black and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2101.00027},
  note = {arXiv:2101.00027}
}

@article{Penedo2023P124,
  title = {The RefinedWeb Dataset for Falcon LLM},
  author = {Guilherme Penedo and Quentin Malartic and Daniel Hesslow and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2306.01116},
  note = {arXiv:2306.01116}
}

@article{Gunasekar2023P125,
  title = {Textbooks Are All You Need (phi)},
  author = {Suriya Gunasekar and Yi Zhang and Jyoti Aneja and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2306.11644},
  note = {arXiv:2306.11644}
}

@article{Shumailov2024P126,
  title = {The Curse of Recursion / Model Collapse},
  author = {Ilia Shumailov and Zakhar Shumaylov and Yiren Zhao and Yarin Gal and Nicolas Papernot and Ross Anderson},
  year = {2024},
  url = {https://arxiv.org/abs/2305.17493},
  note = {arXiv:2305.17493}
}

@article{Xie2023P127,
  title = {DoReMi: Optimizing Data Mixtures with Group DRO},
  author = {Sang Michael Xie and Hieu Pham and Xuanyi Dong and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2305.10429},
  note = {arXiv:2305.10429}
}

@article{Micikevicius2018P130,
  title = {Mixed Precision Training},
  author = {Paulius Micikevicius and Sharan Narang and Jonah Alben and et al.},
  year = {2018},
  url = {https://arxiv.org/abs/1710.03740},
  note = {arXiv:1710.03740}
}

@article{Shoeybi2019P131,
  title = {Megatron-LM: Training Multi-Billion Parameter LMs Using Model Parallelism},
  author = {Mohammad Shoeybi and Mostofa Patwary and Raul Puri and et al.},
  year = {2019},
  url = {https://arxiv.org/abs/1909.08053},
  note = {arXiv:1909.08053}
}

@article{Rajbhandari2020P132,
  title = {ZeRO: Memory Optimizations Toward Training Trillion Parameter Models},
  author = {Samyam Rajbhandari and Jeff Rasley and Olatunji Ruwase and Yuxiong He},
  year = {2020},
  url = {https://arxiv.org/abs/1910.02054},
  note = {arXiv:1910.02054}
}

@article{Huang2019P133,
  title = {GPipe: Efficient Training of Giant Neural Networks Using Pipeline Parallelism},
  author = {Yanping Huang and Youlong Cheng and Ankur Bapna and et al.},
  year = {2019},
  url = {https://arxiv.org/abs/1811.06965},
  note = {arXiv:1811.06965}
}

@misc{Narayanan2019P134,
  title = {PipeDream: Generalized Pipeline Parallelism for DNN Training},
  author = {Deepak Narayanan and Aaron Harlap and Amar Phanishayee and et al.},
  year = {2019},
  url = {https://www.microsoft.com/en-us/research/publication/pipedream-generalized-pipeline-parallelism-for-dnn-training/}
}

@article{Shazeer2018P135,
  title = {Mesh-TensorFlow / GSPMD},
  author = {Noam Shazeer and Youlong Cheng and Niki Parmar and Yuanzhong Xu and HyoukJoong Lee and Blake Hechtman and Zhifeng Chen},
  year = {2018},
  url = {https://arxiv.org/abs/2105.04663},
  note = {arXiv:2105.04663}
}

@article{Zhao2023P136,
  title = {PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel},
  author = {Yanli Zhao and Andrew Gu and Rohan Varma and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2304.11277},
  note = {arXiv:2304.11277}
}

@article{Chen2016P137,
  title = {Training Deep Nets with Sublinear Memory Cost (Activation Checkpointing)},
  author = {Tianqi Chen and Bing Xu and Chiyuan Zhang and Carlos Guestrin},
  year = {2016},
  url = {https://arxiv.org/abs/1604.06174},
  note = {arXiv:1604.06174}
}

@article{Peng2023P138,
  title = {FP8-LM / FP8 Training},
  author = {Houwen Peng and Kan Wu and Han Hu and Peng Cheng and Paulius Micikevicius},
  year = {2023},
  url = {https://arxiv.org/abs/2310.18313},
  note = {arXiv:2310.18313}
}

@article{Christiano2017P200,
  title = {Deep Reinforcement Learning from Human Preferences},
  author = {Paul Christiano and Jan Leike and Tom Brown and Miljan Martic and Shane Legg and Dario Amodei},
  year = {2017},
  url = {https://arxiv.org/abs/1706.03741},
  note = {arXiv:1706.03741}
}

@article{Stiennon2020P201,
  title = {Learning to Summarize from Human Feedback},
  author = {Nisan Stiennon and Long Ouyang and Jeff Wu and Daniel Ziegler and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2009.01325},
  note = {arXiv:2009.01325}
}

@article{Ouyang2022P202,
  title = {InstructGPT: Training LMs to Follow Instructions with Human Feedback},
  author = {Long Ouyang and Jeff Wu and Xu Jiang and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2203.02155},
  note = {arXiv:2203.02155}
}

@article{Wei2022P203,
  title = {Finetuned Language Models Are Zero-Shot Learners (FLAN) / Self-Instruct},
  author = {Jason Wei and Maarten Bosma and Quoc V. Le and Yizhong Wang and Hannaneh Hajishirzi and Noah A. Smith and Daniel Khashabi},
  year = {2022},
  url = {https://arxiv.org/abs/2109.01652},
  note = {arXiv:2109.01652}
}

@article{Bai2022P204,
  title = {Constitutional AI: Harmlessness from AI Feedback (RLAIF)},
  author = {Yuntao Bai and Saurav Kadavath and Sandipan Kundu and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2212.08073},
  note = {arXiv:2212.08073}
}

@article{Rafailov2023P205,
  title = {Direct Preference Optimization (DPO)},
  author = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Stefano Ermon and Christopher Manning and Chelsea Finn},
  year = {2023},
  url = {https://arxiv.org/abs/2305.18290},
  note = {arXiv:2305.18290}
}

@article{Azar2024P206,
  title = {Preference Optimization Family (IPO/KTO/ORPO/SimPO)},
  author = {Mohammad Gheshlaghi Azar and Rémi Munos and Kawin Ethayarajh and Douwe Kiela and Jiwoo Hong and James Thorne and Yu Meng and Danqi Chen},
  year = {2024},
  url = {https://arxiv.org/abs/2402.01306},
  note = {arXiv:2402.01306}
}

@article{Gao2022P207,
  title = {Reward Model Overoptimization / Sycophancy},
  author = {Leo Gao and John Schulman and Jacob Hilton and Mrinank Sharma and Ethan Perez},
  year = {2022},
  url = {https://arxiv.org/abs/2210.10760},
  note = {arXiv:2210.10760}
}

@article{Irving2018P208,
  title = {Scalable Oversight: Debate / Weak-to-Strong Generalization},
  author = {Geoffrey Irving and Paul Christiano and Dario Amodei and Collin Burns and Jan Leike and Ilya Sutskever},
  year = {2018},
  url = {https://arxiv.org/abs/2312.09390},
  note = {arXiv:2312.09390}
}

@article{Wei2022P220,
  title = {Chain-of-Thought Prompting Elicits Reasoning},
  author = {Jason Wei and Xuezhi Wang and Dale Schuurmans and Maarten Bosma and Brian Ichter and Fei Xia and Ed Chi and Quoc Le and Denny Zhou},
  year = {2022},
  url = {https://arxiv.org/abs/2201.11903},
  note = {arXiv:2201.11903}
}

@article{Wang2023P221,
  title = {Self-Consistency Improves Chain-of-Thought Reasoning},
  author = {Xuezhi Wang and Jason Wei and Dale Schuurmans and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2203.11171},
  note = {arXiv:2203.11171}
}

@article{Zhou2023P222,
  title = {Structured Reasoning: Least-to-Most / PoT / Tree of Thoughts},
  author = {Denny Zhou and Quoc V. Le and Ed H. Chi and Wenhu Chen and William W. Cohen and Shunyu Yao and Karthik Narasimhan},
  year = {2023},
  url = {https://arxiv.org/abs/2305.10601},
  note = {arXiv:2305.10601}
}

@article{Cobbe2023P223,
  title = {Verifiers & Process Supervision (GSM8K / Let's Verify Step by Step)},
  author = {Karl Cobbe and John Schulman and Hunter Lightman and Ilya Sutskever and Jan Leike},
  year = {2023},
  url = {https://arxiv.org/abs/2305.20050},
  note = {arXiv:2305.20050}
}

@article{DeepSeekAI2025P224,
  title = {DeepSeek-R1: Incentivizing Reasoning via RL (RLVR)},
  author = {DeepSeek-AI and Daya Guo and Zhihong Shao and Wenfeng Liang},
  year = {2025},
  url = {https://arxiv.org/abs/2501.12948},
  note = {arXiv:2501.12948}
}

@misc{OpenAI2024P225,
  title = {OpenAI o1 / o3 Reasoning Systems},
  author = {OpenAI},
  year = {2024},
  url = {https://openai.com/index/openai-o1-system-card/}
}

@article{Turpin2023P226,
  title = {Reasoning Correctives: Faithfulness / Overthinking / Contamination},
  author = {Miles Turpin and Julian Michael and Ethan Perez and Samuel R. Bowman},
  year = {2023},
  url = {https://arxiv.org/abs/2305.04388},
  note = {arXiv:2305.04388}
}

@article{Yao2023P230,
  title = {ReAct: Synergizing Reasoning and Acting in LMs},
  author = {Shunyu Yao and Jeffrey Zhao and Dian Yu and Nan Du and Izhak Shafran and Karthik Narasimhan and Yuan Cao},
  year = {2023},
  url = {https://arxiv.org/abs/2210.03629},
  note = {arXiv:2210.03629}
}

@article{Schick2023P231,
  title = {Toolformer: LMs Can Teach Themselves to Use Tools},
  author = {Timo Schick and Jane Dwivedi-Yu and Roberto Dessi and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2302.04761},
  note = {arXiv:2302.04761}
}

@article{Karpas2023P232,
  title = {Function Calling / MRKL / Model Context Protocol},
  author = {Ehud Karpas and Opher Lieber and Barak Lenz and Yoav Levine and Yoav Shoham and OpenAI and Anthropic},
  year = {2023},
  url = {https://arxiv.org/abs/2205.00445},
  note = {arXiv:2205.00445}
}

@article{Shinn2023P233,
  title = {Reflexion: Language Agents with Verbal Reinforcement Learning},
  author = {Noah Shinn and Federico Cassano and Edward Berman and Ashwin Gopinath and Karthik Narasimhan and Shunyu Yao},
  year = {2023},
  url = {https://arxiv.org/abs/2303.11366},
  note = {arXiv:2303.11366}
}

@article{Wang2023P234,
  title = {Voyager / Generative Agents},
  author = {Guanzhi Wang and Linxi Fan and Anima Anandkumar and Joon Sung Park and Percy Liang and Michael S. Bernstein},
  year = {2023},
  url = {https://arxiv.org/abs/2305.16291},
  note = {arXiv:2305.16291}
}

@article{Yang2024P235,
  title = {SWE-agent / OpenHands (autonomous software engineering)},
  author = {John Yang and Carlos E. Jimenez and Ofir Press and Karthik Narasimhan and Xingyao Wang and Graham Neubig},
  year = {2024},
  url = {https://arxiv.org/abs/2405.15793},
  note = {arXiv:2405.15793}
}

@article{Packer2023P236,
  title = {Agent Memory & Multi-Agent Systems (MemGPT/AutoGen/CAMEL)},
  author = {Charles Packer and Joseph E. Gonzalez and Qingyun Wu and Chi Wang and Guohao Li and Bernard Ghanem},
  year = {2023},
  url = {https://arxiv.org/abs/2310.08560},
  note = {arXiv:2310.08560}
}

@misc{Richards2023P237,
  title = {AutoGPT / BabyAGI (viral agent software)},
  author = {Toran Bruce Richards and Yohei Nakajima},
  year = {2023},
  url = {https://github.com/Significant-Gravitas/AutoGPT}
}

@article{Dosovitskiy2021P300,
  title = {Vision Transformer (An Image is Worth 16x16 Words)},
  author = {Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2010.11929},
  note = {arXiv:2010.11929}
}

@article{Radford2021P301,
  title = {CLIP: Learning Transferable Visual Models from NL Supervision},
  author = {Alec Radford and Jong Wook Kim and Chris Hallacy and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2103.00020},
  note = {arXiv:2103.00020}
}

@article{Jia2021P302,
  title = {ALIGN: Scaling Up Visual and Vision-Language Representation Learning},
  author = {Chao Jia and Yinfei Yang and Ye Xia and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2102.05918},
  note = {arXiv:2102.05918}
}

@article{Alayrac2022P303,
  title = {Flamingo: a Visual Language Model for Few-Shot Learning},
  author = {Jean-Baptiste Alayrac and Jeff Donahue and Pauline Luc and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2204.14198},
  note = {arXiv:2204.14198}
}

@article{Li2023P304,
  title = {BLIP / BLIP-2 (Q-Former)},
  author = {Junnan Li and Dongxu Li and Silvio Savarese and Steven C. H. Hoi and Caiming Xiong},
  year = {2023},
  url = {https://arxiv.org/abs/2301.12597},
  note = {arXiv:2301.12597}
}

@article{Liu2023P305,
  title = {LLaVA: Visual Instruction Tuning},
  author = {Haotian Liu and Chunyuan Li and Qingyang Wu and Yong Jae Lee},
  year = {2023},
  url = {https://arxiv.org/abs/2304.08485},
  note = {arXiv:2304.08485}
}

@article{Bai2023P306,
  title = {Open Production VLMs (Qwen-VL/InternVL/PaLI/Kosmos)},
  author = {Jinze Bai and Jingren Zhou and Zhe Chen and Jifeng Dai and Xi Chen and Radu Soricut and Shaohan Huang and Furu Wei},
  year = {2023},
  url = {https://arxiv.org/abs/2308.12966},
  note = {arXiv:2308.12966}
}

@article{Kirillov2023P307,
  title = {SAM / Grounding DINO / ImageBind (perception foundation models)},
  author = {Alexander Kirillov and Ross Girshick and Piotr Dollár and Shilong Liu and Lei Zhang and Rohit Girdhar and Ishan Misra},
  year = {2023},
  url = {https://arxiv.org/abs/2304.02643},
  note = {arXiv:2304.02643}
}

@article{Rombach2022P308,
  title = {Latent Diffusion (Stable Diffusion) / Diffusion Transformers (DiT)},
  author = {Robin Rombach and Patrick Esser and Björn Ommer and William Peebles and Saining Xie},
  year = {2022},
  url = {https://arxiv.org/abs/2112.10752},
  note = {arXiv:2112.10752}
}

@article{DeepMind2024P309,
  title = {Native/Omni Multimodality (Gemini / GPT-4V / GPT-4o)},
  author = {Google DeepMind and OpenAI},
  year = {2024},
  url = {https://arxiv.org/abs/2312.11805},
  note = {arXiv:2312.11805}
}

@article{Karpukhin2020P320,
  title = {Dense Passage Retrieval (DPR)},
  author = {Vladimir Karpukhin and Barlas Oguz and Sewon Min and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2004.04906},
  note = {arXiv:2004.04906}
}

@article{Guu2020P321,
  title = {REALM: Retrieval-Augmented LM Pre-Training},
  author = {Kelvin Guu and Kenton Lee and Zora Tung and Panupong Pasupat and Ming-Wei Chang},
  year = {2020},
  url = {https://arxiv.org/abs/2002.08909},
  note = {arXiv:2002.08909}
}

@article{Lewis2020P322,
  title = {Retrieval-Augmented Generation (RAG)},
  author = {Patrick Lewis and Ethan Perez and Aleksandra Piktus and et al.},
  year = {2020},
  url = {https://arxiv.org/abs/2005.11401},
  note = {arXiv:2005.11401}
}

@article{Izacard2021P323,
  title = {FiD / RETRO / Atlas (scaling retrieval)},
  author = {Gautier Izacard and Edouard Grave and Sebastian Borgeaud and Jack W. Rae and Laurent Sifre},
  year = {2021},
  url = {https://arxiv.org/abs/2112.04426},
  note = {arXiv:2112.04426}
}

@article{Khattab2020P324,
  title = {ColBERT / Late Interaction / Rerankers / kNN-LM},
  author = {Omar Khattab and Matei Zaharia and Urvashi Khandelwal and Mike Lewis and Dan Jurafsky},
  year = {2020},
  url = {https://arxiv.org/abs/2004.12832},
  note = {arXiv:2004.12832}
}

@article{Reimers2019P325,
  title = {Contrastive Text Embeddings (Sentence-BERT/SimCSE/E5/BGE)},
  author = {Nils Reimers and Iryna Gurevych and Tianyu Gao and Danqi Chen and Liang Wang and Furu Wei and Shitao Xiao and Zheng Liu},
  year = {2019},
  url = {https://arxiv.org/abs/1908.10084},
  note = {arXiv:1908.10084}
}

@article{Meng2022P326,
  title = {Knowledge Editing (ROME / MEMIT)},
  author = {Kevin Meng and David Bau and Alex Andonian and Yonatan Belinkov},
  year = {2022},
  url = {https://arxiv.org/abs/2202.05262},
  note = {arXiv:2202.05262}
}

@article{Edge2024P327,
  title = {Graph & Structured Retrieval (GraphRAG)},
  author = {Darren Edge and Ha Trinh and et al.},
  year = {2024},
  url = {https://arxiv.org/abs/2404.16130},
  note = {arXiv:2404.16130}
}

@article{Chen2021P340,
  title = {Codex: Evaluating LLMs Trained on Code (HumanEval)},
  author = {Mark Chen and Jerry Tworek and Heewoo Jun and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2107.03374},
  note = {arXiv:2107.03374}
}

@article{Li2022P341,
  title = {AlphaCode (Competition-Level Code Generation)},
  author = {Yujia Li and David Choi and Junyoung Chung and et al.},
  year = {2022},
  url = {https://arxiv.org/abs/2203.07814},
  note = {arXiv:2203.07814}
}

@article{Wang2022P342,
  title = {CodeT5 / CodeGen / InCoder / Fill-in-the-Middle},
  author = {Yue Wang and Steven C.H. Hoi and Erik Nijkamp and Caiming Xiong and Daniel Fried and Mike Lewis and Mohammad Bavarian and Mark Chen},
  year = {2022},
  url = {https://arxiv.org/abs/2207.14255},
  note = {arXiv:2207.14255}
}

@article{Li2023P343,
  title = {Open Code Models (StarCoder/Code Llama/DeepSeek-Coder/Qwen-Coder)},
  author = {Raymond Li and Harm de Vries and Leandro von Werra and Baptiste Rozière and Gabriel Synnaeve and Daya Guo and Binyuan Hui},
  year = {2023},
  url = {https://arxiv.org/abs/2305.06161},
  note = {arXiv:2305.06161}
}

@article{Chen2022P344,
  title = {Execution Feedback / Unit-Test Gen / Program Repair},
  author = {Bei Chen and Jian-Guang Lou and Weizhu Chen and Xinyun Chen and Denny Zhou},
  year = {2022},
  url = {https://arxiv.org/abs/2207.10397},
  note = {arXiv:2207.10397}
}

@article{Touvron2023P360,
  title = {LLaMA: Open and Efficient Foundation Language Models},
  author = {Hugo Touvron and Thibaut Lavril and Gautier Izacard and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2302.13971},
  note = {arXiv:2302.13971}
}

@article{Touvron2024P361,
  title = {Llama 2 & Llama 3},
  author = {Hugo Touvron and Abhimanyu Dubey and et al.},
  year = {2024},
  url = {https://arxiv.org/abs/2407.21783},
  note = {arXiv:2407.21783}
}

@article{Jiang2023P362,
  title = {Mistral 7B / Mixtral},
  author = {Albert Q. Jiang and Alexandre Sablayrolles and Guillaume Lample and Arthur Mensch and Timothée Lacroix and William El Sayed},
  year = {2023},
  url = {https://arxiv.org/abs/2310.06825},
  note = {arXiv:2310.06825}
}

@article{Bai2024P363,
  title = {Qwen (Qwen/Qwen2/Qwen2.5)},
  author = {Jinze Bai and An Yang and Junyang Lin and Jingren Zhou},
  year = {2024},
  url = {https://arxiv.org/abs/2412.15115},
  note = {arXiv:2412.15115}
}

@article{DeepSeekAI2024P364,
  title = {DeepSeek (V2/V3; MLA + efficient MoE + FP8)},
  author = {DeepSeek-AI and Damai Dai and Aixin Liu and Wenfeng Liang},
  year = {2024},
  url = {https://arxiv.org/abs/2412.19437},
  note = {arXiv:2412.19437}
}

@article{Bubeck2024P365,
  title = {Open Ecosystem (Gemma/Phi/OLMo/Falcon/Command R/Yi/GLM/InternLM)},
  author = {Sébastien Bubeck and Marah Abdin and Dirk Groeneveld and Guilherme Penedo and Jie Tang and Gemma Team (Google DeepMind) and Shanghai AI Laboratory},
  year = {2024},
  url = {https://arxiv.org/abs/2402.00838},
  note = {arXiv:2402.00838}
}

@article{Du2021P366,
  title = {GLM: General Language Model Pretraining with Autoregressive Blank Infilling / GLM-130B},
  author = {Zhengxiao Du and Yujie Qian and Xiao Liu and Ming Ding and Zhilin Yang and Jie Tang and Aohan Zeng},
  year = {2021},
  url = {https://arxiv.org/abs/2103.10360},
  note = {arXiv:2103.10360}
}

@article{AI2025P367,
  title = {GLM-4.5 to GLM-5.2 (Zhipu AI / Z.ai open-weight MoE series)},
  author = {Zhipu AI and Z.ai GLM Team and Jie Tang and Aohan Zeng},
  year = {2025},
  url = {https://arxiv.org/abs/2508.06471},
  note = {arXiv:2508.06471}
}

@article{Scao2022P368,
  title = {BLOOM: A 176B-Parameter Open-Access Multilingual Language Model},
  author = {Teven Le Scao and Angela Fan and Christopher Akiki and Thomas Wolf and Stella Biderman},
  year = {2022},
  url = {https://arxiv.org/abs/2211.05100},
  note = {arXiv:2211.05100}
}

@article{Schuhmann2022P369,
  title = {LAION-5B: An Open Large-Scale Dataset for Training Next Generation Image-Text Models},
  author = {Christoph Schuhmann and Romain Beaumont and Richard Vencu and Cade Gordon and Ludwig Schmidt},
  year = {2022},
  url = {https://arxiv.org/abs/2210.08402},
  note = {arXiv:2210.08402}
}

@article{Dao2022P400,
  title = {FlashAttention (1/2/3): IO-Aware Exact Attention},
  author = {Tri Dao and Daniel Y. Fu and Stefano Ermon and Atri Rudra and Christopher Ré and Jay Shah},
  year = {2022},
  url = {https://arxiv.org/abs/2205.14135},
  note = {arXiv:2205.14135}
}

@article{Kwon2023P401,
  title = {PagedAttention / vLLM},
  author = {Woosuk Kwon and Zhuohan Li and Siyuan Zhuang and Ying Sheng and Ion Stoica},
  year = {2023},
  url = {https://arxiv.org/abs/2309.06180},
  note = {arXiv:2309.06180}
}

@misc{Yu2022P402,
  title = {Orca: Continuous (Iteration-Level) Batching},
  author = {Gyeong-In Yu and Joo Seong Jeong and Geon-Woo Kim and et al.},
  year = {2022},
  url = {https://www.usenix.org/conference/osdi22/presentation/yu}
}

@article{Leviathan2023P403,
  title = {Speculative Decoding / Medusa / EAGLE},
  author = {Yaniv Leviathan and Yossi Matias and Charlie Chen and Tianle Cai and Tri Dao and Yuhui Li and Hongyang Zhang},
  year = {2023},
  url = {https://arxiv.org/abs/2211.17192},
  note = {arXiv:2211.17192}
}

@article{Frantar2022P404,
  title = {Inference Quantization (GPTQ/AWQ/SmoothQuant/bitsandbytes)},
  author = {Elias Frantar and Dan Alistarh and Ji Lin and Song Han and Guangxuan Xiao and Tim Dettmers and Luke Zettlemoyer},
  year = {2022},
  url = {https://arxiv.org/abs/2210.17323},
  note = {arXiv:2210.17323}
}

@article{Hu2022P405,
  title = {LoRA / QLoRA (parameter-efficient fine-tuning)},
  author = {Edward J. Hu and Yelong Shen and Weizhu Chen and Tim Dettmers and Luke Zettlemoyer},
  year = {2022},
  url = {https://arxiv.org/abs/2106.09685},
  note = {arXiv:2106.09685}
}

@article{Gerganov2023P406,
  title = {Edge/Format Ecosystem (llama.cpp/GGUF/TensorRT-LLM/SGLang)},
  author = {Georgi Gerganov and Lianmin Zheng and Ying Sheng and Ion Stoica and NVIDIA},
  year = {2023},
  url = {https://arxiv.org/abs/2312.07104},
  note = {arXiv:2312.07104}
}

@article{Beltagy2020P420,
  title = {Sparse Attention (Longformer / BigBird)},
  author = {Iz Beltagy and Matthew E. Peters and Arman Cohan and Manzil Zaheer and Amr Ahmed},
  year = {2020},
  url = {https://arxiv.org/abs/2007.14062},
  note = {arXiv:2007.14062}
}

@article{Kitaev2020P421,
  title = {Linearized Attention (Reformer/Performer/Linear Transformers)},
  author = {Nikita Kitaev and Łukasz Kaiser and Krzysztof Choromanski and Adrian Weller and Angelos Katharopoulos and François Fleuret},
  year = {2020},
  url = {https://arxiv.org/abs/2009.14794},
  note = {arXiv:2009.14794}
}

@article{Liu2024P422,
  title = {Ring Attention / Infini-Attention / MLA},
  author = {Hao Liu and Matei Zaharia and Pieter Abbeel and Tsendsuren Munkhdalai and Manaal Faruqui and DeepSeek-AI},
  year = {2024},
  url = {https://arxiv.org/abs/2310.01889},
  note = {arXiv:2310.01889}
}

@article{Chen2023P423,
  title = {Context Extension (Position Interpolation / NTK / YaRN)},
  author = {Shouyuan Chen and Yuandong Tian and Bowen Peng and Jeffrey Quesnelle and Enrico Shippole},
  year = {2023},
  url = {https://arxiv.org/abs/2309.00071},
  note = {arXiv:2309.00071}
}

@article{Gu2022P424,
  title = {S4: Structured State Space Sequence Models},
  author = {Albert Gu and Karan Goel and Christopher Ré},
  year = {2022},
  url = {https://arxiv.org/abs/2111.00396},
  note = {arXiv:2111.00396}
}

@article{Gu2024P425,
  title = {Mamba / Mamba-2 (Selective State Spaces)},
  author = {Albert Gu and Tri Dao},
  year = {2024},
  url = {https://arxiv.org/abs/2312.00752},
  note = {arXiv:2312.00752}
}

@article{Peng2023P426,
  title = {RWKV / Hyena / Attention-SSM Hybrids (Jamba)},
  author = {Bo Peng and Michael Poli and Stefano Ermon and Christopher Ré and Opher Lieber and Yoav Shoham},
  year = {2023},
  url = {https://arxiv.org/abs/2305.13048},
  note = {arXiv:2305.13048}
}

@article{Liu2023P427,
  title = {Lost in the Middle / Long-Context Reliability (RULER)},
  author = {Nelson F. Liu and Kevin Lin and Percy Liang and Cheng-Ping Hsieh and Boris Ginsburg},
  year = {2023},
  url = {https://arxiv.org/abs/2307.03172},
  note = {arXiv:2307.03172}
}

@article{Hendrycks2021P440,
  title = {MMLU: Massive Multitask Language Understanding},
  author = {Dan Hendrycks and Collin Burns and Steven Basart and et al.},
  year = {2021},
  url = {https://arxiv.org/abs/2009.03300},
  note = {arXiv:2009.03300}
}

@article{Rein2023P441,
  title = {GPQA: Graduate-Level Google-Proof QA},
  author = {David Rein and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2311.12022},
  note = {arXiv:2311.12022}
}

@article{Cobbe2021P442,
  title = {GSM8K / MATH (mathematical reasoning)},
  author = {Karl Cobbe and Vineet Kosaraju and John Schulman and Dan Hendrycks and Jacob Steinhardt},
  year = {2021},
  url = {https://arxiv.org/abs/2103.03874},
  note = {arXiv:2103.03874}
}

@article{Srivastava2022P443,
  title = {BIG-bench / BBH / HELM (holistic eval)},
  author = {Aarohi Srivastava and Mirac Suzgun and Jason Wei and Percy Liang and Rishi Bommasani and Tony Lee},
  year = {2022},
  url = {https://arxiv.org/abs/2211.09110},
  note = {arXiv:2211.09110}
}

@article{Chen2021P444,
  title = {HumanEval / MBPP (function-level code)},
  author = {Mark Chen and Wojciech Zaremba and Jacob Austin and Augustus Odena and Charles Sutton},
  year = {2021},
  url = {https://arxiv.org/abs/2108.07732},
  note = {arXiv:2108.07732}
}

@article{Jimenez2024P445,
  title = {SWE-bench (repository-level SWE)},
  author = {Carlos E. Jimenez and John Yang and et al.},
  year = {2024},
  url = {https://arxiv.org/abs/2310.06770},
  note = {arXiv:2310.06770}
}

@article{Liu2023P446,
  title = {Agent/Web Benchmarks (AgentBench/WebArena/GAIA/tau-bench)},
  author = {Xiao Liu and Jie Tang and Shuyan Zhou and Graham Neubig and Grégoire Mialon and Thomas Scialom and Shunyu Yao and Karthik Narasimhan},
  year = {2023},
  url = {https://arxiv.org/abs/2311.12983},
  note = {arXiv:2311.12983}
}

@article{Zheng2023P447,
  title = {Chatbot Arena / LLM-as-Judge (MT-Bench/AlpacaEval)},
  author = {Lianmin Zheng and Wei-Lin Chiang and Ion Stoica and Xuechen Li and Yann Dubois and Tatsunori B. Hashimoto},
  year = {2023},
  url = {https://arxiv.org/abs/2403.04132},
  note = {arXiv:2403.04132}
}

@article{Lin2021P448,
  title = {Contamination-Resistant Benchmarks (TruthfulQA/LiveBench/SimpleQA)},
  author = {Stephanie Lin and Owain Evans and Colin White and Tom Goldstein and Jason Wei and William Fedus},
  year = {2021},
  url = {https://arxiv.org/abs/2406.19314},
  note = {arXiv:2406.19314}
}

@article{Sainz2023P449,
  title = {The Evaluation Crisis (contamination/saturation/judge-bias)},
  author = {Oscar Sainz and Zhang and et al.},
  year = {2023},
  url = {https://arxiv.org/abs/2310.18018},
  note = {arXiv:2310.18018}
}

@misc{Olah2020P500,
  title = {Zoom In: An Introduction to Circuits},
  author = {Chris Olah and Nick Cammarata and Ludwig Schubert and Gabriel Goh and Michael Petrov and Shan Carter},
  year = {2020},
  url = {https://distill.pub/2020/circuits/zoom-in/}
}

@misc{Elhage2021P501,
  title = {A Mathematical Framework for Transformer Circuits},
  author = {Nelson Elhage and Neel Nanda and Catherine Olsson and Tom Henighan and Chris Olah},
  year = {2021},
  url = {https://transformer-circuits.pub/2021/framework/index.html}
}

@misc{Olsson2022P502,
  title = {In-context Learning and Induction Heads},
  author = {Catherine Olsson and Nelson Elhage and Neel Nanda and Nicholas Joseph and Chris Olah},
  year = {2022},
  url = {https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html}
}

@misc{Elhage2022P503,
  title = {Toy Models of Superposition},
  author = {Nelson Elhage and Tristan Hume and Catherine Olsson and Neel Nanda and Chris Olah},
  year = {2022},
  url = {https://transformer-circuits.pub/2022/toy_model/index.html}
}

@misc{Bricken2023P504,
  title = {Towards Monosemanticity: Decomposing Language Models With Dictionary Learning},
  author = {Trenton Bricken and Adly Templeton and Joshua Batson and Chris Olah},
  year = {2023},
  url = {https://transformer-circuits.pub/2023/monosemantic-features/index.html}
}

@misc{Sutton2019P510,
  title = {The Bitter Lesson},
  author = {Richard Sutton},
  year = {2019},
  url = {http://www.incompleteideas.net/IncIdeas/BitterLesson.html}
}

@misc{Branwen2020P511,
  title = {The Scaling Hypothesis},
  author = {Gwern Branwen},
  year = {2020},
  url = {https://gwern.net/scaling-hypothesis}
}

@misc{Aschenbrenner2024P512,
  title = {Situational Awareness: The Decade Ahead},
  author = {Leopold Aschenbrenner},
  year = {2024},
  url = {https://situational-awareness.ai/}
}

@misc{Halevy2009P513,
  title = {The Unreasonable Effectiveness of Data},
  author = {Alon Halevy and Peter Norvig and Fernando Pereira},
  year = {2009},
  url = {https://ieeexplore.ieee.org/document/4804817}
}

@misc{Amodei2018P514,
  title = {AI and Compute},
  author = {Dario Amodei and Danny Hernandez},
  year = {2018},
  url = {https://openai.com/index/ai-and-compute/}
}

@article{Nakano2021P520,
  title = {WebGPT: Browser-assisted Question-answering with Human Feedback},
  author = {Reiichiro Nakano and Jacob Hilton and Suchir Balaji and John Schulman},
  year = {2021},
  url = {https://arxiv.org/abs/2112.09332},
  note = {arXiv:2112.09332}
}

@article{Ahn2022P521,
  title = {Do As I Can, Not As I Say: Grounding Language in Robotic Affordances (SayCan)},
  author = {Michael Ahn and Anthony Brohan and Noah Brown and Karol Hausman},
  year = {2022},
  url = {https://arxiv.org/abs/2204.01691},
  note = {arXiv:2204.01691}
}

@article{Lewkowycz2022P522,
  title = {Solving Quantitative Reasoning Problems with Language Models (Minerva)},
  author = {Aitor Lewkowycz and Anders Andreassen and David Dohan and Vinay Ramasesh},
  year = {2022},
  url = {https://arxiv.org/abs/2206.14858},
  note = {arXiv:2206.14858}
}

@misc{Bender2021P523,
  title = {On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?},
  author = {Emily M. Bender and Timnit Gebru and Angelina McMillan-Major and Margaret Mitchell},
  year = {2021},
  url = {https://dl.acm.org/doi/10.1145/3442188.3445922}
}

@article{Ramesh2021P530,
  title = {Zero-Shot Text-to-Image Generation (DALL-E)},
  author = {Aditya Ramesh and Mikhail Pavlov and Gabriel Goh and Scott Gray},
  year = {2021},
  url = {https://arxiv.org/abs/2102.12092},
  note = {arXiv:2102.12092}
}

@article{Radford2022P531,
  title = {Robust Speech Recognition via Large-Scale Weak Supervision (Whisper)},
  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman},
  year = {2022},
  url = {https://arxiv.org/abs/2212.04356},
  note = {arXiv:2212.04356}
}

@article{Saharia2022P532,
  title = {Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding (Imagen)},
  author = {Chitwan Saharia and William Chan and Saurabh Saxena and Mohammad Norouzi},
  year = {2022},
  url = {https://arxiv.org/abs/2205.11487},
  note = {arXiv:2205.11487}
}

@misc{Brooks2024P533,
  title = {Video Generation Models as World Simulators (Sora)},
  author = {Tim Brooks and Bill Peebles and OpenAI},
  year = {2024},
  url = {https://openai.com/index/video-generation-models-as-world-simulators/}
}

@misc{Jumper2021P540,
  title = {Highly Accurate Protein Structure Prediction with AlphaFold},
  author = {John Jumper and Richard Evans and Alexander Pritzel and Tim Green and Demis Hassabis},
  year = {2021},
  url = {https://www.nature.com/articles/s41586-021-03819-2}
}

@article{Hinton2015P550,
  title = {Distilling the Knowledge in a Neural Network},
  author = {Geoffrey Hinton and Oriol Vinyals and Jeff Dean},
  year = {2015},
  url = {https://arxiv.org/abs/1503.02531},
  note = {arXiv:1503.02531}
}

@article{Kirkpatrick2017P551,
  title = {Overcoming Catastrophic Forgetting in Neural Networks},
  author = {James Kirkpatrick and Razvan Pascanu and Neil Rabinowitz and Raia Hadsell and Demis Hassabis},
  year = {2017},
  url = {https://arxiv.org/abs/1612.00796},
  note = {arXiv:1612.00796}
}

@article{Behrouz2025P552,
  title = {Titans: Learning to Memorize at Test Time},
  author = {Ali Behrouz and Peilin Zhong and Vahab Mirrokni},
  year = {2025},
  url = {https://arxiv.org/abs/2501.00663},
  note = {arXiv:2501.00663}
}