[ { "id": "A-001", "slug": "backpropagation", "title": "Backpropagation (Learning representations by back-propagating errors)", "shortTitle": "Backpropagation", "authors": [ "David Rumelhart", "Geoffrey Hinton", "Ronald Williams" ], "institutions": [ "UCSD", "CMU" ], "year": 1986, "date": "1986", "venue": null, "artifactType": "foundational_paper", "domain": "pre-transformer", "url": "https://www.nature.com/articles/323533a0", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 40000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the backpropagation algorithm for training multi-layer neural networks by propagating output error backward through the network to compute weight gradients, removing the barrier that networks with hidden layers had no practical learning rule.", "plainSummary": "The paper applies the chain rule to compute, for every weight in a layered network, how much it contributed to the output error, then adjusts weights in the direction that reduces that error. This gave hidden units a way to learn useful internal representations instead of being hand-designed. It made training networks with one or more hidden layers a routine gradient-descent procedure and became the standard method for supervised neural network learning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.nature.com/articles/323533a0" ], "version": 2 }, { "id": "A-002", "slug": "a-neural-probabilistic-language-model", "title": "A Neural Probabilistic Language Model", "shortTitle": "A Neural Probabilistic Language Model", "authors": [ "Yoshua Bengio", "Réjean Ducharme", "Pascal Vincent", "Christian Jauvin" ], "institutions": [ "U. Montreal" ], "year": 2003, "date": "2003", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.jmlr.org/papers/v3/bengio03a.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Proposed learning a distributed vector representation for each word jointly with a neural network that predicts the next word, addressing the curse of dimensionality that made count-based n-gram language models generalize poorly to unseen word combinations.", "plainSummary": "Instead of treating words as discrete symbols, the model maps each word to a real-valued vector and feeds the vectors of the preceding words into a neural network that outputs a probability distribution over the next word. Words used in similar contexts end up with similar vectors, so the model can assign reasonable probability to word sequences it never saw in training. This showed that a single network could learn word features and the probability function together, and set the template later used by neural word embeddings and language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.jmlr.org/papers/v3/bengio03a.html" ], "version": 2 }, { "id": "A-004", "slug": "efficient-estimation-of-word-representations", "title": "Efficient Estimation of Word Representations (Word2Vec)", "shortTitle": "Efficient Estimation of Word Representations", "authors": [ "Tomas Mikolov", "Kai Chen", "Greg Corrado", "Jeffrey Dean" ], "institutions": [ "Google" ], "year": 2013, "date": "2013-01", "venue": null, "artifactType": "preprint", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1301.3781", "arxivId": "1301.3781", "doi": null, "repository": "https://code.google.com/archive/p/word2vec/", "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-influential", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 40000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the continuous bag-of-words and skip-gram models (Word2Vec), simple shallow architectures that learn word vectors from raw text far more cheaply than prior neural language models, removing the compute cost that had limited embedding training to small corpora.", "plainSummary": "The models drop the hidden nonlinear layers of earlier neural language models and instead train a shallow network to predict a word from its surrounding context (CBOW) or the surrounding context from a word (skip-gram). This makes training fast enough to run on billions of words, producing vectors where semantic and syntactic relationships appear as consistent offsets (the vector arithmetic 'king minus man plus woman' lands near 'queen'). It made high-quality word embeddings cheap to compute and widely reusable as input features for other NLP systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1301.3781", "arXiv:1301.3781" ], "version": 2 }, { "id": "A-005", "slug": "glove-global-vectors-for-word-representation", "title": "GloVe: Global Vectors for Word Representation", "shortTitle": "GloVe", "authors": [ "Jeffrey Pennington", "Richard Socher", "Christopher Manning" ], "institutions": [ "Stanford" ], "year": 2014, "date": "2014", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://aclanthology.org/D14-1162/", "arxivId": null, "doi": null, "repository": "https://nlp.stanford.edu/projects/glove/", "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 40000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "GloVe introduced a word-embedding method that learns vectors by factorizing a corpus-wide word co-occurrence matrix, solving the problem of combining global corpus statistics with the useful linear structure of local-context methods.", "plainSummary": "GloVe builds a matrix counting how often each word appears near each other word across the whole corpus, then trains word vectors so that the dot product of two vectors approximates the logarithm of their co-occurrence count, weighting frequent pairs so common words do not dominate. This uses aggregate statistics directly rather than scanning context windows one at a time like word2vec. The resulting vectors placed related words near each other and encoded analogies as vector offsets, and became a widely used pretrained input for later NLP models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://aclanthology.org/D14-1162/" ], "version": 2 }, { "id": "A-006", "slug": "sequence-to-sequence-learning-with-neural-networks", "title": "Sequence to Sequence Learning with Neural Networks", "shortTitle": "Sequence to Sequence Learning with Neural Networks", "authors": [ "Ilya Sutskever", "Oriol Vinyals", "Quoc Le" ], "institutions": [ "Google" ], "year": 2014, "date": "2014-09", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1409.3215", "arxivId": "1409.3215", "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "foundational-superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 30000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced a general end-to-end LSTM encoder-decoder that maps an input sequence to a fixed-length vector and decodes it into an output sequence, removing the need for fixed-alignment, feature-engineered pipelines in sequence transduction.", "plainSummary": "The paper used one multilayer LSTM to read an entire source sentence into a single vector and a second LSTM to generate the target sentence from that vector. A key practical trick was reversing the order of source words, which shortened the average distance between corresponding input and output tokens and made optimization easier, pushing WMT'14 English-to-French BLEU to competitive levels. It showed that a purely neural, general-purpose sequence model could rival phrase-based statistical MT and became the template for later encoder-decoder systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1409.3215", "arXiv:1409.3215" ], "version": 2 }, { "id": "A-007", "slug": "neural-machine-translation-by-jointly-learning-to-align-and-translate", "title": "Neural Machine Translation by Jointly Learning to Align and Translate (Bahdanau attention)", "shortTitle": "Neural Machine Translation by Jointly Learning to Align and Translate", "authors": [ "Dzmitry Bahdanau", "Kyunghyun Cho", "Yoshua Bengio" ], "institutions": [ "U. Montreal" ], "year": 2014, "date": "2014-09", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1409.0473", "arxivId": "1409.0473", "doi": null, "repository": "https://github.com/lisa-groundhog/GroundHog", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-generalized", "confidence": "High", "difficulty": 5, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 35000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced an attention mechanism that lets the decoder compute a weighted combination of all encoder hidden states at each output step, removing the fixed-length bottleneck vector that capped encoder-decoder translation quality on long sentences.", "plainSummary": "Instead of forcing the encoder to compress a whole sentence into one vector, this model learns an alignment: for each output word the decoder scores and softly selects relevant source positions to build a context vector. This kept translation quality from degrading as sentences got longer and produced interpretable soft alignments between source and target words. The mechanism became the core building block that the Transformer later generalized.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1409.0473", "arXiv:1409.0473" ], "version": 2 }, { "id": "A-008", "slug": "effective-approaches-to-attention-based-nmt", "title": "Effective Approaches to Attention-based NMT (Luong attention)", "shortTitle": "Effective Approaches to Attention-based NMT", "authors": [ "Minh-Thang Luong", "Hieu Pham", "Christopher Manning" ], "institutions": [ "Stanford" ], "year": 2015, "date": "2015", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1508.04025", "arxivId": "1508.04025", "doi": null, "repository": null, "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Luong et al. proposed simpler and more general attention mechanisms for neural machine translation, including global and local variants and a cheaper dot-product scoring function, solving the cost and rigidity of the earlier Bahdanau attention design.", "plainSummary": "The paper lets a translation decoder, at each output step, compute a weighted average of encoder hidden states where the weights come from a similarity score between the current decoder state and each source state, then combines that context with the decoder state to predict the next word. It compared scoring functions (dot product, general, concat) and introduced a 'local' attention that only looks at a small window of source positions to save computation. These variants improved translation quality and clarified the design space, and the dot-product scoring it favored became standard in later attention and Transformer work.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1508.04025", "arXiv:1508.04025" ], "version": 2 }, { "id": "A-009", "slug": "deep-residual-learning-for-image-recognition", "title": "Deep Residual Learning for Image Recognition (ResNet)", "shortTitle": "Deep Residual Learning for Image Recognition", "authors": [ "Kaiming He", "Xiangyu Zhang", "Shaoqing Ren", "Jian Sun" ], "institutions": [ "Microsoft Research" ], "year": 2015, "date": "2015-12", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1512.03385", "arxivId": "1512.03385", "doi": null, "repository": "https://github.com/KaimingHe/deep-residual-networks", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 250000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced residual learning with identity skip connections so layers fit a residual function rather than a full mapping, removing the degradation problem that made very deep networks harder to optimize and less accurate.", "plainSummary": "The authors added shortcut connections that add a layer block's input to its output, so each block only has to learn the difference from identity. This made gradients flow through very deep stacks and allowed networks of 50 to 152 layers to train and improve rather than degrade, winning ImageNet 2015. Residual blocks became a default component in most later deep vision and language architectures.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1512.03385", "arXiv:1512.03385" ], "version": 2 }, { "id": "A-010", "slug": "layer-normalization", "title": "Layer Normalization", "shortTitle": "Layer Normalization", "authors": [ "Jimmy Lei Ba", "Jamie Ryan Kiros", "Geoffrey Hinton" ], "institutions": [ "U. Toronto" ], "year": 2016, "date": "2016-07", "venue": null, "artifactType": "preprint", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1607.06450", "arxivId": "1607.06450", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced layer normalization, which normalizes activations across the features of a single training example rather than across the batch, removing batch-size dependence and making normalization usable in recurrent networks and at inference on single examples.", "plainSummary": "The method computes each layer's normalization statistics over the units within one example, so behavior is identical during training and testing and does not depend on batch composition. This made it effective for RNNs and sequence models where batch normalization worked poorly, stabilizing and speeding up training. Layer normalization became standard in Transformer architectures.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1607.06450", "arXiv:1607.06450" ], "version": 2 }, { "id": "A-011", "slug": "adam-a-method-for-stochastic-optimization", "title": "Adam: A Method for Stochastic Optimization", "shortTitle": "Adam", "authors": [ "Diederik Kingma", "Jimmy Ba" ], "institutions": [ "U. Amsterdam", "U. Toronto" ], "year": 2014, "date": "2014-12", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1412.6980", "arxivId": "1412.6980", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 180000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Adam, an optimizer that maintains per-parameter adaptive learning rates from running estimates of the first and second moments of the gradients, removing much of the manual learning-rate tuning and poor conditioning that hampered SGD on sparse or noisy gradients.", "plainSummary": "Adam keeps exponentially decaying averages of past gradients and squared gradients, applies bias correction to those estimates, and uses them to scale each parameter's step. This gives well-behaved step sizes across parameters with different gradient magnitudes and works with little tuning across many problems. Its combination of low memory cost and robust default hyperparameters made it the default optimizer for training deep networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1412.6980", "arXiv:1412.6980" ], "version": 2 }, { "id": "A-012", "slug": "dropout-a-simple-way-to-prevent-neural-networks-from-overfitting", "title": "Dropout: A Simple Way to Prevent Neural Networks from Overfitting", "shortTitle": "Dropout", "authors": [ "Nitish Srivastava", "Geoffrey Hinton", "Alex Krizhevsky", "Ilya Sutskever", "Ruslan Salakhutdinov" ], "institutions": [ "U. Toronto" ], "year": 2014, "date": "2014", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://jmlr.org/papers/v15/srivastava14a.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 3, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 50000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced dropout, a regularization method that randomly deactivates a fraction of units on each training pass, addressing the tendency of large neural networks to overfit by co-adapting their units to the training data.", "plainSummary": "During training, each unit is kept with some probability and dropped otherwise, so the network cannot rely on any specific combination of units and must learn features that are useful on their own. At test time all units are used with their outputs scaled to match, which approximates averaging over the many thinned networks seen during training. This reduced overfitting on a range of vision and speech tasks and became a standard technique for training large networks with limited data.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://jmlr.org/papers/v15/srivastava14a.html" ], "version": 2 }, { "id": "A-013", "slug": "neural-machine-translation-of-rare-words-with-subword-units", "title": "Neural Machine Translation of Rare Words with Subword Units (BPE)", "shortTitle": "Neural Machine Translation of Rare Words with Subword Units", "authors": [ "Rico Sennrich", "Barry Haddow", "Alexandra Birch" ], "institutions": [ "U. Edinburgh" ], "year": 2015, "date": "2015-08", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1508.07909", "arxivId": "1508.07909", "doi": null, "repository": "https://github.com/rsennrich/subword-nmt", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 9000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced byte pair encoding as a subword segmentation method for translation, letting a fixed vocabulary represent rare and unseen words as sequences of learned subword units and removing the out-of-vocabulary problem in open-vocabulary translation.", "plainSummary": "BPE starts from characters and iteratively merges the most frequent adjacent symbol pairs to build a vocabulary of subword units, so any word can be encoded as a sequence of known pieces. This let translation models handle rare words, compounds, and morphology without a huge word vocabulary or a separate back-off mechanism. Subword segmentation via BPE became standard tokenization for most later NMT and language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1508.07909", "arXiv:1508.07909" ], "version": 2 }, { "id": "A-014", "slug": "universal-approximation-theorem", "title": "Universal Approximation Theorem", "shortTitle": "Universal Approximation Theorem", "authors": [ "George Cybenko", "Kurt Hornik" ], "institutions": [ "Multiple" ], "year": 1989, "date": "1989", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://link.springer.com/article/10.1007/BF02551274", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-theory", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 20000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "The Universal Approximation Theorem proved that a feedforward network with a single hidden layer and a suitable nonlinear activation can approximate any continuous function on a bounded domain to arbitrary accuracy, establishing that neural networks are in principle expressive enough to represent essentially any target mapping.", "plainSummary": "Cybenko (1989) showed this for sigmoid-type activations and Hornik (1991) generalized it to broad classes of activation functions, proving that finite sums of scaled, shifted activations are dense in the space of continuous functions. The result is an existence guarantee: for any target accuracy some network with enough hidden units achieves it. It does not say how many units are needed or how to find the weights by training, but it removed the theoretical objection that networks might be fundamentally incapable of representing complex functions.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://link.springer.com/article/10.1007/BF02551274" ], "version": 2 }, { "id": "A-015", "slug": "understanding-the-difficulty-of-training-deep-nets", "title": "Understanding the Difficulty of Training Deep Nets (Xavier/He init)", "shortTitle": "Understanding the Difficulty of Training Deep Nets", "authors": [ "Xavier Glorot", "Yoshua Bengio", "Kaiming He", "Xiangyu Zhang", "Shaoqing Ren", "Jian Sun" ], "institutions": [ "U. Montreal", "MSRA" ], "year": 2010, "date": "2010", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://proceedings.mlr.press/v9/glorot10a.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 30000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Analyzed how activation-function choice and weight-initialization scale cause activations and gradients to shrink or explode across layers, and introduced the Xavier/Glorot initialization scheme that keeps signal variance stable, removing a common cause of failed deep-network training.", "plainSummary": "The paper measures how the variance of activations and back-propagated gradients changes from layer to layer under different setups, showing that poor scaling makes deep networks with saturating activations (like sigmoid) hard to train. It proposes initializing weights with a variance set from the number of incoming and outgoing connections so that signals neither vanish nor blow up as they pass through the layers. This initialization let deeper networks converge more reliably and became a default starting point for many architectures.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://proceedings.mlr.press/v9/glorot10a.html" ], "version": 2 }, { "id": "A-016", "slug": "rectified-linear-units", "title": "Rectified Linear Units (ReLU)", "shortTitle": "Rectified Linear Units", "authors": [ "Vinod Nair", "Geoffrey Hinton", "Xavier Glorot", "Antoine Bordes", "Yoshua Bengio" ], "institutions": [ "U. Toronto", "U. Montreal" ], "year": 2010, "date": "2010", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.cs.toronto.edu/~hinton/absps/reluICML.pdf", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-parent", "confidence": "High", "difficulty": 3, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 25000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Established the rectified linear unit (max of zero and the input) as an activation function for deep networks, addressing the vanishing-gradient and slow-training problems caused by saturating sigmoid and tanh units.", "plainSummary": "A ReLU passes positive inputs through unchanged and outputs zero for negative inputs, so its gradient is a constant one wherever the unit is active instead of shrinking toward zero the way sigmoid and tanh gradients do. This keeps gradients flowing through many layers and produces sparse activations that are cheap to compute. Using ReLUs let deep networks train faster and reach lower error, and it became the default activation for most feedforward and convolutional networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.cs.toronto.edu/~hinton/absps/reluICML.pdf" ], "version": 2 }, { "id": "A-017", "slug": "batch-normalization", "title": "Batch Normalization", "shortTitle": "Batch Normalization", "authors": [ "Sergey Ioffe", "Christian Szegedy" ], "institutions": [ "Google" ], "year": 2015, "date": "2015", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1502.03167", "arxivId": "1502.03167", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 60000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced batch normalization, which standardizes each layer's inputs using the mean and variance of the current mini-batch, addressing the internal covariate shift that forced small learning rates and careful initialization when training deep networks.", "plainSummary": "For each mini-batch, the method normalizes a layer's pre-activation values to zero mean and unit variance, then applies a learned scale and shift so the layer can still represent what it needs. This keeps the distribution of inputs to each layer more stable as the weights below it change, which allows much higher learning rates and less sensitivity to initialization. It sped up training substantially, added a mild regularizing effect, and became a standard component of deep convolutional networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1502.03167", "arXiv:1502.03167" ], "version": 2 }, { "id": "A-018", "slug": "lenet-gradient-based-learning-applied-to-document-recognition", "title": "LeNet: Gradient-Based Learning Applied to Document Recognition", "shortTitle": "LeNet", "authors": [ "Yann LeCun", "Léon Bottou", "Yoshua Bengio", "Patrick Haffner" ], "institutions": [ "Bell Labs" ], "year": 1998, "date": "1998", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "foundational-superseded", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 60000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Presented LeNet, a convolutional neural network trained end to end with backpropagation for handwritten digit and document recognition, showing that learned convolutional features could replace hand-engineered feature extractors for image tasks.", "plainSummary": "The network stacks convolutional layers that apply the same small learned filters across the image with pooling layers that reduce resolution, so it detects local patterns regardless of position while keeping the parameter count low through weight sharing. Trained on labeled digit images, it learns its own feature detectors instead of relying on manually designed ones. It achieved strong accuracy on handwritten digit recognition and was deployed to read amounts on bank checks, demonstrating a practical gradient-trained vision system.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf" ], "version": 2 }, { "id": "A-019", "slug": "alexnet-imagenet-classification-with-deep-cnns", "title": "AlexNet: ImageNet Classification with Deep CNNs", "shortTitle": "AlexNet", "authors": [ "Alex Krizhevsky", "Ilya Sutskever", "Geoffrey Hinton" ], "institutions": [ "U. Toronto" ], "year": 2012, "date": "2012", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://proceedings.neurips.cc/paper/2012/hash/c399862d3b9d6b76c8436e924a68c45b-Abstract.html", "arxivId": null, "doi": null, "repository": "https://github.com/computerhistory/AlexNet-Source-Code", "openness": "unknown", "status": "partially-superseded", "statusRaw": "pivotal-superseded", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 130000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that a large deep convolutional network trained on GPUs could win the ImageNet classification benchmark by a wide margin, demonstrating that scale of data, model, and compute together made deep learning practical for large-scale image recognition.", "plainSummary": "AlexNet is a deep convolutional network with roughly 60 million parameters trained on over a million labeled images across 1,000 categories, using ReLU activations for faster training, dropout to limit overfitting, data augmentation, and two GPUs to fit the model in memory. Combining these let it cut the ImageNet top-5 error far below the previous best. Its result shifted computer vision toward deep learning and drove wide adoption of GPU-trained convolutional networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://proceedings.neurips.cc/paper/2012/hash/c399862d3b9d6b76c8436e924a68c45b-Abstract.html" ], "version": 2 }, { "id": "A-020", "slug": "finding-structure-in-time", "title": "Finding Structure in Time (Elman RNN)", "shortTitle": "Finding Structure in Time", "authors": [ "Jeffrey Elman" ], "institutions": [ "UCSD" ], "year": 1990, "date": "1990", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://onlinelibrary.wiley.com/doi/10.1207/s15516709cog1402_1", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "foundational-superseded", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 18000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Elman introduced the simple recurrent network, which feeds a hidden layer's previous activations back as additional input through a context layer, giving neural networks a working memory of past inputs and solving how to process sequences whose structure unfolds over time.", "plainSummary": "The network copies its hidden-layer state at each step into a 'context' layer that is fed back in alongside the next input, so the hidden units learn representations that depend on prior elements of the sequence. Trained on tasks like predicting the next word or discovering word boundaries in a stream, it developed internal representations that reflected grammatical categories and structure without being told them. This showed that temporal structure could be handled by recurrence rather than by explicit time-delay windows, and the architecture became a foundation for later recurrent networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://onlinelibrary.wiley.com/doi/10.1207/s15516709cog1402_1" ], "version": 2 }, { "id": "A-021", "slug": "learning-long-term-dependencies-is-difficult", "title": "Learning Long-Term Dependencies is Difficult (vanishing gradients)", "shortTitle": "Learning Long-Term Dependencies is Difficult", "authors": [ "Sepp Hochreiter", "Yoshua Bengio", "Patrice Simard", "Paolo Frasconi" ], "institutions": [ "TUM", "U. Montreal" ], "year": 1994, "date": "1994", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://ieeexplore.ieee.org/document/279181", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "niche-but-active", "statusRaw": "defining-problem", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Bengio et al. analyzed why recurrent networks fail to learn dependencies across long time gaps, showing that the gradients propagated back through many steps tend to vanish or explode exponentially, which explained a fundamental obstacle to training networks on long sequences.", "plainSummary": "The paper showed mathematically that for a recurrent network to store information robustly over time its dynamics must contract, but contraction causes error gradients passed backward through many time steps to shrink toward zero, so distant past inputs contribute almost nothing to weight updates. When the dynamics instead expand, gradients blow up. This trade-off means standard gradient descent cannot reliably link causes and effects separated by many steps. The analysis motivated later remedies such as gating (LSTM, GRU), gradient clipping, and better initialization.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://ieeexplore.ieee.org/document/279181" ], "version": 2 }, { "id": "A-022", "slug": "long-short-term-memory", "title": "Long Short-Term Memory (LSTM)", "shortTitle": "Long Short-Term Memory", "authors": [ "Sepp Hochreiter", "Jürgen Schmidhuber", "Felix Gers" ], "institutions": [ "TUM", "IDSIA" ], "year": 1997, "date": "1997", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.bioinf.jku.at/publications/older/2604.pdf", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-parent", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 90000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the Long Short-Term Memory recurrent architecture, which uses a memory cell protected by multiplicative gates to preserve error signals over long time lags, addressing the vanishing and exploding gradients that prevented standard recurrent networks from learning long-range dependencies.", "plainSummary": "Each LSTM unit keeps a memory cell whose value is protected by gates that control when new information is written in, retained, or read out, so the cell can hold information across many time steps. Because the cell passes its state forward with near-constant behavior when the gates allow, gradients do not shrink away over long sequences the way they do in plain recurrent networks. This let recurrent networks learn dependencies spanning hundreds of steps and made them effective for sequence tasks such as speech and language modeling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.bioinf.jku.at/publications/older/2604.pdf" ], "version": 2 }, { "id": "A-023", "slug": "bidirectional-recurrent-neural-networks", "title": "Bidirectional Recurrent Neural Networks", "shortTitle": "Bidirectional Recurrent Neural Networks", "authors": [ "Mike Schuster", "Kuldip Paliwal" ], "institutions": [ "ATR" ], "year": 1997, "date": "1997", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://ieeexplore.ieee.org/document/650093", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-conceptual", "confidence": "High", "difficulty": 4, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 9000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Schuster and Paliwal introduced the bidirectional recurrent network, which runs two recurrent networks over a sequence in opposite directions and combines their states, so each position's representation can use both past and future context, solving the limitation that a forward-only network cannot see later inputs.", "plainSummary": "The architecture processes a sequence left-to-right with one recurrent network and right-to-left with a second, then merges the two hidden states at each position before making a prediction. This gives every output access to the entire input sequence rather than only the elements seen so far. Because it requires the whole sequence up front it suits offline labeling tasks like speech and text tagging, and the bidirectional design later combined with LSTMs to become a standard component for sequence labeling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://ieeexplore.ieee.org/document/650093" ], "version": 2 }, { "id": "A-024", "slug": "gated-recurrent-unit-rnn-encoder-decoder", "title": "Gated Recurrent Unit (GRU) / RNN Encoder-Decoder", "shortTitle": "Gated Recurrent Unit / RNN Encoder-Decoder", "authors": [ "Kyunghyun Cho", "Bart van Merriënboer", "Dzmitry Bahdanau", "Yoshua Bengio" ], "institutions": [ "U. Montreal" ], "year": 2014, "date": "2014", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1406.1078", "arxivId": "1406.1078", "doi": null, "repository": null, "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 25000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Cho et al. introduced the RNN encoder-decoder framework for translating variable-length sequences and the Gated Recurrent Unit, a simplified gated cell, solving how to map one sequence to another while mitigating the vanishing-gradient problem with fewer parameters than an LSTM.", "plainSummary": "One recurrent network (the encoder) reads a source sentence and compresses it into a fixed-length vector, and a second recurrent network (the decoder) generates the target sentence from that vector, trained jointly to maximize the probability of the correct output. To make the recurrence trainable over longer spans the paper introduced the GRU, which uses update and reset gates to control how much past state is kept or overwritten, achieving LSTM-like behavior with a simpler structure. The encoder-decoder setup became the template for neural machine translation and sequence-to-sequence learning, and its learned phrase representations improved a statistical translation system.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1406.1078", "arXiv:1406.1078" ], "version": 2 }, { "id": "A-025", "slug": "connectionist-temporal-classification", "title": "Connectionist Temporal Classification (CTC)", "shortTitle": "Connectionist Temporal Classification", "authors": [ "Alex Graves", "Santiago Fernández", "Faustino Gomez", "Jürgen Schmidhuber" ], "institutions": [ "IDSIA" ], "year": 2006, "date": "2006", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.cs.toronto.edu/~graves/icml_2006.pdf", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "niche-but-active", "statusRaw": "still-used-speech", "confidence": "High", "difficulty": 5, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 8000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Graves et al. introduced Connectionist Temporal Classification, a loss function that lets a recurrent network be trained to map an input sequence to a shorter output label sequence without knowing the alignment between them, solving the need for pre-segmented, frame-by-frame labeled training data in tasks like speech recognition.", "plainSummary": "CTC adds a special 'blank' symbol and defines the probability of a target label sequence as the sum over all frame-level paths that collapse to it (by removing blanks and merging repeats), computed efficiently with a forward-backward dynamic-programming algorithm. This lets the network output a probability distribution over labels at every time step and be trained end-to-end on unsegmented pairs of input and target sequences. It removed the requirement for hand-aligned data and enabled direct recurrent-network sequence transcription in speech and handwriting recognition.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.cs.toronto.edu/~graves/icml_2006.pdf" ], "version": 2 }, { "id": "A-026", "slug": "deep-belief-nets-reducing-dimensionality", "title": "Deep Belief Nets / Reducing Dimensionality", "shortTitle": "Deep Belief Nets / Reducing Dimensionality", "authors": [ "Geoffrey Hinton", "Simon Osindero", "Yee-Whye Teh", "Ruslan Salakhutdinov" ], "institutions": [ "U. Toronto" ], "year": 2006, "date": "2006", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.science.org/doi/10.1126/science.1127647", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "fully-superseded", "statusRaw": "paradigm-superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 30000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that a deep network can be initialized by greedily pre-training one layer at a time as restricted Boltzmann machines and then fine-tuned as a whole, addressing the difficulty of training deep networks from random weights and enabling autoencoders that learn low-dimensional data codes.", "plainSummary": "The method builds a deep belief network by training each layer in turn to model the activations of the layer below, giving the full network a sensible starting point before ordinary gradient fine-tuning. Applied to an autoencoder, this two-stage approach learns a compact code that reconstructs the input better than linear methods like PCA. The work provided a practical way to train deep networks before better initialization and activation functions existed, and helped renew interest in deep learning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.science.org/doi/10.1126/science.1127647" ], "version": 2 }, { "id": "A-027", "slug": "natural-language-processing-from-scratch", "title": "Natural Language Processing (almost) from Scratch", "shortTitle": "Natural Language Processing from Scratch", "authors": [ "Ronan Collobert", "Jason Weston", "Léon Bottou", "et al." ], "institutions": [ "NEC", "Princeton" ], "year": 2011, "date": "2011", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1103.0398", "arxivId": "1103.0398", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "precursor-embeddings", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 10000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Collobert et al. showed that a single convolutional neural network trained on large unlabeled text could handle multiple NLP tagging tasks well using learned word features instead of hand-engineered ones, arguing for a unified, feature-engineering-free approach to language processing.", "plainSummary": "The system maps words to learned vector embeddings, processes sentences with a convolutional network, and is trained jointly across tasks like part-of-speech tagging, chunking, named-entity recognition, and semantic role labeling. Crucially it pretrains word embeddings on large unlabeled corpora using a ranking objective, then reuses them, so the model discovers useful features rather than relying on task-specific hand-crafted inputs. It reached competitive accuracy across tasks with one architecture, demonstrating the transfer value of unsupervised pretrained word representations that later work built on.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1103.0398", "arXiv:1103.0398" ], "version": 2 }, { "id": "A-028", "slug": "the-mathematics-of-statistical-mt", "title": "The Mathematics of Statistical MT (IBM alignment models)", "shortTitle": "The Mathematics of Statistical MT", "authors": [ "Peter F. Brown", "Vincent J. Della Pietra", "Stephen A. Della Pietra", "Robert L. Mercer" ], "institutions": [ "IBM" ], "year": 1993, "date": "1993", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://aclanthology.org/J93-2003/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "alignment-ancestor", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Brown et al. formalized statistical machine translation as a probabilistic model estimated from bilingual text, introducing the IBM Models 1-5 that learn word-to-word translation probabilities and alignments from parallel corpora, solving how to translate by learning from data rather than hand-written rules.", "plainSummary": "The paper frames translation as finding the target sentence most probable given the source, decomposed via Bayes' rule into a translation model and a language model, and introduces a hidden alignment variable saying which source word generated each target word. The five IBM models add successive detail (word translation probabilities, then absolute position, fertility of how many words a source word produces, and distortion of position), and their parameters are estimated from unaligned sentence pairs using the EM algorithm. These alignment models became the statistical foundation of machine translation for roughly two decades and the basis for later phrase-based systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://aclanthology.org/J93-2003/" ], "version": 2 }, { "id": "A-029", "slug": "learning-to-predict-by-the-methods-of-temporal-differences", "title": "Learning to Predict by the Methods of Temporal Differences", "shortTitle": "Learning to Predict by the Methods of Temporal Differences", "authors": [ "Richard S. Sutton" ], "institutions": [ "GTE Labs" ], "year": 1988, "date": "1988", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://link.springer.com/article/10.1007/BF00115009", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-rl", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced temporal-difference (TD) learning, a method that lets a system learn to predict long-run outcomes by adjusting each prediction toward the next prediction rather than waiting for the final result.", "plainSummary": "Sutton showed how to learn multi-step predictions incrementally: instead of comparing a prediction only to the eventual outcome, TD updates each prediction using the difference between successive predictions along the way. This let learning happen online, step by step, and made better use of experience than earlier supervised methods. TD became the computational core of most later reinforcement learning, including value estimation in games and control.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://link.springer.com/article/10.1007/BF00115009" ], "version": 2 }, { "id": "A-030", "slug": "q-learning", "title": "Q-learning", "shortTitle": "Q-learning", "authors": [ "Christopher J. C. H. Watkins", "Peter Dayan" ], "institutions": [ "Cambridge" ], "year": 1992, "date": "1992", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://link.springer.com/article/10.1007/BF00992698", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-rl", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 20000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Q-learning, an off-policy algorithm that learns the value of taking each action in each state and proved it converges to optimal control even while the agent explores with a different policy.", "plainSummary": "Watkins and Dayan defined a rule that updates an estimate of the long-run reward for each state-action pair (the Q-value) from observed transitions and rewards. Because updates use the best available next action rather than the action actually taken, the agent can learn the optimal policy while still exploring. They gave a convergence proof, giving reinforcement learning a model-free method with mathematical guarantees that underpins much later work.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://link.springer.com/article/10.1007/BF00992698" ], "version": 2 }, { "id": "A-031", "slug": "simple-statistical-gradient-following-algorithms", "title": "Simple Statistical Gradient-Following Algorithms (REINFORCE)", "shortTitle": "Simple Statistical Gradient-Following Algorithms", "authors": [ "Ronald J. Williams" ], "institutions": [ "Northeastern" ], "year": 1992, "date": "1992", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://link.springer.com/article/10.1007/BF00992696", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-policy-gradient", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced REINFORCE, a family of algorithms that adjust a policy's parameters directly along an unbiased estimate of the reward gradient, enabling reinforcement learning in networks that output probabilities over actions.", "plainSummary": "Williams derived how to nudge the weights of a stochastic network so that actions leading to higher reward become more likely, using only the reward signal and the probability of the chosen action. The estimate of the gradient is unbiased and needs no model of the environment. This policy-gradient approach became the basis for later actor-critic and deep policy-optimization methods used to train agents and language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://link.springer.com/article/10.1007/BF00992696" ], "version": 2 }, { "id": "A-032", "slug": "td-gammon", "title": "TD-Gammon", "shortTitle": "TD-Gammon", "authors": [ "Gerald Tesauro" ], "institutions": [ "IBM" ], "year": 1995, "date": "1995", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://dl.acm.org/doi/10.1145/203330.203343", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-selfplay", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Tesauro's TD-Gammon showed that a neural network trained by temporal-difference reinforcement learning, largely through self-play, could reach expert-level backgammon, demonstrating that a value function for a complex game could be learned from experience rather than programmed.", "plainSummary": "The program uses a neural network to estimate the expected outcome of a board position and trains it with TD(lambda), which adjusts predictions so that each position's estimated value moves toward the value of positions that follow it during play. By playing hundreds of thousands of games against itself it improved without expert-labeled positions or a hand-tuned evaluation function. It reached strong tournament-level play and even influenced human opening theory, providing an early demonstration that reinforcement learning combined with function approximation could master a large-state-space game.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://dl.acm.org/doi/10.1145/203330.203343" ], "version": 2 }, { "id": "A-033", "slug": "human-level-control-through-deep-rl", "title": "Human-level Control through Deep RL (DQN)", "shortTitle": "Human-level Control through Deep RL", "authors": [ "Volodymyr Mnih", "Koray Kavukcuoglu", "David Silver", "et al." ], "institutions": [ "DeepMind" ], "year": 2015, "date": "2015", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.nature.com/articles/nature14236", "arxivId": null, "doi": null, "repository": "https://github.com/google-deepmind/dqn", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-deep-rl", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 40000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the Deep Q-Network (DQN), which combined Q-learning with a convolutional neural network to learn control policies directly from raw pixels, reaching human-level play across many Atari games with one architecture.", "plainSummary": "Mnih and colleagues trained a network to estimate action values from screen images, using two stabilizing tricks: an experience replay buffer that reuses and decorrelates past transitions, and a periodically updated target network. The same model and hyperparameters learned to play 49 Atari games from pixels and reward alone. It showed that deep networks could serve as reliable function approximators in reinforcement learning, launching the deep RL field.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.nature.com/articles/nature14236" ], "version": 2 }, { "id": "A-034", "slug": "mastering-the-game-of-go", "title": "Mastering the Game of Go (AlphaGo/AlphaZero)", "shortTitle": "Mastering the Game of Go", "authors": [ "David Silver", "Aja Huang", "Chris J. Maddison", "Julian Schrittwieser", "Thomas Hubert", "Demis Hassabis" ], "institutions": [ "DeepMind" ], "year": 2016, "date": "2016", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://www.nature.com/articles/nature16961", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-search-selfplay", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 20000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced AlphaGo and later AlphaZero, systems that combined deep neural networks with Monte Carlo tree search to master Go and other board games, with AlphaZero learning entirely from self-play with no human game data.", "plainSummary": "AlphaGo paired a policy network (which moves to consider) and a value network (who is winning) with tree search to defeat top human Go players. AlphaZero simplified this into a single network trained purely by playing against itself, guided by search, and generalized to chess and shogi. The work showed that self-play plus search could reach superhuman strength without human examples or handcrafted evaluation.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.nature.com/articles/nature16961" ], "version": 2 }, { "id": "A-035", "slug": "auto-encoding-variational-bayes", "title": "Auto-Encoding Variational Bayes (VAE)", "shortTitle": "Auto-Encoding Variational Bayes", "authors": [ "Diederik P. Kingma", "Max Welling" ], "institutions": [ "U. Amsterdam" ], "year": 2013, "date": "2013", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1312.6114", "arxivId": "1312.6114", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-component", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 35000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the variational autoencoder (VAE), which trains a neural network to both encode data into a probabilistic latent space and generate new samples from it, made trainable by the reparameterization trick.", "plainSummary": "Kingma and Welling framed generation as learning a latent variable model and approximating its intractable posterior with a neural encoder. Their key move, the reparameterization trick, rewrote random sampling so gradients could flow through it, allowing the encoder and decoder to be trained together by ordinary backpropagation on a single objective (the evidence lower bound). This gave a scalable way to learn continuous latent representations and generate new data, widely used for images and representation learning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1312.6114", "arXiv:1312.6114" ], "version": 2 }, { "id": "A-036", "slug": "generative-adversarial-networks", "title": "Generative Adversarial Networks (GAN)", "shortTitle": "Generative Adversarial Networks", "authors": [ "Ian J. Goodfellow", "Jean Pouget-Abadie", "Mehdi Mirza", "et al." ], "institutions": [ "U. Montreal" ], "year": 2014, "date": "2014", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1406.2661", "arxivId": "1406.2661", "doi": null, "repository": "https://github.com/goodfeli/adversarial", "openness": "unknown", "status": "partially-superseded", "statusRaw": "partially-superseded", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 70000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced generative adversarial networks (GANs), which train a generator and a discriminator against each other so the generator learns to produce data indistinguishable from real examples.", "plainSummary": "Goodfellow and colleagues set up a game between two networks: a generator that turns noise into samples and a discriminator that tries to tell real data from generated data. Training the two in competition drives the generator to match the real data distribution without needing an explicit likelihood. This adversarial framework produced sharp, realistic image generation and spawned a large family of follow-on models for synthesis and translation.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1406.2661", "arXiv:1406.2661" ], "version": 2 }, { "id": "A-037", "slug": "neural-turing-machines-memory-networks", "title": "Neural Turing Machines / Memory Networks", "shortTitle": "Neural Turing Machines / Memory Networks", "authors": [ "Alex Graves", "Greg Wayne", "Ivo Danihelka", "Jason Weston", "Sumit Chopra", "Antoine Bordes" ], "institutions": [ "DeepMind", "FAIR" ], "year": 2014, "date": "2014", "venue": null, "artifactType": "preprint", "domain": "pre-transformer", "url": "https://arxiv.org/abs/1410.5401", "arxivId": "1410.5401", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "precursor-memory", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Graves et al. introduced the Neural Turing Machine, a neural network coupled to an external memory matrix it can read from and write to using differentiable attention, so the whole system can be trained by gradient descent to learn simple algorithms, addressing standard networks' limited ability to store and manipulate structured data over time.", "plainSummary": "A controller network emits read and write operations addressed to memory locations using soft, differentiable attention weights (based on content similarity and location shifting), so gradients can flow through the memory access and the model learns how to use its memory end-to-end. This separates computation from an addressable storage bank, letting the network learn procedures like copying, sorting, and associative recall and generalize them to longer inputs than seen in training. It, alongside Memory Networks, showed that networks augmented with external memory could learn algorithm-like behavior, influencing later memory-augmented and attention-based models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1410.5401", "arXiv:1410.5401" ], "version": 2 }, { "id": "A-038", "slug": "deep-neural-networks-for-acoustic-modeling-in-speech", "title": "Deep Neural Networks for Acoustic Modeling in Speech", "shortTitle": "Deep Neural Networks for Acoustic Modeling in Speech", "authors": [ "Geoffrey Hinton", "Li Deng", "Dong Yu", "et al." ], "institutions": [ "Multiple" ], "year": 2012, "date": "2012", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://ieeexplore.ieee.org/document/6296526", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This overview paper reported that deep neural networks trained as acoustic models consistently outperformed the long-dominant Gaussian mixture models in speech recognition across four research groups, marking a practical shift to deep learning for acoustic modeling.", "plainSummary": "In a speech recognizer the acoustic model estimates how likely each short audio frame corresponds to each sub-phonetic state; the paper describes replacing the standard Gaussian-mixture estimator with a deep neural network that takes several frames of audio features and outputs those state probabilities, often pretrained layer by layer and then fine-tuned. Reported jointly by teams at Microsoft, Google, IBM, and the University of Toronto, the deep networks lowered word error rates on multiple large-vocabulary benchmarks. The shared results gave the field consistent evidence that deep acoustic models were a general improvement, and they were adopted in commercial speech systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://ieeexplore.ieee.org/document/6296526" ], "version": 2 }, { "id": "A-040", "slug": "imagenet-a-large-scale-hierarchical-image-database", "title": "ImageNet: A Large-Scale Hierarchical Image Database", "shortTitle": "ImageNet", "authors": [ "Jia Deng", "Wei Dong", "Richard Socher", "Li-Jia Li", "Kai Li", "Li Fei-Fei" ], "institutions": [ "Princeton University", "Stanford University" ], "year": 2009, "date": "2009-06", "venue": null, "artifactType": "peer_reviewed", "domain": "pre-transformer", "url": "https://ieeexplore.ieee.org/document/5206848", "arxivId": null, "doi": null, "repository": "https://www.image-net.org/", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 3, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Built a labelled image database of unprecedented scale and hierarchical structure, giving computer vision the large supervised benchmark it needed and creating the competition that deep learning went on to win.", "plainSummary": "ImageNet organised millions of human-labelled images into thousands of categories following the WordNet hierarchy, assembled through large-scale crowdsourcing. It supplied the data and the annual ILSVRC challenge that made it possible to train and fairly compare very large vision models. AlexNets 2012 win on this benchmark is widely taken as the moment deep learning became the dominant approach in vision.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://ieeexplore.ieee.org/document/5206848" ], "version": 2 }, { "id": "O-001", "slug": "boolean-algebra", "title": "Boolean Algebra (The Laws of Thought)", "shortTitle": "Boolean Algebra", "authors": [ "George Boole" ], "institutions": [ "Queen's College Cork" ], "year": 1854, "date": "1854", "venue": null, "artifactType": "foundational_treatise", "domain": "origins", "url": "https://en.wikipedia.org/wiki/The_Laws_of_Thought", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Boole recast logical reasoning as algebra, showing that propositions and their combinations obey formal symbolic laws that can be manipulated like equations.", "plainSummary": "Boole treated statements as variables taking values that behave like 0 and 1, with operations for 'and', 'or', and 'not' following consistent algebraic rules. This turned deductive reasoning into symbol manipulation that could be checked mechanically rather than by intuition. The system later became the mathematical basis for digital logic circuits and for how computers represent and evaluate logical conditions.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/The_Laws_of_Thought" ], "version": 2 }, { "id": "O-002", "slug": "analytical-engine-the-first-algorithm", "title": "Analytical Engine & the First Algorithm", "shortTitle": "Analytical Engine & the First Algorithm", "authors": [ "Charles Babbage", "Ada Lovelace" ], "institutions": [ "London" ], "year": 1843, "date": "1843", "venue": null, "artifactType": "foundational_design", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Analytical_Engine", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Babbage designed a general-purpose mechanical computer controlled by punched cards, and Lovelace's notes on it described how such a machine could be programmed to carry out an arbitrary sequence of operations.", "plainSummary": "The Analytical Engine was a proposed mechanical calculator with a separate store and processing 'mill', able to follow instructions and branch conditionally rather than compute one fixed formula. In her annotations to a paper describing it, Lovelace wrote out a step-by-step procedure for computing Bernoulli numbers, often cited as the first published algorithm intended for a machine. She also argued the engine could operate on symbols beyond numbers, anticipating general-purpose computation, though the machine was never built.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Analytical_Engine" ], "version": 2 }, { "id": "O-003", "slug": "on-computable-numbers", "title": "On Computable Numbers (Turing Machine)", "shortTitle": "On Computable Numbers", "authors": [ "Alan Turing" ], "institutions": [ "Cambridge" ], "year": 1936, "date": "1936", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Turing_machine", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Turing defined a precise abstract model of mechanical computation and used it to characterize which functions can be computed and to prove that some well-defined problems cannot.", "plainSummary": "Turing described a simple imagined device that reads and writes symbols on a tape according to a finite table of rules, and argued this captures anything a human clerk could compute by following steps. Using it he showed that a single 'universal' machine can simulate any other by reading its description, and that the halting problem has no general algorithmic solution. This gave a rigorous definition of 'algorithm' and 'computable', and the universal-machine idea underlies the concept of a programmable general-purpose computer.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Turing_machine" ], "version": 2 }, { "id": "O-004", "slug": "a-symbolic-analysis-of-relay-and-switching-circuits", "title": "A Symbolic Analysis of Relay and Switching Circuits", "shortTitle": "A Symbolic Analysis of Relay and Switching Circuits", "authors": [ "Claude Shannon" ], "institutions": [ "MIT" ], "year": 1937, "date": "1937", "venue": null, "artifactType": "thesis", "domain": "origins", "url": "https://en.wikipedia.org/wiki/A_Symbolic_Analysis_of_Relay_and_Switching_Circuits", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Shannon showed that networks of relay switches correspond directly to expressions in Boolean logic, so circuits can be designed and simplified using algebra instead of trial and error.", "plainSummary": "In his master's thesis Shannon mapped open and closed switches onto Boolean true/false values and series/parallel wiring onto logical operations. This meant a designer could write a switching circuit as a logic equation, reduce it algebraically to use fewer components, and verify it behaves correctly. The result became the standard method for designing the digital logic inside telephone systems and later computers.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/A_Symbolic_Analysis_of_Relay_and_Switching_Circuits" ], "version": 2 }, { "id": "O-005", "slug": "a-logical-calculus-of-ideas-immanent-in-nervous-activity", "title": "A Logical Calculus of Ideas Immanent in Nervous Activity (artificial neuron)", "shortTitle": "A Logical Calculus of Ideas Immanent in Nervous Activity", "authors": [ "Warren McCulloch", "Walter Pitts" ], "institutions": [ "Chicago", "MIT" ], "year": 1943, "date": "1943", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Artificial_neuron", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "McCulloch and Pitts proposed a mathematical model of neurons as simple threshold logic units and showed that networks of them can compute logical functions.", "plainSummary": "They abstracted a neuron as a device that sums weighted inputs and fires if the total crosses a threshold, ignoring biological detail. They proved that networks of such idealized neurons can implement any logical proposition, linking brain-style computation to formal logic and Turing's model. This established the idea that networks of simple connected units can compute, providing the conceptual starting point for later artificial neural networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Artificial_neuron" ], "version": 2 }, { "id": "O-006", "slug": "first-draft-of-a-report-on-the-edvac", "title": "First Draft of a Report on the EDVAC (stored-program architecture)", "shortTitle": "First Draft of a Report on the EDVAC", "authors": [ "von Neumann" ], "institutions": [ "IAS", "Penn" ], "year": 1945, "date": "1945", "venue": null, "artifactType": "architecture_report", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Von_Neumann_architecture", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-current-constraint", "confidence": "High", "difficulty": 5, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Von Neumann's report set out an architecture in which a computer's instructions are stored in the same memory as its data, letting a single machine be reprogrammed without rewiring.", "plainSummary": "The draft described a design with a central arithmetic unit, a control unit, and a shared memory holding both program and data, communicating over common paths. Because instructions live in modifiable memory, the same hardware can run any program just by loading different contents. This 'stored-program' organization became the template for essentially all subsequent general-purpose computers.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Von_Neumann_architecture" ], "version": 2 }, { "id": "O-007", "slug": "as-we-may-think-science-the-endless-frontier", "title": "As We May Think (Memex) + Science the Endless Frontier", "shortTitle": "As We May Think + Science the Endless Frontier", "authors": [ "Vannevar Bush" ], "institutions": [ "MIT", "OSRD" ], "year": 1945, "date": "1945", "venue": null, "artifactType": "vision_essay", "domain": "origins", "url": "https://en.wikipedia.org/wiki/As_We_May_Think", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Bush proposed the Memex, an imagined desk-based device for storing documents and linking them by association, anticipating the idea of navigable connected information.", "plainSummary": "Bush described a hypothetical machine that would hold a person's books, records, and notes on microfilm and let the user create named 'trails' of associative links between items. He argued that organizing knowledge by association rather than rigid indexing would match how people actually think and recall. The essay is widely credited as an early inspiration for hypertext, personal information systems, and the linked structure of the web.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/As_We_May_Think" ], "version": 2 }, { "id": "O-008", "slug": "a-mathematical-theory-of-communication", "title": "A Mathematical Theory of Communication (information theory)", "shortTitle": "A Mathematical Theory of Communication", "authors": [ "Claude Shannon" ], "institutions": [ "Bell Labs" ], "year": 1948, "date": "1948", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/A_Mathematical_Theory_of_Communication", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Shannon founded information theory by defining information quantitatively and proving limits on how much data a channel can carry and how far messages can be compressed.", "plainSummary": "He measured information in bits using entropy, separating a message's content from its meaning, and modeled communication as a source, channel, and receiver subject to noise. He proved that every channel has a maximum reliable rate (its capacity) and that codes exist to approach it with arbitrarily few errors, and gave limits on lossless compression. These results underpin modern data compression, error-correcting codes, and digital communication and storage.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/A_Mathematical_Theory_of_Communication" ], "version": 2 }, { "id": "O-009", "slug": "cybernetics-control-and-communication-in-the-animal-and-the-machine", "title": "Cybernetics: Control and Communication in the Animal and the Machine", "shortTitle": "Cybernetics", "authors": [ "Norbert Wiener" ], "institutions": [ "MIT" ], "year": 1948, "date": "1948", "venue": null, "artifactType": "foundational_book", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Cybernetics", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-influential", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Wiener named and unified the study of feedback and control, arguing that goal-directed behavior in machines and living things can be described by the same mathematics of information and regulation.", "plainSummary": "Cybernetics framed systems as maintaining goals by sensing their output and feeding it back to adjust their input, whether the system is a thermostat, an animal, or a servomechanism. Wiener drew together control theory, feedback, and communication to treat purpose and self-correction as engineering problems. The framework influenced control engineering, early thinking about learning machines, and how researchers modeled adaptive and self-regulating behavior.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Cybernetics" ], "version": 2 }, { "id": "O-010", "slug": "the-organization-of-behavior", "title": "The Organization of Behavior (Hebbian learning)", "shortTitle": "The Organization of Behavior", "authors": [ "Donald Hebb" ], "institutions": [ "McGill" ], "year": 1949, "date": "1949", "venue": null, "artifactType": "foundational_book", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Hebbian_theory", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Hebb proposed that learning happens by strengthening the connection between neurons that are repeatedly active together, giving a physical mechanism for memory and association.", "plainSummary": "Hebb argued that when one neuron persistently helps fire another, the link between them grows stronger, often summarized as 'cells that fire together wire together'. He used this to explain how groups of neurons form 'cell assemblies' that represent learned concepts and associations. The rule gave neuroscience a concrete account of learning and became a foundational principle for training weights in artificial neural networks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Hebbian_theory" ], "version": 2 }, { "id": "O-011", "slug": "computing-machinery-and-intelligence", "title": "Computing Machinery and Intelligence (Turing Test)", "shortTitle": "Computing Machinery and Intelligence", "authors": [ "Alan Turing" ], "institutions": [ "Manchester" ], "year": 1950, "date": "1950", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Computing_Machinery_and_Intelligence", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Turing reframed the question 'can machines think' as an operational test of whether a machine's conversational responses are indistinguishable from a human's.", "plainSummary": "Turing proposed the 'imitation game', in which a judge exchanges typed messages with a hidden human and a hidden machine and tries to tell which is which. He suggested that a machine passing this test should count as intelligent for practical purposes, sidestepping arguments over the definition of thought. The paper also anticipated and answered common objections and outlined machine learning, setting an early behavioral benchmark and framing debates in artificial intelligence.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Computing_Machinery_and_Intelligence" ], "version": 2 }, { "id": "O-012", "slug": "the-manhattan-project-los-alamos", "title": "The Manhattan Project / Los Alamos (big-science template)", "shortTitle": "The Manhattan Project / Los Alamos", "authors": [ "J. Robert Oppenheimer", "Leslie Groves", "John von Neumann", "et al." ], "institutions": [ "Los Alamos" ], "year": 1945, "date": "1945", "venue": null, "artifactType": "institutional_milestone", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Manhattan_Project", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-institutional", "confidence": "High", "difficulty": 3, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "The Manhattan Project organized dispersed scientific talent, industrial-scale engineering, and government funding into a single goal-directed effort, establishing the template for state-backed 'big science' that later underwrote computing, national laboratories, and large research programs.", "plainSummary": "Between 1942 and 1945 the United States gathered physicists, chemists, and engineers at Los Alamos and other sites to build an atomic weapon, coordinating theory, experiment, and mass manufacturing on an unprecedented scale. The effort showed that a well-funded, centrally managed team could compress decades of research into a few years. It created the postwar model of the national laboratory and the pattern of governments financing ambitious technical projects. Many figures and methods from this work, including early electronic computation, carried directly into the founding of modern computer science.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Manhattan_Project" ], "version": 2 }, { "id": "O-013", "slug": "the-monte-carlo-method-metropolis-algorithm", "title": "The Monte Carlo Method / Metropolis Algorithm", "shortTitle": "The Monte Carlo Method / Metropolis Algorithm", "authors": [ "Stanislaw Ulam", "John von Neumann", "Nicholas Metropolis" ], "institutions": [ "Los Alamos" ], "year": 1949, "date": "1949", "venue": null, "artifactType": "method", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Monte_Carlo_method", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "The Monte Carlo method and the Metropolis algorithm introduced the idea of solving hard mathematical and physical problems by drawing large numbers of random samples, giving computers a general way to estimate quantities that cannot be calculated exactly.", "plainSummary": "Working on nuclear physics problems at Los Alamos, Stanislaw Ulam, John von Neumann, Nicholas Metropolis, and colleagues realized that random sampling on a computer could approximate answers to equations too complex to solve directly. The 1953 Metropolis algorithm added a rule for sampling from a probability distribution by accepting or rejecting proposed moves, making it possible to simulate systems in equilibrium. These techniques became standard tools across physics, statistics, and later machine learning. Much of modern probabilistic modeling and Bayesian computation traces back to this sampling idea.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Monte_Carlo_method" ], "version": 2 }, { "id": "O-014", "slug": "enigma-bletchley-park-codebreaking", "title": "Enigma / Bletchley Park codebreaking (Bombe / Colossus)", "shortTitle": "Enigma / Bletchley Park codebreaking", "authors": [ "Alan Turing", "Gordon Welchman", "Tommy Flowers", "Bill Tutte" ], "institutions": [ "Bletchley Park" ], "year": 1943, "date": "1943", "venue": null, "artifactType": "wartime_engineering", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Bletchley_Park", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-institutional", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "The Bletchley Park codebreaking effort, using electromechanical Bombes and the electronic Colossus, showed that machines could automate large-scale symbolic reasoning and search, and it drove practical advances in programmable computing hardware.", "plainSummary": "During the Second World War, British codebreakers built machines to break the German Enigma and Lorenz ciphers by mechanizing the search through vast numbers of possible settings. The Bombe automated the logical elimination of impossible key configurations, while Colossus used electronic valves to process teleprinter traffic at high speed. This work demonstrated that computation could replace human labor on structured reasoning tasks. It also produced engineering experience and people, including Alan Turing, who shaped the first general-purpose computers.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Bletchley_Park" ], "version": 2 }, { "id": "O-015", "slug": "theory-of-games-and-economic-behavior", "title": "Theory of Games and Economic Behavior (game theory)", "shortTitle": "Theory of Games and Economic Behavior", "authors": [ "John von Neumann", "Oskar Morgenstern" ], "institutions": [ "IAS", "Princeton" ], "year": 1944, "date": "1944", "venue": null, "artifactType": "foundational_book", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Theory_of_Games_and_Economic_Behavior", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "'Theory of Games and Economic Behavior' founded game theory as a formal framework for analyzing decisions among competing agents, providing the mathematical language of strategies, payoffs, and equilibria used across economics, AI, and multi-agent systems.", "plainSummary": "In 1944 John von Neumann and Oskar Morgenstern set out a mathematical theory of how rational agents should act when their outcomes depend on each other's choices. They formalized games, strategies, and payoffs, and proved results such as the minimax theorem for zero-sum games, along with a theory of utility for decisions under uncertainty. The book gave economics and later computer science a rigorous way to reason about competition and cooperation. Its concepts underpin work on reinforcement learning, mechanism design, and multi-agent AI.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Theory_of_Games_and_Economic_Behavior" ], "version": 2 }, { "id": "O-016", "slug": "the-dartmouth-workshop", "title": "The Dartmouth Workshop (birth of Artificial Intelligence)", "shortTitle": "The Dartmouth Workshop", "authors": [ "John McCarthy", "Marvin Minsky", "Claude Shannon", "Nathaniel Rochester" ], "institutions": [ "Dartmouth" ], "year": 1956, "date": "1956", "venue": null, "artifactType": "founding_event", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Dartmouth_workshop", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-founding", "confidence": "High", "difficulty": 3, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "The 1956 Dartmouth Summer Research Project named the field of artificial intelligence and set its founding agenda, proposing that learning and intelligence could be described precisely enough for a machine to simulate them.", "plainSummary": "In the summer of 1956, John McCarthy, Marvin Minsky, Nathaniel Rochester, and Claude Shannon organized a workshop at Dartmouth College to study whether machines could be made to reason, learn, and use language. Their proposal coined the term 'artificial intelligence' and argued that every aspect of intelligence could in principle be specified well enough to be automated. The meeting gathered the researchers who would lead the field for decades. It set the research directions, from problem solving to language and learning, that defined AI's first generation.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Dartmouth_workshop" ], "version": 2 }, { "id": "O-017", "slug": "the-perceptron", "title": "The Perceptron", "shortTitle": "The Perceptron", "authors": [ "Frank Rosenblatt" ], "institutions": [ "Cornell Aeronautical Lab" ], "year": 1958, "date": "1958", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Perceptron", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-foundational", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Rosenblatt's Perceptron introduced a trainable model that adjusts numerical weights from examples to classify inputs, giving a concrete mechanism for machine learning and serving as the direct ancestor of neural networks.", "plainSummary": "In 1958 Frank Rosenblatt described the perceptron, a simple network that combines weighted inputs and produces a decision, and a procedure that updates those weights whenever it makes a mistake. This gave a working example of a machine that improves its performance by seeing labeled data rather than being explicitly programmed. The perceptron could learn to separate patterns that are linearly separable, though later work showed its limits on harder problems. Its learning rule and layered structure are the foundation of today's deep learning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Perceptron" ], "version": 2 }, { "id": "O-018", "slug": "perceptrons", "title": "Perceptrons (Minsky-Papert critique)", "shortTitle": "Perceptrons", "authors": [ "Marvin Minsky", "Seymour Papert" ], "institutions": [ "MIT" ], "year": 1969, "date": "1969", "venue": null, "artifactType": "critique", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Perceptrons_(book)", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "corrective-result", "statusRaw": "origin-negative-result", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Minsky and Papert's book analyzed the mathematical capabilities of single-layer perceptrons, proving they cannot compute functions that are not linearly separable, such as XOR, which clarified the fundamental limits of that model.", "plainSummary": "Through geometric and algebraic analysis the authors characterized exactly which predicates a single-layer perceptron can and cannot represent, showing that some require information the local, limited-order perceptron cannot combine, with XOR and connectedness as prominent examples of failures. They noted that multilayer networks could in principle overcome these limits but that no effective training method for them was then known. The rigorous negative results are widely credited with cooling early enthusiasm and funding for neural network research until multilayer training via backpropagation was popularized in the 1980s.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Perceptrons_(book)" ], "version": 2 }, { "id": "O-019", "slug": "algorithmic-information-theory", "title": "Algorithmic Information Theory (Solomonoff / Kolmogorov)", "shortTitle": "Algorithmic Information Theory", "authors": [ "Ray Solomonoff", "Andrey Kolmogorov", "Gregory Chaitin" ], "institutions": [ "Multiple" ], "year": 1964, "date": "1964", "venue": null, "artifactType": "foundational_paper", "domain": "origins", "url": "https://en.wikipedia.org/wiki/Kolmogorov_complexity", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "historically-important", "statusRaw": "origin-theoretical", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": "n/a-historical", "retrievalDate": "not-applicable", "verified": false }, "verificationTier": "verified", "thesis": "Algorithmic information theory defined the complexity of an object as the length of the shortest program that produces it, giving a formal account of randomness, compression, and simplicity that underlies modern ideas about learning and inductive inference.", "plainSummary": "In the 1960s Ray Solomonoff, Andrey Kolmogorov, and Gregory Chaitin independently proposed measuring how complex a string of data is by the size of the smallest computer program that can generate it. A string that needs a long program is effectively random, while a compressible one is simple. Solomonoff used this idea to build a theory of prediction that favors the simplest explanation consistent with the data. These concepts connect compression, probability, and learning, and they inform how researchers reason about generalization and Occam's razor in machine learning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://en.wikipedia.org/wiki/Kolmogorov_complexity" ], "version": 2 }, { "id": "P-001", "slug": "attention-is-all-you-need", "title": "Attention Is All You Need", "shortTitle": "Attention Is All You Need", "authors": [ "Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit", "Llion Jones", "Aidan N. Gomez", "Lukasz Kaiser", "Illia Polosukhin" ], "institutions": [ "Google Brain", "Google Research" ], "year": 2017, "date": "2017-06", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/1706.03762", "arxivId": "1706.03762", "doi": null, "repository": "https://github.com/tensorflow/tensor2tensor", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 10, "engineering": 9, "industrial": 9, "openSource": 7, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 130000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the Transformer, a sequence model built entirely on self-attention that removed the sequential recurrence of RNNs and enabled full parallelization of training across positions.", "plainSummary": "The paper replaces recurrent and convolutional encoders with stacked multi-head self-attention and feed-forward layers, using positional encodings to retain word order. Because attention relates all positions at once rather than stepping through a sequence, training parallelizes across the sequence and long-range dependencies are captured in a constant number of operations. It set new machine-translation results on WMT English-German and English-French at lower training cost, and became the base architecture for essentially all later large language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1706.03762", "arXiv:1706.03762" ], "version": 2 }, { "id": "P-002", "slug": "roformer-rotary-position-embedding", "title": "RoFormer: Rotary Position Embedding (RoPE)", "shortTitle": "RoFormer", "authors": [ "Jianlin Su", "Yu Lu", "Shengfeng Pan", "Ahmed Murtadha", "Bo Wen", "Yunfeng Liu" ], "institutions": [ "Zhuiyi Technology" ], "year": 2021, "date": "2021-04", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/2104.09864", "arxivId": "2104.09864", "doi": null, "repository": "https://github.com/ZhuiyiTechnology/roformer", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced rotary position embedding, which encodes position by rotating query and key vectors so their dot product depends only on relative distance, removing the need for additive position embeddings that generalize poorly to longer sequences.", "plainSummary": "RoPE multiplies each pair of embedding dimensions by a position-dependent rotation before computing attention, so a token's position is baked into the query/key geometry rather than added as a separate vector. Because two rotated vectors' dot product is a function of their offset, attention scores naturally become relative-position aware. This gives cleaner extrapolation to sequence lengths not seen in training and is now the default position scheme in most open large language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2104.09864", "arXiv:2104.09864" ], "version": 2 }, { "id": "P-003", "slug": "root-mean-square-layer-normalization", "title": "Root Mean Square Layer Normalization (RMSNorm)", "shortTitle": "Root Mean Square Layer Normalization", "authors": [ "Biao Zhang", "Rico Sennrich" ], "institutions": [ "U. Edinburgh" ], "year": 2019, "date": "2019-10", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/1910.07467", "arxivId": "1910.07467", "doi": null, "repository": "https://github.com/bzhangGo/rmsnorm", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced RMSNorm, which normalizes activations by their root-mean-square alone and drops the mean-centering and bias of LayerNorm, cutting normalization cost while keeping training stability.", "plainSummary": "RMSNorm rescales each activation vector by its root-mean-square magnitude and applies a learned gain, skipping the mean-subtraction step that standard LayerNorm performs. The authors argue the re-centering in LayerNorm contributes little and that re-scaling is what actually stabilizes training. The result is fewer operations and less compute per layer at comparable accuracy, which is why it replaced LayerNorm in LLaMA and many later transformer stacks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1910.07467", "arXiv:1910.07467" ], "version": 2 }, { "id": "P-004", "slug": "glu-variants-improve-transformer", "title": "GLU Variants Improve Transformer (SwiGLU)", "shortTitle": "GLU Variants Improve Transformer", "authors": [ "Noam Shazeer" ], "institutions": [ "Google" ], "year": 2020, "date": "2020-02", "venue": null, "artifactType": "preprint", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/2002.05202", "arxivId": "2002.05202", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 1200, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that replacing the transformer feed-forward layer's ReLU with gated linear unit variants, especially SwiGLU, yields better quality at matched compute.", "plainSummary": "The paper swaps the standard two-matrix ReLU feed-forward block for GLU variants, where one linear projection is elementwise-multiplied by a gated (sigmoid, GELU, or Swish) version of another projection. In controlled pretraining and fine-tuning comparisons, these gated blocks lower perplexity and improve downstream scores over the plain ReLU feed-forward. SwiGLU became the standard feed-forward design in PaLM, LLaMA, and most subsequent large models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2002.05202", "arXiv:2002.05202" ], "version": 2 }, { "id": "P-005", "slug": "train-short-test-long-attention-with-linear-biases", "title": "Train Short Test Long: Attention with Linear Biases (ALiBi)", "shortTitle": "Train Short Test Long", "authors": [ "Ofir Press", "Noah A. Smith", "Mike Lewis" ], "institutions": [ "UW", "FAIR", "AI2" ], "year": 2021, "date": "2021-08", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/2108.12409", "arxivId": "2108.12409", "doi": null, "repository": "https://github.com/ofirpress/attention_with_linear_biases", "openness": "unknown", "status": "partially-superseded", "statusRaw": "partially-superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "ALiBi replaces learned or sinusoidal position embeddings with a fixed linear penalty added to attention scores based on key-query distance, letting a model trained on short sequences run on much longer ones without retraining.", "plainSummary": "Instead of adding position vectors to token embeddings, ALiBi biases each attention score downward in proportion to how far apart the two tokens are, with a per-head slope. Because this bias is a simple distance function rather than a lookup limited to training lengths, a model trained on sequences of, say, 1024 tokens keeps working when evaluated on far longer inputs. This made length extrapolation cheap and removed the need to train at the target context length to get usable long-context behavior.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2108.12409", "arXiv:2108.12409" ], "version": 2 }, { "id": "P-006", "slug": "self-attention-with-relative-position-representations-transformer-xl", "title": "Self-Attention with Relative Position Representations / Transformer-XL", "shortTitle": "Self-Attention with Relative Position Representations / Transformer-XL", "authors": [ "Peter Shaw", "Jakob Uszkoreit", "Ashish Vaswani", "Zihang Dai", "Zhilin Yang", "Quoc V. Le", "Ruslan Salakhutdinov" ], "institutions": [ "Google", "CMU+Google" ], "year": 2018, "date": "2018-2019", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/1901.02860", "arxivId": "1901.02860", "doi": null, "repository": "https://github.com/kimiyoung/transformer-xl", "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-conceptual", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 5000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Transformer-XL adds a recurrence mechanism that reuses hidden states from prior segments plus a relative positional encoding, removing the fixed-length context limit and the context fragmentation caused by processing text in independent chunks.", "plainSummary": "Standard Transformers split long text into fixed segments processed independently, so the model never sees dependencies crossing a segment boundary. Transformer-XL caches the hidden states of the previous segment and lets the current segment attend back into them, while switching from absolute to relative position encodings so the reused states stay positionally consistent. This extended the effective context well beyond a single segment and sped up evaluation, enabling coherent modeling of longer-range dependencies.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1901.02860", "arXiv:1901.02860" ], "version": 2 }, { "id": "P-007", "slug": "on-layer-normalization-in-the-transformer-architecture", "title": "On Layer Normalization in the Transformer Architecture (pre-norm)", "shortTitle": "On Layer Normalization in the Transformer Architecture", "authors": [ "Ruibin Xiong", "Yunchang Yang", "Di He", "et al." ], "institutions": [ "Microsoft Research Asia" ], "year": 2020, "date": "2020-02", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/2002.04745", "arxivId": "2002.04745", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed analytically and empirically that placing layer normalization inside the residual branch (Pre-LN) instead of after it (Post-LN) keeps gradients well-scaled at initialization, removing the need for the learning-rate warm-up stage that Post-LN Transformers required for stable training.", "plainSummary": "The paper studies where the layer-normalization step sits relative to the residual connection in a Transformer block. In the original Post-LN design, gradients near the output layer are large at initialization, so training diverges unless you start with a small learning rate and slowly warm it up. By moving the normalization to the input of each sub-layer (Pre-LN), gradients become bounded and roughly uniform across depth, so models train stably without warm-up, tolerate larger learning rates, and converge faster. This made deep Transformers easier and cheaper to train and is why most later large models adopt the Pre-LN arrangement.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2002.04745", "arXiv:2002.04745" ], "version": 2 }, { "id": "P-008", "slug": "fast-transformer-decoding-one-write-head-is-all-you-need", "title": "Fast Transformer Decoding: One Write-Head is All You Need (MQA)", "shortTitle": "Fast Transformer Decoding", "authors": [ "Noam Shazeer" ], "institutions": [ "Google" ], "year": 2019, "date": "2019-11", "venue": null, "artifactType": "preprint", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/1911.02150", "arxivId": "1911.02150", "doi": null, "repository": null, "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded-by-GQA", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 700, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced multi-query attention, which shares a single key and value head across all query heads so autoregressive decoding needs far less memory bandwidth for the attention cache.", "plainSummary": "In standard multi-head attention each head has its own key and value projections, so the per-step key/value cache that dominates incremental decoding is large and bandwidth-bound. MQA keeps multiple query heads but collapses to one shared key/value head, shrinking that cache by the number of heads and making token-by-token generation much faster. The trade-off is a small quality drop, but the decoding speedup made MQA a common choice for inference-heavy models and set up the later GQA compromise.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1911.02150", "arXiv:1911.02150" ], "version": 2 }, { "id": "P-009", "slug": "gqa-training-generalized-multi-query-transformer-models", "title": "GQA: Training Generalized Multi-Query Transformer Models", "shortTitle": "GQA", "authors": [ "Joshua Ainslie", "James Lee-Thorp", "Michiel de Jong", "Yury Zemlyanskiy", "Federico Lebron", "Sumit Sanghai" ], "institutions": [ "Google" ], "year": 2023, "date": "2023-05", "venue": null, "artifactType": "peer_reviewed", "domain": "transformer-architecture", "url": "https://arxiv.org/abs/2305.13245", "arxivId": "2305.13245", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 900, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced grouped-query attention, an interpolation between multi-head and multi-query attention that shares each key/value head across a small group of query heads, plus a recipe to uptrain existing multi-head checkpoints into this form.", "plainSummary": "GQA divides query heads into groups that each share one key/value head, giving a tunable point between full multi-head attention (best quality, large cache) and multi-query attention (smallest cache, some quality loss). The authors also show you can convert an already-trained multi-head model to GQA by mean-pooling its key/value heads and briefly continuing training. This recovers most of MQA's inference speedup while keeping quality close to multi-head, and it became the standard attention layout in LLaMA 2/3-scale models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.13245", "arXiv:2305.13245" ], "version": 2 }, { "id": "P-010", "slug": "improving-language-understanding-by-generative-pre-training", "title": "Improving Language Understanding by Generative Pre-Training (GPT)", "shortTitle": "Improving Language Understanding by Generative Pre-Training", "authors": [ "Alec Radford", "Karthik Narasimhan", "Tim Salimans", "Ilya Sutskever" ], "institutions": [ "OpenAI" ], "year": 2018, "date": "2018-06", "venue": null, "artifactType": "technical_report", "domain": "decoder-only", "url": "https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf", "arxivId": null, "doi": null, "repository": "https://github.com/openai/finetune-transformer-lm", "openness": "unknown", "status": "historically-important", "statusRaw": "paradigm-foundational", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 13000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that a single Transformer decoder pre-trained on unlabeled text with a language-modeling objective, then fine-tuned per task, could beat task-specific architectures across diverse NLP benchmarks.", "plainSummary": "GPT-1 pre-trains a left-to-right Transformer on a large corpus of books to predict the next token, then adapts to each downstream task with a small supervised fine-tuning step and task-specific input formatting. This two-stage recipe removed the need to hand-design a separate model per task and to rely on scarce labeled data. It improved results on entailment, question answering, semantic similarity, and classification, establishing generative pre-training plus fine-tuning as a general NLP method.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf" ], "version": 2 }, { "id": "P-011", "slug": "language-models-are-unsupervised-multitask-learners", "title": "Language Models are Unsupervised Multitask Learners (GPT-2)", "shortTitle": "Language Models are Unsupervised Multitask Learners", "authors": [ "Alec Radford", "Jeffrey Wu", "Rewon Child", "David Luan", "Dario Amodei", "Ilya Sutskever" ], "institutions": [ "OpenAI" ], "year": 2019, "date": "2019-02", "venue": null, "artifactType": "technical_report", "domain": "decoder-only", "url": "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "arxivId": null, "doi": null, "repository": "https://github.com/openai/gpt-2", "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-pivotal", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Demonstrated that scaling a decoder-only language model and its training corpus lets it perform many NLP tasks with no gradient updates, purely by conditioning on a prompt.", "plainSummary": "GPT-2 trains a 1.5B-parameter Transformer on WebText, a large corpus scraped from outbound Reddit links, using the same next-token objective as GPT-1. The paper shows the model handles reading comprehension, translation, summarization, and question answering in a zero-shot setting when the task is phrased as text, without any task-specific training. This reframed NLP tasks as special cases of language modeling and gave early evidence that capability scales with model and data size.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf" ], "version": 2 }, { "id": "P-012", "slug": "language-models-are-few-shot-learners", "title": "Language Models are Few-Shot Learners (GPT-3)", "shortTitle": "Language Models are Few-Shot Learners", "authors": [ "Tom B. Brown", "Benjamin Mann", "Nick Ryder", "Melanie Subbiah", "et al." ], "institutions": [ "OpenAI" ], "year": 2020, "date": "2020-05", "venue": null, "artifactType": "peer_reviewed", "domain": "decoder-only", "url": "https://arxiv.org/abs/2005.14165", "arxivId": "2005.14165", "doi": null, "repository": "https://github.com/openai/gpt-3", "openness": "unknown", "status": "partially-superseded", "statusRaw": "foundational-superseded-as-model", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 35000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that scaling an autoregressive language model to 175B parameters yields in-context few-shot learning, where tasks are specified through examples in the prompt rather than by fine-tuning weights.", "plainSummary": "GPT-3 trains a 175-billion-parameter Transformer on a filtered Common Crawl plus other corpora, keeping the next-token objective but scaling roughly 100x over GPT-2. Given a natural-language instruction and a handful of demonstrations in its context window, it performs translation, question answering, arithmetic, and other tasks without weight updates, with accuracy generally rising as more examples are shown. This removed the per-task fine-tuning and labeled-data requirement for many uses and made prompting the primary interface to large models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2005.14165", "arXiv:2005.14165" ], "version": 2 }, { "id": "P-013", "slug": "bert-pre-training-of-deep-bidirectional-transformers", "title": "BERT: Pre-training of Deep Bidirectional Transformers", "shortTitle": "BERT", "authors": [ "Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova" ], "institutions": [ "Google AI Language" ], "year": 2018, "date": "2018-10", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-only", "url": "https://arxiv.org/abs/1810.04805", "arxivId": "1810.04805", "doi": null, "repository": "https://github.com/google-research/bert", "openness": "unknown", "status": "niche-but-active", "statusRaw": "essential-in-niche", "confidence": "High", "difficulty": 5, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 110000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced BERT, a Transformer encoder pre-trained with masked-language-modeling and next-sentence prediction to produce deeply bidirectional representations usable across tasks via light fine-tuning.", "plainSummary": "BERT pre-trains an encoder by masking random tokens and predicting them from both left and right context, unlike the left-to-right models of the GPT line, plus a next-sentence-prediction objective. The resulting representations condition each token on the full surrounding sentence, and a single added output layer fine-tunes the model for classification, tagging, or span-based question answering. It set new results on GLUE and SQuAD and became the standard encoder for understanding-oriented NLP tasks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1810.04805", "arXiv:1810.04805" ], "version": 2 }, { "id": "P-014", "slug": "roberta-a-robustly-optimized-bert-pretraining-approach", "title": "RoBERTa: A Robustly Optimized BERT Pretraining Approach", "shortTitle": "RoBERTa", "authors": [ "Yinhan Liu", "Myle Ott", "Naman Goyal", "Jingfei Du", "et al." ], "institutions": [ "FAIR", "UW" ], "year": 2019, "date": "2019-07", "venue": null, "artifactType": "preprint", "domain": "encoder-only", "url": "https://arxiv.org/abs/1907.11692", "arxivId": "1907.11692", "doi": null, "repository": "https://github.com/facebookresearch/fairseq", "openness": "unknown", "status": "current-standard", "statusRaw": "standard-encoder", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 30000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that BERT was substantially undertrained and that with more data, longer training, larger batches, removal of the next-sentence-prediction objective, and dynamic masking, the same architecture reaches much higher accuracy, establishing that pretraining recipe choices, not architecture, drove most of the remaining gains.", "plainSummary": "RoBERTa is a careful re-examination of how BERT was trained rather than a new model architecture. The authors trained longer on about ten times more text, used bigger batches, generated the masking pattern fresh each time an example is seen (dynamic masking), dropped the next-sentence-prediction task, and trained on full-length sentence sequences. These changes alone pushed performance above the original BERT and matched or beat later models on GLUE, SQuAD, and RACE. The main lesson was methodological: much of what looked like architectural progress was actually the result of undertraining, and controlled comparisons require fixing the training budget.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1907.11692", "arXiv:1907.11692" ], "version": 2 }, { "id": "P-015", "slug": "albert-a-lite-bert", "title": "ALBERT: A Lite BERT", "shortTitle": "ALBERT", "authors": [ "Zhenzhong Lan", "Mingda Chen", "Sebastian Goodman", "Kevin Gimpel", "Piyush Sharma", "Radu Soricut" ], "institutions": [ "Google", "TTIC" ], "year": 2019, "date": "2019-09", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-only", "url": "https://arxiv.org/abs/1909.11942", "arxivId": "1909.11942", "doi": null, "repository": "https://github.com/google-research/albert", "openness": "unknown", "status": "fully-superseded", "statusRaw": "superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 9000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "ALBERT cuts BERT's parameter count through factorized embedding parameterization and cross-layer parameter sharing, and swaps next-sentence prediction for a sentence-order objective, letting larger-capacity configurations train within the same memory budget.", "plainSummary": "ALBERT decouples the vocabulary embedding size from the hidden size (factorizing that large matrix) and shares the same parameters across all Transformer layers, drastically reducing the model's parameter footprint. It also replaces BERT's next-sentence prediction with a sentence-order prediction task that forces the model to learn inter-sentence coherence rather than topic overlap. Together these let ALBERT scale hidden dimensions and depth without a proportional memory blowup, reaching stronger results than BERT-large at a fraction of the stored parameters.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1909.11942", "arXiv:1909.11942" ], "version": 2 }, { "id": "P-016", "slug": "electra-pre-training-text-encoders-as-discriminators", "title": "ELECTRA: Pre-training Text Encoders as Discriminators", "shortTitle": "ELECTRA", "authors": [ "Kevin Clark", "Minh-Thang Luong", "Quoc V. Le", "Christopher D. Manning" ], "institutions": [ "Stanford", "Google" ], "year": 2020, "date": "2020-03", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-only", "url": "https://arxiv.org/abs/2003.10555", "arxivId": "2003.10555", "doi": null, "repository": "https://github.com/google-research/electra", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 5000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "ELECTRA trains an encoder with replaced-token detection, a discriminator that classifies every token as original or generator-substituted, extracting learning signal from all positions instead of only the ~15% masked ones and cutting pretraining compute.", "plainSummary": "Rather than masking tokens and predicting them, ELECTRA uses a small generator to swap some tokens for plausible alternatives, then trains the main model to decide, for every token, whether it was replaced. Because the loss covers all positions rather than the small masked subset, each training step yields more signal per example. This let ELECTRA match or exceed BERT-scale accuracy using substantially less pretraining compute, making strong encoders reachable on smaller budgets.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2003.10555", "arXiv:2003.10555" ], "version": 2 }, { "id": "P-020", "slug": "exploring-the-limits-of-transfer-learning-with-a-unified-text-to-text-transformer", "title": "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer (T5)", "shortTitle": "Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer", "authors": [ "Colin Raffel", "Noam Shazeer", "Adam Roberts", "Katherine Lee", "Sharan Narang", "Michael Matena", "Yanqi Zhou", "Wei Li", "Peter J. Liu" ], "institutions": [ "Google" ], "year": 2020, "date": "2019-10", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-decoder", "url": "https://arxiv.org/abs/1910.10683", "arxivId": "1910.10683", "doi": null, "repository": "https://github.com/google-research/text-to-text-transfer-transformer", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-framing", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 20000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced T5, which casts every NLP task as text-to-text generation, allowing one model, objective, and decoding procedure to cover classification, translation, summarization, and question answering.", "plainSummary": "T5 uses an encoder-decoder Transformer trained with a span-corruption denoising objective and represents each task, including ones with numeric or label outputs, as producing a target string from an input string. The paper runs a controlled comparison of architectures, objectives, corpora, and transfer strategies, and pre-trains on the C4 Common Crawl corpus the authors assembled and released. The unified format removed per-task output heads and enabled systematic study of what drives transfer, with strong results across many benchmarks at scale.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1910.10683", "arXiv:1910.10683" ], "version": 2 }, { "id": "P-021", "slug": "bart-denoising-seq2seq-pre-training", "title": "BART: Denoising Seq2Seq Pre-training", "shortTitle": "BART", "authors": [ "Mike Lewis", "Yinhan Liu", "Naman Goyal", "Marjan Ghazvininejad", "Abdelrahman Mohamed", "Omer Levy", "Ves Stoyanov", "Luke Zettlemoyer" ], "institutions": [ "FAIR" ], "year": 2020, "date": "2019-10", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-decoder", "url": "https://arxiv.org/abs/1910.13461", "arxivId": "1910.13461", "doi": null, "repository": "https://github.com/facebookresearch/fairseq", "openness": "unknown", "status": "niche-but-active", "statusRaw": "still-used", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 13000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "BART pretrains a full encoder-decoder Transformer as a denoising autoencoder that reconstructs original text from arbitrarily corrupted input, unifying BERT-style bidirectional encoding with GPT-style autoregressive generation for both understanding and generation tasks.", "plainSummary": "BART corrupts documents with noise such as token masking, deletion, sentence shuffling, and text-span infilling, then trains a bidirectional encoder plus autoregressive decoder to reconstruct the original. This seq2seq setup means the same pretrained model handles classification (via the encoder) and generation like summarization and translation (via the decoder), rather than needing separate architectures. Text infilling in particular proved effective, and BART reached strong results on generation benchmarks while remaining competitive on discriminative tasks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1910.13461", "arXiv:1910.13461" ], "version": 2 }, { "id": "P-022", "slug": "ul2-unifying-language-learning-paradigms", "title": "UL2: Unifying Language Learning Paradigms", "shortTitle": "UL2", "authors": [ "Yi Tay", "Mostafa Dehghani", "Vinh Q. Tran", "Xavier Garcia", "et al." ], "institutions": [ "Google" ], "year": 2023, "date": "2022-05", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-decoder", "url": "https://arxiv.org/abs/2205.05131", "arxivId": "2205.05131", "doi": null, "repository": "https://github.com/google-research/google-research/tree/master/ul2", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-idea", "confidence": "Medium", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 900, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "UL2 introduces Mixture-of-Denoisers, a pretraining scheme that blends several span-corruption and prefix-language-modeling objectives with mode-switching tokens, producing a single model that works across both fine-tuning and in-context few-shot settings regardless of architecture.", "plainSummary": "UL2 trains on a mixture of denoising objectives (short spans, long spans, and sequential/prefix-LM corruption) and prepends a mode token that tells the model which denoising regime applies, letting it be switched between modes at inference. This bridges the gap where masked-denoising models excelled at fine-tuning while causal LMs excelled at few-shot prompting, giving one recipe that performs on both. The framework is architecture-agnostic and let a single pretrained model be adapted to supervised fine-tuning and prompting without choosing an objective in advance.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2205.05131", "arXiv:2205.05131" ], "version": 2 }, { "id": "P-024", "slug": "deberta-decoding-enhanced-bert-with-disentangled-attention", "title": "DeBERTa: Decoding-enhanced BERT with Disentangled Attention", "shortTitle": "DeBERTa", "authors": [ "Pengcheng He", "Xiaodong Liu", "Jianfeng Gao", "Weizhu Chen" ], "institutions": [ "Microsoft" ], "year": 2021, "date": "2020-06", "venue": null, "artifactType": "peer_reviewed", "domain": "encoder-only", "url": "https://arxiv.org/abs/2006.03654", "arxivId": "2006.03654", "doi": null, "repository": "https://github.com/microsoft/DeBERTa", "openness": "unknown", "status": "niche-but-active", "statusRaw": "essential-in-niche", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 5000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "DeBERTa introduces disentangled attention, representing each token by separate content and relative-position vectors whose attention is computed across both, plus an enhanced mask decoder that reinjects absolute positions, improving over BERT and RoBERTa at equal or smaller scale.", "plainSummary": "DeBERTa keeps a token's content and its relative position as two distinct vectors and computes attention weights from content-to-content, content-to-position, and position-to-content terms, capturing that a word's relationship to another depends on their distance. Because relative encoding alone loses absolute placement needed for tasks like masked-word prediction, it adds an enhanced mask decoder that folds absolute positions back in just before the output layer. These changes improved sample efficiency and accuracy on language-understanding benchmarks relative to BERT and RoBERTa at comparable model sizes.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2006.03654", "arXiv:2006.03654" ], "version": 2 }, { "id": "P-100", "slug": "scaling-laws-for-neural-language-models", "title": "Scaling Laws for Neural Language Models", "shortTitle": "Scaling Laws for Neural Language Models", "authors": [ "Jared Kaplan", "Sam McCandlish", "Tom Henighan", "Tom B. Brown", "et al." ], "institutions": [ "OpenAI", "JHU" ], "year": 2020, "date": "2020-01", "venue": null, "artifactType": "preprint", "domain": "scaling", "url": "https://arxiv.org/abs/2001.08361", "arxivId": "2001.08361", "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "framework-current-prescription-superseded", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 8000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Established that language-model test loss follows smooth power laws in model size, dataset size, and training compute, turning architecture-and-scale guesswork into predictable extrapolation.", "plainSummary": "The authors trained many Transformer language models across orders of magnitude in parameters, data, and compute, then fit power laws to the loss curves. They showed loss scales predictably with each factor when the others are not bottlenecked, and that within their observed range larger models are more sample-efficient, so given fixed compute it was better to train very large models on comparatively less data and stop early. This gave labs a quantitative basis to forecast returns from more compute and allocate budget before committing to a run.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2001.08361", "arXiv:2001.08361" ], "version": 2 }, { "id": "P-101", "slug": "training-compute-optimal-large-language-models", "title": "Training Compute-Optimal Large Language Models (Chinchilla)", "shortTitle": "Training Compute-Optimal Large Language Models", "authors": [ "Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch", "et al." ], "institutions": [ "DeepMind" ], "year": 2022, "date": "2022-03", "venue": null, "artifactType": "peer_reviewed", "domain": "scaling", "url": "https://arxiv.org/abs/2203.15556", "arxivId": "2203.15556", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Corrected the earlier scaling prescription by showing parameters and training tokens should be scaled in roughly equal proportion, revealing that then-current large models were badly undertrained for their compute budgets.", "plainSummary": "DeepMind trained over 400 models at varied size and token counts and re-estimated the compute-optimal frontier, finding that for a given compute budget parameters and data should grow together at about a 1:1 ratio rather than favoring size. To demonstrate it they trained Chinchilla, a 70B model on 1.4 trillion tokens, which outperformed the 280B Gopher and other larger models while being cheaper to run at inference. The result redirected the field toward training smaller models on far more data.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2203.15556", "arXiv:2203.15556" ], "version": 2 }, { "id": "P-102", "slug": "emergent-abilities-of-large-language-models", "title": "Emergent Abilities of Large Language Models", "shortTitle": "Emergent Abilities of Large Language Models", "authors": [ "Jason Wei", "Yi Tay", "Rishi Bommasani", "et al." ], "institutions": [ "Google", "Stanford", "DeepMind" ], "year": 2022, "date": "2022-06", "venue": null, "artifactType": "peer_reviewed", "domain": "scaling", "url": "https://arxiv.org/abs/2206.07682", "arxivId": "2206.07682", "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "contested", "confidence": "Medium", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This paper documents that certain capabilities are absent in smaller language models yet appear once scale crosses a threshold, arguing such abilities are not predictable by extrapolating small-model performance and framing scale itself as unlocking qualitatively new behavior.", "plainSummary": "The authors survey tasks where accuracy stays near chance for small models and then rises sharply beyond a certain parameter or compute scale, calling these emergent abilities. They catalog examples across few-shot prompting and augmented-prompting settings, showing the jumps are not evident from the performance trend of smaller models. The claim shaped how the field reasoned about scaling, suggesting some capabilities cannot be forecast and appear only after crossing a size threshold.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2206.07682", "arXiv:2206.07682" ], "version": 2 }, { "id": "P-103", "slug": "are-emergent-abilities-of-llms-a-mirage", "title": "Are Emergent Abilities of LLMs a Mirage?", "shortTitle": "Are Emergent Abilities of LLMs a Mirage?", "authors": [ "Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo" ], "institutions": [ "Stanford" ], "year": 2023, "date": "2023-04", "venue": null, "artifactType": "peer_reviewed", "domain": "scaling", "url": "https://arxiv.org/abs/2304.15004", "arxivId": "2304.15004", "doi": null, "repository": null, "openness": "unknown", "status": "corrective-result", "statusRaw": "essential-corrective", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This paper argues apparent emergent abilities are largely an artifact of discontinuous or nonlinear evaluation metrics rather than fundamental jumps in model capability, showing that switching to smooth, continuous metrics turns the sharp jumps into gradual, predictable improvement.", "plainSummary": "Schaeffer et al. show that metrics like exact-match or multiple-choice accuracy are harsh and nonlinear: a model's underlying per-token performance can improve smoothly while the reported score stays flat then spikes once the whole answer becomes correct. Re-scoring the same models and tasks with continuous or partial-credit metrics converts many reported emergent curves into steady, extrapolable trends. The work reframes much claimed emergence as a measurement choice, cautioning that metric selection, not new capability, can manufacture the appearance of a threshold.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2304.15004", "arXiv:2304.15004" ], "version": 2 }, { "id": "P-104", "slug": "scaling-data-constrained-language-models", "title": "Scaling Data-Constrained Language Models", "shortTitle": "Scaling Data-Constrained Language Models", "authors": [ "Niklas Muennighoff", "Alexander M. Rush", "Boaz Barak", "et al." ], "institutions": [ "HuggingFace", "Harvard" ], "year": 2023, "date": "2023-05", "venue": null, "artifactType": "peer_reviewed", "domain": "scaling", "url": "https://arxiv.org/abs/2305.16264", "arxivId": "2305.16264", "doi": null, "repository": "https://github.com/huggingface/datablations", "openness": "unknown", "status": "current-standard", "statusRaw": "current", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 700, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It measured how repeating a fixed corpus trades off against adding parameters, showing that under a data budget you can reuse tokens for several epochs at near-fresh-data value and derive scaling laws that account for repetition.", "plainSummary": "The authors trained hundreds of models while holding the amount of unique text fixed and varying how many times it was repeated, then fit scaling laws that treat repeated tokens as worth less than fresh ones. They found that repeating data for up to about four epochs yields loss nearly identical to using that much new data, with returns decaying quickly after roughly 16 epochs. This gave data-limited teams a principled way to allocate compute between more epochs and more parameters, and quantified when scraping or generating more data stops helping.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.16264", "arXiv:2305.16264" ], "version": 2 }, { "id": "P-110", "slug": "outrageously-large-neural-networks-sparsely-gated-moe", "title": "Outrageously Large Neural Networks: Sparsely-Gated MoE", "shortTitle": "Outrageously Large Neural Networks", "authors": [ "Noam Shazeer", "Azalia Mirhoseini", "Krzysztof Maziarz", "Andy Davis", "Quoc Le", "Geoffrey Hinton", "Jeff Dean" ], "institutions": [ "Google Brain" ], "year": 2017, "date": "2017-01", "venue": null, "artifactType": "peer_reviewed", "domain": "moe", "url": "https://arxiv.org/abs/1701.06538", "arxivId": "1701.06538", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the sparsely-gated mixture-of-experts layer, where a trainable gating network routes each input to a few of thousands of expert sub-networks, letting capacity grow enormously without a proportional increase in per-example compute.", "plainSummary": "The layer holds up to thousands of expert feed-forward networks and a gating function that selects only the top-k experts per token, so only a small fraction of parameters activate per example. The authors add noisy top-k gating and load-balancing losses to keep experts from collapsing onto a few favorites, and apply it to stacked LSTM language and translation models. This showed conditional computation could scale parameter count by orders of magnitude on real tasks, and it is the direct ancestor of transformer MoE models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1701.06538", "arXiv:1701.06538" ], "version": 2 }, { "id": "P-111", "slug": "gshard-scaling-giant-models-with-conditional-computation-and-automatic-sharding", "title": "GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding", "shortTitle": "GShard", "authors": [ "Dmitry Lepikhin", "HyoukJoong Lee", "Yuanzhong Xu", "et al." ], "institutions": [ "Google" ], "year": 2021, "date": "2020-06", "venue": null, "artifactType": "peer_reviewed", "domain": "moe", "url": "https://arxiv.org/abs/2006.16668", "arxivId": "2006.16668", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-infra", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It introduced a sparsely-gated mixture-of-experts layer combined with automatic sharding annotations (the GShard/XLA compiler system) so that models with hundreds of billions of parameters could be split across thousands of accelerators, removing the memory and engineering bottleneck that blocked training beyond a few billion parameters.", "plainSummary": "GShard scales Transformers by replacing some dense feed-forward layers with a large set of expert sub-networks, where a learned gating function routes each token to only a couple of experts, so total parameters grow without a proportional growth in compute per token. The other half of the contribution is a lightweight annotation API that lets the programmer mark how tensors should be partitioned, after which the compiler automatically shards the computation and inserts the needed cross-device communication. Together these let the authors train a 600-billion-parameter multilingual translation model across 2048 TPU cores in a few days. The work made conditional computation and large-scale model parallelism practical without rewriting model code per device layout.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2006.16668", "arXiv:2006.16668" ], "version": 2 }, { "id": "P-112", "slug": "switch-transformer", "title": "Switch Transformer", "shortTitle": "Switch Transformer", "authors": [ "William Fedus", "Barret Zoph", "Noam Shazeer" ], "institutions": [ "Google" ], "year": 2022, "date": "2021-01", "venue": null, "artifactType": "peer_reviewed", "domain": "moe", "url": "https://arxiv.org/abs/2101.03961", "arxivId": "2101.03961", "doi": null, "repository": "https://github.com/google-research/t5x", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the Switch Transformer, which simplifies mixture-of-experts routing to a single expert per token and pairs it with stability and distributed-training techniques to train sparse models up to a trillion parameters.", "plainSummary": "Switch replaces the top-k MoE gate with top-1 routing so each token goes to exactly one expert, which cuts routing computation and communication while keeping a fixed compute budget per token. The paper adds selective precision, capacity factors and an auxiliary load-balancing loss, and expert-parallel sharding to make this stable at scale. The result trains far faster than dense baselines at equal FLOPs and reaches trillion-parameter counts, giving a practical template for sparse scaling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2101.03961", "arXiv:2101.03961" ], "version": 2 }, { "id": "P-113", "slug": "glam-efficient-scaling-with-mixture-of-experts", "title": "GLaM: Efficient Scaling with Mixture-of-Experts", "shortTitle": "GLaM", "authors": [ "Nan Du", "Yanping Huang", "Andrew M. Dai", "et al." ], "institutions": [ "Google" ], "year": 2022, "date": "2021-12", "venue": null, "artifactType": "peer_reviewed", "domain": "moe", "url": "https://arxiv.org/abs/2112.06905", "arxivId": "2112.06905", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It applied a sparsely-gated mixture-of-experts to decoder language models so that each token activates only two experts, letting a 1.2T-parameter model match or beat a dense GPT-3 while using roughly a third of the training energy and half the inference FLOPs.", "plainSummary": "GLaM replaces the feed-forward block in every other transformer layer with a set of 64 experts and a learned gate that routes each token to the top two, so total capacity grows without activating all weights per token. This decoupled model size from per-token compute, allowing a much larger parameter count at fixed inference cost. The result was competitive or better zero/one/few-shot accuracy than a dense 175B model at substantially lower compute and energy, demonstrating sparse MoE as a practical scaling path for large LMs.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2112.06905", "arXiv:2112.06905" ], "version": 2 }, { "id": "P-114", "slug": "mixture-of-experts-with-expert-choice-routing", "title": "Mixture-of-Experts with Expert Choice Routing", "shortTitle": "Mixture-of-Experts with Expert Choice Routing", "authors": [ "Yanqi Zhou", "Tao Lei", "Hanxiao Liu", "et al." ], "institutions": [ "Google" ], "year": 2022, "date": "2022-02", "venue": null, "artifactType": "peer_reviewed", "domain": "moe", "url": "https://arxiv.org/abs/2202.09368", "arxivId": "2202.09368", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-alternative", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It replaced token-chooses-expert routing with expert-chooses-token routing, letting each expert select a fixed number of tokens so load is balanced by construction and no auxiliary balancing loss or token dropping is needed.", "plainSummary": "In standard MoE each token picks its top experts, which causes uneven loads where some experts overflow and drop tokens. Expert Choice inverts this: every expert picks a fixed quota of the tokens it scores highest, so all experts stay exactly full and a token may be handled by a variable number of experts. This removed the need for load-balancing loss terms and capacity-factor tuning, and trained faster to a given quality (reported over 2x convergence speedup) while improving downstream results at matched compute.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2202.09368", "arXiv:2202.09368" ], "version": 2 }, { "id": "P-115", "slug": "st-moe-designing-stable-and-transferable-sparse-expert-models", "title": "ST-MoE: Designing Stable and Transferable Sparse Expert Models", "shortTitle": "ST-MoE", "authors": [ "Barret Zoph", "Irwan Bello", "Sameer Kumar", "et al." ], "institutions": [ "Google" ], "year": 2022, "date": "2022-02", "venue": null, "artifactType": "preprint", "domain": "moe", "url": "https://arxiv.org/abs/2202.08906", "arxivId": "2202.08906", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-practitioner", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It diagnosed why sparse expert models are unstable and transfer poorly, and introduced the router z-loss plus fine-tuning fixes that let a 269B-parameter sparse model train reliably and reach then-state-of-the-art results on SuperGLUE and reasoning benchmarks.", "plainSummary": "The authors traced MoE training instabilities to large router logits and showed that a z-loss penalizing those logit magnitudes stabilizes training without hurting quality. They also identified fine-tuning pitfalls, such as sparse and dense layers preferring different hyperparameters and expert dropout settings, and gave concrete recipes to address them. Combining these, their ST-MoE-32B model trained stably and transferred well across many NLP tasks, turning sparse experts from a finicky research artifact into a reproducible design.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2202.08906", "arXiv:2202.08906" ], "version": 2 }, { "id": "P-116", "slug": "mixtral-of-experts", "title": "Mixtral of Experts", "shortTitle": "Mixtral of Experts", "authors": [ "Albert Q. Jiang", "Alexandre Sablayrolles", "Antoine Roux", "et al." ], "institutions": [ "Mistral AI" ], "year": 2024, "date": "2024-01", "venue": null, "artifactType": "technical_report", "domain": "moe", "url": "https://arxiv.org/abs/2401.04088", "arxivId": "2401.04088", "doi": null, "repository": "https://github.com/mistralai/mistral-src", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "landmark-open-weight", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It released an open-weight sparse mixture-of-experts language model in which a router picks 2 of 8 expert feed-forward blocks per token per layer, giving the quality of a large (~47B-parameter) model while only activating about 13B parameters per token, cutting the compute and latency cost of high-quality inference.", "plainSummary": "Mixtral 8x7B is a decoder-only Transformer where each feed-forward layer is replaced by eight expert networks and a small router that selects two experts for every token. Because only two of eight experts run per token, the model holds roughly 47 billion parameters in memory but does the compute of a roughly 13-billion-parameter model at inference. Released under an open license, it matched or exceeded much larger dense models such as Llama 2 70B on most benchmarks while being faster to serve. It demonstrated that sparse expert routing could deliver a strong, openly available model with a favorable quality-to-active-compute ratio.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2401.04088", "arXiv:2401.04088" ], "version": 2 }, { "id": "P-117", "slug": "deepseekmoe-towards-ultimate-expert-specialization", "title": "DeepSeekMoE: Towards Ultimate Expert Specialization", "shortTitle": "DeepSeekMoE", "authors": [ "Damai Dai", "Chengqi Deng", "Chenggang Zhao", "et al." ], "institutions": [ "DeepSeek-AI" ], "year": 2024, "date": "2024-01", "venue": null, "artifactType": "technical_report", "domain": "moe", "url": "https://arxiv.org/abs/2401.06066", "arxivId": "2401.06066", "doi": null, "repository": "https://github.com/deepseek-ai/DeepSeek-MoE", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-frontier", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 600, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It restructured MoE by using many finer-grained experts plus a few always-on shared experts, so routed experts can specialize on distinct knowledge while shared ones absorb common patterns, improving quality at equal activated parameters.", "plainSummary": "DeepSeekMoE splits each expert into smaller pieces and increases their number so the router can compose more precise combinations, and it isolates a handful of shared experts that every token uses to capture redundant, general knowledge. This reduces the parameter redundancy and poor specialization seen in conventional MoE with a few large experts. At matched activated compute it matched or beat larger dense and standard-MoE baselines, and the design underpinned the efficiency of later DeepSeek models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2401.06066", "arXiv:2401.06066" ], "version": 2 }, { "id": "P-120", "slug": "sentencepiece", "title": "SentencePiece", "shortTitle": "SentencePiece", "authors": [ "Taku Kudo", "John Richardson" ], "institutions": [ "Google" ], "year": 2018, "date": "2018-08", "venue": null, "artifactType": "software_release", "domain": "data-tokenization", "url": "https://arxiv.org/abs/1808.06226", "arxivId": "1808.06226", "doi": null, "repository": "https://github.com/google/sentencepiece", "openness": "source-available", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It packaged subword tokenization (BPE and unigram language-model segmentation) into a single self-contained tool that trains directly from raw untokenized text and treats input as a reversible byte/character stream, removing the dependence on language-specific pre-tokenizers and making tokenization reproducible and fully invertible across languages.", "plainSummary": "SentencePiece is a tokenizer that learns a subword vocabulary straight from raw text, without needing a separate word-splitting step that most earlier pipelines assumed. It escapes whitespace as a normal symbol (the underscore marker) so that tokenizing and detokenizing are exactly reversible, which matters for languages like Japanese or Chinese that do not put spaces between words. It supports both byte-pair-encoding and unigram-language-model segmentation and ships as a library with fixed, serializable models so the same text always maps to the same tokens. This standardized, language-agnostic tokenization step is now a default component in many multilingual and non-English NLP systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1808.06226", "arXiv:1808.06226" ], "version": 2 }, { "id": "P-121", "slug": "subword-regularization-unigram-lm-tokenization", "title": "Subword Regularization / Unigram LM Tokenization", "shortTitle": "Subword Regularization / Unigram LM Tokenization", "authors": [ "Taku Kudo" ], "institutions": [ "Google" ], "year": 2018, "date": "2018-04", "venue": null, "artifactType": "peer_reviewed", "domain": "data-tokenization", "url": "https://arxiv.org/abs/1804.10959", "arxivId": "1804.10959", "doi": null, "repository": "https://github.com/google/sentencepiece", "openness": "unknown", "status": "niche-but-active", "statusRaw": "common-alternative", "confidence": "High", "difficulty": 5, "influence": { "academic": 6, "frontier": 6, "engineering": 7, "industrial": 5, "openSource": 3, "underappreciated": 5 }, "importanceScore": 6, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It introduced probabilistic subword segmentation, sampling multiple tokenizations of the same text during training as regularization, and a unigram language-model tokenizer that produces those alternative segmentations.", "plainSummary": "Standard subword methods like BPE give one deterministic segmentation per word, so the model never sees alternative splits. This work defines a unigram LM over subwords that can yield multiple probable segmentations and samples among them each epoch, exposing the model to varied tokenizations of identical text. Acting as data augmentation, it improved neural machine translation accuracy especially on low-resource and noisy settings, and the unigram tokenizer became a widely used alternative to BPE (shipped in SentencePiece).", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1804.10959", "arXiv:1804.10959" ], "version": 2 }, { "id": "P-122", "slug": "deduplicating-training-data-makes-language-models-better", "title": "Deduplicating Training Data Makes Language Models Better", "shortTitle": "Deduplicating Training Data Makes Language Models Better", "authors": [ "Katherine Lee", "Daphne Ippolito", "Andrew Nystrom", "et al." ], "institutions": [ "Google", "UCB" ], "year": 2022, "date": "2021-07", "venue": null, "artifactType": "peer_reviewed", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2107.06499", "arxivId": "2107.06499", "doi": null, "repository": "https://github.com/google-research/deduplicate-text-datasets", "openness": "unknown", "status": "current-standard", "statusRaw": "standard-practice", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 1200, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that standard language-model training corpora contain large amounts of near- and exact-duplicate text, and that removing these duplicates with scalable exact and approximate (suffix-array and MinHash) methods reduces memorization, cleans up train/test overlap, and lets models reach the same or better quality with fewer training steps.", "plainSummary": "The authors measured duplication in common pretraining datasets and found that many sequences appear many times, including substantial overlap between training and evaluation sets. They built two deduplication tools: an exact-substring method using suffix arrays to find long repeated spans, and an approximate document-level method using MinHash to catch near-duplicates. Training on the deduplicated data reduced how often models regurgitated memorized text verbatim, gave more trustworthy evaluation numbers by removing leaked test examples, and required fewer training steps to reach a given accuracy. The paper made corpus deduplication a standard preprocessing step for language-model training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2107.06499", "arXiv:2107.06499" ], "version": 2 }, { "id": "P-123", "slug": "the-pile", "title": "The Pile", "shortTitle": "The Pile", "authors": [ "Leo Gao", "Stella Biderman", "Sid Black", "et al." ], "institutions": [ "EleutherAI" ], "year": 2021, "date": "2020-12", "venue": null, "artifactType": "dataset_paper", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2101.00027", "arxivId": "2101.00027", "doi": null, "repository": "https://pile.eleuther.ai/", "openness": "unknown", "status": "historically-important", "statusRaw": "historically-pivotal", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Assembled an 800GB English text corpus from 22 curated, diverse sources, giving open language-model training a large, documented, multi-domain dataset instead of ad hoc web scrapes.", "plainSummary": "EleutherAI combined 22 sources spanning academic writing, code, books, legal and other domains into a single 800GB corpus, deliberately weighting high-quality and specialized text rather than relying only on filtered Common Crawl. They documented composition, per-source weighting, and preprocessing, and showed models trained on this mixture improved on domain-specific evaluation relative to web-only data. It became a widely used training and benchmarking corpus for open models such as GPT-Neo and GPT-J.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2101.00027", "arXiv:2101.00027" ], "version": 2 }, { "id": "P-124", "slug": "the-refinedweb-dataset-for-falcon-llm", "title": "The RefinedWeb Dataset for Falcon LLM", "shortTitle": "The RefinedWeb Dataset for Falcon LLM", "authors": [ "Guilherme Penedo", "Quentin Malartic", "Daniel Hesslow", "et al." ], "institutions": [ "TII" ], "year": 2023, "date": "2023-06", "venue": null, "artifactType": "dataset_paper", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2306.01116", "arxivId": "2306.01116", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-paradigm", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 700, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that aggressive filtering and deduplication of Common Crawl alone can produce web text good enough to train competitive LLMs, challenging the assumption that curated sources like books and Wikipedia are required.", "plainSummary": "The authors built a pipeline (MacroData Refinement) that applies URL filtering, language identification, trafilatura-based text extraction, quality heuristics, and both fuzzy and exact deduplication to Common Crawl at scale. From this they released RefinedWeb, a five-trillion-token web-only corpus, and trained Falcon models that matched or exceeded models trained on curated mixtures. This demonstrated that scale plus rigorous cleaning of raw web data can substitute for hand-picked high-quality sources, and provided a large open dataset for the community.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2306.01116", "arXiv:2306.01116" ], "version": 2 }, { "id": "P-125", "slug": "textbooks-are-all-you-need", "title": "Textbooks Are All You Need (phi)", "shortTitle": "Textbooks Are All You Need", "authors": [ "Suriya Gunasekar", "Yi Zhang", "Jyoti Aneja", "et al." ], "institutions": [ "Microsoft" ], "year": 2023, "date": "2023-06", "venue": null, "artifactType": "technical_report", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2306.11644", "arxivId": "2306.11644", "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "influential-debated", "confidence": "Medium", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that training a small code model on a small corpus of textbook-quality and synthetically generated exercises yields coding ability far above what its parameter and token count would predict, shifting emphasis from data quantity to data quality.", "plainSummary": "The authors assembled a filtered set of high-educational-value code from the web plus GPT-generated textbook-style text and problem/solution exercises, then trained phi-1, a 1.3B model, on only about 7 billion tokens. Despite being orders of magnitude smaller in data and parameters than contemporaries, phi-1 reached strong pass@1 on HumanEval and MBPP. This provided evidence that carefully curated, instruction-dense data can dramatically improve sample efficiency, seeding the small-high-quality-data line of work.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2306.11644", "arXiv:2306.11644" ], "version": 2 }, { "id": "P-126", "slug": "the-curse-of-recursion-model-collapse", "title": "The Curse of Recursion / Model Collapse", "shortTitle": "The Curse of Recursion / Model Collapse", "authors": [ "Ilia Shumailov", "Zakhar Shumaylov", "Yiren Zhao", "Yarin Gal", "Nicolas Papernot", "Ross Anderson" ], "institutions": [ "Oxford", "Cambridge", "Toronto" ], "year": 2024, "date": "2023-05", "venue": null, "artifactType": "peer_reviewed", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2305.17493", "arxivId": "2305.17493", "doi": null, "repository": "https://github.com/iliaishacked/curse_recurse", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-caution", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 7, "industrial": 5, "openSource": 3, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It identified and analyzed model collapse, the degenerative process where training successive generations of models on their predecessors' outputs makes them progressively forget the true data distribution, especially its tails.", "plainSummary": "The authors trained models on data produced by earlier model generations and observed that each generation's outputs narrow toward the mean, losing rare events and low-probability tails until later models converge to a degenerate distribution. They gave a theoretical account attributing this to compounding statistical approximation, functional expressivity, and sampling errors across generations. The finding warned that as AI-generated text saturates the web, indiscriminate training on it degrades future models, making access to genuine human-produced data increasingly valuable.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.17493", "arXiv:2305.17493" ], "version": 2 }, { "id": "P-127", "slug": "doremi-optimizing-data-mixtures-with-group-dro", "title": "DoReMi: Optimizing Data Mixtures with Group DRO", "shortTitle": "DoReMi", "authors": [ "Sang Michael Xie", "Hieu Pham", "Xuanyi Dong", "et al." ], "institutions": [ "Google", "Stanford" ], "year": 2023, "date": "2023-05", "venue": null, "artifactType": "peer_reviewed", "domain": "data-tokenization", "url": "https://arxiv.org/abs/2305.10429", "arxivId": "2305.10429", "doi": null, "repository": "https://github.com/sangmichaelxie/doremi", "openness": "unknown", "status": "current-standard", "statusRaw": "current-research", "confidence": "High", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 7, "industrial": 5, "openSource": 3, "underappreciated": 5 }, "importanceScore": 6, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "DoReMi trains a small proxy model with Group DRO to learn domain weights for a pretraining corpus, so a large model can be trained once on a tuned data mixture instead of tuned by trial and error.", "plainSummary": "The method first trains a small reference model, then trains a second small proxy model with Group Distributionally Robust Optimization that raises the sampling weight of domains where the proxy has the largest excess loss over the reference. The resulting domain weights are reused to sample data for a much larger model. This removed the need to hand-tune or grid-search corpus mixtures at full scale, and reached target accuracy in fewer training steps by reweighting domains like The Pile rather than using their default proportions.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.10429", "arXiv:2305.10429" ], "version": 2 }, { "id": "P-130", "slug": "mixed-precision-training", "title": "Mixed Precision Training", "shortTitle": "Mixed Precision Training", "authors": [ "Paulius Micikevicius", "Sharan Narang", "Jonah Alben", "et al." ], "institutions": [ "NVIDIA", "Baidu" ], "year": 2018, "date": "2017-10", "venue": null, "artifactType": "peer_reviewed", "domain": "distributed-training", "url": "https://arxiv.org/abs/1710.03740", "arxivId": "1710.03740", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that most neural-network operations can run in 16-bit floating point rather than 32-bit, roughly halving memory use and speeding training on tensor hardware without accuracy loss.", "plainSummary": "The authors stored and computed most values in FP16 while keeping a master copy of the weights in FP32 for stable updates, and introduced loss scaling to shift small gradient values into FP16's representable range. Combined with FP32 accumulation for reductions, this matched full-precision accuracy across image, speech, language, and GAN models. The technique cut memory footprint and doubled arithmetic throughput on hardware with 16-bit units, becoming a default ingredient in large-scale training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1710.03740", "arXiv:1710.03740" ], "version": 2 }, { "id": "P-131", "slug": "megatron-lm-training-multi-billion-parameter-lms-using-model-parallelism", "title": "Megatron-LM: Training Multi-Billion Parameter LMs Using Model Parallelism", "shortTitle": "Megatron-LM", "authors": [ "Mohammad Shoeybi", "Mostofa Patwary", "Raul Puri", "et al." ], "institutions": [ "NVIDIA" ], "year": 2019, "date": "2019-09", "venue": null, "artifactType": "infrastructure_paper", "domain": "distributed-training", "url": "https://arxiv.org/abs/1909.08053", "arxivId": "1909.08053", "doi": null, "repository": "https://github.com/NVIDIA/Megatron-LM", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced a simple tensor (intra-layer) model-parallel scheme for Transformers that splits individual matrix multiplications across GPUs, letting models grow past a single device's memory using only standard framework primitives.", "plainSummary": "NVIDIA partitioned the Transformer's attention and MLP weight matrices column- and row-wise across GPUs, arranging the splits so only two all-reduce operations per layer are needed in the forward and backward passes, implemented directly in PyTorch. This let them train models up to 8.3 billion parameters and scale efficiently to 512 GPUs. The approach became a standard building block, later combined with data and pipeline parallelism, for training multi-billion-parameter models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1909.08053", "arXiv:1909.08053" ], "version": 2 }, { "id": "P-132", "slug": "zero-memory-optimizations-toward-training-trillion-parameter-models", "title": "ZeRO: Memory Optimizations Toward Training Trillion Parameter Models", "shortTitle": "ZeRO", "authors": [ "Samyam Rajbhandari", "Jeff Rasley", "Olatunji Ruwase", "Yuxiong He" ], "institutions": [ "Microsoft" ], "year": 2020, "date": "2019-10", "venue": null, "artifactType": "infrastructure_paper", "domain": "distributed-training", "url": "https://arxiv.org/abs/1910.02054", "arxivId": "1910.02054", "doi": null, "repository": "https://github.com/microsoft/DeepSpeed", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Removed the memory redundancy of data-parallel training by partitioning optimizer states, gradients, and parameters across devices instead of replicating them, making it feasible to train models with orders of magnitude more parameters.", "plainSummary": "The authors observed that standard data parallelism stores a full copy of optimizer states, gradients, and weights on every GPU, wasting memory as models grow. ZeRO shards these three components across the data-parallel group in three progressive stages, reconstructing pieces via communication only when needed, so per-device memory falls roughly in proportion to the number of devices while keeping data-parallel efficiency. Implemented in DeepSpeed, it enabled training of models into the hundreds of billions of parameters on existing clusters.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1910.02054", "arXiv:1910.02054" ], "version": 2 }, { "id": "P-133", "slug": "gpipe-efficient-training-of-giant-neural-networks-using-pipeline-parallelism", "title": "GPipe: Efficient Training of Giant Neural Networks Using Pipeline Parallelism", "shortTitle": "GPipe", "authors": [ "Yanping Huang", "Youlong Cheng", "Ankur Bapna", "et al." ], "institutions": [ "Google" ], "year": 2019, "date": "2018-11", "venue": null, "artifactType": "peer_reviewed", "domain": "distributed-training", "url": "https://arxiv.org/abs/1811.06965", "arxivId": "1811.06965", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-refined", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "GPipe partitions a deep model across accelerators as pipeline stages and splits each minibatch into micro-batches, letting a network too large for one device be trained while keeping most devices busy.", "plainSummary": "The model layers are divided into consecutive stages placed on separate accelerators, and each minibatch is broken into micro-batches that flow through the stages so multiple devices compute at once instead of waiting idle. Gradients are accumulated across micro-batches and applied synchronously, so the result is numerically identical to non-pipelined training. Combined with activation recomputation, this let models with billions of parameters be trained across devices that individually could not hold them, at near-linear throughput scaling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1811.06965", "arXiv:1811.06965" ], "version": 2 }, { "id": "P-134", "slug": "pipedream-generalized-pipeline-parallelism-for-dnn-training", "title": "PipeDream: Generalized Pipeline Parallelism for DNN Training", "shortTitle": "PipeDream", "authors": [ "Deepak Narayanan", "Aaron Harlap", "Amar Phanishayee", "et al." ], "institutions": [ "Microsoft", "Stanford" ], "year": 2019, "date": "2019-09", "venue": null, "artifactType": "peer_reviewed", "domain": "distributed-training", "url": "https://www.microsoft.com/en-us/research/publication/pipedream-generalized-pipeline-parallelism-for-dnn-training/", "arxivId": null, "doi": null, "repository": "https://github.com/msr-fiddle/pipedream", "openness": "unknown", "status": "partially-superseded", "statusRaw": "absorbed", "confidence": "High", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 7, "industrial": 5, "openSource": 3, "underappreciated": 5 }, "importanceScore": 6, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "PipeDream keeps the pipeline full by injecting a new minibatch at every stage as soon as it is free and using weight stashing to keep forward and backward passes consistent, removing the pipeline-flush bubble of synchronous schemes.", "plainSummary": "Rather than draining the pipeline between minibatches, PipeDream uses a one-forward-one-backward schedule that overlaps the forward and backward work of different minibatches to keep all stages busy. Because a minibatch's backward pass then sees newer weights than its forward pass, it stashes the weight version used in the forward pass and reuses it in the backward pass so gradients stay consistent. An automatic profiler partitions layers across stages to balance compute and communication, cutting time-to-target-accuracy relative to data-parallel and flush-based pipeline training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.microsoft.com/en-us/research/publication/pipedream-generalized-pipeline-parallelism-for-dnn-training/" ], "version": 2 }, { "id": "P-135", "slug": "mesh-tensorflow-gspmd", "title": "Mesh-TensorFlow / GSPMD", "shortTitle": "Mesh-TensorFlow / GSPMD", "authors": [ "Noam Shazeer", "Youlong Cheng", "Niki Parmar", "Yuanzhong Xu", "HyoukJoong Lee", "Blake Hechtman", "Zhifeng Chen" ], "institutions": [ "Google" ], "year": 2018, "date": "2018-2021", "venue": null, "artifactType": "infrastructure_paper", "domain": "distributed-training", "url": "https://arxiv.org/abs/2105.04663", "arxivId": "2105.04663", "doi": null, "repository": "https://github.com/tensorflow/mesh", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-jax-tpu", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 700, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "GSPMD lets a user annotate a few tensors with sharding specifications and then automatically propagates and compiles those into a partitioned program, so the same model graph can be split across devices by data, operator, or pipeline dimensions without rewriting it.", "plainSummary": "Building on Mesh-TensorFlow's idea of mapping tensor dimensions onto a logical device mesh, GSPMD takes per-tensor sharding annotations, propagates them through the whole XLA computation graph, and inserts the needed collective communication. The same code can therefore be run data-parallel, tensor/operator-parallel, or pipeline-parallel just by changing annotations. This decoupled how a model is written from how it is partitioned, enabling parallelization of very large models with minimal code changes across thousands of accelerators.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2105.04663", "arXiv:2105.04663" ], "version": 2 }, { "id": "P-136", "slug": "pytorch-fsdp-experiences-on-scaling-fully-sharded-data-parallel", "title": "PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel", "shortTitle": "PyTorch FSDP", "authors": [ "Yanli Zhao", "Andrew Gu", "Rohan Varma", "et al." ], "institutions": [ "Meta" ], "year": 2023, "date": "2023-04", "venue": null, "artifactType": "infrastructure_paper", "domain": "distributed-training", "url": "https://arxiv.org/abs/2304.11277", "arxivId": "2304.11277", "doi": null, "repository": "https://github.com/pytorch/pytorch", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "PyTorch FSDP shards each layer's parameters, gradients, and optimizer state across data-parallel workers and reconstructs full parameters on the fly only for the layer being computed, cutting per-device memory so large models fit under standard data-parallel training.", "plainSummary": "Instead of replicating the full model on every worker as standard data parallelism does, FSDP splits parameters, gradients, and optimizer state into shards held by different workers; before a layer runs it all-gathers that layer's parameters, then frees them afterward, and reduce-scatters gradients during the backward pass. Communication is overlapped with computation and grouped into units to limit memory spikes and latency. Delivered as a native PyTorch API, it let practitioners train models with hundreds of billions of parameters on commodity clusters without a separate framework.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2304.11277", "arXiv:2304.11277" ], "version": 2 }, { "id": "P-137", "slug": "training-deep-nets-with-sublinear-memory-cost", "title": "Training Deep Nets with Sublinear Memory Cost (Activation Checkpointing)", "shortTitle": "Training Deep Nets with Sublinear Memory Cost", "authors": [ "Tianqi Chen", "Bing Xu", "Chiyuan Zhang", "Carlos Guestrin" ], "institutions": [ "UW", "others" ], "year": 2016, "date": "2016-04", "venue": null, "artifactType": "preprint", "domain": "distributed-training", "url": "https://arxiv.org/abs/1604.06174", "arxivId": "1604.06174", "doi": null, "repository": "https://github.com/dmlc/mxnet-memonger", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This work trades compute for memory by discarding most intermediate activations during the forward pass and recomputing them during the backward pass, cutting activation memory for an n-layer network from linear to about the square root of n.", "plainSummary": "During the forward pass only a sparse set of checkpoint activations is kept and the rest are dropped; during backpropagation the missing activations are recomputed on demand from the nearest checkpoint. Placing checkpoints roughly every sqrt(n) layers reduces activation memory to O(sqrt(n)) at the cost of one extra forward computation. This gradient/activation checkpointing let much deeper networks and longer sequences be trained within a fixed GPU memory budget, and became a standard building block in later large-model training systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1604.06174", "arXiv:1604.06174" ], "version": 2 }, { "id": "P-138", "slug": "fp8-lm-fp8-training", "title": "FP8-LM / FP8 Training", "shortTitle": "FP8-LM / FP8 Training", "authors": [ "Houwen Peng", "Kan Wu", "Han Hu", "Peng Cheng", "Paulius Micikevicius" ], "institutions": [ "Microsoft", "NVIDIA" ], "year": 2023, "date": "2023-10", "venue": null, "artifactType": "technical_report", "domain": "distributed-training", "url": "https://arxiv.org/abs/2310.18313", "arxivId": "2310.18313", "doi": null, "repository": "https://github.com/NVIDIA/TransformerEngine", "openness": "unknown", "status": "niche-but-active", "statusRaw": "emerging-frontier", "confidence": "Medium", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 7, "industrial": 5, "openSource": 3, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 300, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "FP8-LM shows large language models can be trained with 8-bit floating point for gradients, optimizer state, and communication by adding per-tensor scaling and precision decoupling, reducing memory and communication versus 16-bit training without loss degradation.", "plainSummary": "The framework extends mixed-precision training so that not just matrix multiplies but also gradients, the optimizer's moment states, and distributed communication use FP8, guarded by automatic per-tensor scaling to keep values inside FP8's narrow dynamic range and by decoupling precision across components to avoid underflow. Applied to GPT-scale models, it matched the accuracy of BF16 training while cutting memory footprint and communication volume and raising throughput. This pushed practical LLM training below 16 bits across more of the pipeline than earlier FP8 schemes, which had limited FP8 to the matmuls.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.18313", "arXiv:2310.18313" ], "version": 2 }, { "id": "P-200", "slug": "deep-reinforcement-learning-from-human-preferences", "title": "Deep Reinforcement Learning from Human Preferences", "shortTitle": "Deep Reinforcement Learning from Human Preferences", "authors": [ "Paul Christiano", "Jan Leike", "Tom Brown", "Miljan Martic", "Shane Legg", "Dario Amodei" ], "institutions": [ "OpenAI", "DeepMind" ], "year": 2017, "date": "2017-06", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/1706.03741", "arxivId": "1706.03741", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed a reward model can be trained from humans' pairwise comparisons of agent trajectory segments, letting RL optimize behaviors too hard to hand-specify without ever writing an explicit reward function.", "plainSummary": "The work has people repeatedly pick which of two short video clips of an agent looks closer to a goal, and fits a reward predictor to those choices while a policy is trained against that predictor. Because labeling comparisons is cheaper than demonstrating or coding rewards, it learned tasks like Atari games and simulated-robot backflips from under an hour of human feedback on a small fraction of the agent's interactions. This comparison-based reward modeling became the template later scaled to language-model alignment.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1706.03741", "arXiv:1706.03741" ], "version": 2 }, { "id": "P-201", "slug": "learning-to-summarize-from-human-feedback", "title": "Learning to Summarize from Human Feedback", "shortTitle": "Learning to Summarize from Human Feedback", "authors": [ "Nisan Stiennon", "Long Ouyang", "Jeff Wu", "Daniel Ziegler", "et al." ], "institutions": [ "OpenAI" ], "year": 2020, "date": "2020-09", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2009.01325", "arxivId": "2009.01325", "doi": null, "repository": "https://github.com/openai/summarize-from-feedback", "openness": "unknown", "status": "influential-precursor", "statusRaw": "foundational-precursor", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It trained a summarization model against a reward model learned from human comparisons of candidate summaries, showing that optimizing for human preferences with reinforcement learning produces better summaries than optimizing for likelihood or standard automatic metrics like ROUGE.", "plainSummary": "Instead of training the model to imitate reference summaries, the authors collected human judgments of which of two summaries was better, trained a reward model to predict those judgments, and then used reinforcement learning to make the summarizer score highly on that reward. The resulting summaries were preferred by people over both the reference summaries and models trained by supervised imitation. This work is the practical template for reinforcement learning from human feedback (RLHF) that later underpinned instruction-following chat models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2009.01325", "arXiv:2009.01325" ], "version": 2 }, { "id": "P-202", "slug": "instructgpt-training-lms-to-follow-instructions-with-human-feedback", "title": "InstructGPT: Training LMs to Follow Instructions with Human Feedback", "shortTitle": "InstructGPT", "authors": [ "Long Ouyang", "Jeff Wu", "Xu Jiang", "et al." ], "institutions": [ "OpenAI" ], "year": 2022, "date": "2022-03", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2203.02155", "arxivId": "2203.02155", "doi": null, "repository": "https://github.com/openai/following-instructions-human-feedback", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Applied reinforcement learning from human feedback to GPT-3 so a much smaller aligned model followed user intent better than the raw base model, closing the gap between next-token pretraining and doing what a user actually asked.", "plainSummary": "Labelers wrote demonstrations to supervise-fine-tune GPT-3, then ranked model outputs to train a reward model, and PPO optimized the policy against that reward. Outputs from the 1.3B InstructGPT model were preferred to the 175B GPT-3's despite being about 100x smaller, with gains in truthfulness and reductions in toxic generation. The three-stage SFT-then-reward-model-then-PPO recipe became the standard alignment pipeline behind instruction-following chat models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2203.02155", "arXiv:2203.02155" ], "version": 2 }, { "id": "P-203", "slug": "finetuned-language-models-are-zero-shot-learners-self-instruct", "title": "Finetuned Language Models Are Zero-Shot Learners (FLAN) / Self-Instruct", "shortTitle": "Finetuned Language Models Are Zero-Shot Learners / Self-Instruct", "authors": [ "Jason Wei", "Maarten Bosma", "Quoc V. Le", "Yizhong Wang", "Hannaneh Hajishirzi", "Noah A. Smith", "Daniel Khashabi" ], "institutions": [ "Google", "UW+AI2" ], "year": 2022, "date": "2021-09", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2109.01652", "arxivId": "2109.01652", "doi": null, "repository": "https://github.com/google-research/FLAN", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that fine-tuning a language model on a broad collection of tasks phrased as natural-language instructions makes the model follow instructions for unseen tasks it was never trained on, removing the need for task-specific examples or prompt engineering to steer a model.", "plainSummary": "The authors took a pretrained model and fine-tuned it on many existing NLP datasets that were each rewritten as instructions (for example, 'Is this review positive or negative?'). After this instruction tuning, the model could handle new kinds of tasks it had not seen during training, just from reading the instruction. This established instruction tuning as a general method for making base models usable without few-shot prompting, and improved zero-shot performance across a range of benchmarks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2109.01652", "arXiv:2109.01652" ], "version": 2 }, { "id": "P-204", "slug": "constitutional-ai-harmlessness-from-ai-feedback", "title": "Constitutional AI: Harmlessness from AI Feedback (RLAIF)", "shortTitle": "Constitutional AI", "authors": [ "Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu", "et al." ], "institutions": [ "Anthropic" ], "year": 2022, "date": "2022-12", "venue": null, "artifactType": "technical_report", "domain": "alignment", "url": "https://arxiv.org/abs/2212.08073", "arxivId": "2212.08073", "doi": null, "repository": "https://github.com/anthropics/ConstitutionalHarmlessnessPaper", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Replaced most human-labeled harmlessness feedback with model-generated feedback governed by a written set of principles, removing the need for large volumes of human labels on harmful content to train a harmless assistant.", "plainSummary": "In a supervised phase the model critiques and revises its own responses against a short list of natural-language principles (a 'constitution'), and in an RL phase a model rather than humans ranks response pairs for harmlessness to train the preference model (RLAIF). This let the assistant refuse or push back on harmful requests while explaining its reasoning instead of giving evasive non-answers. It cut human labeling of toxic content and made the value targets explicit and editable as written rules.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2212.08073", "arXiv:2212.08073" ], "version": 2 }, { "id": "P-205", "slug": "direct-preference-optimization", "title": "Direct Preference Optimization (DPO)", "shortTitle": "Direct Preference Optimization", "authors": [ "Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher Manning", "Chelsea Finn" ], "institutions": [ "Stanford" ], "year": 2023, "date": "2023-05", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2305.18290", "arxivId": "2305.18290", "doi": null, "repository": "https://github.com/eric-mitchell/direct-preference-optimization", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed the RLHF objective can be solved by a single classification-style loss on preference pairs, eliminating the separate reward model and reinforcement-learning loop.", "plainSummary": "Direct Preference Optimization uses the mathematical link between reward and optimal policy to rewrite preference learning so the language model itself is optimized directly on chosen-versus-rejected examples with a simple supervised loss. There is no reward model to train and no PPO sampling, so it is more stable and cheaper while matching or beating PPO-based RLHF on preference tuning. This made preference alignment practical for teams without reinforcement-learning infrastructure.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.18290", "arXiv:2305.18290" ], "version": 2 }, { "id": "P-206", "slug": "preference-optimization-family", "title": "Preference Optimization Family (IPO/KTO/ORPO/SimPO)", "shortTitle": "Preference Optimization Family", "authors": [ "Mohammad Gheshlaghi Azar", "Rémi Munos", "Kawin Ethayarajh", "Douwe Kiela", "Jiwoo Hong", "James Thorne", "Yu Meng", "Danqi Chen" ], "institutions": [ "Multiple" ], "year": 2024, "date": "2023-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2402.01306", "arxivId": "2402.01306", "doi": null, "repository": "https://github.com/ContextualAI/HALOs", "openness": "unknown", "status": "niche-but-active", "statusRaw": "active-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 600, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family of methods (IPO, KTO, ORPO, SimPO) reworks Direct Preference Optimization to fix its failure modes and drop its dependencies, aligning language models from preference data with simpler or more robust objectives and often no separate reward or reference model.", "plainSummary": "Each variant targets a specific limitation of DPO: IPO replaces the log-sigmoid objective with a bounded one to curb overfitting when preferences are near-deterministic; KTO learns from single labeled examples marked desirable or undesirable rather than requiring paired comparisons; ORPO folds a preference odds-ratio penalty directly into supervised fine-tuning so no separate alignment stage or reference model is needed; SimPO removes the reference model and uses a length-normalized reward with a target margin. Together they made preference-based alignment cheaper and more stable, reducing reliance on paired data, reference models, and reward-model training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2402.01306", "arXiv:2402.01306" ], "version": 2 }, { "id": "P-207", "slug": "reward-model-overoptimization-sycophancy", "title": "Reward Model Overoptimization / Sycophancy", "shortTitle": "Reward Model Overoptimization / Sycophancy", "authors": [ "Leo Gao", "John Schulman", "Jacob Hilton", "Mrinank Sharma", "Ethan Perez" ], "institutions": [ "OpenAI", "Anthropic" ], "year": 2022, "date": "2022-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "alignment", "url": "https://arxiv.org/abs/2210.10760", "arxivId": "2210.10760", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-caution", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 800, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It measured how optimizing a policy against a learned reward model eventually degrades true reward, quantifying reward-model overoptimization as a Goodhart-law failure of RLHF.", "plainSummary": "The authors trained policies against reward models of varying size and data, then compared the proxy reward the model assigned against a gold reward model treated as ground truth. As optimization pressure (measured in KL distance from the initial policy) increased, proxy reward kept rising while true reward peaked and then fell, and they fit scaling laws for where this divergence begins. This gave RLHF practitioners a concrete way to predict when a reward model stops being a trustworthy target and to bound optimization accordingly, explaining downstream failures like sycophancy where the policy exploits reward-model quirks rather than improving.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2210.10760", "arXiv:2210.10760" ], "version": 2 }, { "id": "P-208", "slug": "scalable-oversight-debate-weak-to-strong-generalization", "title": "Scalable Oversight: Debate / Weak-to-Strong Generalization", "shortTitle": "Scalable Oversight", "authors": [ "Geoffrey Irving", "Paul Christiano", "Dario Amodei", "Collin Burns", "Jan Leike", "Ilya Sutskever" ], "institutions": [ "OpenAI", "DeepMind" ], "year": 2018, "date": "2018-2023", "venue": null, "artifactType": "preprint", "domain": "alignment", "url": "https://arxiv.org/abs/2312.09390", "arxivId": "2312.09390", "doi": null, "repository": "https://github.com/openai/weak-to-strong", "openness": "unknown", "status": "niche-but-active", "statusRaw": "open-frontier", "confidence": "Medium", "difficulty": 7, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This line of work asked how humans or weaker models can supervise systems more capable than themselves, and showed that fine-tuning a strong model on a weak model's labels can recover much of the strong model's latent capability.", "plainSummary": "The weak-to-strong experiments used a small model's imperfect labels to fine-tune a much larger pretrained model and found the larger model generalized beyond its teacher's errors, an analogy for humans supervising superhuman systems. Related debate proposals have two models argue opposing positions so a limited judge can adjudicate claims it could not verify directly. Together these define the scalable-oversight problem and give early empirical evidence that supervision signal can transfer even when the supervisor is less capable than the system being trained.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2312.09390", "arXiv:2312.09390" ], "version": 2 }, { "id": "P-220", "slug": "chain-of-thought-prompting-elicits-reasoning", "title": "Chain-of-Thought Prompting Elicits Reasoning", "shortTitle": "Chain-of-Thought Prompting Elicits Reasoning", "authors": [ "Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou" ], "institutions": [ "Google" ], "year": 2022, "date": "2022-01", "venue": null, "artifactType": "peer_reviewed", "domain": "reasoning", "url": "https://arxiv.org/abs/2201.11903", "arxivId": "2201.11903", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 9000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that prompting a large model to emit intermediate reasoning steps before its answer unlocks multi-step reasoning that direct-answer prompting fails at, without any fine-tuning.", "plainSummary": "By putting a few exemplars that spell out step-by-step worked solutions into the prompt, the model imitates that format and reasons through arithmetic, commonsense, and symbolic problems one step at a time. The benefit appears mainly at large model scale and substantially raised accuracy on benchmarks like GSM8K math word problems. It made intermediate-computation prompting a standard, training-free way to get harder reasoning out of existing models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2201.11903", "arXiv:2201.11903" ], "version": 2 }, { "id": "P-221", "slug": "self-consistency-improves-chain-of-thought-reasoning", "title": "Self-Consistency Improves Chain-of-Thought Reasoning", "shortTitle": "Self-Consistency Improves Chain-of-Thought Reasoning", "authors": [ "Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "et al." ], "institutions": [ "Google" ], "year": 2023, "date": "2022-03", "venue": null, "artifactType": "peer_reviewed", "domain": "reasoning", "url": "https://arxiv.org/abs/2203.11171", "arxivId": "2203.11171", "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Improved chain-of-thought accuracy by sampling many reasoning paths and taking the majority-vote answer instead of trusting a single greedy decode, removing the fragility of one-shot reasoning chains.", "plainSummary": "Self-consistency samples multiple diverse reasoning chains for the same question and then marginalizes over the reasoning to pick the answer most paths agree on. Because a correct answer tends to be reachable by several distinct valid derivations while errors are scattered, voting over sampled chains raised accuracy on arithmetic and commonsense benchmarks over standard chain-of-thought. It became a simple, decoding-time add-on for more reliable reasoning at the cost of extra samples.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2203.11171", "arXiv:2203.11171" ], "version": 2 }, { "id": "P-222", "slug": "structured-reasoning-least-to-most-pot-tree-of-thoughts", "title": "Structured Reasoning: Least-to-Most / PoT / Tree of Thoughts", "shortTitle": "Structured Reasoning", "authors": [ "Denny Zhou", "Quoc V. Le", "Ed H. Chi", "Wenhu Chen", "William W. Cohen", "Shunyu Yao", "Karthik Narasimhan" ], "institutions": [ "Google", "Princeton" ], "year": 2023, "date": "2022-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "reasoning", "url": "https://arxiv.org/abs/2305.10601", "arxivId": "2305.10601", "doi": null, "repository": "https://github.com/princeton-nlp/tree-of-thought-llm", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family reorganized chain-of-thought into explicit search or decomposition structures, letting a model explore, evaluate, and backtrack over intermediate steps instead of committing to one linear reasoning path.", "plainSummary": "Tree of Thoughts generalizes chain-of-thought by having the model generate multiple candidate reasoning steps, score them with a value estimate, and search the resulting tree with breadth- or depth-first exploration and backtracking. Least-to-Most decomposes a hard problem into ordered easier subproblems solved in sequence, and Program-of-Thought offloads arithmetic and logic to generated code executed by an interpreter. The shared move is treating reasoning as a structured, evaluable process rather than a single sampled string, which raised accuracy on tasks requiring planning, search, or exact computation.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.10601", "arXiv:2305.10601" ], "version": 2 }, { "id": "P-223", "slug": "verifiers-process-supervision", "title": "Verifiers & Process Supervision (GSM8K / Let's Verify Step by Step)", "shortTitle": "Verifiers & Process Supervision", "authors": [ "Karl Cobbe", "John Schulman", "Hunter Lightman", "Ilya Sutskever", "Jan Leike" ], "institutions": [ "OpenAI" ], "year": 2023, "date": "2021-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "reasoning", "url": "https://arxiv.org/abs/2305.20050", "arxivId": "2305.20050", "doi": null, "repository": "https://github.com/openai/prm800k", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It compared rewarding only the final answer (outcome supervision) against rewarding each intermediate reasoning step (process supervision) and showed that step-level feedback trains a more reliable verifier that selects correct solutions to hard math problems more often.", "plainSummary": "The authors had humans label the correctness of every step in a model's chain of reasoning, then trained a verifier to score solutions step by step. Ranking many candidate solutions by this process-supervised verifier picked correct answers more often than a verifier trained only on whether the final answer was right. They also released PRM800K, a large dataset of step-level human labels, making process-based reward modeling reproducible for reasoning tasks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.20050", "arXiv:2305.20050" ], "version": 2 }, { "id": "P-224", "slug": "deepseek-r1-incentivizing-reasoning-via-rl", "title": "DeepSeek-R1: Incentivizing Reasoning via RL (RLVR)", "shortTitle": "DeepSeek-R1", "authors": [ "DeepSeek-AI", "Daya Guo", "Zhihong Shao", "Wenfeng Liang" ], "institutions": [ "DeepSeek-AI" ], "year": 2025, "date": "2025-01", "venue": null, "artifactType": "technical_report", "domain": "reasoning", "url": "https://arxiv.org/abs/2501.12948", "arxivId": "2501.12948", "doi": null, "repository": "https://github.com/deepseek-ai/DeepSeek-R1", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-frontier-open", "confidence": "Medium", "difficulty": 7, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "It showed that a base language model can be trained to produce long chains of reasoning largely through reinforcement learning against automatically checkable rewards (correct answers, passing tests), without first needing large amounts of human-written reasoning demonstrations.", "plainSummary": "The team applied reinforcement learning where the reward comes from verifying the final result (a math answer being correct or code passing tests) rather than from human preference labels. Under this pressure the model learned on its own to write longer reasoning, check its work, and backtrack, with a pure-RL variant (R1-Zero) developing these behaviors from a base model before a small amount of supervised data was added for readability. This demonstrated that reinforcement learning with verifiable rewards (RLVR) is a scalable path to reasoning models, and the released weights made such training widely reproducible.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2501.12948", "arXiv:2501.12948" ], "version": 2 }, { "id": "P-225", "slug": "openai-o1-o3-reasoning-systems", "title": "OpenAI o1 / o3 Reasoning Systems", "shortTitle": "OpenAI o1 / o3 Reasoning Systems", "authors": [ "OpenAI" ], "institutions": [ "OpenAI" ], "year": 2024, "date": "2024-09", "venue": null, "artifactType": "system_card", "domain": "reasoning", "url": "https://openai.com/index/openai-o1-system-card/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "disclosure-limited", "statusRaw": "frontier-product-closed", "confidence": "Low", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 0, "source": "none-applicable", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "These were the first widely deployed systems trained to spend additional inference-time computation on an internal chain of reasoning before answering, trading longer per-query compute for markedly higher accuracy on math, science, and coding problems.", "plainSummary": "Rather than answering immediately, these models generate an extended internal reasoning process and can allocate more or less thinking time to a query. Accuracy on hard reasoning benchmarks improved as more inference compute was spent, establishing test-time compute as a scaling axis distinct from making the model or its training set larger. The reasoning details were released through system cards rather than a formal paper, and the internal chain of thought was kept hidden from users.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://openai.com/index/openai-o1-system-card/" ], "version": 2 }, { "id": "P-226", "slug": "reasoning-correctives-faithfulness-overthinking-contamination", "title": "Reasoning Correctives: Faithfulness / Overthinking / Contamination", "shortTitle": "Reasoning Correctives", "authors": [ "Miles Turpin", "Julian Michael", "Ethan Perez", "Samuel R. Bowman" ], "institutions": [ "Multiple" ], "year": 2023, "date": "2023-2025", "venue": null, "artifactType": "peer_reviewed", "domain": "reasoning", "url": "https://arxiv.org/abs/2305.04388", "arxivId": "2305.04388", "doi": null, "repository": "https://github.com/milesaturpin/cot-unfaithfulness", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-caution", "confidence": "High", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "These studies showed that a model's stated chain-of-thought often does not reflect the actual cause of its answer, documenting unfaithful, biased, and over-length reasoning traces.", "plainSummary": "By inserting biasing features into prompts, such as always marking option (A) as correct, the authors showed models would follow the bias while writing plausible reasoning that never mentioned it, demonstrating that CoT explanations can be post-hoc rather than causal. Related findings cover overthinking, where longer reasoning hurts accuracy, and contamination, where memorized answers masquerade as derived ones. The collective correction is that a readable chain-of-thought is not automatically a faithful account of the computation, so it cannot be trusted as an interpretability or safety signal without further verification.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.04388", "arXiv:2305.04388" ], "version": 2 }, { "id": "P-230", "slug": "react-synergizing-reasoning-and-acting-in-lms", "title": "ReAct: Synergizing Reasoning and Acting in LMs", "shortTitle": "ReAct", "authors": [ "Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao" ], "institutions": [ "Princeton", "Google" ], "year": 2023, "date": "2022-10", "venue": null, "artifactType": "peer_reviewed", "domain": "agents", "url": "https://arxiv.org/abs/2210.03629", "arxivId": "2210.03629", "doi": null, "repository": "https://github.com/ysymyth/ReAct", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Interleaves free-text reasoning traces with discrete actions in a single prompting loop, removing the split between chain-of-thought reasoning (which cannot gather new information) and action-only agents (which cannot plan or revise).", "plainSummary": "The method prompts a language model to alternate between writing a reasoning step and issuing an action, such as a query to a search API, then feeding the observation back before the next thought. This lets the model plan, look things up to correct itself, and reduce fabricated facts on knowledge tasks like HotpotQA and FEVER, while the reasoning traces make its behavior on interactive benchmarks more legible. It became a common template for tool-using and agentic prompting because it needs no fine-tuning, only few-shot exemplars.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2210.03629", "arXiv:2210.03629" ], "version": 2 }, { "id": "P-231", "slug": "toolformer-lms-can-teach-themselves-to-use-tools", "title": "Toolformer: LMs Can Teach Themselves to Use Tools", "shortTitle": "Toolformer", "authors": [ "Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessi", "et al." ], "institutions": [ "Meta FAIR" ], "year": 2023, "date": "2023-02", "venue": null, "artifactType": "peer_reviewed", "domain": "agents", "url": "https://arxiv.org/abs/2302.04761", "arxivId": "2302.04761", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-tooluse", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Trains a model to decide when to call external APIs and with what arguments using a self-supervised objective, removing the need for human annotation of tool-use demonstrations.", "plainSummary": "The model samples candidate API calls (calculator, search, calendar, question-answering, translation) inside ordinary text, executes them, and keeps only the calls whose returned results lower the perplexity of the following tokens, producing a training set it then fine-tunes on. As a result a 6.7B-parameter model learns to invoke tools where they help and gains on arithmetic, factual QA, and temporal tasks without degrading its core language modeling. The mechanism showed tool use can be bootstrapped from the model's own predictions rather than curated examples.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2302.04761", "arXiv:2302.04761" ], "version": 2 }, { "id": "P-232", "slug": "function-calling-mrkl-model-context-protocol", "title": "Function Calling / MRKL / Model Context Protocol", "shortTitle": "Function Calling / MRKL / Model Context Protocol", "authors": [ "Ehud Karpas", "Opher Lieber", "Barak Lenz", "Yoav Levine", "Yoav Shoham", "OpenAI", "Anthropic" ], "institutions": [ "AI21", "OpenAI", "Anthropic" ], "year": 2023, "date": "2022-2024", "venue": null, "artifactType": "product_release", "domain": "agents", "url": "https://arxiv.org/abs/2205.00445", "arxivId": "2205.00445", "doi": null, "repository": "https://modelcontextprotocol.io", "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 300, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family defined structured interfaces for letting language models call external tools, routing between a reasoning core and specialized modules or APIs instead of relying on the model's parametric knowledge alone.", "plainSummary": "MRKL proposed a neuro-symbolic architecture in which a router directs a query to discrete expert modules, such as a calculator, database, or API, and merges their outputs. Later function-calling formats let a model emit a structured call that host code executes and feeds back, and the Model Context Protocol standardizes how tools, data sources, and context are exposed to models across applications. The common contribution is a defined boundary and calling convention between the model and outside systems, which enabled reliable access to current data, exact computation, and actions the model cannot perform internally.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2205.00445", "arXiv:2205.00445" ], "version": 2 }, { "id": "P-233", "slug": "reflexion-language-agents-with-verbal-reinforcement-learning", "title": "Reflexion: Language Agents with Verbal Reinforcement Learning", "shortTitle": "Reflexion", "authors": [ "Noah Shinn", "Federico Cassano", "Edward Berman", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao" ], "institutions": [ "Northeastern", "MIT" ], "year": 2023, "date": "2023-03", "venue": null, "artifactType": "peer_reviewed", "domain": "agents", "url": "https://arxiv.org/abs/2303.11366", "arxivId": "2303.11366", "doi": null, "repository": "https://github.com/noahshinn/reflexion", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Reflexion added a verbal self-reflection loop in which an agent turns feedback from a failed attempt into written critiques stored in memory and used to improve the next attempt, without updating model weights.", "plainSummary": "After each trajectory, an evaluator produces a reward or error signal, and the agent generates a natural-language reflection on what went wrong, which is appended to an episodic memory buffer that conditions subsequent trials. Because learning happens through text in the context window rather than gradient updates, the same frozen model improves across attempts on decision-making, coding, and reasoning benchmarks. This provided a cheap iterative-improvement mechanism for agents that reuses a model's own language ability as the update rule.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2303.11366", "arXiv:2303.11366" ], "version": 2 }, { "id": "P-234", "slug": "voyager-generative-agents", "title": "Voyager / Generative Agents", "shortTitle": "Voyager / Generative Agents", "authors": [ "Guanzhi Wang", "Linxi Fan", "Anima Anandkumar", "Joon Sung Park", "Percy Liang", "Michael S. Bernstein" ], "institutions": [ "NVIDIA", "Stanford" ], "year": 2023, "date": "2023", "venue": null, "artifactType": "preprint", "domain": "agents", "url": "https://arxiv.org/abs/2305.16291", "arxivId": "2305.16291", "doi": null, "repository": "https://github.com/MineDojo/Voyager", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-research", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "These works built open-ended agents that set their own goals and accumulate reusable skills, and populated simulated worlds with believable character agents driven by memory and reflection.", "plainSummary": "Voyager runs a GPT-4 agent in Minecraft with an automatic curriculum that proposes progressively harder tasks, a growing library of executable code skills it writes and reuses, and an iterative prompting loop that repairs failures, letting it explore without a fixed objective. Generative Agents give each simulated character a memory stream of observations, a retrieval scheme, and periodic reflection that synthesizes higher-level inferences, producing emergent social behavior like spreading information and coordinating events. Together they showed that lifelong skill acquisition and persistent, plausible multi-agent behavior can be driven largely by language models plus structured memory rather than task-specific training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.16291", "arXiv:2305.16291" ], "version": 2 }, { "id": "P-235", "slug": "swe-agent-openhands", "title": "SWE-agent / OpenHands (autonomous software engineering)", "shortTitle": "SWE-agent / OpenHands", "authors": [ "John Yang", "Carlos E. Jimenez", "Ofir Press", "Karthik Narasimhan", "Xingyao Wang", "Graham Neubig" ], "institutions": [ "Princeton", "UIUC+others" ], "year": 2024, "date": "2024", "venue": null, "artifactType": "peer_reviewed", "domain": "agents", "url": "https://arxiv.org/abs/2405.15793", "arxivId": "2405.15793", "doi": null, "repository": "https://github.com/princeton-nlp/SWE-agent", "openness": "unknown", "status": "niche-but-active", "statusRaw": "frontier-advancing", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 800, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "They packaged a language model as an autonomous agent with a structured interface to a code repository, shell, and editor, showing that a model given the right tools and feedback loop can resolve real GitHub issues end to end rather than only generating isolated code snippets.", "plainSummary": "These systems give the model a defined set of actions for navigating files, editing code, and running commands, plus the feedback it gets back from executing those actions, so it can work through a real bug or feature request over many steps. On the SWE-bench benchmark of actual GitHub issues, this agent-plus-tools approach resolved a meaningful fraction of tasks that require reading a codebase, making changes, and checking them. The work showed that the design of the agent's action interface, not just the underlying model, strongly affects how many software tasks get solved, and the released open frameworks made this style of coding agent broadly available.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2405.15793", "arXiv:2405.15793" ], "version": 2 }, { "id": "P-236", "slug": "agent-memory-multi-agent-systems", "title": "Agent Memory & Multi-Agent Systems (MemGPT/AutoGen/CAMEL)", "shortTitle": "Agent Memory & Multi-Agent Systems", "authors": [ "Charles Packer", "Joseph E. Gonzalez", "Qingyun Wu", "Chi Wang", "Guohao Li", "Bernard Ghanem" ], "institutions": [ "UC Berkeley", "Microsoft", "KAUST" ], "year": 2023, "date": "2023", "venue": null, "artifactType": "preprint", "domain": "agents", "url": "https://arxiv.org/abs/2310.08560", "arxivId": "2310.08560", "doi": null, "repository": "https://github.com/letta-ai/letta", "openness": "unknown", "status": "niche-but-active", "statusRaw": "active-unsolved", "confidence": "High", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family gave agents persistent, paged memory and frameworks for structured multi-agent collaboration, letting systems exceed the fixed context window and divide work across role-specialized agents.", "plainSummary": "MemGPT borrows virtual-memory ideas, treating the context window as limited RAM and external storage as disk, so the agent pages information in and out under its own control to maintain long-running conversations and documents beyond the token limit. AutoGen provides a framework for defining multiple conversable agents that message each other and call tools to solve a task jointly, and CAMEL uses role-playing prompts to make two agents cooperate through dialogue with minimal human steering. The shared contribution is infrastructure for memory that outlives a single context and for coordinating several agents, enabling longer and more complex workflows than a single stateless call.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.08560", "arXiv:2310.08560" ], "version": 2 }, { "id": "P-237", "slug": "autogpt-babyagi", "title": "AutoGPT / BabyAGI (viral agent software)", "shortTitle": "AutoGPT / BabyAGI", "authors": [ "Toran Bruce Richards", "Yohei Nakajima" ], "institutions": [ "Community" ], "year": 2023, "date": "2023", "venue": null, "artifactType": "software_release", "domain": "agents", "url": "https://github.com/Significant-Gravitas/AutoGPT", "arxivId": null, "doi": null, "repository": "https://github.com/yoheinakajima/babyagi", "openness": "source-available", "status": "current-standard", "statusRaw": "software-not-research", "confidence": "High", "difficulty": 3, "influence": { "academic": 4, "frontier": 4, "engineering": 3, "industrial": 3, "openSource": 1, "underappreciated": 4 }, "importanceScore": 4, "citations": { "approx": 0, "source": "none-applicable", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "AutoGPT and BabyAGI were widely-copied open-source programs that wrapped a language model in an autonomous loop of goal decomposition, task queuing, and self-prompted execution toward a user objective.", "plainSummary": "Given a high-level goal, these programs prompt the model to generate a task list, execute tasks one at a time using tools like web search and file access, and feed results back to reprioritize and spawn new tasks, running with little human intervention. BabyAGI centered on a compact task-creation and prioritization loop backed by a vector store, while AutoGPT added tool integrations and persistence around a similar cycle. They were engineering demonstrations rather than research papers, and their viral spread popularized the autonomous-agent pattern while also exposing its practical limits in reliability, looping, and cost.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://github.com/Significant-Gravitas/AutoGPT" ], "version": 2 }, { "id": "P-300", "slug": "vision-transformer", "title": "Vision Transformer (An Image is Worth 16x16 Words)", "shortTitle": "Vision Transformer", "authors": [ "Alexey Dosovitskiy", "Lucas Beyer", "Alexander Kolesnikov", "et al." ], "institutions": [ "Google Brain" ], "year": 2021, "date": "2020-10", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2010.11929", "arxivId": "2010.11929", "doi": null, "repository": "https://github.com/google-research/vision_transformer", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 50000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced the Vision Transformer, showing that a plain Transformer applied to fixed-size image patches can match or beat convolutional networks on image classification, removing the assumption that vision required convolution-specific inductive biases.", "plainSummary": "The model splits an image into a grid of small patches (e.g. 16x16 pixels), linearly embeds each patch as if it were a word token, adds position embeddings, and feeds the sequence into a standard Transformer encoder. With no convolutions, it learns spatial structure from data alone, which works well only when pre-trained on very large image datasets. This let the same architecture used for language be reused for vision and scaled up with more data and compute rather than hand-designed vision components.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2010.11929", "arXiv:2010.11929" ], "version": 2 }, { "id": "P-301", "slug": "clip-learning-transferable-visual-models-from-nl-supervision", "title": "CLIP: Learning Transferable Visual Models from NL Supervision", "shortTitle": "CLIP", "authors": [ "Alec Radford", "Jong Wook Kim", "Chris Hallacy", "et al." ], "institutions": [ "OpenAI" ], "year": 2021, "date": "2021-02", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2103.00020", "arxivId": "2103.00020", "doi": null, "repository": "https://github.com/openai/CLIP", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 10, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 10, "citations": { "approx": 30000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced CLIP, which trains image and text encoders jointly on 400 million web image-caption pairs with a contrastive objective, removing the need for fixed labeled classification datasets and enabling zero-shot recognition from natural-language prompts.", "plainSummary": "Two encoders (one for images, one for text) are trained so that a photo and its matching caption land close together in a shared embedding space while mismatched pairs are pushed apart. To classify a new image with no task-specific training, you write candidate labels as text prompts and pick the one whose embedding is closest to the image. Because the labels are open-ended text rather than a fixed category list, one model transfers to many recognition tasks and became a reusable component for retrieval and for later image-generation and vision-language systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2103.00020", "arXiv:2103.00020" ], "version": 2 }, { "id": "P-302", "slug": "align-scaling-up-visual-and-vision-language-representation-learning", "title": "ALIGN: Scaling Up Visual and Vision-Language Representation Learning", "shortTitle": "ALIGN", "authors": [ "Chao Jia", "Yinfei Yang", "Ye Xia", "et al." ], "institutions": [ "Google" ], "year": 2021, "date": "2021-02", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2102.05918", "arxivId": "2102.05918", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-parallel", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "ALIGN showed that a dual-encoder image-text model trained with contrastive learning on a billion-scale noisy web alt-text dataset matches curated-data methods, removing the dependency on expensive human-labeled or filtered training pairs.", "plainSummary": "ALIGN trains separate image and text encoders to map matched image-caption pairs close together in a shared embedding space using a contrastive loss, learned over ~1.8 billion raw web image/alt-text pairs with only minimal frequency-based cleaning. By tolerating noisy supervision at scale, it removed the bottleneck of building large curated vision-language datasets and enabled strong zero-shot classification and image-text retrieval from web data alone.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2102.05918", "arXiv:2102.05918" ], "version": 2 }, { "id": "P-303", "slug": "flamingo-a-visual-language-model-for-few-shot-learning", "title": "Flamingo: a Visual Language Model for Few-Shot Learning", "shortTitle": "Flamingo", "authors": [ "Jean-Baptiste Alayrac", "Jeff Donahue", "Pauline Luc", "et al." ], "institutions": [ "DeepMind" ], "year": 2022, "date": "2022-04", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2204.14198", "arxivId": "2204.14198", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-adapter", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 3500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Flamingo, a vision-language model that connects a frozen pretrained image encoder to a frozen pretrained language model via trainable cross-attention layers, letting a single model handle interleaved image-and-text prompts and learn new tasks from a few in-context examples.", "plainSummary": "Frozen vision and language backbones are bridged by a Perceiver-style resampler that turns variable numbers of image features into a fixed set of tokens, plus gated cross-attention layers inserted into the language model so text generation can attend to images. Training on large interleaved image-text web data teaches the bridge without retraining the expensive backbones. This allowed few-shot performance on captioning and visual question answering by showing examples in the prompt, rather than fine-tuning a separate model per task.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2204.14198", "arXiv:2204.14198" ], "version": 2 }, { "id": "P-304", "slug": "blip-blip-2", "title": "BLIP / BLIP-2 (Q-Former)", "shortTitle": "BLIP / BLIP-2", "authors": [ "Junnan Li", "Dongxu Li", "Silvio Savarese", "Steven C. H. Hoi", "Caiming Xiong" ], "institutions": [ "Salesforce" ], "year": 2023, "date": "2022-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2301.12597", "arxivId": "2301.12597", "doi": null, "repository": "https://github.com/salesforce/LAVIS", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "BLIP-2 introduced a lightweight Querying Transformer (Q-Former) that connects a frozen image encoder to a frozen large language model, removing the need to train either backbone to build a vision-language model.", "plainSummary": "BLIP-2 keeps a pretrained image encoder and a pretrained LLM frozen and trains only a small Q-Former, a set of learnable query tokens that extract the visual features most relevant to text and feed them into the language model. This two-stage bridging cuts the trainable parameters and compute needed for vision-language pretraining by orders of magnitude, letting new image encoders and LLMs be combined into captioning and visual question-answering systems without full end-to-end retraining.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2301.12597", "arXiv:2301.12597" ], "version": 2 }, { "id": "P-305", "slug": "llava-visual-instruction-tuning", "title": "LLaVA: Visual Instruction Tuning", "shortTitle": "LLaVA", "authors": [ "Haotian Liu", "Chunyuan Li", "Qingyang Wu", "Yong Jae Lee" ], "institutions": [ "UW-Madison", "Microsoft" ], "year": 2023, "date": "2023-04", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2304.08485", "arxivId": "2304.08485", "doi": null, "repository": "https://github.com/haotian-liu/LLaVA", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-open-vlm", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced LLaVA, which connects a CLIP vision encoder to an open language model through a simple projection layer and fine-tunes on GPT-generated visual instruction data, providing a low-cost recipe for building conversational vision-language assistants.", "plainSummary": "Image features from a frozen CLIP encoder are mapped into the language model's token space by a single trainable projection, so the model can treat an image as extra context alongside a text prompt. The team used a text-only GPT model to generate multimodal instruction-following examples (questions, descriptions, and reasoning about images) and fine-tuned on them. This showed that a capable image-chat assistant could be built cheaply from existing open components plus synthetic instruction data, and it became a common baseline architecture for open multimodal models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2304.08485", "arXiv:2304.08485" ], "version": 2 }, { "id": "P-306", "slug": "open-production-vlms", "title": "Open Production VLMs (Qwen-VL/InternVL/PaLI/Kosmos)", "shortTitle": "Open Production VLMs", "authors": [ "Jinze Bai", "Jingren Zhou", "Zhe Chen", "Jifeng Dai", "Xi Chen", "Radu Soricut", "Shaohan Huang", "Furu Wei" ], "institutions": [ "Alibaba", "Shanghai AI Lab", "Google", "Microsoft" ], "year": 2023, "date": "2022-2024", "venue": null, "artifactType": "technical_report", "domain": "multimodal", "url": "https://arxiv.org/abs/2308.12966", "arxivId": "2308.12966", "doi": null, "repository": "https://github.com/QwenLM/Qwen-VL", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-open-frontier", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of open production vision-language models (represented by Qwen-VL, with InternVL, PaLI, and Kosmos) packaged image encoders plus LLMs into openly released, instruction-tuned multimodal assistants, removing the reliance on closed APIs for capable general-purpose VLMs.", "plainSummary": "These models connect a strong vision encoder to a language model and train through staged image-text pretraining followed by multimodal instruction tuning, adding capabilities such as higher-resolution input, region grounding, OCR, and multilingual support. Represented by Qwen-VL, the family shipped weights, tooling, and reproducible recipes that made competitive image understanding, document reading, and grounded visual dialogue available to build on directly rather than only behind proprietary services.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2308.12966", "arXiv:2308.12966" ], "version": 2 }, { "id": "P-307", "slug": "sam-grounding-dino-imagebind", "title": "SAM / Grounding DINO / ImageBind (perception foundation models)", "shortTitle": "SAM / Grounding DINO / ImageBind", "authors": [ "Alexander Kirillov", "Ross Girshick", "Piotr Dollár", "Shilong Liu", "Lei Zhang", "Rohit Girdhar", "Ishan Misra" ], "institutions": [ "Meta" ], "year": 2023, "date": "2023", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2304.02643", "arxivId": "2304.02643", "doi": null, "repository": "https://github.com/facebookresearch/segment-anything", "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 8000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A set of perception foundation models (represented by Segment Anything, with Grounding DINO and ImageBind) produced promptable, general-purpose perception backbones that generalize to new objects and tasks without per-task retraining.", "plainSummary": "Segment Anything trains a promptable segmentation model on a billion-mask dataset so that a point, box, or text-linked prompt yields object masks for categories never explicitly labeled; Grounding DINO adds open-vocabulary detection from text queries, and ImageBind aligns six modalities into one embedding space. Represented by SAM, this family removed the need to collect labels and train a fresh model for each new segmentation or detection target, enabling zero-shot and interactive perception that downstream systems can prompt directly.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2304.02643", "arXiv:2304.02643" ], "version": 2 }, { "id": "P-308", "slug": "latent-diffusion-diffusion-transformers", "title": "Latent Diffusion (Stable Diffusion) / Diffusion Transformers (DiT)", "shortTitle": "Latent Diffusion / Diffusion Transformers", "authors": [ "Robin Rombach", "Patrick Esser", "Björn Ommer", "William Peebles", "Saining Xie" ], "institutions": [ "LMU Munich", "UC Berkeley" ], "year": 2022, "date": "2021-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://arxiv.org/abs/2112.10752", "arxivId": "2112.10752", "doi": null, "repository": "https://github.com/CompVis/latent-diffusion", "openness": "unknown", "status": "current-standard", "statusRaw": "current-generation-backbone", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 15000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced latent diffusion (the basis of Stable Diffusion), which runs the diffusion process in a compressed autoencoder latent space instead of pixel space, cutting the compute cost of training and sampling high-resolution image generators by a large factor.", "plainSummary": "An autoencoder first compresses images into a smaller latent representation, and the denoising diffusion model learns to generate in that latent space rather than over millions of raw pixels. A cross-attention mechanism injects conditioning such as text prompts, and the decoder turns the final latent back into a full-resolution image. Working in the smaller space made training and inference cheap enough to run on consumer GPUs, which enabled the open release of a general-purpose text-to-image model.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2112.10752", "arXiv:2112.10752" ], "version": 2 }, { "id": "P-309", "slug": "native-omni-multimodality", "title": "Native/Omni Multimodality (Gemini / GPT-4V / GPT-4o)", "shortTitle": "Native/Omni Multimodality", "authors": [ "Google DeepMind", "OpenAI" ], "institutions": [ "Google DeepMind", "OpenAI" ], "year": 2024, "date": "2023-2024", "venue": null, "artifactType": "system_card", "domain": "multimodal", "url": "https://arxiv.org/abs/2312.11805", "arxivId": "2312.11805", "doi": null, "repository": null, "openness": "unknown", "status": "disclosure-limited", "statusRaw": "frontier-product-closed", "confidence": "Low", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 0, "source": "none-applicable", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Presented Gemini, a family of models trained from the start on interleaved text, images, audio, and video rather than bolting separate modality models together, aiming to remove the boundaries between modality-specific systems in a single model.", "plainSummary": "The report describes models built to accept and reason over mixed inputs (text, images, audio, and video) natively and to produce text and image outputs, trained jointly across modalities and offered in sizes from on-device to datacenter scale. Native multimodal training is meant to avoid the information loss of connecting independently trained encoders after the fact. The report reports benchmark results across language, coding, reasoning, and multimodal tasks; the underlying weights and full training details are not public, and related closed systems like GPT-4V and GPT-4o pursue similar goals.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2312.11805", "arXiv:2312.11805" ], "version": 2 }, { "id": "P-320", "slug": "dense-passage-retrieval", "title": "Dense Passage Retrieval (DPR)", "shortTitle": "Dense Passage Retrieval", "authors": [ "Vladimir Karpukhin", "Barlas Oguz", "Sewon Min", "et al." ], "institutions": [ "Meta FAIR" ], "year": 2020, "date": "2020-04", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2004.04906", "arxivId": "2004.04906", "doi": null, "repository": "https://github.com/facebookresearch/DPR", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 5000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Dense Passage Retrieval, showing that a BERT-based dual-encoder trained with contrastive learning retrieves relevant passages more accurately than keyword search like BM25, removing sparse lexical matching as the default first stage for open-domain question answering.", "plainSummary": "Two BERT encoders map questions and passages into a shared vector space, trained so a question sits near passages that answer it, using in-batch negatives to make training efficient. At query time the question is embedded once and nearest passages are found by fast vector similarity search over a precomputed index. Replacing term-overlap retrieval with learned semantic matching improved retrieval and downstream answer accuracy, and established the dense-retriever design widely used in retrieval-augmented pipelines.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2004.04906", "arXiv:2004.04906" ], "version": 2 }, { "id": "P-321", "slug": "realm-retrieval-augmented-lm-pre-training", "title": "REALM: Retrieval-Augmented LM Pre-Training", "shortTitle": "REALM", "authors": [ "Kelvin Guu", "Kenton Lee", "Zora Tung", "Panupong Pasupat", "Ming-Wei Chang" ], "institutions": [ "Google" ], "year": 2020, "date": "2020-02", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2002.08909", "arxivId": "2002.08909", "doi": null, "repository": "https://github.com/google-research/language/tree/master/language/realm", "openness": "unknown", "status": "influential-precursor", "statusRaw": "foundational-precursor", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "REALM introduced end-to-end pretraining of a language model jointly with a neural retriever over a text corpus, removing the need to store all world knowledge implicitly in model parameters.", "plainSummary": "REALM augments masked language-model pretraining with a learned retriever that fetches relevant documents from a corpus and conditions predictions on them, backpropagating the language-modeling signal through the retrieval step so the retriever learns which passages help. This let a relatively small model consult an external, updatable knowledge store instead of memorizing facts in its weights, improving open-domain question answering and making the knowledge source inspectable and swappable.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2002.08909", "arXiv:2002.08909" ], "version": 2 }, { "id": "P-322", "slug": "retrieval-augmented-generation", "title": "Retrieval-Augmented Generation (RAG)", "shortTitle": "Retrieval-Augmented Generation", "authors": [ "Patrick Lewis", "Ethan Perez", "Aleksandra Piktus", "et al." ], "institutions": [ "Meta FAIR", "UCL" ], "year": 2020, "date": "2020-05", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2005.11401", "arxivId": "2005.11401", "doi": null, "repository": "https://github.com/huggingface/transformers", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced retrieval-augmented generation, coupling a parametric sequence-to-sequence model with a dense-retrieval index so a language model can pull in external documents at generation time instead of relying only on facts baked into its weights.", "plainSummary": "RAG pairs a trained generator with a retriever that fetches relevant passages from an external corpus (a Wikipedia index searched with dense vectors), then conditions the output on those passages. Both the retriever's query encoder and the generator are trained together, and retrieved documents can be swapped or updated without retraining the model. This lets a model cite and use up-to-date or domain-specific knowledge, reducing fabricated answers on knowledge-intensive tasks like open-domain question answering.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2005.11401", "arXiv:2005.11401" ], "version": 2 }, { "id": "P-323", "slug": "fid-retro-atlas", "title": "FiD / RETRO / Atlas (scaling retrieval)", "shortTitle": "FiD / RETRO / Atlas", "authors": [ "Gautier Izacard", "Edouard Grave", "Sebastian Borgeaud", "Jack W. Rae", "Laurent Sifre" ], "institutions": [ "Meta", "DeepMind" ], "year": 2021, "date": "2020-2022", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2112.04426", "arxivId": "2112.04426", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A line of retrieval-scaling methods (represented by RETRO, with Fusion-in-Decoder and Atlas) showed that conditioning generation on large retrieved text databases lets smaller models match much larger ones, removing the need to scale parameters to hold more knowledge.", "plainSummary": "These methods retrieve many relevant passages from a large corpus and fuse them into the generator: Fusion-in-Decoder encodes each passage separately and combines them in the decoder, RETRO cross-attends to chunk-level neighbors from a trillion-token database during pretraining, and Atlas jointly trains retriever and reader for few-shot knowledge tasks. Represented by RETRO, the family let models trade parameters for an external datastore, reaching strong language-modeling and question-answering results at lower model size while keeping the knowledge base updatable.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2112.04426", "arXiv:2112.04426" ], "version": 2 }, { "id": "P-324", "slug": "colbert-late-interaction-rerankers-knn-lm", "title": "ColBERT / Late Interaction / Rerankers / kNN-LM", "shortTitle": "ColBERT / Late Interaction / Rerankers / kNN-LM", "authors": [ "Omar Khattab", "Matei Zaharia", "Urvashi Khandelwal", "Mike Lewis", "Dan Jurafsky" ], "institutions": [ "Stanford", "others" ], "year": 2020, "date": "2019-2021", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2004.12832", "arxivId": "2004.12832", "doi": null, "repository": "https://github.com/stanford-futuredata/ColBERT", "openness": "unknown", "status": "current-standard", "statusRaw": "current-best-practice", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "ColBERT introduced late interaction, encoding queries and documents into token-level embeddings and scoring them with a cheap MaxSim operation, removing the trade-off between the accuracy of full cross-encoders and the speed of single-vector retrieval.", "plainSummary": "ColBERT represents each query and document as a bag of contextual token embeddings and computes relevance by summing, for each query token, its maximum similarity to any document token, so most document computation is precomputed offline and only lightweight matching happens at query time. This late-interaction design, alongside related rerankers and nearest-neighbor language models, delivered near cross-encoder ranking quality at a scale and latency close to bag-of-vectors search, making high-quality dense retrieval practical over large collections.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2004.12832", "arXiv:2004.12832" ], "version": 2 }, { "id": "P-325", "slug": "contrastive-text-embeddings", "title": "Contrastive Text Embeddings (Sentence-BERT/SimCSE/E5/BGE)", "shortTitle": "Contrastive Text Embeddings", "authors": [ "Nils Reimers", "Iryna Gurevych", "Tianyu Gao", "Danqi Chen", "Liang Wang", "Furu Wei", "Shitao Xiao", "Zheng Liu" ], "institutions": [ "UKP", "Princeton", "Microsoft" ], "year": 2019, "date": "2019-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/1908.10084", "arxivId": "1908.10084", "doi": null, "repository": "https://github.com/UKPLab/sentence-transformers", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 8000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Sentence-BERT, which fine-tunes BERT in a siamese setup to produce sentence embeddings comparable with cosine similarity, removing the need to run BERT on every sentence pair and making large-scale semantic similarity and search practical.", "plainSummary": "Standard BERT compares two sentences by feeding them together, so finding the best match in a large collection needs a separate forward pass per pair, which is far too slow. SBERT instead passes each sentence through a shared BERT with pooling to get one fixed vector per sentence, trained on natural-language-inference and similarity data so that similar meanings give nearby vectors. This lets you embed a corpus once and then compare sentences with cheap vector operations, enabling clustering, semantic search, and duplicate detection at scale.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1908.10084", "arXiv:1908.10084" ], "version": 2 }, { "id": "P-326", "slug": "knowledge-editing", "title": "Knowledge Editing (ROME / MEMIT)", "shortTitle": "Knowledge Editing", "authors": [ "Kevin Meng", "David Bau", "Alex Andonian", "Yonatan Belinkov" ], "institutions": [ "MIT", "Northeastern" ], "year": 2022, "date": "2022", "venue": null, "artifactType": "peer_reviewed", "domain": "retrieval", "url": "https://arxiv.org/abs/2202.05262", "arxivId": "2202.05262", "doi": null, "repository": "https://github.com/kmeng01/rome", "openness": "unknown", "status": "niche-but-active", "statusRaw": "active-research", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Knowledge editing methods (ROME and MEMIT) located where specific facts are stored in a transformer's feed-forward layers and edited them by a direct weight update, removing the need to retrain or fine-tune to change or add individual facts.", "plainSummary": "ROME uses causal tracing to identify the mid-layer feed-forward modules that store a given factual association, then applies a rank-one weight edit that rewrites that fact while leaving unrelated knowledge intact; MEMIT extends the same mechanism to insert thousands of edits at once. This gave a targeted, low-cost way to update or correct specific facts in a trained model and provided evidence for where and how factual knowledge is stored inside transformers.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2202.05262", "arXiv:2202.05262" ], "version": 2 }, { "id": "P-327", "slug": "graph-structured-retrieval", "title": "Graph & Structured Retrieval (GraphRAG)", "shortTitle": "Graph & Structured Retrieval", "authors": [ "Darren Edge", "Ha Trinh", "et al." ], "institutions": [ "Microsoft" ], "year": 2024, "date": "2024", "venue": null, "artifactType": "technical_report", "domain": "retrieval", "url": "https://arxiv.org/abs/2404.16130", "arxivId": "2404.16130", "doi": null, "repository": "https://github.com/microsoft/graphrag", "openness": "unknown", "status": "niche-but-active", "statusRaw": "emerging", "confidence": "Medium", "difficulty": 5, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 300, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "GraphRAG builds a knowledge graph and hierarchical community summaries from a corpus so retrieval can operate over structured, aggregated entities, removing standard RAG's inability to answer global questions that span an entire dataset.", "plainSummary": "GraphRAG uses a language model to extract entities and relationships into a graph, clusters the graph into communities, and precomputes summaries for each community; queries are then answered by combining these community summaries rather than only retrieving isolated text chunks. This structured approach lets the system answer broad, whole-corpus sensemaking questions (such as overarching themes) that flat vector retrieval handles poorly, while keeping evidence traceable to source entities and relations.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2404.16130", "arXiv:2404.16130" ], "version": 2 }, { "id": "P-340", "slug": "codex-evaluating-llms-trained-on-code", "title": "Codex: Evaluating LLMs Trained on Code (HumanEval)", "shortTitle": "Codex", "authors": [ "Mark Chen", "Jerry Tworek", "Heewoo Jun", "et al." ], "institutions": [ "OpenAI" ], "year": 2021, "date": "2021-07", "venue": null, "artifactType": "technical_report", "domain": "code", "url": "https://arxiv.org/abs/2107.03374", "arxivId": "2107.03374", "doi": null, "repository": "https://github.com/openai/human-eval", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Codex and the HumanEval benchmark, evaluating a GPT model fine-tuned on public code for generating programs from docstrings and measuring correctness by actually running unit tests, establishing execution-based functional evaluation instead of text-overlap metrics.", "plainSummary": "The model is a GPT language model further trained on a large corpus of GitHub code so it can turn a function signature and docstring into a working implementation. Rather than scoring generated code by similarity to a reference, HumanEval defines 164 hand-written problems with unit tests and reports pass@k, the chance that at least one of k sampled solutions passes all tests. This functional metric, plus the finding that sampling many candidates raises success rates, shaped how code-generation models are evaluated and underpinned tools like GitHub Copilot.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2107.03374", "arXiv:2107.03374" ], "version": 2 }, { "id": "P-341", "slug": "alphacode", "title": "AlphaCode (Competition-Level Code Generation)", "shortTitle": "AlphaCode", "authors": [ "Yujia Li", "David Choi", "Junyoung Chung", "et al." ], "institutions": [ "DeepMind" ], "year": 2022, "date": "2022-03", "venue": null, "artifactType": "peer_reviewed", "domain": "code", "url": "https://arxiv.org/abs/2203.07814", "arxivId": "2203.07814", "doi": null, "repository": "https://github.com/google-deepmind/code_contests", "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "AlphaCode showed that a transformer trained on public code plus massive sampling and test-based filtering could solve competitive programming problems at roughly median-human level, turning code generation from single-shot guessing into a search-and-filter pipeline.", "plainSummary": "AlphaCode generates a very large pool of candidate programs per problem (up to millions), then filters them by running the provided example tests and clusters the survivors by their behavior on generated inputs to pick a small number to submit. This sample-filter-cluster loop, rather than a single accurate decode, is what let it reach an average ranking around the top 54% of participants on Codeforces contests. It demonstrated that hard, multi-step reasoning problems could be approached by trading inference-time compute for correctness when an execution oracle is available.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2203.07814", "arXiv:2203.07814" ], "version": 2 }, { "id": "P-342", "slug": "codet5-codegen-incoder-fill-in-the-middle", "title": "CodeT5 / CodeGen / InCoder / Fill-in-the-Middle", "shortTitle": "CodeT5 / CodeGen / InCoder / Fill-in-the-Middle", "authors": [ "Yue Wang", "Steven C.H. Hoi", "Erik Nijkamp", "Caiming Xiong", "Daniel Fried", "Mike Lewis", "Mohammad Bavarian", "Mark Chen" ], "institutions": [ "Salesforce", "OpenAI", "Meta" ], "year": 2022, "date": "2021-2022", "venue": null, "artifactType": "peer_reviewed", "domain": "code", "url": "https://arxiv.org/abs/2207.14255", "arxivId": "2207.14255", "doi": null, "repository": "https://github.com/salesforce/CodeT5", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family established the pretraining objectives specific to code, culminating in Fill-in-the-Middle, which showed that reordering training documents lets a left-to-right model infill arbitrary spans given both left and right context at no loss to ordinary generation.", "plainSummary": "CodeT5 (encoder-decoder, identifier-aware), CodeGen (autoregressive, multi-turn program synthesis), and InCoder (masked-span infilling) explored how objective and architecture shape code models. The FIM paper showed a simple data transformation: move a randomly chosen middle span to the end with sentinel tokens, so a standard causal model learns to fill gaps using surrounding code while retaining normal left-to-right ability. This removed the need for a separate bidirectional model to power editor-style completions like inserting a function body between existing lines.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2207.14255", "arXiv:2207.14255" ], "version": 2 }, { "id": "P-343", "slug": "open-code-models", "title": "Open Code Models (StarCoder/Code Llama/DeepSeek-Coder/Qwen-Coder)", "shortTitle": "Open Code Models", "authors": [ "Raymond Li", "Harm de Vries", "Leandro von Werra", "Baptiste Rozière", "Gabriel Synnaeve", "Daya Guo", "Binyuan Hui" ], "institutions": [ "BigCode", "Meta", "DeepSeek", "Alibaba" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "technical_report", "domain": "code", "url": "https://arxiv.org/abs/2305.06161", "arxivId": "2305.06161", "doi": null, "repository": "https://github.com/bigcode-project", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-open-frontier", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family delivered openly released, permissively licensed code LLMs trained on large curated code corpora, with StarCoder pairing a 15B model and an 8K context with documented, opt-out, license-filtered training data to make capable code models usable and inspectable outside the big labs.", "plainSummary": "StarCoder was trained on The Stack, a permissively licensed GitHub corpus with PII redaction and an opt-out mechanism, and supports fill-in-the-middle plus an 8K-token window for repository-scale context. Code Llama (continued-pretraining of Llama 2 on code, long-context and instruct variants) and DeepSeek-Coder (repo-level pretraining with strong data curation) extended the same open, code-specialized line. Together they gave practitioners high-quality open-weight coding models with transparent data provenance, enabling local deployment, fine-tuning, and reproducible evaluation.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.06161", "arXiv:2305.06161" ], "version": 2 }, { "id": "P-344", "slug": "execution-feedback-unit-test-gen-program-repair", "title": "Execution Feedback / Unit-Test Gen / Program Repair", "shortTitle": "Execution Feedback / Unit-Test Gen / Program Repair", "authors": [ "Bei Chen", "Jian-Guang Lou", "Weizhu Chen", "Xinyun Chen", "Denny Zhou" ], "institutions": [ "Multiple" ], "year": 2022, "date": "2022-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "code", "url": "https://arxiv.org/abs/2207.10397", "arxivId": "2207.10397", "doi": null, "repository": "https://github.com/microsoft/CodeT", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-frontier", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This line closed the loop between generation and correctness by using program execution and unit tests as a feedback signal, so models could generate tests, run candidate code, read the errors, and repair themselves instead of relying on the first output.", "plainSummary": "Rather than treating a generated program as final, these methods execute it against unit tests (often model-generated) and feed the resulting pass/fail signals and error traces back to guide selection or iterative repair. Approaches include ranking candidates by test agreement, self-debugging from stack traces, and generating tests to expose faults. This grounds code models in an objective oracle, letting them fix functional bugs that pure next-token likelihood cannot detect.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2207.10397", "arXiv:2207.10397" ], "version": 2 }, { "id": "P-360", "slug": "llama-open-and-efficient-foundation-language-models", "title": "LLaMA: Open and Efficient Foundation Language Models", "shortTitle": "LLaMA", "authors": [ "Hugo Touvron", "Thibaut Lavril", "Gautier Izacard", "et al." ], "institutions": [ "Meta AI" ], "year": 2023, "date": "2023-02", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2302.13971", "arxivId": "2302.13971", "doi": null, "repository": "https://github.com/meta-llama", "openness": "open-weight", "status": "foundational-and-current", "statusRaw": "foundational-open", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 9, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Showed that training smaller models on far more publicly available tokens produces compute-efficient foundation models, removing the assumption that strong performance required proprietary data or the largest parameter counts.", "plainSummary": "Meta trained a family from 7B to 65B parameters exclusively on public datasets, pushing token counts well past Chinchilla-optimal points to favor cheaper inference. The 13B model matched or beat GPT-3 (175B) on many benchmarks and the 65B was competitive with Chinchilla and PaLM. Its release under a research license, and the subsequent weight availability, seeded a large open ecosystem of fine-tuned derivatives.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2302.13971", "arXiv:2302.13971" ], "version": 2 }, { "id": "P-361", "slug": "llama-2-llama-3", "title": "Llama 2 & Llama 3", "shortTitle": "Llama 2 & Llama 3", "authors": [ "Hugo Touvron", "Abhimanyu Dubey", "et al." ], "institutions": [ "Meta AI" ], "year": 2024, "date": "2023-2024", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2407.21783", "arxivId": "2407.21783", "doi": null, "repository": "https://github.com/meta-llama", "openness": "open-weight", "status": "current-standard", "statusRaw": "current-baseline", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 8, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 12000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A detailed engineering report on Meta's Llama family of open-weight dense transformer models, documenting the data, scaling, and post-training recipe that made a widely downloadable model competitive with the strongest closed models of its time.", "plainSummary": "The paper describes how the Llama 3 models (up to 405B parameters) were built: a large curated pretraining corpus, a standard dense transformer architecture scaled up, and a post-training pipeline of supervised fine-tuning and preference optimization for instruction following. It also covers extended context length, multilingual and code capabilities, and safety tuning. Because the weights are downloadable under a community license, it gave researchers and companies a strong base model to run and fine-tune locally, though the license and undisclosed training data mean it is open-weight rather than fully open source.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2407.21783", "arXiv:2407.21783" ], "version": 2 }, { "id": "P-362", "slug": "mistral-7b-mixtral", "title": "Mistral 7B / Mixtral", "shortTitle": "Mistral 7B / Mixtral", "authors": [ "Albert Q. Jiang", "Alexandre Sablayrolles", "Guillaume Lample", "Arthur Mensch", "Timothée Lacroix", "William El Sayed" ], "institutions": [ "Mistral AI" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2310.06825", "arxivId": "2310.06825", "doi": null, "repository": "https://github.com/mistralai/mistral-src", "openness": "open-weight", "status": "current-standard", "statusRaw": "current-deployed", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 7, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A compact 7-billion-parameter open-weight model that used grouped-query attention and sliding-window attention to match or beat larger models, showing that architectural efficiency could substitute for raw parameter count.", "plainSummary": "Mistral 7B combines grouped-query attention (which shares key/value projections across query heads to cut memory and speed up inference) with sliding-window attention (which limits each token's attention to a fixed recent window so longer sequences stay cheap). With these changes it performed on par with or better than models roughly twice its size on common benchmarks. Released with downloadable weights under a permissive Apache 2.0 license, it became a practical base for local deployment and fine-tuning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.06825", "arXiv:2310.06825" ], "version": 2 }, { "id": "P-363", "slug": "qwen", "title": "Qwen (Qwen/Qwen2/Qwen2.5)", "shortTitle": "Qwen", "authors": [ "Jinze Bai", "An Yang", "Junyang Lin", "Jingren Zhou" ], "institutions": [ "Alibaba" ], "year": 2024, "date": "2023-2024", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2412.15115", "arxivId": "2412.15115", "doi": null, "repository": "https://github.com/QwenLM", "openness": "open-weight", "status": "niche-but-active", "statusRaw": "current-open-frontier", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 8, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A technical report on Alibaba's Qwen2.5 family, a set of open-weight models spanning many sizes trained on a much larger pretraining corpus, with strong multilingual, coding, math, and long-context performance.", "plainSummary": "Qwen2.5 scales up the pretraining data to roughly 18 trillion tokens and refines the post-training pipeline (supervised fine-tuning plus reinforcement learning from preferences) across a range of model sizes from small to large. It targets improvements in instruction following, structured output, mathematics, and coding, and supports long context windows and many languages. The base and instruction-tuned weights are downloadable, giving practitioners a broad menu of sizes to fit different hardware budgets; most sizes are released under a permissive license while some larger ones use a more restrictive one, so it is open-weight.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2412.15115", "arXiv:2412.15115" ], "version": 2 }, { "id": "P-364", "slug": "deepseek", "title": "DeepSeek (V2/V3; MLA + efficient MoE + FP8)", "shortTitle": "DeepSeek", "authors": [ "DeepSeek-AI", "Damai Dai", "Aixin Liu", "Wenfeng Liang" ], "institutions": [ "DeepSeek-AI" ], "year": 2024, "date": "2024-2025", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2412.19437", "arxivId": "2412.19437", "doi": null, "repository": "https://github.com/deepseek-ai", "openness": "open-weight", "status": "niche-but-active", "statusRaw": "current-frontier-open", "confidence": "Medium", "difficulty": 6, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 8, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A technical report on DeepSeek-V3, a large mixture-of-experts model that combined multi-head latent attention, a load-balanced sparse expert design, and FP8 mixed-precision training to reach frontier-level quality at substantially lower training cost.", "plainSummary": "DeepSeek-V3 is a mixture-of-experts model where only a small fraction of its total parameters activate per token, keeping compute per token low. It introduces multi-head latent attention, which compresses the attention key/value cache into a smaller latent representation to reduce memory during inference, and pairs it with an auxiliary-loss-free scheme for balancing which experts get used. Trained in FP8 mixed precision, it reached quality comparable to leading models while using far fewer GPU-hours, and the weights are downloadable, making it open-weight.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2412.19437", "arXiv:2412.19437" ], "version": 2 }, { "id": "P-365", "slug": "open-ecosystem", "title": "Open Ecosystem (Gemma/Phi/OLMo/Falcon/Command R/Yi/GLM/InternLM)", "shortTitle": "Open Ecosystem", "authors": [ "Sébastien Bubeck", "Marah Abdin", "Dirk Groeneveld", "Guilherme Penedo", "Jie Tang", "Gemma Team (Google DeepMind)", "Shanghai AI Laboratory" ], "institutions": [ "Multiple" ], "year": 2024, "date": "2023-2025", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2402.00838", "arxivId": "2402.00838", "doi": null, "repository": "https://github.com/allenai/OLMo", "openness": "open-weight", "status": "current-standard", "statusRaw": "current-broad-ecosystem", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 7, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This open ecosystem produced strong sub-frontier base models under varied openness levels, with OLMo going furthest by releasing not just weights but the full training data, code, logs, and checkpoints so the entire training process is reproducible.", "plainSummary": "Gemma, Phi (small models trained on curated/synthetic 'textbook-quality' data), Falcon, Command R (retrieval and tool-use oriented), Yi, and GLM each shipped capable open-weight models with differing licenses and disclosure. OLMo is fully open: alongside weights it releases the Dolma training corpus, the training and evaluation code, and intermediate checkpoints, making it a scientific artifact rather than only a usable model. The distinction matters because most 'open' models share only weights, whereas OLMo lets researchers study and reproduce how a given capability arose during training.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2402.00838", "arXiv:2402.00838" ], "version": 2 }, { "id": "P-366", "slug": "glm-general-language-model-pretraining-with-autoregressive-blank-infilling-glm-130b", "title": "GLM: General Language Model Pretraining with Autoregressive Blank Infilling / GLM-130B", "shortTitle": "GLM", "authors": [ "Zhengxiao Du", "Yujie Qian", "Xiao Liu", "Ming Ding", "Zhilin Yang", "Jie Tang", "Aohan Zeng" ], "institutions": [ "Tsinghua University", "Zhipu AI" ], "year": 2021, "date": "2021-03", "venue": null, "artifactType": "peer_reviewed", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2103.10360", "arxivId": "2103.10360", "doi": null, "repository": "https://github.com/THUDM/GLM", "openness": "open-weight", "status": "niche-but-active", "statusRaw": "current-open-frontier", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 7, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Introduced the GLM pretraining objective — autoregressive blank infilling — which unifies bidirectional context and left-to-right generation in a single model, and scaled it into GLM-130B, an openly released English–Chinese bilingual foundation model.", "plainSummary": "GLM masks spans of text and trains the model to regenerate them autoregressively while attending bidirectionally to the surrounding context, giving one model both the understanding strengths of masked encoders like BERT and the generation ability of decoder-only models. The team scaled the recipe to GLM-130B, a 130-billion-parameter bilingual model released with open weights and INT4 inference support so it could run on modest hardware. It anchored the GLM/ChatGLM family from Zhipu AI and Tsinghua and became one of the more capable early open bilingual foundation models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2103.10360", "arXiv:2103.10360" ], "version": 2 }, { "id": "P-367", "slug": "glm-45-to-glm-52", "title": "GLM-4.5 to GLM-5.2 (Zhipu AI / Z.ai open-weight MoE series)", "shortTitle": "GLM-4.5 to GLM-5.2", "authors": [ "Zhipu AI", "Z.ai GLM Team", "Jie Tang", "Aohan Zeng" ], "institutions": [ "Zhipu AI", "Z.ai" ], "year": 2025, "date": "2025-07", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2508.06471", "arxivId": "2508.06471", "doi": null, "repository": "https://huggingface.co/zai-org", "openness": "open-weight", "status": "niche-but-active", "statusRaw": "current-frontier-open", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 7, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Put a frontier line of large mixture-of-experts models into the open-weight ecosystem: GLM-4.5 through the GLM-5.2 flagship, released by Zhipu AI / Z.ai under an MIT license with downloadable weights rather than API-only access.", "plainSummary": "From GLM-4.5 (a 355B-parameter sparse mixture-of-experts model with 32B active, July 2025) through the GLM-5 family and the GLM-5.2 flagship (a roughly 750B-parameter MoE with sparse attention and a million-token context, 2026), Zhipu AI released the weights on Hugging Face under MIT, with FP8 variants for cheaper serving. The series targets reasoning and agentic software engineering and is also served through the z.ai API, but the checkpoints are genuinely public and self-hostable. It is among the most capable open-weight lines outside the Llama, Qwen, and DeepSeek families. (GLM-5.x release dates are approximate, pinned from repository timestamps rather than day-precise announcements.)", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2508.06471", "arXiv:2508.06471" ], "version": 2 }, { "id": "P-368", "slug": "bloom-a-176b-parameter-open-access-multilingual-language-model", "title": "BLOOM: A 176B-Parameter Open-Access Multilingual Language Model", "shortTitle": "BLOOM", "authors": [ "Teven Le Scao", "Angela Fan", "Christopher Akiki", "Thomas Wolf", "Stella Biderman" ], "institutions": [ "BigScience Workshop" ], "year": 2022, "date": "2022-11", "venue": null, "artifactType": "technical_report", "domain": "open-weight-family", "url": "https://arxiv.org/abs/2211.05100", "arxivId": "2211.05100", "doi": null, "repository": "https://huggingface.co/bigscience/bloom", "openness": "open-weight", "status": "influential-precursor", "statusRaw": "influential-precursor", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 7, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Produced a 176-billion-parameter multilingual language model through an open 1000-researcher collaboration, releasing the weights, training code, and data so a frontier-scale model was no longer the preserve of a single well-resourced lab.", "plainSummary": "BLOOM was trained by the BigScience workshop on the ROOTS corpus spanning 46 natural and 13 programming languages, with the model, code, and data documentation released openly under a responsible-AI licence. It brought many languages underserved by English-centric models into a large open model and made the full training process transparent. It was an early proof that open, multi-institution collaboration could build models at frontier scale.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2211.05100", "arXiv:2211.05100" ], "version": 2 }, { "id": "P-369", "slug": "laion-5b-an-open-large-scale-dataset-for-training-next-generation-image-text-models", "title": "LAION-5B: An Open Large-Scale Dataset for Training Next Generation Image-Text Models", "shortTitle": "LAION-5B", "authors": [ "Christoph Schuhmann", "Romain Beaumont", "Richard Vencu", "Cade Gordon", "Ludwig Schmidt" ], "institutions": [ "LAION" ], "year": 2022, "date": "2022-10", "venue": null, "artifactType": "dataset_paper", "domain": "multimodal", "url": "https://arxiv.org/abs/2210.08402", "arxivId": "2210.08402", "doi": null, "repository": "https://laion.ai/blog/laion-5b/", "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Released an openly available dataset of 5.85 billion image-text pairs, giving the research community the web-scale multimodal data that had previously been locked inside a few large labs and enabling open text-to-image and vision-language models.", "plainSummary": "LAION-5B is a filtered, CLIP-scored collection of billions of image-caption pairs scraped from the web and released openly by the LAION non-profit. It supplied the training data behind Stable Diffusion and many open CLIP and vision-language models, shifting a major bottleneck -- data -- from proprietary to public. It also foregrounded the governance, consent, and safety questions that come with web-scale datasets.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2210.08402", "arXiv:2210.08402" ], "version": 2 }, { "id": "P-400", "slug": "flashattention-io-aware-exact-attention", "title": "FlashAttention (1/2/3): IO-Aware Exact Attention", "shortTitle": "FlashAttention", "authors": [ "Tri Dao", "Daniel Y. Fu", "Stefano Ermon", "Atri Rudra", "Christopher Ré", "Jay Shah" ], "institutions": [ "Stanford" ], "year": 2022, "date": "2022-05", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://arxiv.org/abs/2205.14135", "arxivId": "2205.14135", "doi": null, "repository": "https://github.com/Dao-AILab/flash-attention", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 9, "frontier": 9, "engineering": 10, "industrial": 8, "openSource": 6, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Computes exact attention with an IO-aware tiling and online-softmax algorithm that avoids materializing the full attention matrix in GPU high-bandwidth memory, removing the memory and bandwidth bottleneck that made long sequences slow and costly.", "plainSummary": "The kernel splits queries, keys, and values into blocks that fit in fast on-chip SRAM, computes softmax incrementally with running statistics, and recomputes intermediates during the backward pass instead of storing them, so memory scales linearly rather than quadratically in sequence length. This yields wall-clock speedups and lower memory use without approximation, giving identical results to standard attention. It enabled training and serving with longer context windows and became a default attention implementation in mainstream frameworks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2205.14135", "arXiv:2205.14135" ], "version": 2 }, { "id": "P-401", "slug": "pagedattention-vllm", "title": "PagedAttention / vLLM", "shortTitle": "PagedAttention / vLLM", "authors": [ "Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang", "Ying Sheng", "Ion Stoica" ], "institutions": [ "UC Berkeley" ], "year": 2023, "date": "2023-09", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://arxiv.org/abs/2309.06180", "arxivId": "2309.06180", "doi": null, "repository": "https://github.com/vllm-project/vllm", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Manages the key-value cache in non-contiguous fixed-size blocks like virtual-memory paging, removing the fragmentation that wasted most KV-cache memory under contiguous per-request allocation.", "plainSummary": "The KV cache for each sequence is stored in small blocks tracked by a lookup table, so memory is allocated on demand and blocks can be shared across sequences that share a prefix, with copy-on-write on divergence. This cuts memory waste to a few percent and raises the number of requests that can be batched together, and it is the core of the vLLM serving engine. The result was substantially higher serving throughput at the same latency compared to prior systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2309.06180", "arXiv:2309.06180" ], "version": 2 }, { "id": "P-402", "slug": "orca-continuous-batching", "title": "Orca: Continuous (Iteration-Level) Batching", "shortTitle": "Orca", "authors": [ "Gyeong-In Yu", "Joo Seong Jeong", "Geon-Woo Kim", "et al." ], "institutions": [ "Seoul National Univ" ], "year": 2022, "date": "2022", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://www.usenix.org/conference/osdi22/presentation/yu", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 600, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced iteration-level (continuous) batching for transformer serving, letting the server add and retire requests at each generation step instead of processing a batch to completion, which sharply raised GPU throughput for LLM inference.", "plainSummary": "Traditional batching runs a whole batch of requests together until all finish, so short requests wait for long ones and the GPU sits underused. Orca schedules at the granularity of a single decoding iteration: after each token step it can inject newly arrived requests and remove completed ones, keeping the batch continuously full. Combined with selective batching that handles operations with different sequence lengths correctly, this delivered large throughput and latency gains and became the standard serving pattern later adopted by systems like vLLM.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.usenix.org/conference/osdi22/presentation/yu" ], "version": 2 }, { "id": "P-403", "slug": "speculative-decoding-medusa-eagle", "title": "Speculative Decoding / Medusa / EAGLE", "shortTitle": "Speculative Decoding / Medusa / EAGLE", "authors": [ "Yaniv Leviathan", "Yossi Matias", "Charlie Chen", "Tianle Cai", "Tri Dao", "Yuhui Li", "Hongyang Zhang" ], "institutions": [ "Google", "DeepMind", "Princeton" ], "year": 2023, "date": "2022-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://arxiv.org/abs/2211.17192", "arxivId": "2211.17192", "doi": null, "repository": "https://github.com/FasterDecoding/Medusa", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced speculative decoding, which uses a small fast draft model to propose several tokens that the large target model verifies in parallel, cutting the number of expensive sequential steps without changing the output distribution.", "plainSummary": "Autoregressive generation is slow because each token requires a full forward pass of the large model, run one at a time. Speculative decoding runs a cheap draft model to guess a short run of upcoming tokens, then the large model checks all of them in a single parallel pass and accepts the longest correct prefix, resampling at the first mismatch. A modified acceptance rule guarantees the result matches what the large model would have produced on its own, so it speeds up inference (often 2-3x) with no loss in output quality.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2211.17192", "arXiv:2211.17192" ], "version": 2 }, { "id": "P-404", "slug": "inference-quantization", "title": "Inference Quantization (GPTQ/AWQ/SmoothQuant/bitsandbytes)", "shortTitle": "Inference Quantization", "authors": [ "Elias Frantar", "Dan Alistarh", "Ji Lin", "Song Han", "Guangxuan Xiao", "Tim Dettmers", "Luke Zettlemoyer" ], "institutions": [ "ISTA", "MIT", "Meta", "UW" ], "year": 2022, "date": "2022-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://arxiv.org/abs/2210.17323", "arxivId": "2210.17323", "doi": null, "repository": "https://github.com/IST-DASLab/gptq", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced GPTQ, a one-shot post-training weight quantization method that compresses large language model weights to about 3-4 bits using approximate second-order (Hessian) information, letting big models run on far less memory with little accuracy loss.", "plainSummary": "GPTQ quantizes a pretrained model's weights layer by layer after training, using an approximation of the layer's Hessian to decide how to round each weight so that the overall error is minimized, and updating the remaining weights to compensate as it goes. This brings models down to roughly 3-4 bits per weight in one pass without retraining, shrinking memory enough to fit very large models on a single GPU. Related methods such as SmoothQuant (which shifts activation outliers into weights) and AWQ (which protects the weights most important to activations) address the same goal of low-bit inference through different trade-offs.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2210.17323", "arXiv:2210.17323" ], "version": 2 }, { "id": "P-405", "slug": "lora-qlora", "title": "LoRA / QLoRA (parameter-efficient fine-tuning)", "shortTitle": "LoRA / QLoRA", "authors": [ "Edward J. Hu", "Yelong Shen", "Weizhu Chen", "Tim Dettmers", "Luke Zettlemoyer" ], "institutions": [ "Microsoft", "UW" ], "year": 2022, "date": "2021-2023", "venue": null, "artifactType": "peer_reviewed", "domain": "inference-serving", "url": "https://arxiv.org/abs/2106.09685", "arxivId": "2106.09685", "doi": null, "repository": "https://github.com/microsoft/LoRA", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": 8000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Freezes the pretrained weights and injects trainable low-rank matrices into each layer, removing the storage and memory cost of full fine-tuning while adding no inference latency.", "plainSummary": "Instead of updating a weight matrix directly, the method learns a low-rank product added to it, so only a tiny fraction of parameters are trained and the small adapters can be stored per task and swapped in. Because the update can be merged back into the base weights after training, there is no extra latency at inference, unlike adapter layers. On GPT-3 175B it reduced trainable parameters by roughly ten-thousand-fold while matching full fine-tuning quality, making per-task customization of large models cheap and modular.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2106.09685", "arXiv:2106.09685" ], "version": 2 }, { "id": "P-406", "slug": "edge-format-ecosystem", "title": "Edge/Format Ecosystem (llama.cpp/GGUF/TensorRT-LLM/SGLang)", "shortTitle": "Edge/Format Ecosystem", "authors": [ "Georgi Gerganov", "Lianmin Zheng", "Ying Sheng", "Ion Stoica", "NVIDIA" ], "institutions": [ "Community", "NVIDIA", "Stanford" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "software_release", "domain": "inference-serving", "url": "https://arxiv.org/abs/2312.07104", "arxivId": "2312.07104", "doi": null, "repository": "https://github.com/ggerganov/llama.cpp", "openness": "source-available", "status": "current-standard", "statusRaw": "current-fast-moving", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 8, "industrial": 6, "openSource": 4, "underappreciated": 4 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This ecosystem built the runtimes and file formats that let LLMs run efficiently on consumer and production hardware, with SGLang contributing RadixAttention, which reuses shared prefix key-value cache across requests via a radix tree to speed up structured and multi-call generation.", "plainSummary": "llama.cpp plus the GGUF format made quantized models runnable on laptops and CPUs; TensorRT-LLM optimizes inference on NVIDIA GPUs; SGLang targets programs that make many related LLM calls. SGLang's RadixAttention keeps a radix tree of previously computed key-value cache so that requests sharing a prompt prefix (few-shot templates, agent loops, branching decodes) skip recomputation, and its language lets developers express constrained, multi-step generations. The result is substantially higher throughput for the structured, repetitive call patterns that agents and serving systems generate.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2312.07104", "arXiv:2312.07104" ], "version": 2 }, { "id": "P-420", "slug": "sparse-attention", "title": "Sparse Attention (Longformer / BigBird)", "shortTitle": "Sparse Attention", "authors": [ "Iz Beltagy", "Matthew E. Peters", "Arman Cohan", "Manzil Zaheer", "Amr Ahmed" ], "institutions": [ "AI2", "Google" ], "year": 2020, "date": "2020", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2007.14062", "arxivId": "2007.14062", "doi": null, "repository": "https://github.com/allenai/longformer", "openness": "unknown", "status": "partially-superseded", "statusRaw": "foundational-partly-superseded", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 3500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Sparse-attention transformers cut the quadratic cost of full attention to linear in sequence length by attending over a fixed sparse pattern, with BigBird combining local, global, and random attention while proving the pattern still preserves the transformer's expressive and theoretical properties.", "plainSummary": "Full self-attention costs grow with the square of sequence length, capping practical context. Longformer uses sliding-window local attention plus a few global tokens; BigBird adds random connections and shows this local+global+random combination is a universal approximator and Turing-complete, so the sparsity does not sacrifice power. By making attention scale linearly, these models pushed usable context from around 512 into the thousands of tokens for tasks like long-document QA and genomics.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2007.14062", "arXiv:2007.14062" ], "version": 2 }, { "id": "P-421", "slug": "linearized-attention", "title": "Linearized Attention (Reformer/Performer/Linear Transformers)", "shortTitle": "Linearized Attention", "authors": [ "Nikita Kitaev", "Łukasz Kaiser", "Krzysztof Choromanski", "Adrian Weller", "Angelos Katharopoulos", "François Fleuret" ], "institutions": [ "Google", "EPFL" ], "year": 2020, "date": "2020-2021", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2009.14794", "arxivId": "2009.14794", "doi": null, "repository": "https://github.com/google-research/google-research/tree/master/performer", "openness": "unknown", "status": "partially-superseded", "statusRaw": "superseded-influential", "confidence": "High", "difficulty": 6, "influence": { "academic": 6, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 2 }, "importanceScore": 6, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Linearized-attention models replaced the softmax attention matrix with kernel feature maps or hashing so attention could be computed in linear time and memory, with Performer using random features (FAVOR+) to approximate softmax attention without ever forming the full N-by-N matrix.", "plainSummary": "Softmax attention requires materializing an N-by-N similarity matrix; these methods avoid it. Performer's FAVOR+ maps queries and keys through random feature functions whose dot products approximate the softmax kernel unbiasedly, letting attention be reassociated into linear-time operations. Reformer used locality-sensitive hashing and reversible layers to similar ends, and Linear Transformers recast attention as a kernelized RNN. This traded exact attention for near-linear scaling, enabling much longer sequences on fixed memory, though with some approximation cost that limited adoption at frontier scale.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2009.14794", "arXiv:2009.14794" ], "version": 2 }, { "id": "P-422", "slug": "ring-attention-infini-attention-mla", "title": "Ring Attention / Infini-Attention / MLA", "shortTitle": "Ring Attention / Infini-Attention / MLA", "authors": [ "Hao Liu", "Matei Zaharia", "Pieter Abbeel", "Tsendsuren Munkhdalai", "Manaal Faruqui", "DeepSeek-AI" ], "institutions": [ "UC Berkeley", "Google", "DeepSeek" ], "year": 2024, "date": "2023-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2310.01889", "arxivId": "2310.01889", "doi": null, "repository": "https://github.com/haoliuhl/ringattention", "openness": "unknown", "status": "niche-but-active", "statusRaw": "current-frontier", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "This family scaled context length by distributing or compressing the attention computation rather than approximating it, with Ring Attention sharding the sequence across devices and passing key-value blocks in a ring so exact attention over near-arbitrarily long contexts fits in aggregate memory.", "plainSummary": "Ring Attention splits a long sequence across multiple devices, and each device computes its local block of attention while key-value chunks circulate around the ring, overlapping communication with computation so the effective context grows with the number of devices without approximation. Infini-Attention instead keeps a bounded compressive memory that accumulates old key-value information, letting a fixed-size model attend to unbounded history. Multi-head Latent Attention (MLA, from DeepSeek) compresses the key-value cache into a low-rank latent to shrink the memory each token costs at inference. Together they attack long context from three angles: distribute it, compress the history, or compress the cache.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.01889", "arXiv:2310.01889" ], "version": 2 }, { "id": "P-423", "slug": "context-extension", "title": "Context Extension (Position Interpolation / NTK / YaRN)", "shortTitle": "Context Extension", "authors": [ "Shouyuan Chen", "Yuandong Tian", "Bowen Peng", "Jeffrey Quesnelle", "Enrico Shippole" ], "institutions": [ "Meta", "Nous" ], "year": 2023, "date": "2023", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2309.00071", "arxivId": "2309.00071", "doi": null, "repository": "https://github.com/jquesnelle/yarn", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 700, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of RoPE-rescaling methods (Position Interpolation, NTK-aware scaling, YaRN) that extend a pretrained transformer's usable context window with little or no retraining by remapping rotary position frequencies instead of pretraining longer.", "plainSummary": "Transformers trained with rotary position embeddings degrade sharply when run past their training context length because they encounter position rotations never seen in training. These methods rescale the RoPE frequencies so longer positions map back into the range the model already learned: Position Interpolation linearly compresses positions, NTK-aware scaling adjusts per-frequency to preserve high-frequency detail, and YaRN combines frequency-selective interpolation with an attention-temperature correction. The result is context windows extended by large factors (e.g. 4x-16x) after only brief fine-tuning or none at all, avoiding the cost of pretraining from scratch on long sequences.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2309.00071", "arXiv:2309.00071" ], "version": 2 }, { "id": "P-424", "slug": "s4-structured-state-space-sequence-models", "title": "S4: Structured State Space Sequence Models", "shortTitle": "S4", "authors": [ "Albert Gu", "Karan Goel", "Christopher Ré" ], "institutions": [ "Stanford" ], "year": 2022, "date": "2021-11", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2111.00396", "arxivId": "2111.00396", "doi": null, "repository": "https://github.com/state-spaces/s4", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-ssm", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced S4, a structured state space sequence model whose special (diagonal-plus-low-rank) parameterization made state space models trainable and efficient, giving a sub-quadratic alternative to attention that handles very long sequences and long-range dependencies.", "plainSummary": "State space models describe a sequence through a continuous linear recurrence, but naive versions are numerically unstable and too slow to train. S4 reparameterizes the state matrix in a structured diagonal-plus-low-rank form that can be computed as a convolution, making training stable and efficient while scaling near-linearly with sequence length. This let a single architecture model dependencies over tens of thousands of steps, strongly outperforming transformers on long-range benchmarks and seeding the later line of work leading to Mamba.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2111.00396", "arXiv:2111.00396" ], "version": 2 }, { "id": "P-425", "slug": "mamba-mamba-2", "title": "Mamba / Mamba-2 (Selective State Spaces)", "shortTitle": "Mamba / Mamba-2", "authors": [ "Albert Gu", "Tri Dao" ], "institutions": [ "CMU", "Princeton" ], "year": 2024, "date": "2023-12", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2312.00752", "arxivId": "2312.00752", "doi": null, "repository": "https://github.com/state-spaces/mamba", "openness": "unknown", "status": "niche-but-active", "statusRaw": "active-frontier-hybrid", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 2500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced a selective state-space sequence model whose parameters depend on the input, matching Transformer quality on language while scaling linearly in sequence length and avoiding the quadratic attention bottleneck.", "plainSummary": "The authors made structured state-space model parameters input-dependent so the model can selectively remember or forget information along a sequence, then designed a hardware-aware parallel scan that computes the recurrence efficiently on GPUs without materializing the full state. Mamba runs in linear time with constant memory per step at inference and handles very long sequences, while matching or exceeding Transformers of similar size on language, audio, and genomics. It offered an attention-free architecture with higher inference throughput for long-context modeling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2312.00752", "arXiv:2312.00752" ], "version": 2 }, { "id": "P-426", "slug": "rwkv-hyena-attention-ssm-hybrids", "title": "RWKV / Hyena / Attention-SSM Hybrids (Jamba)", "shortTitle": "RWKV / Hyena / Attention-SSM Hybrids", "authors": [ "Bo Peng", "Michael Poli", "Stefano Ermon", "Christopher Ré", "Opher Lieber", "Yoav Shoham" ], "institutions": [ "EleutherAI", "Stanford", "AI21" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2305.13048", "arxivId": "2305.13048", "doi": null, "repository": "https://github.com/BlinkDL/RWKV-LM", "openness": "unknown", "status": "current-standard", "statusRaw": "current-growing", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of sub-quadratic sequence models (RWKV, Hyena, and attention-SSM hybrids like Jamba) that replace or interleave softmax attention with recurrent or convolutional/state-space mixing to remove attention's quadratic-cost and growing-KV-cache bottleneck at long sequence lengths.", "plainSummary": "Standard attention costs compute and memory that grow quadratically with sequence length and requires a KV cache that grows linearly, making long sequences expensive. This line of work reformulates sequence mixing as linear-cost operations: RWKV casts a transformer-like model as an RNN with constant per-token state, Hyena uses long implicit convolutions with gating, and hybrids like Jamba interleave state-space (Mamba) layers with a few attention layers. These architectures achieve near-linear scaling in sequence length and constant or bounded inference memory while remaining competitive with transformers on language modeling.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2305.13048", "arXiv:2305.13048" ], "version": 2 }, { "id": "P-427", "slug": "lost-in-the-middle-long-context-reliability", "title": "Lost in the Middle / Long-Context Reliability (RULER)", "shortTitle": "Lost in the Middle / Long-Context Reliability", "authors": [ "Nelson F. Liu", "Kevin Lin", "Percy Liang", "Cheng-Ping Hsieh", "Boris Ginsburg" ], "institutions": [ "Stanford", "NVIDIA" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "long-context", "url": "https://arxiv.org/abs/2307.03172", "arxivId": "2307.03172", "doi": null, "repository": "https://github.com/nelson-liu/lost-in-the-middle", "openness": "unknown", "status": "current-standard", "statusRaw": "essential-caution", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 6, "engineering": 5, "industrial": 5, "openSource": 3, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "An empirical study showing that long-context models retrieve information reliably only when it sits near the start or end of the input and degrade sharply for content in the middle, exposing that a large context window does not guarantee uniform use of it (with RULER later giving a systematic length-vs-reliability probe).", "plainSummary": "The work tested models on multi-document QA and key-value retrieval while varying where the relevant information was placed in a long input. Accuracy followed a U-shaped curve: high when the needed fact was at the beginning or end, substantially lower when buried in the middle, even for models nominally supporting the full length. This demonstrated that advertised context length overstates usable context, motivating position-robustness work and synthetic stress benchmarks like RULER that measure at what effective length a model still retrieves and reasons reliably.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2307.03172", "arXiv:2307.03172" ], "version": 2 }, { "id": "P-440", "slug": "mmlu-massive-multitask-language-understanding", "title": "MMLU: Massive Multitask Language Understanding", "shortTitle": "MMLU", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "et al." ], "institutions": [ "UC Berkeley" ], "year": 2021, "date": "2020-09", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2009.03300", "arxivId": "2009.03300", "doi": null, "repository": "https://github.com/hendrycks/test", "openness": "unknown", "status": "current-but-contested", "statusRaw": "saturated", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 4000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A 57-subject multiple-choice benchmark spanning STEM, humanities, social sciences, and professional exams that measures broad pretrained knowledge in zero- and few-shot settings, removing reliance on narrow single-task metrics for judging general capability.", "plainSummary": "The benchmark collects roughly 16,000 exam-style questions at difficulties from high school to professional level and scores models without task-specific training. At release most models scored near the 25% random-chance baseline while the largest GPT-3 reached about 44%, with especially weak and poorly calibrated performance on subjects like law and morality. It became a standard headline number for comparing the general knowledge of frontier models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2009.03300", "arXiv:2009.03300" ], "version": 2 }, { "id": "P-441", "slug": "gpqa-graduate-level-google-proof-qa", "title": "GPQA: Graduate-Level Google-Proof QA", "shortTitle": "GPQA", "authors": [ "David Rein", "et al." ], "institutions": [ "NYU", "Anthropic", "Cohere" ], "year": 2023, "date": "2023-11", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2311.12022", "arxivId": "2311.12022", "doi": null, "repository": "https://github.com/idavidrein/gpqa", "openness": "unknown", "status": "current-standard", "statusRaw": "current-headline", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 600, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "GPQA, a benchmark of several hundred graduate-level science multiple-choice questions written and validated by domain experts to be hard even for skilled non-experts with web access, giving a contamination-resistant measure of expert-level reasoning rather than lookup.", "plainSummary": "Existing knowledge benchmarks had largely saturated and were answerable by search, so they no longer separated genuine reasoning from retrieval. GPQA collects questions in biology, physics, and chemistry authored by PhD-level experts, then filters to items where other experts agree on the answer but skilled non-experts fail even after extended Googling (validated 'Google-proof'). This yields a small, high-difficulty set that measures whether a model can reason at graduate level and serves as a hard target and a testbed for scalable-oversight research.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2311.12022", "arXiv:2311.12022" ], "version": 2 }, { "id": "P-442", "slug": "gsm8k-math", "title": "GSM8K / MATH (mathematical reasoning)", "shortTitle": "GSM8K / MATH", "authors": [ "Karl Cobbe", "Vineet Kosaraju", "John Schulman", "Dan Hendrycks", "Jacob Steinhardt" ], "institutions": [ "OpenAI", "UC Berkeley" ], "year": 2021, "date": "2021", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2103.03874", "arxivId": "2103.03874", "doi": null, "repository": "https://github.com/hendrycks/math", "openness": "unknown", "status": "current-but-contested", "statusRaw": "partly-saturated", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Released the MATH dataset of competition mathematics problems with full step-by-step solutions, providing a hard benchmark that exposed how weak language models were at multi-step mathematical reasoning.", "plainSummary": "MATH contains 12,500 competition-level problems, each with a worked solution, spanning subjects like algebra, geometry, and number theory at varying difficulty. Because answers require multi-step derivations rather than lookup, early large models scored very low, making it a clear measure of reasoning progress. It is often paired with GSM8K, a grade-school word-problem set, and together they became standard yardsticks for tracking improvements in chain-of-thought and mathematical reasoning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2103.03874", "arXiv:2103.03874" ], "version": 2 }, { "id": "P-443", "slug": "big-bench-bbh-helm", "title": "BIG-bench / BBH / HELM (holistic eval)", "shortTitle": "BIG-bench / BBH / HELM", "authors": [ "Aarohi Srivastava", "Mirac Suzgun", "Jason Wei", "Percy Liang", "Rishi Bommasani", "Tony Lee" ], "institutions": [ "Multiple", "Stanford" ], "year": 2022, "date": "2022", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2211.09110", "arxivId": "2211.09110", "doi": null, "repository": "https://github.com/stanford-crfm/helm", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "framework-enduring", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 2000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of broad-coverage evaluation efforts (BIG-bench, BIG-bench Hard, and HELM) that shifted LLM assessment from single-task accuracy to standardized measurement across many tasks and multiple metrics, exposing where scaling helps, where it doesn't, and trade-offs beyond accuracy.", "plainSummary": "As models grew, single-benchmark scores gave a narrow and often misleading picture of capability. BIG-bench assembled hundreds of diverse community tasks (with BBH isolating the subset where models then lagged humans), and HELM evaluated many models on a common set of scenarios reporting not just accuracy but calibration, robustness, fairness, bias, toxicity, and efficiency side by side. Together they made evaluation multi-dimensional and comparable across models, revealing capability gaps and cost/quality trade-offs that a lone accuracy number hides.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2211.09110", "arXiv:2211.09110" ], "version": 2 }, { "id": "P-444", "slug": "humaneval-mbpp", "title": "HumanEval / MBPP (function-level code)", "shortTitle": "HumanEval / MBPP", "authors": [ "Mark Chen", "Wojciech Zaremba", "Jacob Austin", "Augustus Odena", "Charles Sutton" ], "institutions": [ "OpenAI", "Google" ], "year": 2021, "date": "2021", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2108.07732", "arxivId": "2108.07732", "doi": null, "repository": "https://github.com/openai/human-eval", "openness": "unknown", "status": "current-but-contested", "statusRaw": "saturating", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 6000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Function-level program-synthesis benchmarks (HumanEval and MBPP) that score generated Python by executing it against hidden unit tests, removing the mismatch between surface-similarity text metrics and whether code actually runs correctly.", "plainSummary": "HumanEval (164 problems, from the Codex paper, arXiv 2107.03374) and MBPP (about 970 crowd-sourced entry-level problems, arXiv 2108.07732) each pair a natural-language prompt with test cases, and evaluate a model by whether sampled completions pass all tests. They introduced the pass@k metric, which estimates the chance that at least one of k sampled programs is correct, giving a functional rather than lexical measure of code generation. The two became the default yardsticks for code LLMs and the basis for later, harder coding benchmarks.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2108.07732", "arXiv:2108.07732" ], "version": 2 }, { "id": "P-445", "slug": "swe-bench", "title": "SWE-bench (repository-level SWE)", "shortTitle": "SWE-bench", "authors": [ "Carlos E. Jimenez", "John Yang", "et al." ], "institutions": [ "Princeton" ], "year": 2024, "date": "2023-10", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2310.06770", "arxivId": "2310.06770", "doi": null, "repository": "https://github.com/princeton-nlp/SWE-bench", "openness": "unknown", "status": "current-standard", "statusRaw": "current-critical", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": 1000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Evaluates language models on resolving real GitHub issues by requiring a repository-wide code patch that makes the project's actual test suite pass, removing the artificiality of self-contained single-function coding tasks.", "plainSummary": "The benchmark draws over 2,000 issue-and-pull-request pairs from popular Python repositories, giving the model an issue description and the codebase and asking it to produce a patch, then judging it by running the maintainers' fail-to-pass and pass-to-pass tests. Success demands navigating a large codebase, editing multiple files, and understanding project context, and at release even strong models solved only a low single-digit percentage. It became a leading measure of practical software-engineering ability and drove work on agentic coding systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.06770", "arXiv:2310.06770" ], "version": 2 }, { "id": "P-446", "slug": "agent-web-benchmarks", "title": "Agent/Web Benchmarks (AgentBench/WebArena/GAIA/tau-bench)", "shortTitle": "Agent/Web Benchmarks", "authors": [ "Xiao Liu", "Jie Tang", "Shuyan Zhou", "Graham Neubig", "Grégoire Mialon", "Thomas Scialom", "Shunyu Yao", "Karthik Narasimhan" ], "institutions": [ "Tsinghua", "CMU", "Meta" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2311.12983", "arxivId": "2311.12983", "doi": null, "repository": "https://github.com/web-arena-x/webarena", "openness": "unknown", "status": "niche-but-active", "statusRaw": "rising", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of interactive agent benchmarks (WebArena, AgentBench, GAIA, tau-bench) that evaluate models on multi-step tasks in realistic executable environments (live websites, tools, APIs, simulated customer service) rather than static prompts, measuring whether an agent actually completes a goal.", "plainSummary": "Static QA benchmarks cannot measure whether a model can plan and act over many steps against a stateful environment. WebArena provides self-hosted realistic websites where agents must navigate and complete tasks judged by end-state, AgentBench spans multiple environments (OS, database, web, games), GAIA poses real-world questions needing tool use and multi-hop reasoning, and tau-bench simulates tool-and-user customer-service dialogues. By scoring task completion in executable settings, they exposed a large gap between chat competence and reliable agentic execution.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2311.12983", "arXiv:2311.12983" ], "version": 2 }, { "id": "P-447", "slug": "chatbot-arena-llm-as-judge", "title": "Chatbot Arena / LLM-as-Judge (MT-Bench/AlpacaEval)", "shortTitle": "Chatbot Arena / LLM-as-Judge", "authors": [ "Lianmin Zheng", "Wei-Lin Chiang", "Ion Stoica", "Xuechen Li", "Yann Dubois", "Tatsunori B. Hashimoto" ], "institutions": [ "LMSYS", "UC Berkeley" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2403.04132", "arxivId": "2403.04132", "doi": null, "repository": "https://github.com/lm-sys/FastChat", "openness": "unknown", "status": "current-but-contested", "statusRaw": "high-but-biased", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 3000, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Introduced Chatbot Arena and MT-Bench, establishing crowd-sourced pairwise human voting and a strong-model-as-judge protocol as scalable ways to rank chat models on open-ended quality where fixed benchmarks fall short.", "plainSummary": "Chatbot Arena collects anonymous head-to-head comparisons: users chat with two unnamed models, vote for the better response, and the votes feed an Elo-style rating that ranks models. MT-Bench is a set of multi-turn questions scored by a strong model acting as an automated judge, which the paper validates against human preferences while also documenting judge biases such as favoring longer answers or the first response shown. Together they gave the field a way to evaluate conversational quality and instruction following that static multiple-choice benchmarks cannot capture.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2403.04132", "arXiv:2403.04132" ], "version": 2 }, { "id": "P-448", "slug": "contamination-resistant-benchmarks", "title": "Contamination-Resistant Benchmarks (TruthfulQA/LiveBench/SimpleQA)", "shortTitle": "Contamination-Resistant Benchmarks", "authors": [ "Stephanie Lin", "Owain Evans", "Colin White", "Tom Goldstein", "Jason Wei", "William Fedus" ], "institutions": [ "Oxford", "Abacus", "OpenAI" ], "year": 2021, "date": "2021-2024", "venue": null, "artifactType": "benchmark_paper", "domain": "evaluation", "url": "https://arxiv.org/abs/2406.19314", "arxivId": "2406.19314", "doi": null, "repository": "https://github.com/livebench/livebench", "openness": "unknown", "status": "current-standard", "statusRaw": "current-response", "confidence": "High", "difficulty": 4, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": 1500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "A family of contamination-resistant benchmarks (TruthfulQA, LiveBench, SWE-bench-verified) designed so scores are not inflated by training-set memorization or leaked test data, using adversarial questions, continuously refreshed recent items, or human-verified solvable tasks.", "plainSummary": "Because static public benchmarks leak into training corpora, high scores can reflect memorization rather than capability. These efforts attack that differently: TruthfulQA targets questions where imitating human text produces confident falsehoods, LiveBench continuously draws fresh questions from recent sources (papers, news, competitions) with objective ground truth and rotates them out to limit contamination, and SWE-bench-verified is a human-filtered subset of real GitHub issues confirmed to be well-specified and solvable. The shared idea is to keep evaluation valid over time by making the test set hard to have already seen or trivially matched.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2406.19314", "arXiv:2406.19314" ], "version": 2 }, { "id": "P-449", "slug": "the-evaluation-crisis", "title": "The Evaluation Crisis (contamination/saturation/judge-bias)", "shortTitle": "The Evaluation Crisis", "authors": [ "Oscar Sainz", "Zhang", "et al." ], "institutions": [ "Multiple" ], "year": 2023, "date": "2023-2024", "venue": null, "artifactType": "peer_reviewed", "domain": "evaluation", "url": "https://arxiv.org/abs/2310.18018", "arxivId": "2310.18018", "doi": null, "repository": "https://github.com/hitz-zentroa/lm-contamination", "openness": "unknown", "status": "corrective-result", "statusRaw": "essential-negative-result", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": 500, "source": "Google Scholar (est)", "retrievalDate": "not-live-verified", "verified": false }, "verificationTier": "verified", "thesis": "Analyzed the reliability problems undermining language model evaluation, showing how benchmark contamination, saturation, and biased automated judges inflate reported scores and distort model comparisons.", "plainSummary": "The work documents three linked failures in how models are scored: contamination, where test items leak into training data so results overstate ability; saturation, where top models cluster near the ceiling of older benchmarks so the numbers no longer separate them; and judge bias, where using a model as an automated grader introduces systematic errors like preferring verbose or self-similar answers. By characterizing these effects, it argues that many headline evaluation numbers are unreliable and motivates contamination checks, harder benchmarks, and more careful judging protocols.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2310.18018", "arXiv:2310.18018" ], "version": 2 }, { "id": "P-500", "slug": "zoom-in-an-introduction-to-circuits", "title": "Zoom In: An Introduction to Circuits", "shortTitle": "Zoom In", "authors": [ "Chris Olah", "Nick Cammarata", "Ludwig Schubert", "Gabriel Goh", "Michael Petrov", "Shan Carter" ], "institutions": [ "OpenAI" ], "year": 2020, "date": "2020-03", "venue": null, "artifactType": "technical_report", "domain": "interpretability", "url": "https://distill.pub/2020/circuits/zoom-in/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Argued that neural networks can be reverse-engineered into human-understandable circuits — features connected by weights — turning interpretability from vague intuition into a study of concrete mechanisms.", "plainSummary": "The essay lays out three claims: networks contain features (directions that detect meaningful concepts), features are connected by weighted circuits that implement algorithms, and analogous features and circuits recur across models. Working through vision networks, it shows curve detectors and other units that can be read and tested. It set the agenda and vocabulary for the mechanistic-interpretability program that followed.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://distill.pub/2020/circuits/zoom-in/" ], "version": 2 }, { "id": "P-501", "slug": "a-mathematical-framework-for-transformer-circuits", "title": "A Mathematical Framework for Transformer Circuits", "shortTitle": "A Mathematical Framework for Transformer Circuits", "authors": [ "Nelson Elhage", "Neel Nanda", "Catherine Olsson", "Tom Henighan", "Chris Olah" ], "institutions": [ "Anthropic" ], "year": 2021, "date": "2021-12", "venue": null, "artifactType": "technical_report", "domain": "interpretability", "url": "https://transformer-circuits.pub/2021/framework/index.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Provided a mathematical account of attention-only Transformers as compositions of interpretable operations, making the attention mechanism analyzable rather than opaque.", "plainSummary": "The report decomposes a Transformer into a residual stream that attention heads read from and write to, and shows how heads compose across layers to move and combine information. It introduces tools like the QK and OV circuits and identifies simple mechanisms in small models. This gave mechanistic interpretability a concrete framework for studying language models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://transformer-circuits.pub/2021/framework/index.html" ], "version": 2 }, { "id": "P-502", "slug": "in-context-learning-and-induction-heads", "title": "In-context Learning and Induction Heads", "shortTitle": "In-context Learning and Induction Heads", "authors": [ "Catherine Olsson", "Nelson Elhage", "Neel Nanda", "Nicholas Joseph", "Chris Olah" ], "institutions": [ "Anthropic" ], "year": 2022, "date": "2022-03", "venue": null, "artifactType": "technical_report", "domain": "interpretability", "url": "https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Identified induction heads — attention heads that complete repeated patterns — and argued they are a primary mechanism behind in-context learning in Transformers.", "plainSummary": "By tracking model behavior over training, the authors found that a phase change in in-context learning coincides with the formation of induction heads that look back for a previous occurrence of the current token and copy what followed. They present converging evidence that these heads drive much of a model’s ability to learn from its prompt. It connected a specific circuit to an emergent capability.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html" ], "version": 2 }, { "id": "P-503", "slug": "toy-models-of-superposition", "title": "Toy Models of Superposition", "shortTitle": "Toy Models of Superposition", "authors": [ "Nelson Elhage", "Tristan Hume", "Catherine Olsson", "Neel Nanda", "Chris Olah" ], "institutions": [ "Anthropic" ], "year": 2022, "date": "2022-09", "venue": null, "artifactType": "technical_report", "domain": "interpretability", "url": "https://transformer-circuits.pub/2022/toy_model/index.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that neural networks pack more features than they have dimensions by putting them in superposition, explaining why individual neurons are often polysemantic and hard to read.", "plainSummary": "Using small toy models, the authors demonstrate that when features are sparse a network represents many of them as overlapping directions rather than one-per-neuron, trading interference for capacity. They map when superposition appears and how it organizes into geometric structures. This reframed why interpretability is hard and motivated methods to pull features apart.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://transformer-circuits.pub/2022/toy_model/index.html" ], "version": 2 }, { "id": "P-504", "slug": "towards-monosemanticity-decomposing-language-models-with-dictionary-learning", "title": "Towards Monosemanticity: Decomposing Language Models With Dictionary Learning", "shortTitle": "Towards Monosemanticity", "authors": [ "Trenton Bricken", "Adly Templeton", "Joshua Batson", "Chris Olah" ], "institutions": [ "Anthropic" ], "year": 2023, "date": "2023-10", "venue": null, "artifactType": "technical_report", "domain": "interpretability", "url": "https://transformer-circuits.pub/2023/monosemantic-features/index.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 7, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Used sparse dictionary learning to decompose a language model’s activations into many monosemantic features, giving a scalable way to extract human-interpretable concepts from superposed representations.", "plainSummary": "The work trains a sparse autoencoder on the activations of a small language model and recovers thousands of features that each correspond to a single, nameable concept, resolving the superposition problem in practice. The features are interpretable, can be steered, and generalize. It became the template for the sparse-autoencoder interpretability work now applied to frontier models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://transformer-circuits.pub/2023/monosemantic-features/index.html" ], "version": 2 }, { "id": "P-510", "slug": "the-bitter-lesson", "title": "The Bitter Lesson", "shortTitle": "The Bitter Lesson", "authors": [ "Richard Sutton" ], "institutions": [ "University of Alberta" ], "year": 2019, "date": "2019-03", "venue": null, "artifactType": "essay", "domain": "scaling", "url": "http://www.incompleteideas.net/IncIdeas/BitterLesson.html", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 3, "influence": { "academic": 9, "frontier": 9, "engineering": 8, "industrial": 8, "openSource": 6, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Argued from 70 years of AI history that general methods leveraging more computation reliably beat approaches built on human-designed knowledge, a lesson repeatedly relearned and central to the scaling era.", "plainSummary": "Sutton observes that in game-playing, speech, and vision, hand-crafted domain knowledge gave early wins but was eventually overtaken by search and learning methods that scale with compute. The uncomfortable conclusion is that human insight matters less than the ability to exploit growing computation. The essay became a rallying text for scaling-first research.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "http://www.incompleteideas.net/IncIdeas/BitterLesson.html" ], "version": 2 }, { "id": "P-511", "slug": "the-scaling-hypothesis", "title": "The Scaling Hypothesis", "shortTitle": "The Scaling Hypothesis", "authors": [ "Gwern Branwen" ], "institutions": [ "Independent" ], "year": 2020, "date": "2020-05", "venue": null, "artifactType": "essay", "domain": "scaling", "url": "https://gwern.net/scaling-hypothesis", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "current-but-contested", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Argued that the capabilities of large language models are largely a smooth function of scale, so continuing to scale compute and data should keep producing gains without new architectural ideas.", "plainSummary": "Written after GPT-3, the essay collects the evidence that loss and many capabilities improve predictably with size, and contends that the field had underrated how far pure scaling would go. It frames scaling as a hypothesis to be taken seriously rather than a curiosity. It is one of the defining statements of the scaling-era worldview and its stance is debated.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://gwern.net/scaling-hypothesis" ], "version": 2 }, { "id": "P-512", "slug": "situational-awareness-the-decade-ahead", "title": "Situational Awareness: The Decade Ahead", "shortTitle": "Situational Awareness", "authors": [ "Leopold Aschenbrenner" ], "institutions": [ "Independent" ], "year": 2024, "date": "2024-06", "venue": null, "artifactType": "vision_essay", "domain": "scaling", "url": "https://situational-awareness.ai/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "current-but-contested", "confidence": "High", "difficulty": 3, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Extrapolated compute, algorithmic-efficiency, and investment trendlines to argue that broadly human-level AI could arrive within a few years, and drew out the national-security implications.", "plainSummary": "The long essay projects the straight lines on graphs of training compute and efficiency forward, argues that the gap from current models to human-level research assistants is a few more orders of magnitude, and discusses security and governance consequences. It is a prediction and advocacy piece, not an empirical result, and its timelines are contested. It captured a strand of frontier-lab thinking in 2024.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://situational-awareness.ai/" ], "version": 2 }, { "id": "P-513", "slug": "the-unreasonable-effectiveness-of-data", "title": "The Unreasonable Effectiveness of Data", "shortTitle": "The Unreasonable Effectiveness of Data", "authors": [ "Alon Halevy", "Peter Norvig", "Fernando Pereira" ], "institutions": [ "Google" ], "year": 2009, "date": "2009-03", "venue": null, "artifactType": "peer_reviewed", "domain": "data-tokenization", "url": "https://ieeexplore.ieee.org/document/4804817", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-precursor", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 9, "industrial": 7, "openSource": 5, "underappreciated": 3 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Argued that for many hard problems, simple models trained on far more data outperform sophisticated models on less, shifting emphasis from clever algorithms to data scale.", "plainSummary": "Drawing on web-scale examples, the authors contend that unreasonably large corpora let comparatively simple methods succeed where elaborate ones with little data fail, and urge researchers to embrace data. Written years before deep learning’s dominance, it foreshadowed the data-centric logic later formalized by scaling laws. It is a conceptual ancestor of the scaling era.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://ieeexplore.ieee.org/document/4804817" ], "version": 2 }, { "id": "P-514", "slug": "ai-and-compute", "title": "AI and Compute", "shortTitle": "AI and Compute", "authors": [ "Dario Amodei", "Danny Hernandez" ], "institutions": [ "OpenAI" ], "year": 2018, "date": "2018-05", "venue": null, "artifactType": "technical_report", "domain": "scaling", "url": "https://openai.com/index/ai-and-compute/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-precursor", "confidence": "High", "difficulty": 3, "influence": { "academic": 7, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Documented that the compute used in the largest AI training runs had grown roughly exponentially, quantifying the trend that later scaling laws and frontier models would ride.", "plainSummary": "The analysis charts training compute across landmark systems and finds a doubling time far faster than Moore’s law over the studied period. It made the compute-growth trend legible and widely cited as evidence that scaling, not just architecture, was driving progress. It is an early empirical anchor for the scaling narrative.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://openai.com/index/ai-and-compute/" ], "version": 2 }, { "id": "P-520", "slug": "webgpt-browser-assisted-question-answering-with-human-feedback", "title": "WebGPT: Browser-assisted Question-answering with Human Feedback", "shortTitle": "WebGPT", "authors": [ "Reiichiro Nakano", "Jacob Hilton", "Suchir Balaji", "John Schulman" ], "institutions": [ "OpenAI" ], "year": 2021, "date": "2021-12", "venue": null, "artifactType": "preprint", "domain": "agents", "url": "https://arxiv.org/abs/2112.09332", "arxivId": "2112.09332", "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "partially-superseded", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Fine-tuned a language model to answer questions by browsing the web and citing sources, using human feedback to reward well-supported answers and reducing unsupported claims.", "plainSummary": "WebGPT gives GPT-3 a text-based browser and trains it with human comparisons to search, navigate, and quote evidence, then optimizes against a learned reward model. The result answers long-form questions with references that raters prefer, at the cost of sometimes over-trusting sources. It was an early demonstration of tool use and retrieval-augmented answering with RLHF.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2112.09332", "arXiv:2112.09332" ], "version": 2 }, { "id": "P-521", "slug": "do-as-i-can-not-as-i-say-grounding-language-in-robotic-affordances", "title": "Do As I Can, Not As I Say: Grounding Language in Robotic Affordances (SayCan)", "shortTitle": "Do As I Can, Not As I Say", "authors": [ "Michael Ahn", "Anthony Brohan", "Noah Brown", "Karol Hausman" ], "institutions": [ "Google" ], "year": 2022, "date": "2022-04", "venue": null, "artifactType": "preprint", "domain": "agents", "url": "https://arxiv.org/abs/2204.01691", "arxivId": "2204.01691", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-precursor", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Grounded a language model’s plans in what a robot can actually do by combining the model’s task knowledge with learned affordance values, letting an LLM direct real-world robotic behavior.", "plainSummary": "SayCan scores candidate skills by multiplying an LLM’s estimate that a skill helps achieve the instruction with a value function’s estimate that the skill is currently feasible, so the robot picks useful and possible actions. This let a robot carry out long natural-language instructions in a kitchen. It was an influential early bridge from language models to embodied action.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2204.01691", "arXiv:2204.01691" ], "version": 2 }, { "id": "P-522", "slug": "solving-quantitative-reasoning-problems-with-language-models", "title": "Solving Quantitative Reasoning Problems with Language Models (Minerva)", "shortTitle": "Solving Quantitative Reasoning Problems with Language Models", "authors": [ "Aitor Lewkowycz", "Anders Andreassen", "David Dohan", "Vinay Ramasesh" ], "institutions": [ "Google" ], "year": 2022, "date": "2022-06", "venue": null, "artifactType": "preprint", "domain": "reasoning", "url": "https://arxiv.org/abs/2206.14858", "arxivId": "2206.14858", "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "partially-superseded", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that a large model trained on mathematical and scientific text, with chain-of-thought and majority voting, could solve quantitative reasoning problems far better than prior models without external tools.", "plainSummary": "Minerva continues pretraining a large language model on math-heavy web and arXiv data, then uses step-by-step prompting and samples many solutions to vote on the answer. It substantially raised scores on MATH and STEM problem sets using only the model’s own reasoning. It marked how far scaled models plus test-time sampling could push mathematical reasoning.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2206.14858", "arXiv:2206.14858" ], "version": 2 }, { "id": "P-523", "slug": "on-the-dangers-of-stochastic-parrots-can-language-models-be-too-big", "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?", "shortTitle": "On the Dangers of Stochastic Parrots", "authors": [ "Emily M. Bender", "Timnit Gebru", "Angelina McMillan-Major", "Margaret Mitchell" ], "institutions": [ "University of Washington", "Google" ], "year": 2021, "date": "2021-03", "venue": null, "artifactType": "critique", "domain": "evaluation", "url": "https://dl.acm.org/doi/10.1145/3442188.3445922", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "current-but-contested", "confidence": "High", "difficulty": 4, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Argued that ever-larger language models carry underexamined costs — environmental, financial, and social — and that fluency without grounding risks mistaking pattern completion for understanding.", "plainSummary": "The paper contends that scaling training data and compute concentrates benefits, raises environmental and labor costs, encodes and amplifies bias from unaudited web text, and produces systems that stitch together form without meaning (the “stochastic parrot”). It calls for documentation, curation, and restraint. It became the defining critical counterpoint to the scale-first paradigm and remains contested.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://dl.acm.org/doi/10.1145/3442188.3445922" ], "version": 2 }, { "id": "P-530", "slug": "zero-shot-text-to-image-generation", "title": "Zero-Shot Text-to-Image Generation (DALL-E)", "shortTitle": "Zero-Shot Text-to-Image Generation", "authors": [ "Aditya Ramesh", "Mikhail Pavlov", "Gabriel Goh", "Scott Gray" ], "institutions": [ "OpenAI" ], "year": 2021, "date": "2021-02", "venue": null, "artifactType": "preprint", "domain": "multimodal", "url": "https://arxiv.org/abs/2102.12092", "arxivId": "2102.12092", "doi": null, "repository": null, "openness": "unknown", "status": "partially-superseded", "statusRaw": "partially-superseded", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that a single autoregressive Transformer over combined text and image tokens could generate coherent images from natural-language captions, demonstrating zero-shot text-to-image generation at scale.", "plainSummary": "DALL-E encodes images as discrete tokens and trains a large Transformer to model the sequence of text followed by image tokens, then samples images from captions and reranks them with CLIP. It produced novel, compositional images without task-specific training. It opened the modern text-to-image line later dominated by diffusion methods.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2102.12092", "arXiv:2102.12092" ], "version": 2 }, { "id": "P-531", "slug": "robust-speech-recognition-via-large-scale-weak-supervision", "title": "Robust Speech Recognition via Large-Scale Weak Supervision (Whisper)", "shortTitle": "Robust Speech Recognition via Large-Scale Weak Supervision", "authors": [ "Alec Radford", "Jong Wook Kim", "Tao Xu", "Greg Brockman" ], "institutions": [ "OpenAI" ], "year": 2022, "date": "2022-12", "venue": null, "artifactType": "preprint", "domain": "multimodal", "url": "https://arxiv.org/abs/2212.04356", "arxivId": "2212.04356", "doi": null, "repository": "https://github.com/openai/whisper", "openness": "unknown", "status": "current-standard", "statusRaw": "current-standard", "confidence": "High", "difficulty": 5, "influence": { "academic": 8, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Trained a single speech model on a very large, weakly-labeled multilingual corpus, achieving robust zero-shot transcription and translation without dataset-specific fine-tuning.", "plainSummary": "Whisper is an encoder-decoder Transformer trained on hundreds of thousands of hours of diverse audio-text pairs scraped from the web, covering many languages and tasks in one model. It approaches specialized systems zero-shot and is resilient to accents and noise. Its open release made strong speech recognition broadly available.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2212.04356", "arXiv:2212.04356" ], "version": 2 }, { "id": "P-532", "slug": "photorealistic-text-to-image-diffusion-models-with-deep-language-understanding", "title": "Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding (Imagen)", "shortTitle": "Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding", "authors": [ "Chitwan Saharia", "William Chan", "Saurabh Saxena", "Mohammad Norouzi" ], "institutions": [ "Google" ], "year": 2022, "date": "2022-05", "venue": null, "artifactType": "preprint", "domain": "multimodal", "url": "https://arxiv.org/abs/2205.11487", "arxivId": "2205.11487", "doi": null, "repository": null, "openness": "unknown", "status": "influential-precursor", "statusRaw": "influential-precursor", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that conditioning an image-diffusion model on a large frozen text encoder yields high-fidelity, well-aligned text-to-image generation, highlighting the text encoder as the key lever.", "plainSummary": "Imagen pairs a frozen T5-XXL language-model text encoder with a cascade of diffusion models and finds that scaling the text encoder improves image-text alignment more than scaling the image model. It reached strong photorealism and prompt fidelity on standard evaluations. It clarified the role of language understanding in text-to-image systems.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2205.11487", "arXiv:2205.11487" ], "version": 2 }, { "id": "P-533", "slug": "video-generation-models-as-world-simulators", "title": "Video Generation Models as World Simulators (Sora)", "shortTitle": "Video Generation Models as World Simulators", "authors": [ "Tim Brooks", "Bill Peebles", "OpenAI" ], "institutions": [ "OpenAI" ], "year": 2024, "date": "2024-02", "venue": null, "artifactType": "technical_report", "domain": "multimodal", "url": "https://openai.com/index/video-generation-models-as-world-simulators/", "arxivId": null, "doi": null, "repository": null, "openness": "unknown", "status": "current-but-contested", "statusRaw": "current-but-contested", "confidence": "High", "difficulty": 5, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Framed video generation as training a diffusion Transformer on patches of video and image data at scale, producing minute-long coherent videos and evidence of emergent world-simulation behavior.", "plainSummary": "Sora represents videos as spacetime patches and trains a diffusion Transformer over them, scaling data and compute rather than baking in physics. The technical report shows long, high-resolution, temporally consistent clips and argues that scale yields rudimentary simulation of objects and dynamics. It signaled diffusion-Transformer scaling as a path to video and world models.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://openai.com/index/video-generation-models-as-world-simulators/" ], "version": 2 }, { "id": "P-540", "slug": "highly-accurate-protein-structure-prediction-with-alphafold", "title": "Highly Accurate Protein Structure Prediction with AlphaFold", "shortTitle": "Highly Accurate Protein Structure Prediction with AlphaFold", "authors": [ "John Jumper", "Richard Evans", "Alexander Pritzel", "Tim Green", "Demis Hassabis" ], "institutions": [ "DeepMind" ], "year": 2021, "date": "2021-07", "venue": null, "artifactType": "peer_reviewed", "domain": "multimodal", "url": "https://www.nature.com/articles/s41586-021-03819-2", "arxivId": null, "doi": null, "repository": "https://github.com/google-deepmind/alphafold", "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 7, "influence": { "academic": 9, "frontier": 8, "engineering": 7, "industrial": 7, "openSource": 5, "underappreciated": 0 }, "importanceScore": 9, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that a deep-learning system built on attention could predict a protein 3D structure from its amino-acid sequence at near-experimental accuracy, solving a decades-old grand challenge in biology and demonstrating that attention-based models generalize far beyond language.", "plainSummary": "AlphaFold pairs an attention-based Evoformer that reasons jointly over multiple-sequence alignments and residue-pair representations with a structure module that directly predicts atomic coordinates, trained end-to-end with protein geometry built in. At the CASP14 assessment it predicted many structures to within experimental error, a step change over prior methods. DeepMind released the code and a public database of predicted structures, reshaping computational structural biology.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://www.nature.com/articles/s41586-021-03819-2" ], "version": 2 }, { "id": "P-550", "slug": "distilling-the-knowledge-in-a-neural-network", "title": "Distilling the Knowledge in a Neural Network", "shortTitle": "Distilling the Knowledge in a Neural Network", "authors": [ "Geoffrey Hinton", "Oriol Vinyals", "Jeff Dean" ], "institutions": [ "Google" ], "year": 2015, "date": "2015-03", "venue": null, "artifactType": "preprint", "domain": "inference-serving", "url": "https://arxiv.org/abs/1503.02531", "arxivId": "1503.02531", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 5, "influence": { "academic": 9, "frontier": 9, "engineering": 10, "industrial": 8, "openSource": 6, "underappreciated": 2 }, "importanceScore": 9, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Showed that a small student network can be trained to match a large teacher’s softened output distribution, transferring most of the teacher’s accuracy into a far cheaper model and turning model size into a deployment choice rather than a fixed cost.", "plainSummary": "Instead of training only on hard labels, the student is trained on the teacher’s full probability distribution softened by a temperature, so it inherits the relative confidences the teacher assigns across classes — information the hard label throws away. Students recover much of the teacher’s performance at a fraction of the parameters and compute. Distillation became a standard compression tool and, later, the way strong small language models are trained from frontier teachers.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1503.02531", "arXiv:1503.02531" ], "version": 2 }, { "id": "P-551", "slug": "overcoming-catastrophic-forgetting-in-neural-networks", "title": "Overcoming Catastrophic Forgetting in Neural Networks", "shortTitle": "Overcoming Catastrophic Forgetting in Neural Networks", "authors": [ "James Kirkpatrick", "Razvan Pascanu", "Neil Rabinowitz", "Raia Hadsell", "Demis Hassabis" ], "institutions": [ "DeepMind" ], "year": 2017, "date": "2017-03", "venue": null, "artifactType": "peer_reviewed", "domain": "continual-learning", "url": "https://arxiv.org/abs/1612.00796", "arxivId": "1612.00796", "doi": null, "repository": null, "openness": "unknown", "status": "foundational-and-current", "statusRaw": "foundational-and-current", "confidence": "High", "difficulty": 6, "influence": { "academic": 8, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 0 }, "importanceScore": 8, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Introduced elastic weight consolidation, which slows learning on the parameters most important to previously learned tasks, letting one network learn new tasks without catastrophically forgetting the old ones.", "plainSummary": "The method estimates how sensitive prior-task performance is to each weight using the Fisher information, then adds a quadratic penalty anchoring the important weights while leaving the rest free to adapt. A single network could learn a sequence of tasks and retain earlier ones where ordinary training would overwrite them. It turned catastrophic forgetting from a vague failure into a measurable, tractable problem and anchored the continual-learning literature.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/1612.00796", "arXiv:1612.00796" ], "version": 2 }, { "id": "P-552", "slug": "titans-learning-to-memorize-at-test-time", "title": "Titans: Learning to Memorize at Test Time", "shortTitle": "Titans", "authors": [ "Ali Behrouz", "Peilin Zhong", "Vahab Mirrokni" ], "institutions": [ "Google Research" ], "year": 2025, "date": "2025-01", "venue": null, "artifactType": "preprint", "domain": "continual-learning", "url": "https://arxiv.org/abs/2501.00663", "arxivId": "2501.00663", "doi": null, "repository": "https://github.com/ABehrouz/Titans", "openness": "unknown", "status": "niche-but-active", "statusRaw": "niche-but-active", "confidence": "High", "difficulty": 6, "influence": { "academic": 7, "frontier": 7, "engineering": 6, "industrial": 6, "openSource": 4, "underappreciated": 1 }, "importanceScore": 7, "citations": { "approx": null, "source": null, "retrievalDate": null, "verified": false }, "verificationTier": "verified", "thesis": "Introduced a neural long-term memory module that learns what to memorize at test time, letting a model write new information into parameters during inference rather than staying frozen after training.", "plainSummary": "Titans pairs short-term attention with a deep neural memory updated online by a surprise-based gradient signal, so salient new information is written into the memory as the model reads, with selective forgetting. This yields effective context well beyond the attention window while keeping inference cost bounded. It is a leading example of the test-time-learning branch, where memory lives in parameters updated at inference rather than only in the context window.", "createdDate": "2026-07-13", "lastReviewed": "2026-07-14", "sources": [ "https://arxiv.org/abs/2501.00663", "arXiv:2501.00663" ], "version": 2 } ]