ref.bib

@inproceedings{jawahar2019does,
	title={What does BERT learn about the structure of language?},
	author={Jawahar, Ganesh and Sagot, Beno{\^\i}t and Seddah, Djam{\'e}},
	year={2019}
}
@article{tenney2019bert,
	title={Bert rediscovers the classical nlp pipeline},
	author={Tenney, Ian and Das, Dipanjan and Pavlick, Ellie},
	journal={arXiv preprint arXiv:1905.05950},
	year={2019}
}
@article{klafka2020spying,
	title={Spying on your neighbors: Fine-grained probing of contextual embeddings for information about surrounding words},
	author={Klafka, Josef and Ettinger, Allyson},
	journal={arXiv preprint arXiv:2005.01810},
	year={2020}
}
@article{liu2019linguistic,
	title={Linguistic knowledge and transferability of contextual representations},
	author={Liu, Nelson F and Gardner, Matt and Belinkov, Yonatan and Peters, Matthew and Smith, Noah A},
	journal={arXiv preprint arXiv:1903.08855},
	year={2019}
}
@inproceedings{lin2019open,
  title={Open Sesame: Getting inside BERT’s Linguistic Knowledge},
  author={Lin, Yongjie and Tan, Yi Chern and Frank, Robert},
  booktitle={Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP},
  pages={241--253},
  year={2019}
}
@article{tenney2019you,
	title={What do you learn from context? probing for sentence structure in contextualized word representations},
	author={Tenney, Ian and Xia, Patrick and Chen, Berlin and Wang, Alex and Poliak, Adam and McCoy, R Thomas and Kim, Najoung and Van Durme, Benjamin and Bowman, Samuel R and Das, Dipanjan and others},
	journal={arXiv preprint arXiv:1905.06316},
	year={2019}
}
@article{devlin2018bert,
	title={Bert: Pre-training of deep bidirectional transformers for language understanding},
	author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
	journal={arXiv preprint arXiv:1810.04805},
	year={2018}
}
@article{goldberg2019assessing,
	title={Assessing BERT's Syntactic Abilities},
	author={Goldberg, Yoav},
	journal={arXiv preprint arXiv:1901.05287},
	year={2019}
}
@article{coenen2019visualizing,
	title={Visualizing and measuring the geometry of bert},
	author={Coenen, Andy and Reif, Emily and Yuan, Ann and Kim, Been and Pearce, Adam and Vi{\'e}gas, Fernanda and Wattenberg, Martin},
	journal={arXiv preprint arXiv:1906.02715},
	year={2019}
}
@inproceedings{hewitt2019structural,
	title={A structural probe for finding syntax in word representations},
	author={Hewitt, John and Manning, Christopher D},
	booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
	pages={4129--4138},
	year={2019}
}
@article{vig2019analyzing,
	title={Analyzing the structure of attention in a transformer language model},
	author={Vig, Jesse and Belinkov, Yonatan},
	journal={arXiv preprint arXiv:1906.04284},
	year={2019}
}
@article{clark2019does,
	title={What Does BERT Look At? An Analysis of BERT's Attention},
	author={Clark, Kevin and Khandelwal, Urvashi and Levy, Omer and Manning, Christopher D},
	journal={arXiv preprint arXiv:1906.04341},
	year={2019}
}

@inproceedings{
brunner2020identifiability,
title={On Identifiability in Transformers},
author={Gino Brunner and Yang Liu and Damian Pascual and Oliver Richter and Massimiliano Ciaramita and Roger Wattenhofer},
booktitle={International Conference on Learning Representations},
year={2020},
}

@inproceedings{kovaleva2019revealing,
    title = "Revealing the Dark Secrets of {BERT}",
    author = "Kovaleva, Olga  and
      Romanov, Alexey  and
      Rogers, Anna  and
      Rumshisky, Anna",
    publisher = "Association for Computational Linguistics",

}

@inproceedings{voita2019analyzing,
  author={Elena Voita and David Talbot and Fedor Moiseev and Rico Sennrich and Ivan Titov},
  title={Analyzing Multi-Head Self-Attention: Specialized Heads Do the Heavy Lifting, the Rest Can Be Pruned},
  year={2019},
  booktitle={ACL (1)},
}
@inproceedings{michel2019sixteen,
	title={Are Sixteen Heads Really Better than One?},
	author={Michel, Paul and Levy, Omer and Neubig, Graham},
	booktitle={Advances in Neural Information Processing Systems},
	pages={14014--14024},
	year={2019}
}

@inproceedings{serrano2019attention,
  title={Is Attention Interpretable?},
  author={Sofia Serrano and Noah A. Smith},
  booktitle={ACL},
  year={2019}
}

@article{de2020decisions,
	title={How do decisions emerge across layers in neural models? interpretation with differentiable masking},
	author={De Cao, Nicola and Schlichtkrull, Michael and Aziz, Wilker and Titov, Ivan},
	journal={arXiv preprint arXiv:2004.14992},
	year={2020}
}

@inproceedings{
jin2019towards,
title={Towards Hierarchical Importance Attribution: Explaining Compositional Semantics for Neural Sequence Models},
author={Xisen Jin and Zhongyu Wei and Junyi Du and Xiangyang Xue and Xiang Ren},
booktitle={International Conference on Learning Representations},
year={2020},
}

@inproceedings{
dhamdhere2018important,
title={How Important is a Neuron},
author={Kedar Dhamdhere and Mukund Sundararajan and Qiqi Yan},
booktitle={International Conference on Learning Representations},
year={2019},
}

@inproceedings{Vaswani2017AttentionIA,
author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N. and Kaiser, undefinedukasz and Polosukhin, Illia},
title = {Attention is All You Need},
year = {2017},
booktitle = {Proceedings of the 31st International Conference on Neural Information Processing Systems},
}


@inproceedings{datta2016algorithmic,
	title={Algorithmic transparency via quantitative input influence: Theory and experiments with learning systems},
	author={Datta, Anupam and Sen, Shayak and Zick, Yair},
	booktitle={2016 IEEE symposium on security and privacy (SP)},
	pages={598--617},
	year={2016},
	organization={IEEE}
}
@article{omlin1996constructing,
	title={Constructing deterministic finite-state automata in recurrent neural networks},
	author={Omlin, Christian W and Giles, C Lee},
	journal={Journal of the ACM (JACM)},
	volume={43},
	number={6},
	pages={937--972},
	year={1996},
	publisher={ACM New York, NY, USA}
}
@article{gers2001context,
	author = {Gers, F. A. and Schmidhuber, E.},
	title = {LSTM Recurrent Networks Learn Simple Context-free and Context-sensitive Languages},
	journal = {Trans. Neur. Netw.},
	year = {2001},
	address = {Piscataway, NJ, USA},
}
@article{linzen2016assessing,
	title={Assessing the ability of LSTMs to learn syntax-sensitive dependencies},
	author={Linzen, Tal and Dupoux, Emmanuel and Goldberg, Yoav},
	journal={Transactions of the Association for Computational Linguistics},
	year={2016},
}
@article{bach2015pixel,
	title={On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation},
	author={Bach, Sebastian and Binder, Alexander and Montavon, Gr{\'e}goire and Klauschen, Frederick and M{\"u}ller, Klaus-Robert and Samek, Wojciech},
	journal={PloS one},
	year={2015},
	publisher={Public Library of Science}
}

%repeated:
article{dhamdhere2018important,
	title={How important is a neuron?},
	author={Dhamdhere, Kedar and Sundararajan, Mukund and Yan, Qiqi},
	journal={arXiv preprint arXiv:1805.12233},
	year={2018}
}

@article{arenas2015make,
	title={How to make a midbrain dopaminergic neuron},
	author={Arenas, Ernest and Denham, Mark and Villaescusa, J Carlos},
	journal={Development},
	volume={142},
	number={11},
	pages={1918--1936},
	year={2015},
	publisher={Oxford University Press for The Company of Biologists Limited}
}
@article{alishahi2019analyzing,
	title =	 {Analyzing and Interpreting Neural Networks for NLP: A Report on the First
	BlackboxNLP Workshop},
	author =	 {Alishahi, Afra and Chrupa{\l}a, Grzegorz and Linzen, Tal},
	journal =	 {arXiv preprint arXiv:1904.04063},
	year =	 {2019}
}
@inproceedings{
murdoch2018beyond,
title={Beyond Word Importance:  Contextual Decomposition to Extract Interactions from {LSTM}s},
author={W. James Murdoch and Peter J. Liu and Bin Yu},
booktitle={International Conference on Learning Representations},
year={2018},
}

@article{hochreiter1997long,
	title={Long short-term memory},
	author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
	journal={Neural computation},
	volume={9},
	number={8},
	pages={1735--1780},
	year={1997},
	publisher={MIT Press}
}
@inproceedings{sundararajan2017axiomatic,
	title={Axiomatic attribution for deep networks},
	author={Sundararajan, Mukund and Taly, Ankur and Yan, Qiqi},
	booktitle={Proceedings of the 34th International Conference on Machine Learning-Volume 70},
	pages={3319--3328},
	year={2017},
	organization={JMLR. org}
}
@phdthesis{lei2017interpretable,
	title={Interpretable neural models for natural language processing},
	author={Lei, Tao and others},
	year={2017},
	school={Massachusetts Institute of Technology}
}
@inproceedings{leino2018influence,
	title={Influence-directed explanations for deep convolutional networks},
	author={Leino, Klas and Sen, Shayak and Datta, Anupam and Fredrikson, Matt and Li, Linyi},
	booktitle={2018 IEEE International Test Conference (ITC)},
	pages={1--8},
	year={2018},
	organization={IEEE}
}
@article{simonyan13saliency,
	title={Deep Inside Convolutional Networks: Visualising Image Classification Models and Saliency Maps},
	author={Karen Simonyan and Andrea Vedaldi and Andrew Zisserman},
	year={2013},
	eprint={1312.6034},
	archivePrefix={arXiv},
	primaryClass={cs.CV}
}

@article{selvaraju16gradcam,
	author    = {Ramprasaath R. Selvaraju and
		Abhishek Das and
		Ramakrishna Vedantam and
		Michael Cogswell and
		Devi Parikh and
	Dhruv Batra},
	title     = {Grad-CAM: Why did you say that? Visual Explanations from Deep Networks
	via Gradient-based Localization},
	journal   = {CoRR},

}
@article{ribeiro16lime,
	author    = {Marco T{\'{u}}lio Ribeiro and
		Sameer Singh and
	Carlos Guestrin},
	title     = {"Why Should {I} Trust You?": Explaining the Predictions of Any Classifier},
	journal   = {CoRR},
	year      = {2016},
}
@inproceedings{fiacco2019deep,
	title={Deep neural model inspection and comparison via functional neuron pathways},
	author={Fiacco, James and Choudhary, Samridhi and Rose, Carolyn},
	booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
	pages={5754--5764},
	year={2019}
}
@article{karpathy2015visualizing,
	title={Visualizing and understanding recurrent networks},
	author={Karpathy, Andrej and Johnson, Justin and Fei-Fei, Li},
	journal={arXiv preprint arXiv:1506.02078},
	year={2015}
}
@article{hochreiter1998vanishing,
	title={The vanishing gradient problem during learning recurrent neural nets and problem solutions},
	author={Hochreiter, Sepp},
	journal={International Journal of Uncertainty, Fuzziness and Knowledge-Based Systems},
}
@article{greff2017lstm,
	title={LSTM: A search space odyssey},
	author={Greff, Klaus and Srivastava, Rupesh K and Koutn{\'\i}k, Jan and Steunebrink, Bas R and Schmidhuber, J{\"u}rgen},
	journal={IEEE transactions on neural networks and learning systems},
	year={2017},
	publisher={IEEE}
}
@article{lakretz2019emergence,
	title={The emergence of number and syntax units in LSTM language models},
	author={Lakretz, Yair and Kruszewski, German and Desbordes, Theo and Hupkes, Dieuwke and Dehaene, Stanislas and Baroni, Marco},
	journal={arXiv preprint arXiv:1903.07435},
	year={2019}
}
@article{hupkes2018visualisation,
	title={Visualisation and'diagnostic classifiers' reveal how recurrent and recursive neural networks process hierarchical structure},
	author={Hupkes, Dieuwke and Veldhoen, Sara and Zuidema, Willem},
	journal={Journal of Artificial Intelligence Research},
	year={2018}
}
@article{verwimp2018state,
	title={State gradients for RNN memory analysis},
	author={Verwimp, Lyan and Renkens, Vincent and Wambacq, Patrick and others},
	journal={arXiv preprint arXiv:1805.04264},
	year={2018}
}
@article{Gulordavaa,
	archivePrefix = {arXiv},
	arxivId = {1803.11138v1},
	author = {Gulordava, Kristina and Bojanowski, Piotr and Grave, Edouard and Linzen, Tal and Baroni, Marco},
	eprint = {1803.11138v1},
}
@techreport{Alishahi2019,
	archivePrefix = {arXiv},
	arxivId = {1904.04063v1},
	author = {Alishahi, Afra and Chrupa{\l}a, Grzegorz and Linzen, Tal},
	title = {{Analyzing and Interpreting Neural Networks for NLP: A Report on the First BlackboxNLP Workshop}},
	year = {2019}
}
@inproceedings{marvin2018targeted,
  title={Targeted Syntactic Evaluation of Language Models},
  author={Marvin, Rebecca and Linzen, Tal},
  booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
  pages={1192--1202},
  year={2018}
}
@article{turc2019,
  title={Well-Read Students Learn Better: On the Importance of Pre-training Compact Models},
  author={Turc, Iulia and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1908.08962v2 },
  year={2019}
}

@inproceedings{hewitt2019designing,
  title={Designing and Interpreting Probes with Control Tasks},
  author={Hewitt, John and Liang, Percy},
  booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
  pages={2733--2743},
  year={2019}
}
@techreport{Kuncoro,
	abstract = {Language exhibits hierarchical structure, but recent work using a subject-verb agreement diagnostic argued that state-of-the-art language models, LSTMs, fail to learn long-range syntax-sensitive dependencies. Using the same diagnostic, we show that, in fact, LSTMs do succeed in learning such dependencies-provided they have enough capacity. We then explore whether models that have access to explicit syntactic information learn agreement more effectively, and how the way in which this structural information is incorporated into the model impacts performance. We find that the mere presence of syntactic information does not improve accuracy , but when model architecture is determined by syntax, number agreement is improved. Further, we find that the choice of how syntactic structure is built affects how well number agreement is learned: top-down construction outperforms left-corner and bottom-up variants in capturing long-distance structural dependencies.},
	author = {Kuncoro, Adhiguna and Dyer, Chris and Hale, John and Yogatama, Dani and Clark, Stephen and Blunsom, Phil},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Kuncoro et al. - Unknown - LSTMs Can Learn Syntax-Sensitive Dependencies Well, But Modeling Structure Makes Them Better.pdf:pdf},
	pages = {1426--1436},
	publisher = {Association for Computational Linguistics},
	title = {{LSTMs Can Learn Syntax-Sensitive Dependencies Well, But Modeling Structure Makes Them Better}},
	url = {https://github.com/tensorflow/models/}
}
@techreport{Linzen,
	abstract = {The success of long short-term memory (LSTM) neural networks in language processing is typically attributed to their ability to capture long-distance statistical regularities. Linguistic regularities are often sensitive to syntactic structure; can such dependencies be captured by LSTMs, which do not have explicit structural representations? We begin addressing this question using number agreement in English subject-verb dependencies. We probe the architecture's grammatical competence both using training objectives with an explicit grammatical target (number prediction, grammaticality judgments) and using language models. In the strongly supervised settings, the LSTM achieved very high overall accuracy (less than 1{\%} errors), but errors increased when sequential and structural information conflicted. The frequency of such errors rose sharply in the language-modeling setting. We conclude that LSTMs can capture a non-trivial amount of grammatical structure given targeted supervision, but stronger architectures may be required to further reduce errors; furthermore, the language modeling signal is insufficient for capturing syntax-sensitive dependencies, and should be supplemented with more direct supervision if such dependencies need to be captured.},
	archivePrefix = {arXiv},
	arxivId = {1611.01368v1},
	author = {Linzen, Tal and Dupoux, Emmanuel and Goldberg, Yoav},
	eprint = {1611.01368v1},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Linzen, Dupoux, Goldberg - Unknown - Assessing the Ability of LSTMs to Learn Syntax-Sensitive Dependencies(2).pdf:pdf},
	title = {{Assessing the Ability of LSTMs to Learn Syntax-Sensitive Dependencies}},
	url = {https://arxiv.org/pdf/1611.01368.pdf}
}
@techreport{Lakretz,
	abstract = {Recent work has shown that LSTMs trained on a generic language modeling objective capture syntax-sensitive generalizations such as long-distance number agreement. We have however no mechanistic understanding of how they accomplish this remarkable feat. Some have conjectured it depends on heuristics that do not truly take hierarchical structure into account. We present here a detailed study of the inner mechanics of number tracking in LSTMs at the single neuron level. We discover that long-distance number information is largely managed by two "number units". Importantly, the behaviour of these units is partially controlled by other units independently shown to track syntactic structure. We conclude that LSTMs are, to some extent, implementing genuinely syntactic processing mechanisms, paving the way to a more general understanding of grammatical encoding in LSTMs.},
	annote = {This paper high lights},
	author = {Lakretz, Yair and Kruszewski, German and Desbordes, Theo and Hupkes, Dieuwke and Dehaene, Stanislas and Baroni, Marco},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Lakretz et al. - Unknown - The emergence of number and syntax units in LSTM language models.pdf:pdf},
	title = {{The emergence of number and syntax units in LSTM language models}},
	url = {https://research.fb.com/wp-content/uploads/2019/03/The-emergence-of-number-and-syntax-units-in-LSTM-language-models.pdf?}
}
@article{Giulianelli2018,
	abstract = {How do neural language models keep track of number agreement between subject and verb? We show that `diagnostic classifiers', trained to predict number from the internal states of a language model, provide a detailed understanding of how, when, and where this information is represented. Moreover, they give us insight into when and where number information is corrupted in cases where the language model ends up making agreement errors. To demonstrate the causal role played by the representations we find, we then use agreement information to influence the course of the LSTM during the processing of difficult sentences. Results from such an intervention reveal a large increase in the language model's accuracy. Together, these results show that diagnostic classifiers give us an unrivalled detailed look into the representation of linguistic information in neural models, and demonstrate that this knowledge can be used to improve their performance.},
	archivePrefix = {arXiv},
	arxivId = {1808.08079},
	author = {Giulianelli, Mario and Harding, Jack and Mohnert, Florian and Hupkes, Dieuwke and Zuidema, Willem},
	eprint = {1808.08079},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Giulianelli et al. - 2018 - Under the Hood Using Diagnostic Classifiers to Investigate and Improve how Language Models Track Agreement I.pdf:pdf},
	month = {aug},
	title = {{Under the Hood: Using Diagnostic Classifiers to Investigate and Improve how Language Models Track Agreement Information}},
	url = {http://arxiv.org/abs/1808.08079},
	year = {2018}
}
@techreport{Gulordava,
	abstract = {Recurrent neural networks (RNNs) have achieved impressive results in a variety of linguistic processing tasks, suggesting that they can induce non-trivial properties of language. We investigate here to what extent RNNs learn to track abstract hierarchical syntactic structure. We test whether RNNs trained with a generic language modeling objective in four languages (Italian, English, Hebrew, Russian) can predict long-distance number agreement in various constructions. We include in our evaluation nonsensical sentences where RNNs cannot rely on semantic or lexical cues ("The colorless green ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas ideas I ate with the chair sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep sleep furiously"), and, for Italian, we compare model performance to human intuitions. Our language-model-trained RNNs make reliable predictions about long-distance agreement , and do not lag much behind human performance. We thus bring support to the hypothesis that RNNs are not just shallow-pattern extractors, but they also acquire deeper grammatical competence.},
	archivePrefix = {arXiv},
	arxivId = {1803.11138v1},
	author = {Gulordava, Kristina and Bojanowski, Piotr and Grave, Edouard and Linzen, Tal and Baroni, Marco},
	eprint = {1803.11138v1},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Gulordava et al. - Unknown - Colorless green recurrent networks dream hierarchically.pdf:pdf},
	title = {{Colorless green recurrent networks dream hierarchically}},
	url = {https://github.com/}
}
@article{Hupkes2018,
	abstract = {{\textless}p{\textgreater}We investigate how neural networks can learn and process languages with hierarchical, compositional semantics. To this end, we define the artificial task of processing nested arithmetic expressions, and study whether different types of neural networks can learn to compute their meaning. We find that recursive neural networks can implement a generalising solution to this problem, and we visualise this solution by breaking it up in three steps: project, sum and squash. As a next step, we investigate recurrent neural networks, and show that a gated recurrent unit, that processes its input incrementally, also performs very well on this task: the network learns to predict the outcome of the arithmetic expressions with high accuracy, although performance deteriorates somewhat with increasing length. To develop an understanding of what the recurrent network encodes, visualisation techniques alone do not suffice. Therefore, we develop an approach where we formulate and test multiple hypotheses on the information encoded and processed by the network. For each hypothesis, we derive predictions about features of the hidden state representations at each time step, and train 'diagnostic classifiers' to test those predictions. Our results indicate that the networks follow a strategy similar to our hypothesised 'cumulative strategy', which explains the high accuracy of the network on novel expressions, the generalisation to longer expressions than seen in training, and the mild deterioration with increasing length. This in turn shows that diagnostic classifiers can be a useful technique for opening up the black box of neural networks. We argue that diagnostic classification, unlike most visualisation techniques, does scale up from small networks in a toy domain, to larger and deeper recurrent networks dealing with real-life data, and may therefore contribute to a better understanding of the internal dynamics of current state-of-the-art models in natural language processing.{\textless}/p{\textgreater}},
	author = {Hupkes, Dieuwke and Veldhoen, Sara and Zuidema, Willem},
	doi = {10.1613/jair.1.11196},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Hupkes, Veldhoen, Zuidema - 2018 - Visualisation and 'Diagnostic Classifiers' Reveal How Recurrent and Recursive Neural Networks Proc(2).pdf:pdf},
	issn = {1076-9757},
	journal = {Journal of Artificial Intelligence Research},
	month = {apr},
	pages = {907--926},
	title = {{Visualisation and 'Diagnostic Classifiers' Reveal How Recurrent and Recursive Neural Networks Process Hierarchical Structure}},
	url = {https://jair.org/index.php/jair/article/view/11196},
	volume = {61},
	year = {2018}
}
@inproceedings{jain2019attention,
  title={Attention is not Explanation},
  author={Jain, Sarthak and Wallace, Byron C},
  booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
  pages={3543--3556},
  year={2019}
}
@techreport{Yogatama,
	abstract = {We compare and analyze sequential, random access, and stack memory architec-tures for recurrent neural network language models. Our experiments on the Penn Treebank and Wikitext-2 datasets show that stack-based memory architectures consistently achieve the best performance in terms of held out perplexity. We also propose a generalization to existing continuous stack models (Joulin {\&} Mikolov, 2015; Grefenstette et al., 2015) to allow a variable number of pop operations more naturally that further improves performance. We further evaluate these language models in terms of their ability to capture non-local syntactic dependencies on a subject-verb agreement dataset (Linzen et al., 2016) and establish new state of the art results using memory augmented language models. Our results demonstrate the value of stack-structured memory for explaining the distribution of words in natural language, in line with linguistic theories claiming a context-free backbone for natural language.},
	author = {Yogatama, Dani and Miao, Yishu and Melis, Gabor and Ling, Wang and Kuncoro, Adhiguna and Dyer, Chris and Blunsom, Phil},
	file = {:Users/CalebKaijiLu/Library/Application Support/Mendeley Desktop/Downloaded/Yogatama et al. - Unknown - MEMORY ARCHITECTURES IN RECURRENT NEURAL NETWORK LANGUAGE MODELS.pdf:pdf},
	title = {{MEMORY ARCHITECTURES IN RECURRENT NEURAL NETWORK LANGUAGE MODELS}},
	url = {https://pdfs.semanticscholar.org/2798/1998aaef92952eabef2c1490b926f9150c4f.pdf}
}
@inproceedings{mikolov2013distributed,
	title={Distributed representations of words and phrases and their compositionality},
	author={Mikolov, Tomas and Sutskever, Ilya and Chen, Kai and Corrado, Greg S and Dean, Jeff},
	booktitle={Advances in neural information processing systems},
	pages={3111--3119},
	year={2013}
}


@inproceedings{DBLP:conf/acl/LuMLFD20,
  author    = {Kaiji Lu and
               Piotr Mardziel and
               Klas Leino and
               Matt Fredrikson and
               Anupam Datta},
  title     = {Influence Paths for Characterizing Subject-Verb Number Agreement in
               {LSTM} Language Models},
  publisher = {Association for Computational Linguistics},
  year      = {2020},
}

@misc{smilkov2017smoothgrad,
      title={SmoothGrad: removing noise by adding noise},
      author={Daniel Smilkov and Nikhil Thorat and Been Kim and Fernanda Viégas and Martin Wattenberg},
      year={2017},
      eprint={1706.03825},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{wang2020smoothed,
      title={Smoothed Geometry for Robust Attribution},
      author={Zifan Wang and Haofan Wang and Shakul Ramkumar and Matt Fredrikson and Piotr Mardziel and Anupam Datta},
      year={2020},
      eprint={2006.06643},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}
@article{elazar2020bert,
  title={When Bert Forgets How To POS: Amnesic Probing of Linguistic Properties and MLM Predictions},
  author={Elazar, Yanai and Ravfogel, Shauli and Jacovi, Alon and Goldberg, Yoav},
  journal={arXiv preprint arXiv:2006.00995},
  year={2020}
}
@article{prasanna2020bert,
  title={When BERT Plays the Lottery, All Tickets Are Winning},
  author={Prasanna, Sai and Rogers, Anna and Rumshisky, Anna},
  journal={arXiv preprint arXiv:2005.00561},
  year={2020}
}
@article{sanhdistilbert,
  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},
  author={SANH, Victor and DEBUT, Lysandre and CHAUMOND, Julien and WOLF, Thomas and Face, Hugging}
}
@article{jiao2019tinybert,
  title={Tinybert: Distilling bert for natural language understanding},
  author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
  journal={arXiv preprint arXiv:1909.10351},
  year={2019}
}

@misc{45503,
title	= {Calculus on Computational Graphs: Backpropagation},
author	= {Christopher Olah},
year	= {2015},
URL	= {http://colah.github.io/posts/2015-08-Backprop/}
}

@article{baehrens2010explain,
  title={How to explain individual classification decisions},
  author={Baehrens, David and Schroeter, Timon and Harmeling, Stefan and Kawanabe, Motoaki and Hansen, Katja and M{\"u}ller, Klaus-Robert},
  journal={The Journal of Machine Learning Research},
  volume={11},
  pages={1803--1831},
  year={2010},
  publisher={JMLR. org}
}