main.bib

@article{mehta_sparsity-based_nodate,
	title        = {Sparsity-{Based} {Generalization} {Bounds} for {Predictive} {Sparse} {Coding}},
	author       = {Mehta, Nishant A and Gray, Alexander G},
	abstract     = {The goal of predictive sparse coding is to learn a representation of examples as sparse linear combinations of elements from a dictionary, such that a learned hypothesis linear in the new representation performs well on a predictive task. Predictive sparse coding has demonstrated impressive performance on a variety of supervised tasks, but its generalization properties have not been studied. We establish the first generalization error bounds for predictive sparse coding, in the overcomplete setting, where the number of features k exceeds the original dimensionality d. The learning bound decays as O˜( dk/m) with respect to d, k, and the size m of the training sample. It depends intimately on stability properties of the learned sparse encoder, as measured on the training sample. Consequently, we also present a fundamental stability result for the LASSO, a result that characterizes the stability of the sparse codes with respect to dictionary perturbations.},
	language     = {en},
}
@article{horn1962eigenvalues,
	title        = {Eigenvalues of sums of Hermitian matrices},
	author       = {Horn, Alfred},
	year         = 1962,
	journal      = {Pacific Journal of Mathematics},
	publisher    = {Mathematical Sciences Publishers},
	volume       = 12,
	number       = 1,
}
@article{hubel_receptive_1962,
	title        = {Receptive fields, binocular interaction and functional architecture in the cat's visual cortex},
	author       = {Hubel, D. H. and Wiesel, T. N.},
	year         = 1962,
	month        = jan,
	journal      = {The Journal of Physiology},
	volume       = 160,
	number       = 1,
	issn         = {0022-3751},
	url          = {https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1359523/},
	urldate      = {2022-09-21},
	abstract     = {Images null},
	pmid         = 14449617,
	pmcid        = {PMC1359523},
}
@article{matheron1963principles,
	title        = {Principles of geostatistics},
	author       = {Matheron, Georges},
	year         = 1963,
	journal      = {Economic geology},
	publisher    = {Society of Economic Geologists},
	volume       = 58,
	number       = 8,
}
@article{marvcenko1967distribution,
	title        = {Distribution of eigenvalues for some sets of random matrices},
	author       = {{Mar{\v{c}}enko}, Vladimir A and Pastur, Leonid Andreevich},
	year         = 1967,
	journal      = {Mathematics of the USSR-Sbornik},
	publisher    = {IOP Publishing},
	volume       = 1,
	number       = 4,
}
@article{thompson1971eigenvalues,
	title        = {On the eigenvalues of sums of Hermitian matrices},
	author       = {Thompson, Robert C and Freede, Linda J},
	year         = 1971,
	journal      = {Linear Algebra and Its Applications},
	publisher    = {Elsevier},
	volume       = 4,
	number       = 4,
}
@book{crank1979mathematics,
	title        = {The mathematics of diffusion},
	author       = {Crank, John},
	year         = 1979,
	publisher    = {Oxford university press},
}
@article{marr1979computational,
	title        = {A computational theory of human stereo vision},
	author       = {Marr, David and Poggio, Tomaso},
	year         = 1979,
	journal      = {Proceedings of the Royal Society of London. Series B. Biological Sciences},
	publisher    = {The Royal Society London},
	volume       = 204,
	number       = 1156,
}
@book{Phillips81,
	title        = {Amorphous Solids: Low Temperature Properties},
	author       = {Anderson, A.C.},
	year         = 1981,
	publisher    = {Springer, Berlin},
	series       = {Topics in Current Physics},
	volume       = 24,
	editor       = {W. A. Phillips},
}
@book{Mezard87,
	title        = {Spin glass theory and beyond: An Introduction to the Replica Method and Its Applications},
	author       = {{M{\'e}zard}, Marc and Parisi, Giorgio and Virasoro, Miguel},
	year         = 1987,
	publisher    = {World Scientific Publishing Company},
	volume       = 9,
}
@article{Baum88,
	title        = {On the capabilities of multilayer perceptrons},
	author       = {Baum, Eric B},
	year         = 1988,
	journal      = {Journal of complexity},
	publisher    = {Academic Press},
	volume       = 4,
	number       = 3,
}
@article{Gardner88,
	title        = {The space of interactions in neural network models},
	author       = {Gardner, Elizabeth},
	year         = 1988,
	journal      = {Journal of physics A: Mathematical and general},
	publisher    = {IOP Publishing},
	volume       = 21,
	number       = 1,
}
@article{bourrely1989parallelization,
	title        = {Parallelization of a neural network learning algorithm on a hypercube},
	author       = {Bourrely, J},
	year         = 1989,
	journal      = {Hypercube and distributed computers. Elsiever Science Publishing},
}
@article{lecun_backpropagation_1989,
	title        = {Backpropagation {Applied} to {Handwritten} {Zip} {Code} {Recognition}},
	author       = {LeCun, Y. and Boser, B. and Denker, J. S. and Henderson, D. and Howard, R. E. and Hubbard, W. and Jackel, L. D.},
	year         = 1989,
	month        = dec,
	journal      = {Neural Computation},
	volume       = 1,
	number       = 4,
	doi          = {10.1162/neco.1989.1.4.541},
	issn         = {0899-7667},
	url          = {https://doi.org/10.1162/neco.1989.1.4.541},
	urldate      = {2021-03-24},
	abstract     = {The ability of learning networks to generalize can be greatly enhanced by providing constraints from the task domain. This paper demonstrates how such constraints can be integrated into a backpropagation network through the architecture of the network. This approach has been successfully applied to the recognition of handwritten zip code digits provided by the U.S. Postal Service. A single network learns the entire recognition operation, going from the normalized image of the character to the final classification.},
}
@article{piela1989multiple,
	title        = {On the multiple-minima problem in the conformational analysis of molecules: deformation of the potential energy hypersurface by the diffusion equation method},
	author       = {Piela, Lucjan and Kostrowicki, Jaroslaw and Scheraga, Harold A},
	year         = 1989,
	journal      = {The Journal of Physical Chemistry},
	publisher    = {ACS Publications},
	volume       = 93,
	number       = 8,
}
@inproceedings{lecun1990optimal,
	title        = {Optimal brain damage},
	author       = {LeCun, Yann and Denker, John S and Solla, Sara A},
	year         = 1990,
	booktitle    = {Advances in neural information processing systems},
}
@incollection{lecun90,
	title        = {Optimal Brain Damage},
	author       = {LeCun, Yann and John S. Denker and Sara A. Solla},
	year         = 1990,
	booktitle    = {Advances in Neural Information Processing Systems 2},
	publisher    = {Morgan-Kaufmann},
	url          = {http://papers.nips.cc/paper/250-optimal-brain-damage.pdf},
	editor       = {D. S. Touretzky},
}
@article{bottou1991stochastic,
	title        = {Stochastic gradient learning in neural networks},
	author       = {Bottou, {L{\'e}on}},
	year         = 1991,
	journal      = {Proceedings of Neuro-{N{\i}mes}},
	volume       = 91,
	number       = 8,
}
@article{le1991eigenvalues,
	title        = {Eigenvalues of covariance matrices: Application to neural-network learning},
	author       = {Le Cun, Yann and Kanter, Ido and Solla, Sara A},
	year         = 1991,
	journal      = {Physical Review Letters},
	publisher    = {APS},
	volume       = 66,
	number       = 18,
}
@article{amari1992four,
	title        = {Four types of learning curves},
	author       = {Amari, Shun-ichi and Fujita, Naotake and Shinomoto, Shigeru},
	year         = 1992,
	journal      = {Neural Computation},
	publisher    = {MIT Press},
	volume       = 4,
	number       = 4,
}
@inproceedings{krogh1992simple,
	title        = {A simple weight decay can improve generalization},
	author       = {Krogh, Anders and Hertz, John A},
	year         = 1992,
	booktitle    = {Advances in neural information processing systems},
}
@article{moller1993exact,
	title        = {Exact Calculation of the Product of the Hessian Matrix of Feed-Forward Network Error Functions and a Vector in 0 (N) Time},
	author       = {{M{\o}ller}, Martin F},
	year         = 1993,
	journal      = {DAIMI Report Series},
	volume       = 22,
	number       = 432,
}
@article{amari1993universal,
	title        = {A universal theorem on learning curves},
	author       = {Amari, Shun-Ichi},
	year         = 1993,
	journal      = {Neural networks},
	publisher    = {Elsevier},
	volume       = 6,
	number       = 2,
}
@article{cuku,
	title        = {Analytical solution of the off-equilibrium dynamics of a long-range spin-glass model},
	author       = {Cugliandolo, Leticia F and Kurchan, Jorge},
	year         = 1993,
	journal      = {Physical Review Letters},
	publisher    = {APS},
	volume       = 71,
	number       = 1,
}
@article{hassibi1993second,
	title        = {Second order derivatives for network pruning: Optimal brain surgeon},
	author       = {Hassibi, Babak and Stork, David G and others},
	year         = 1993,
	journal      = {Advances in neural information processing systems},
	publisher    = {Morgan Kaufmann Publishers},
}
@article{pardalos1994optimization,
	title        = {Optimization methods for computing global minima of nonconvex potential energy functions},
	author       = {Pardalos, Panos M and Shalloway, David and Xue, Guoliang},
	year         = 1994,
	journal      = {Journal of Global Optimization},
	publisher    = {Springer},
	volume       = 4,
	number       = 2,
}
@article{pearlmutter1994fast,
	title        = {Fast exact multiplication by the Hessian},
	author       = {Pearlmutter, Barak A},
	year         = 1994,
	journal      = {Neural computation},
	publisher    = {MIT Press},
	volume       = 6,
	number       = 1,
}
@article{Lecun95,
	title        = {Convolutional networks for images, speech, and time series},
	author       = {LeCun, Yann and Bengio, Yoshua and others},
	year         = 1995,
	journal      = {The handbook of brain theory and neural networks},
	volume       = 3361,
	number       = 10,
}
@string{epje = {Eur.\ Phys.\ J E}}
@string{epl = {Europhys.\ Lett.}}
@string{pnas = {Proc.\ Natl.\ Acad.\ Sci.\ U.S.A.}}
@string{prb = {Phys.\ Rev.\ B}}
@string{pre = {Phys.\ Rev.\ E}}
@string{prl = {Phys.\ Rev.\ Lett.}}
@string{rmp = {Rev.\ Mod.\ Phys.}}
@article{Monasson95,
	title        = {Weight space structure and internal representations: a direct approach to learning and generalization in multilayer neural networks},
	author       = {Monasson, {R{\'e}mi} and Zecchina, Riccardo},
	year         = 1995,
	journal      = {Physical review letters},
	publisher    = {APS},
	volume       = 75,
	number       = 12,
}
@article{monasson1995weight,
	title        = {Weight space structure and internal representations: a direct approach to learning and generalization in multilayer neural networks},
	author       = {Monasson, R{\'e}mi and Zecchina, Riccardo},
	year         = 1995,
	journal      = {Physical review letters},
	publisher    = {APS},
	volume       = 75,
	number       = 12,
}
@article{saad1995line,
	title        = {On-line learning in soft committee machines},
	author       = {Saad, David and Solla, Sara A},
	year         = 1995,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 52,
	number       = 4,
}
@book{Statistical_Mechanics,
	title        = {Statistical Mechanics},
	author       = {Beale, P.D.},
	year         = 1996,
	publisher    = {Elsevier Science},
	isbn         = 9780080541716,
	url          = {https://books.google.ch/books?id=PIk9sF9j2oUC},
}
@book{Risken1996,
	title        = {The Fokker-Planck Equation Springer Series in Synergetics},
	author       = {Hannes Risken},
	year         = 1996,
	journal      = {Dynamical Systems},
	abstract     = {© 2015 Zbigniew Otremba and Eugeniusz Andrulewicz.The article discusses an important issue of technical pressure exerted on the marine environment during construction and operation of maritime wind farms (MFW) on waters of the Polish Exclusive Economic Zone. A motivation for analysing this issue is the need for attracting attention to the aspect of physical field modification as the factor which links large scale technical activity at sea with the existence and functioning of the marine ecosystem, including further consequences to its economic benefits. Based on current knowledge and authors' analyses, the scale of modifications (disturbances) of physical fields expected to take place during MFW construction and operation was assessed.},
}
@article{kurchanlaloux,
	title        = {Phase space geometry and slow dynamics},
	author       = {Kurchan, Jorge and Laloux, Laurent},
	year         = 1996,
	journal      = {Journal of Physics A: Mathematical and General},
	publisher    = {IOP Publishing},
	volume       = 29,
	number       = 9,
}
@book{Neal1996,
	title        = {Bayesian Learning for Neural Networks},
	author       = {Neal, Radford M.},
	year         = 1996,
	publisher    = {Springer-Verlag New York, Inc.},
	address      = {Secaucus, NJ, USA},
	isbn         = {0387947248},
}
@article{olshausen_emergence_1996,
	title        = {Emergence of simple-cell receptive field properties by learning a sparse code for natural images},
	author       = {Olshausen, Bruno A. and Field, David J.},
	year         = 1996,
	month        = jun,
	journal      = {Nature},
	volume       = 381,
	number       = 6583,
	doi          = {10.1038/381607a0},
	issn         = {1476-4687},
	url          = {https://www.nature.com/articles/381607a0},
	urldate      = {2022-09-29},
	copyright    = {1996 Nature Publishing Group},
	note         = {Number: 6583 Publisher: Nature Publishing Group},
	abstract     = {THE receptive fields of simple cells in mammalian primary visual cortex can be characterized as being spatially localized, oriented1–4 and bandpass (selective to structure at different spatial scales), comparable to the basis functions of wavelet transforms5,6. One approach to understanding such response properties of visual neurons has been to consider their relationship to the statistical structure of natural images in terms of efficient coding7–12. Along these lines, a number of studies have attempted to train unsupervised learning algorithms on natural images in the hope of developing receptive fields with similar properties13–18, but none has succeeded in producing a full set that spans the image space and contains all three of the above properties. Here we investigate the proposal8,12 that a coding strategy that maximizes sparseness is sufficient to account for these properties. We show that a learning algorithm that attempts to find sparse linear codes for natural scenes will develop a complete family of localized, oriented, bandpass receptive fields, similar to those found in the primary visual cortex. The resulting sparse image code provides a more efficient representation for later stages of processing because it possesses a higher degree of statistical independence among its outputs.},
	language     = {en},
	keywords     = {Humanities and Social Sciences, multidisciplinary, Science},
}
@inproceedings{bos1997dynamics,
	title        = {Dynamics of training},
	author       = {{B{\"o}s}, Siegfried and Opper, Manfred},
	year         = 1997,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{hochreiter1997flat,
	title        = {Flat minima},
	author       = {Hochreiter, Sepp and Schmidhuber, {J{\"u}rgen}},
	year         = 1997,
	journal      = {Neural Computation},
	publisher    = {MIT Press},
	volume       = 9,
	number       = 1,
}
@article{Hochreiter97,
	title        = {Flat minima},
	author       = {Hochreiter, Sepp and Schmidhuber, {J{\"u}rgen}},
	year         = 1997,
	journal      = {Neural Computation},
	publisher    = {MIT Press},
	volume       = 9,
	number       = 1,
}
@inproceedings{williams1997computing,
	title        = {Computing with infinite networks},
	author       = {Williams, Christopher KI},
	year         = 1997,
	booktitle    = {Advances in neural information processing systems},
}
@article{reviewBCKM,
	title        = {Out of equilibrium dynamics in spin-glasses and other glassy systems},
	author       = {Bouchaud, Jean-Philippe and Cugliandolo, Leticia F and Kurchan, Jorge and Mezard, Marc},
	year         = 1998,
	journal      = {Spin glasses and random fields},
	publisher    = {World Scientific, Singapore},
}
@article{chen1998atomic,
	title        = {Atomic Decomposition by Basis Pursuit},
	author       = {Chen, Scott Shaobing and Donoho, David L. and Saunders, Michael A.},
	year         = 1998,
	journal      = {SIAM Journal on Scientific Computing},
	volume       = 20,
	number       = 1,
	doi          = {10.1137/S1064827596304010},
}
@article{lecun_gradient-based_1998,
	title        = {Gradient-based learning applied to document recognition},
	author       = {Lecun, Y. and Bottou, L. and Bengio, Y. and Haffner, P.},
	year         = 1998,
	month        = nov,
	journal      = {Proceedings of the IEEE},
	volume       = 86,
	number       = 11,
	doi          = {10.1109/5.726791},
	issn         = {1558-2256},
	note         = {Conference Name: Proceedings of the IEEE},
	abstract     = {Multilayer neural networks trained with the back-propagation algorithm constitute the best example of a successful gradient based learning technique. Given an appropriate network architecture, gradient-based learning algorithms can be used to synthesize a complex decision surface that can classify high-dimensional patterns, such as handwritten characters, with minimal preprocessing. This paper reviews various methods applied to handwritten character recognition and compares them on a standard handwritten digit recognition task. Convolutional neural networks, which are specifically designed to deal with the variability of 2D shapes, are shown to outperform all other techniques. Real-life document recognition systems are composed of multiple modules including field extraction, segmentation recognition, and language modeling. A new learning paradigm, called graph transformer networks (GTN), allows such multimodule systems to be trained globally using gradient-based methods so as to minimize an overall performance measure. Two systems for online handwriting recognition are described. Experiments demonstrate the advantage of global training, and the flexibility of graph transformer networks. A graph transformer network for reading a bank cheque is also described. It uses convolutional neural network character recognizers combined with global training techniques to provide record accuracy on business and personal cheques. It is deployed commercially and reads several million cheques per day.},
	keywords     = {2D shape variability, back-propagation, backpropagation, Character recognition, cheque reading, complex decision surface synthesis, convolution, convolutional neural network character recognizers, document recognition, document recognition systems, Feature extraction, field extraction, gradient based learning technique, gradient-based learning, graph transformer networks, GTN, handwritten character recognition, handwritten digit recognition task, Hidden Markov models, high-dimensional patterns, language modeling, Machine learning, Multi-layer neural network, multilayer neural networks, multilayer perceptrons, multimodule systems, Neural networks, optical character recognition, Optical character recognition software, Optical computing, Pattern recognition, performance measure minimization, Principal component analysis, segmentation recognition},
}
@article{lecun1998gradient,
	title        = {Gradient-based learning applied to document recognition},
	author       = {LeCun, Yann and Bottou, {L{\'e}on} and Bengio, Yoshua and Haffner, Patrick},
	year         = 1998,
	journal      = {Proceedings of the IEEE},
	publisher    = {IEEE},
	volume       = 86,
	number       = 11,
}
@article{lecun1998efficient,
	title        = {Efficient backprop},
	author       = {LeCun, Yann and Bottou, {L\'eon} and Orr, GB and {M{\"u}ller}, K-R},
	year         = 1998,
	journal      = {Lecture notes in computer science},
	publisher    = {Springer},
}
@article{niyogi_incorporating_1998,
	title        = {Incorporating prior information in machine learning by creating virtual examples},
	author       = {Niyogi, P. and Girosi, F. and Poggio, T.},
	year         = 1998,
	month        = nov,
	journal      = {Proceedings of the IEEE},
	volume       = 86,
	number       = 11,
	doi          = {10.1109/5.726787},
	issn         = {1558-2256},
	note         = {Conference Name: Proceedings of the IEEE},
	abstract     = {One of the key problems in supervised learning is the insufficient size of the training set. The natural way for an intelligent learner to counter this problem and successfully generalize is to exploit prior information that may be available about the domain or that can be learned from prototypical examples. We discuss the notion of using prior knowledge by creating virtual examples and thereby expanding the effective training-set size. We show that in some contexts this idea is mathematically equivalent to incorporating the prior knowledge as a regularizer, suggesting that the strategy is well motivated. The process of creating virtual examples in real-world pattern recognition tasks is highly nontrivial. We provide demonstrative examples from object recognition and speech recognition to illustrate the idea.},
	keywords     = {Associate members, Counting circuits, Function approximation, Knowledge acquisition, Learning systems, Machine learning, Pattern recognition, Prototypes, Speech, Supervised learning},
}
@incollection{prechelt1998early,
	title        = {Early stopping-but when?},
	author       = {Prechelt, Lutz},
	year         = 1998,
	booktitle    = {Neural Networks: Tricks of the trade},
	publisher    = {Springer},
}
@article{smola1998connection,
	title        = {The connection between regularization operators and support vector kernels},
	author       = {Smola, Alex J and Sch{\"o}lkopf, Bernhard and M{\"u}ller, Klaus-Robert},
	year         = 1998,
	journal      = {Neural networks},
	publisher    = {Elsevier},
	volume       = 11,
	number       = 4,
}
@article{monasson1999determining,
	title        = {Determining computational complexity from characteristic phase transitions},
	author       = {Monasson, {R{\'e}mi} and Zecchina, Riccardo and Kirkpatrick, Scott and Selman, Bart and Troyansky, Lidror},
	year         = 1999,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 400,
	number       = 6740,
}
@inproceedings{scholkopf_kernel_1999,
	title        = {Kernel principal component analysis},
	author       = {Scholkopf, Bernhard and Smola, Alexander and Müller, Klaus-Robert},
	year         = 1999,
	booktitle    = {Advances in {Kernel} {Methods} - {Support} {Vector} {Learning}},
	publisher    = {MIT Press},
	abstract     = {A new method for performing a nonlinear form of Principal Component Analysis is proposed. By the use of integral operator kernel functions, one can efficiently compute principal components in high-dimensional feature spaces, related to input space by some nonlinear map; for instance the space of all possible d-pixel products in images. We give the derivation of the method and present experimental results on polynomial feature extraction for pattern recognition.},
}
@article{stein1999predicting,
	title        = {Predicting random fields with increasing dense observations},
	author       = {Stein, Michael L and others},
	year         = 1999,
	journal      = {The Annals of Applied Probability},
	publisher    = {Institute of Mathematical Statistics},
	volume       = 9,
	number       = 1,
}
@article{Tkachenko99,
	title        = {Stress propagation through frictionless granular material},
	author       = {Tkachenko, Alexei V. and Witten, Thomas A.},
	year         = 1999,
	month        = {Jul},
	journal      = {Phys. Rev. E},
	publisher    = {American Physical Society},
	volume       = 60,
	number       = 1,
	doi          = {10.1103/PhysRevE.60.687},
	numpages     = 9,
	bdsk-url-1   = {http://dx.doi.org/10.1103/PhysRevE.60.687},
}
@article{dacey2000center,
	title        = {Center surround receptive field structure of cone bipolar cells in primate retina},
	author       = {Dacey, Dennis and Packer, Orin S and Diller, Lisa and Brainard, David and Peterson, Beth and Lee, Barry},
	year         = 2000,
	journal      = {Vision research},
	publisher    = {Elsevier},
	volume       = 40,
	number       = 14,
}
@inproceedings{domingos00,
	title        = {A unified bias-variance decomposition},
	author       = {Domingos, Pedro},
	year         = 2000,
	booktitle    = {Proceedings of 17th International Conference on Machine Learning},
}
@article{Bremaud2000,
	title        = {Markov Chains: Gibbs Fields, Monte Carlo Simulation, and Queues},
	author       = {Laurent Saloff-Coste and Pierre Bremaud},
	year         = 2000,
	journal      = {Journal of the American Statistical Association},
	volume       = 95,
	doi          = {10.2307/2669802},
	issn         = {01621459},
	abstract     = {This book discusses both the theory and applications of Markov chains. The author studies both discrete-time and continuous-time chains and connected topics such as finite Gibbs fields, non-homogeneous Markov chains, discrete time regenerative processes, Monte Carlo simulation, simulated annealing, and queueing networks are also developed in this accessible and self-contained text. The text is firstly an introduction to the theory of stochastic processes at the undergraduate or beginning graduate level. Its primary objective is to initiate the student to the art of stochastic modelling. The treatment is mathematical, with definitions, theorems, proofs and a number of classroom examples which help the student to fully grasp the content of the main results. Problems of varying difficulty are proposed at the close of each chapter. The text is motivated by significant applications and progressively brings the student to the borders of contemporary research. Students and researchers in operations research and electrical engineering as well as in physics, biology and the social sciences will find this book of interest.},
	issue        = 452,
}
@article{smola2000regularization,
	title        = {Regularization with dot-product kernels},
	author       = {Smola, Alex and Ov{\'a}ri, Zolt{\'a}n and Williamson, Robert C.},
	year         = 2000,
	journal      = {Advances in neural information processing systems},
	volume       = 13,
}
@inproceedings{caruana2001overfitting,
	title        = {Overfitting in neural nets: Backpropagation, conjugate gradient, and early stopping},
	author       = {Caruana, Rich and Lawrence, Steve and Giles, C Lee},
	year         = 2001,
	booktitle    = {Advances in neural information processing systems},
}
@book{engel2001statistical,
	title        = {Statistical mechanics of learning},
	author       = {Engel, Andreas and Van den Broeck, Christian},
	year         = 2001,
	publisher    = {Cambridge University Press},
}
@article{knutson2001honeycombs,
	title        = {Honeycombs and sums of Hermitian matrices},
	author       = {Knutson, Allen and Tao, Terence},
	year         = 2001,
	journal      = {Notices Amer. Math. Soc},
	volume       = 48,
	number       = 2,
}
@book{opper2001advanced,
	title        = {Advanced mean field methods: Theory and practice},
	author       = {Opper, Manfred and Saad, David},
	year         = 2001,
	publisher    = {MIT press},
}
@book{scholkopf2001learning,
	title        = {Learning with kernels: support vector machines, regularization, optimization, and beyond},
	author       = {Scholkopf, Bernhard and Smola, Alexander J},
	year         = 2001,
	publisher    = {MIT press},
}
@article{reviewbray,
	title        = {Theory of phase-ordering kinetics},
	author       = {Bray, Alan J},
	year         = 2002,
	journal      = {Advances in Physics},
	publisher    = {Taylor {\&} Francis},
	volume       = 51,
	number       = 2,
}
@article{mezard2002analytic,
	title        = {Analytic and algorithmic solution of random satisfiability problems},
	author       = {M{\'e}zard, Marc and Parisi, Giorgio and Zecchina, Riccardo},
	year         = 2002,
	journal      = {Science},
	publisher    = {American Association for the Advancement of Science},
	volume       = 297,
	number       = 5582,
}
@incollection{cugliandololeshouches,
	title        = {Course 7: Dynamics of glassy systems},
	author       = {Cugliandolo, Leticia F},
	year         = 2003,
	booktitle    = {Slow Relaxations and nonequilibrium dynamics in condensed matter},
	publisher    = {Springer},
}
@article{Ohern03,
	title        = {Jamming at zero temperature and zero applied stress: The epitome of disorder},
	author       = {O'Hern, Corey S. and Silbert, Leonardo E. and Liu, Andrea J. and Nagel, Sidney R.},
	year         = 2003,
	month        = {Jul},
	journal      = {Phys. Rev. E},
	publisher    = {American Physical Society},
	volume       = 68,
	number       = 1,
	doi          = {10.1103/PhysRevE.68.011306},
	bdsk-url-1   = {http://link.aps.org/doi/10.1103/PhysRevE.68.011306},
	bdsk-url-2   = {http://dx.doi.org/10.1103/PhysRevE.68.011306},
}
@article{Donev04a,
	title        = {Improving the Density of Jammed Disordered Packings Using Ellipsoids},
	author       = {Donev, Aleksandar and Cisse, Ibrahim and Sachs, David and Variano, Evan A. and Stillinger, Frank H. and Connelly, Robert and Torquato, Salvatore and Chaikin, P. M.},
	year         = 2004,
	journal      = {Science},
	volume       = 303,
	number       = 5660,
	doi          = {10.1126/science.1093010},
	bdsk-url-1   = {http://dx.doi.org/10.1126/science.1093010},
	abstract     = {Packing problems, such as how densely objects can fill a volume, are among the most ancient and persistent problems in mathematics and science. For equal spheres, it has only recently been proved that the face-centered cubic lattice has the highest possible packing fraction . It is also well known that certain random (amorphous) jammed packings have {{\oe}{\"U}} ?{{\^a}{\`a}} 0.64. Here, we show experimentally and with a new simulation algorithm that ellipsoids can randomly pack more densely?{{\"A}{\^\i}}up to {{\oe}{\"U}}= 0.68 to 0.71for spheroids with an aspect ratio close to that of M&M's Candies?{{\"A}{\^\i}}and even approach {{\oe}{\"U}} ?{{\^a}{\`a}} 0.74 for ellipsoids with other aspect ratios. We suggest that the higher density is directly related to the higher number of degrees of freedom per particle and thus the larger number of particle contacts required to mechanically stabilize the packing. We measured the number of contacts per particle Z ?{{\^a}{\`a}} 10 for our spheroids, as compared to Z ?{{\^a}{\`a}} 6 for spheres. Our results have implications for a broad range of scientific disciplines, including the properties of granular media and ceramics, glass formation, and discrete geometry.},
}
@article{lowe2004distinctive,
	title        = {Distinctive image features from scale-invariant keypoints},
	author       = {Lowe, David G},
	year         = 2004,
	journal      = {International journal of computer vision},
	publisher    = {Springer},
	volume       = 60,
	number       = 2,
}
@article{luxburg2004distance,
	title        = {Distance-based classification with Lipschitz functions},
	author       = {Luxburg, Ulrike von and Bousquet, Olivier},
	year         = 2004,
	journal      = {Journal of Machine Learning Research},
	volume       = 5,
	number       = {Jun},
}
@article{baik2005phase,
	title        = {Phase transition of the largest eigenvalue for nonnull complex sample covariance matrices},
	author       = {Baik, Jinho and Ben Arous, {G{\'e}rard} and {P{\'e}ch{\'e}}, Sandrine and others},
	year         = 2005,
	journal      = {The Annals of Probability},
	publisher    = {Institute of Mathematical Statistics},
	volume       = 33,
	number       = 5,
}
@article{cavagnaSGpedestrians,
	title        = {Spin-glass theory for pedestrians},
	author       = {Castellani, Tommaso and Cavagna, Andrea},
	year         = 2005,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	publisher    = {IOP Publishing},
	volume       = 2005,
	number       = {05},
}
@article{ikeda2005asymptotic,
	title        = {An asymptotic statistical analysis of support vector machines with soft margins},
	author       = {Ikeda, Kazushi and Aoishi, Tsutomu},
	year         = 2005,
	journal      = {Neural Networks},
	publisher    = {Elsevier},
	volume       = 18,
	number       = 3,
}
@article{Silbert05,
	title        = {Vibrations and Diverging Length Scales Near the Unjamming Transition},
	author       = {L. E. Silbert and A. J. Liu and S. R. Nagel},
	year         = 2005,
	journal      = PRL,
	volume       = 95,
}
@article{Wyart05b,
	title        = {On the Rigidity of Amorphous Solids},
	author       = {M. Wyart},
	year         = 2005,
	journal      = {Annales de Phys},
	volume       = 30,
	number       = 3,
}
@article{Wyart05a,
	title        = {Effects of compression on the vibrational modes of marginally jammed solids},
	author       = {Wyart, Matthieu and Silbert, Leonardo E and Nagel, Sidney R and Witten, Thomas A},
	year         = 2005,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 72,
	number       = 5,
	date         = 2005,
}
@article{eqBAetal,
	title        = {Cugliandolo-Kurchan equations for dynamics of Spin-Glasses},
	author       = {Ben Arous, {G\'erard} and Dembo, Amir and Guionnet, Alice},
	year         = 2006,
	journal      = {Probability theory and related fields},
	publisher    = {Springer},
	volume       = 136,
	number       = 4,
}
@article{montanarisemerjian,
	title        = {Rigorous inequalities between length and time scales in glassy systems},
	author       = {Montanari, Andrea and Semerjian, Guilhem},
	year         = 2006,
	journal      = {Journal of statistical physics},
	publisher    = {Springer},
	volume       = 125,
	number       = 1,
}
@article{nocedal2006numerical,
	title        = {Numerical Optimization, Second Edition},
	author       = {Nocedal, Jorge and Wright, Stephen J},
	year         = 2006,
	journal      = {Numerical optimization},
	publisher    = {Springer New York},
}
@book{williams2006gaussian,
	title        = {Gaussian processes for machine learning},
	author       = {Williams, Christopher KI and Rasmussen, Carl Edward},
	year         = 2006,
	publisher    = {MIT Press Cambridge, MA},
	volume       = 2,
	number       = 3,
}
@article{pnasmontanariksat,
	title        = {Gibbs states and the set of solutions of random constraint satisfaction problems},
	author       = {{Krzaka{\l}a}, Florent and Montanari, Andrea and Ricci-Tersenghi, Federico and Semerjian, Guilhem and {Zdeborov{\'a}}, Lenka},
	year         = 2007,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Academy of Sciences},
	volume       = 104,
	number       = 25,
	doi          = {10.1073/pnas.0703685104},
	issn         = {0027-8424},
	abstract     = {An instance of a random constraint satisfaction problem defines a random subset ?? (the set of solutions) of a large product space X N (the set of assignments). We consider two prototypical problem ensembles (random k-satisfiability and q-coloring of random regular graphs) and study the uniform measure with support on S. As the number of constraints per variable increases, this measure first decomposes into an exponential number of pure states ({\textquotedblleft}clusters{\textquotedblright}) and subsequently condensates over the largest such states. Above the condensation point, the mass carried by the n largest states follows a Poisson-Dirichlet process. For typical large instances, the two transitions are sharp. We determine their precise location. Further, we provide a formal definition of each phase transition in terms of different notions of correlation between distinct variables in the problem. The degree of correlation naturally affects the performances of many search/sampling algorithms. Empirical evidence suggests that local Monte Carlo Markov chain strategies are effective up to the clustering phase transition and belief propagation up to the condensation point. Finally, refined message passing techniques (such as survey propagation) may also beat this threshold.},
}
@article{Zdeborova07,
	title        = {Phase transitions in the coloring of random graphs},
	author       = {{Zdeborov{\'a}}, Lenka and Krzakala, Florent},
	year         = 2007,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 76,
	number       = 3,
}
@book{kardar2007statistical,
	title        = {Statistical physics of fields},
	author       = {Kardar, Mehran},
	year         = 2007,
	publisher    = {Cambridge University Press},
}
@article{Krzakala07,
	title        = {Landscape analysis of constraint satisfaction problems},
	author       = {Krzakala, Florent and Kurchan, Jorge},
	year         = 2007,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 76,
	number       = 2,
}
@inproceedings{watanabe2007almost,
	title        = {Almost all learning machines are singular},
	author       = {Watanabe, Sumio},
	year         = 2007,
	booktitle    = {Foundations of Computational Intelligence, 2007. FOCI 2007. IEEE Symposium on},
	organization = {IEEE},
}
@inproceedings{achlioptas2008algorithmic,
	title        = {Algorithmic barriers from phase transitions},
	author       = {Achlioptas, Dimitris and Coja-Oghlan, Amin},
	year         = 2008,
	booktitle    = {Foundations of Computer Science, 2008. FOCS'08. IEEE 49th Annual IEEE Symposium on},
	organization = {IEEE},
}
@inproceedings{mairal_supervised_2008,
	title        = {Supervised {Dictionary} {Learning}},
	author       = {Mairal, Julien and Ponce, Jean and Sapiro, Guillermo and Zisserman, Andrew and Bach, Francis},
	year         = 2008,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher    = {Curran Associates, Inc.},
	volume       = 21,
	url          = {https://papers.nips.cc/paper/2008/hash/c0f168ce8900fa56e57789e2a2f2c9d0-Abstract.html},
	urldate      = {2022-09-29},
	abstract     = {It is now well established that sparse signal models are well suited to restoration tasks and can effectively be learned from audio, image, and video data. Recent research has been aimed at learning discriminative sparse models instead of purely reconstructive ones. This paper proposes a new step in that direction with a novel sparse representation for signals belonging to different classes in terms of a shared dictionary and multiple decision functions. It is shown that the linear variant of the model admits a simple probabilistic interpretation, and that its most general variant also admits a simple interpretation in terms of kernels. An optimization framework for learning all the components of the proposed model is presented, along with experiments on standard handwritten digit and texture classification tasks.},
}
@article{bengio2009learning,
	title        = {Learning deep architectures for AI},
	author       = {Bengio, Yoshua and others},
	year         = 2009,
	journal      = {Foundations and trends{\textregistered} in Machine Learning},
	publisher    = {Now Publishers, Inc.},
	volume       = 2,
	number       = 1,
}
@inproceedings{deng_imagenet_2009,
	title        = {{ImageNet}: {A} large-scale hierarchical image database},
	author       = {Deng, J. and Dong, W. and Socher, R. and Li, L. and {Kai Li} and {Li Fei-Fei}},
	year         = 2009,
	month        = jun,
	booktitle    = {2009 {IEEE} {Conference} on {Computer} {Vision} and {Pattern} {Recognition}},
	doi          = {10.1109/CVPR.2009.5206848},
	note         = {ISSN: 1063-6919},
	abstract     = {The explosion of image data on the Internet has the potential to foster more sophisticated and robust models and algorithms to index, retrieve, organize and interact with images and multimedia data. But exactly how such data can be harnessed and organized remains a critical problem. We introduce here a new database called “ImageNet”, a large-scale ontology of images built upon the backbone of the WordNet structure. ImageNet aims to populate the majority of the 80,000 synsets of WordNet with an average of 500-1000 clean and full resolution images. This will result in tens of millions of annotated images organized by the semantic hierarchy of WordNet. This paper offers a detailed analysis of ImageNet in its current state: 12 subtrees with 5247 synsets and 3.2 million images in total. We show that ImageNet is much larger in scale and diversity and much more accurate than the current image datasets. Constructing such a large-scale database is a challenging task. We describe the data collection scheme with Amazon Mechanical Turk. Lastly, we illustrate the usefulness of ImageNet through three simple applications in object recognition, image classification and automatic object clustering. We hope that the scale, accuracy, diversity and hierarchical structure of ImageNet can offer unparalleled opportunities to researchers in the computer vision community and beyond.},
	keywords     = {computer vision, Explosions, Image databases, image resolution, image retrieval, Image retrieval, ImageNet database, Information retrieval, Internet, large-scale hierarchical image database, large-scale ontology, Large-scale systems, multimedia computing, multimedia data, Multimedia databases, Ontologies, ontologies (artificial intelligence), Robustness, Spine, subtree, trees (mathematics), very large databases, visual databases, wordNet structure},
}
@article{krizhevsky_learning_2009,
	title        = {Learning multiple layers of features from tiny images},
	author       = {Krizhevsky, Alex},
	year         = 2009,
	journal      = {.},
	abstract     = {Groups at MIT and NYU have collected a dataset of millions of tiny colour images from the web. It is, in principle, an excellent dataset for unsupervised training of deep generative models, but previous researchers who have tried this have found it difficult to learn a good set of filters from the images. We show how to train a multi-layer generative model that learns to extract meaningful features which resemble those found in the human visual cortex. Using a novel parallelization algorithm to distribute the work among multiple machines connected on a network, we show how training such a model can be done in reasonable time. A second problematic aspect of the tiny images dataset is that there are no reliable class labels which makes it hard to use for object recognition experiments. We created two sets of reliable labels. The CIFAR-10 set has 6000 examples of each of 10 classes and the CIFAR-100 set has 600 examples of each of 100 non-overlapping classes. Using these labels, we show that object recognition is significantly},
}
@article{Mailman09,
	title        = {Jamming in Systems Composed of Frictionless Ellipse-Shaped Particles},
	author       = {Mailman, Mitch and Schreck, Carl F. and O'Hern, Corey S. and Chakraborty, Bulbul},
	year         = 2009,
	month        = {Jun},
	journal      = {Phys. Rev. Lett.},
	publisher    = {American Physical Society},
	volume       = 102,
	number       = 25,
	doi          = {10.1103/PhysRevLett.102.255501},
	numpages     = 4,
	bdsk-url-1   = {http://dx.doi.org/10.1103/PhysRevLett.102.255501},
}
@incollection{Cho2009,
	title        = {Kernel Methods for Deep Learning},
	author       = {Youngmin Cho and Lawrence K. Saul},
	year         = 2009,
	booktitle    = {Advances in Neural Information Processing Systems 22},
	publisher    = {Curran Associates, Inc.},
	url          = {http://papers.nips.cc/paper/3628-kernel-methods-for-deep-learning.pdf},
}
@article{Zeravcic09,
	title        = {Excitations of ellipsoid packings near jamming},
	author       = {Z. Zeravcic and N. Xu and A. J. Liu and S. R. Nagel and W. van Saarloos},
	year         = 2009,
	journal      = EPL,
	volume       = 87,
	number       = 2,
	abstract     = {We study the vibrational modes of three-dimensional jammed packings of soft ellipsoids of revolution as a function of particle aspect ratio {{\OE}µ} and packing fraction. At the jamming transition for ellipsoids, as distinct from the idealized case using spheres where {{\OE}µ=1}, there are many unconstrained and nontrivial rotational degrees of freedom. These constitute a set of zero-frequency modes that are gradually mobilized into a new rotational band as {|{\OE}µ-1|} increases. Quite surprisingly, as this new band is separated from zero frequency by a gap, and lies below the onset frequency for translational vibrations, {{\oe}{\^a} * }, the presence of these new degrees of freedom leaves unaltered the basic scenario that the translational spectrum is determined only by the average contact number. Indeed, {{\oe}{\^a} *} depends solely on coordination as it does for compressed packings of spheres. We also discuss the regime of large {|{\OE}µ-1|}, where the two bands merge.},
}
@incollection{bottou2010large,
	title        = {Large-scale machine learning with stochastic gradient descent},
	author       = {Bottou, {L{\'e}on}},
	year         = 2010,
	booktitle    = {Proceedings of COMPSTAT'2010},
	publisher    = {Physica-Verlag HD},
}
@book{Liu10,
	title        = {The jamming scenario - an introduction and outlook},
	author       = {J Liu, Andrea and R Nagel, Sidney and Saarloos, W and Wyart, Matthieu},
	year         = 2010,
	month        = {06},
	booktitle    = {Dynamical Heterogeneities in Glasses, Colloids, and Granular Media},
	publisher    = {OUP Oxford},
}
@article{lecun-mnist,
	title        = {{MNIST} handwritten digit database},
	author       = {LeCun, Yann and Cortes, Corinna},
	year         = 2010,
	journal      = {},
	url          = {http://yann.lecun.com/exdb/mnist/},
	added-at     = {2010-06-28T21:16:30.000+0200},
	biburl       = {https://www.bibsonomy.org/bibtex/2935bad99fa1f65e03c25b315aa3c1032/mhwombat},
	groups       = {public},
	howpublished = {http://yann.lecun.com/exdb/mnist/},
	interhash    = {21b9d0558bd66279df9452562df6e6f3},
	intrahash    = {935bad99fa1f65e03c25b315aa3c1032},
	lastchecked  = {2016-01-14 14:24:11},
	timestamp    = {2016-07-12T19:25:30.000+0200},
	username     = {mhwombat},
	keywords     = {MSc _checked character_recognition mnist network neural},
}
@inproceedings{martens2010deep,
	title        = {Deep learning via Hessian-free optimization},
	author       = {Martens, James},
	year         = 2010,
	booktitle    = {Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
}
@article{wyart2010scaling,
	title        = {Scaling of phononic transport with connectivity in amorphous solids},
	author       = {Wyart, Matthieu},
	year         = 2010,
	journal      = {EPL (Europhysics Letters)},
	publisher    = {IOP Publishing},
	volume       = 89,
	number       = 6,
}
@article{reviewBB,
	title        = {Theoretical perspective on the glass transition and amorphous materials},
	author       = {Berthier, Ludovic and Biroli, Giulio},
	year         = 2011,
	journal      = {Reviews of Modern Physics},
	publisher    = {APS},
	volume       = 83,
	number       = 2,
}
@incollection{alexnet,
	title        = {ImageNet Classification with Deep Convolutional Neural Networks},
	author       = {Alex Krizhevsky and Sutskever, Ilya and Hinton, Geoffrey E},
	year         = 2012,
	booktitle    = {Advances in Neural Information Processing Systems 25},
	publisher    = {Curran Associates, Inc.},
	url          = {http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf},
	editor       = {F. Pereira and C. J. C. Burges and L. Bottou and K. Q. Weinberger},
}
@book{atkinson2012spherical,
	title        = {Spherical harmonics and approximations on the unit sphere: an introduction},
	author       = {Atkinson, Kendall and Han, Weimin},
	year         = 2012,
	publisher    = {Springer Science \& Business Media},
	volume       = 2044,
}
@article{Charbonneau12,
	title        = {Universal Microstructure and Mechanical Stability of Jammed Packings},
	author       = {Charbonneau, Patrick and Corwin, Eric I. and Parisi, Giorgio and Zamponi, Francesco},
	year         = 2012,
	month        = 11,
	day          = 13,
	journal      = {Physical Review Letters},
	publisher    = {American Physical Society},
	volume       = 109,
	number       = 20,
	url          = {http://link.aps.org/doi/10.1103/PhysRevLett.109.205501},
	date         = {2012/11/13/},
	id           = {10.1103/PhysRevLett.109.205501},
	j1           = {PRL},
	journal1     = {Phys. Rev. Lett.},
	ty           = {JOUR},
	bdsk-url-1   = {http://link.aps.org/doi/10.1103/PhysRevLett.109.205501},
}
@article{Lerner12,
	title        = {Toward a microscopic description of flow near the jamming threshold},
	author       = {E. Lerner and G. {D\"uring} and M. Wyart},
	year         = 2012,
	journal      = {EPL (Europhysics Letters)},
	volume       = 99,
	number       = 5,
	bdsk-url-1   = {http://stacks.iop.org/0295-5075/99/i=5/a=58003},
}
@article{Hinton12,
	title        = {Deep neural networks for acoustic modeling in speech recognition: The shared views of four research groups},
	author       = {Hinton, Geoffrey and Deng, Li and Yu, Dong and Dahl, George E and Mohamed, Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Sainath, Tara N and others},
	year         = 2012,
	journal      = {IEEE Signal processing magazine},
	publisher    = {IEEE},
	volume       = 29,
	number       = 6,
}
@inproceedings{Krizhevsky12,
	title        = {Imagenet classification with deep convolutional neural networks},
	author       = {Krizhevsky, Alex and Sutskever, Ilya and Hinton, Geoffrey E},
	year         = 2012,
	booktitle    = {Advances in neural information processing systems},
}
@article{lerner2012unified,
	title        = {A unified framework for non-Brownian suspension flows and soft amorphous solids},
	author       = {Lerner, Edan and D{\"u}ring, Gustavo and Wyart, Matthieu},
	year         = 2012,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 109,
	number       = 13,
}
@book{stein2012interpolation,
	title        = {Interpolation of spatial data: some theory for kriging},
	author       = {Stein, Michael L},
	year         = 2012,
	publisher    = {Springer Science \& Business Media},
}
@article{wyart2012marginal,
	title        = {Marginal stability constrains force and pair distributions at random close packing},
	author       = {Wyart, Matthieu},
	year         = 2012,
	journal      = {Physical review letters},
	publisher    = {APS},
	volume       = 109,
	number       = 12,
}
@article{Wyart12,
	title        = {Marginal Stability Constrains Force and Pair Distributions at Random Close Packing},
	author       = {Wyart, Matthieu},
	year         = 2012,
	month        = {Sep},
	journal      = {Phys. Rev. Lett.},
	publisher    = {American Physical Society},
	volume       = 109,
	doi          = {10.1103/PhysRevLett.109.125502},
	issue        = 12,
	numpages     = 5,
	bdsk-url-1   = {http://link.aps.org/doi/10.1103/PhysRevLett.109.125502},
	bdsk-url-2   = {http://dx.doi.org/10.1103/PhysRevLett.109.125502},
}
@article{During13,
	title        = {Phonon gap and localization lengths in floppy materials},
	author       = {{D{\"u}ring}, Gustavo and Lerner, Edan and Wyart, Matthieu},
	year         = 2013,
	journal      = {Soft Matter},
	publisher    = {Royal Society of Chemistry},
	volume       = 9,
	number       = 1,
	date         = 2013,
}
@article{auffinger2013random,
	title        = {Random matrices and complexity of spin glasses},
	author       = {Auffinger, Antonio and Ben Arous, {G{\'e}rard} and {\v{C}}ern{\`y}, Ji{\v{r}}{\'\i}},
	year         = 2013,
	journal      = {Communications on Pure and Applied Mathematics},
	publisher    = {Wiley Online Library},
	volume       = 66,
	number       = 2,
}
@article{bruna2013invariant,
	title        = {Invariant scattering convolution networks},
	author       = {Bruna, Joan and Mallat, St{\'e}phane},
	year         = 2013,
	journal      = {IEEE transactions on pattern analysis and machine intelligence},
	publisher    = {IEEE},
	volume       = 35,
	number       = 8,
}
@inproceedings{le2013building,
	title        = {Building high-level features using large scale unsupervised learning},
	author       = {Le, Quoc V},
	year         = 2013,
	booktitle    = {2013 IEEE international conference on acoustics, speech and signal processing},
	organization = {IEEE},
}
@article{Lerner13a,
	title        = {Low-energy non-linear excitations in sphere packings},
	author       = {Lerner, Edan and During, Gustavo and Wyart, Matthieu},
	year         = 2013,
	journal      = {Soft Matter},
	publisher    = {The Royal Society of Chemistry},
	volume       = 9,
	doi          = {10.1039/C3SM50515D},
	issue        = 34,
	bdsk-url-1   = {http://dx.doi.org/10.1039/C3SM50515D},
	abstract     = {We study theoretically and numerically how hard frictionless particles in random packings can rearrange. We demonstrate the existence of two distinct unstable non-linear modes of rearrangement{,} both associated with the opening and the closing of contacts. The first mode{,} whose density is characterized by some exponent [small theta][prime or minute]{,} corresponds to motions of particles extending throughout the entire system. The second mode{,} whose density is characterized by an exponent [small theta] [not equal] [small theta][prime or minute]{,} corresponds to the local buckling of a few particles. Extended modes are shown to yield at a much higher rate than local modes when a stress is applied. We show that the distribution of contact forces follows P(f) [similar] fmin([small theta][prime or minute]{,}[small theta]){,} and that imposing the restriction that the packing cannot be densified further leads to the bounds and {,} where [gamma] characterizes the singularity of the pair distribution function g(r) at contact. These results extend the theoretical analysis of [Wyart{,} Phys. Rev. Lett.{,} 2012{,} 109{,} 125502] where the existence of local modes was not considered. We perform numerics that support that these bounds are saturated with [gamma] [approximate] 0.38{,} [small theta] [approximate] 0.17 and [small theta][prime or minute] [approximate] 0.44. We measure systematically the stability of all such modes in packings{,} and confirm their marginal stability. The principle of marginal stability thus allows us to make clearcut predictions on the ensemble of configurations visited in these out-of-equilibrium systems{,} and on the contact forces and pair distribution functions. It also reveals the excitations that need to be included in a description of plasticity or flow near jamming{,} and suggests a new path to study two-level systems and soft spots in simple amorphous solids of repulsive particles.},
}
@article{mnih2013playing,
	title        = {Playing atari with deep reinforcement learning},
	author       = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
	year         = 2013,
	journal      = {arXiv preprint arXiv:1312.5602},
}
@article{schaul2013no,
	title        = {No more pesky learning rates.},
	author       = {Schaul, Tom and Zhang, Sixin and LeCun, Yann},
	year         = 2013,
	journal      = {ICML (3)},
	volume       = 28,
}
@inproceedings{sutskever2013importance,
	title        = {On the importance of initialization and momentum in deep learning},
	author       = {Sutskever, Ilya and Martens, James and Dahl, George and Hinton, Geoffrey},
	year         = 2013,
	booktitle    = {International conference on machine learning},
}
@article{tibshirani_lasso_2013,
	title        = {The lasso problem and uniqueness},
	author       = {Tibshirani, Ryan J.},
	year         = 2013,
	month        = jan,
	journal      = {Electronic Journal of Statistics},
	volume       = 7,
	number       = {none},
	doi          = {10.1214/13-EJS815},
	issn         = {1935-7524},
	url          = {https://projecteuclid.org/journals/electronic-journal-of-statistics/volume-7/issue-none/The-lasso-problem-and-uniqueness/10.1214/13-EJS815.full},
	urldate      = {2022-05-20},
	language     = {en},
}
@article{Bianchini14,
	title        = {On the complexity of neural network classifiers: A comparison between shallow and deep architectures},
	author       = {Bianchini, Monica and Scarselli, Franco},
	year         = 2014,
	journal      = {IEEE transactions on neural networks and learning systems},
	publisher    = {IEEE},
	volume       = 25,
	number       = 8,
}
@article{Charbonneau14a,
	title        = {Exact theory of dense amorphous hard spheres in high dimension. III. The full replica symmetry breaking solution},
	author       = {Charbonneau, Patrick and Kurchan, Jorge and Parisi, Giorgio and Urbani, Pierfrancesco and Zamponi, Francesco},
	year         = 2014,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	publisher    = {IOP Publishing},
	volume       = 2014,
	number       = 10,
	date         = 2014,
}
@article{Charbonneau14,
	title        = {Fractal free energy landscapes in structural glasses},
	author       = {Charbonneau, Patrick and Kurchan, Jorge and Parisi, Giorgio and Urbani, Pierfrancesco and Zamponi, Francesco},
	year         = 2014,
	journal      = {Nature Communications},
	publisher    = {Nature Publishing Group},
	volume       = 5,
	number       = 3725,
	date         = 2014,
}
@inproceedings{dauphin2014identifying,
	title        = {Identifying and attacking the saddle point problem in high-dimensional non-convex optimization},
	author       = {Dauphin, Yann N and Pascanu, Razvan and Gulcehre, Caglar and Cho, Kyunghyun and Ganguli, Surya and Bengio, Yoshua},
	year         = 2014,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{DeGiuli14,
	title        = {Effects of coordination and pressure on sound attenuation, boson peak and elasticity in amorphous solids},
	author       = {DeGiuli, Eric and Laversanne-Finot, Adrien and {D\"uring}, Gustavo Alberto and Lerner, Edan and Wyart, Matthieu},
	year         = 2014,
	journal      = {Soft Matter},
	publisher    = {Royal Society of Chemistry},
	volume       = 10,
	number       = 30,
	date         = 2014,
}
@inproceedings{denton14,
	title        = {Exploiting Linear Structure Within Convolutional Networks for Efficient Evaluation},
	author       = {Denton, Emily and Zaremba, Wojciech and Bruna, Joan and LeCun, Yann and Fergus, Rob},
	year         = 2014,
	booktitle    = {Proceedings of the 27th International Conference on Neural Information Processing Systems},
	location     = {Montreal, Canada},
	publisher    = {MIT Press},
	address      = {Cambridge, MA, USA},
	series       = {NIPS'14},
	url          = {http://dl.acm.org/citation.cfm?id=2968826.2968968},
	numpages     = 9,
	acmid        = 2968968,
}
@book{efthimiou2014spherical,
	title        = {Spherical harmonics in p dimensions},
	author       = {Efthimiou, Costas and Frye, Christopher},
	year         = 2014,
	publisher    = {World Scientific},
}
@inproceedings{Montufar14,
	title        = {On the number of linear regions of deep neural networks},
	author       = {Montufar, Guido F and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua},
	year         = 2014,
	booktitle    = {Advances in neural information processing systems},
}
@article{Saxe13,
	title        = {Exact solutions to the nonlinear dynamics of learning in deep linear neural networks},
	author       = {Saxe, Andrew M and McClelland, James L and Ganguli, Surya},
	year         = 2014,
	journal      = {International Conference on Learning Representations},
}
@book{shalev2014understanding,
	title        = {Understanding machine learning: From theory to algorithms},
	author       = {Shalev-Shwartz, Shai and Ben-David, Shai},
	year         = 2014,
	publisher    = {Cambridge university press},
}
@article{srivastava2014dropout,
	title        = {Dropout: a simple way to prevent neural networks from overfitting},
	author       = {Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
	year         = 2014,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 15,
	number       = 1,
}
@article{zhou2014object,
	title        = {Object detectors emerge in deep scene cnns},
	author       = {Zhou, Bolei and Khosla, Aditya and Lapedriza, Agata and Oliva, Aude and Torralba, Antonio},
	year         = 2014,
	journal      = {arXiv preprint arXiv:1412.6856},
}
@inproceedings{liu2015,
	title        = {Sparse Convolutional Neural Networks},
	author       = {Baoyuan Liu and Min Wang and H. Foroosh and M. Tappen and M. Penksy},
	year         = 2015,
	month        = {June},
	booktitle    = {2015 IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
	doi          = {10.1109/CVPR.2015.7298681},
	issn         = {1063-6919},
	keywords     = {matrix decomposition;matrix multiplication;neural nets;object detection;SCNN model;cascade model;object detection problem;sparse convolutional neural networks;sparse decomposition;sparse fully connected layers;sparse matrix multiplication algorithm;Accuracy;Convolutional codes;Kernel;Matrix decomposition;Neural networks;Redundancy;Sparse matrices},
}
@article{Charbonneau15,
	title        = {Jamming Criticality Revealed by Removing Localized Buckling Excitations},
	author       = {Charbonneau, Patrick and Corwin, Eric I and Parisi, Giorgio and Zamponi, Francesco},
	year         = 2015,
	journal      = {Physical Review Letters},
	publisher    = {APS},
	volume       = 114,
	number       = 12,
	date         = 2015,
}
@inproceedings{Choromanska15,
	title        = {The loss surfaces of multilayer networks},
	author       = {Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Ben Arous, G{\'e}rard and LeCun, Yann},
	year         = 2015,
	booktitle    = {Artificial Intelligence and Statistics},
}
@article{degiuli2015unified,
	title        = {Unified theory of inertial granular flows and non-Brownian suspensions},
	author       = {DeGiuli, E and D{\"u}ring, G and Lerner, E and Wyart, M},
	year         = 2015,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 91,
	number       = 6,
}
@article{degiuli2015theory,
	title        = {Theory of the jamming transition at finite temperature},
	author       = {Degiuli, Eric and Lerner, E and Wyart, M},
	year         = 2015,
	journal      = {The Journal of chemical physics},
	publisher    = {AIP Publishing LLC},
	volume       = 142,
	number       = 16,
}
@software{autograd15,
	title        = {Autograd: Reverse-mode differentiation of native {P}ython},
	author       = {Dougal Maclaurin and David Duvenaud and Matthew Johnson and Ryan P. Adams},
	year         = 2015,
	url          = {http://github.com/HIPS/autograd},
	version      = {1.1.2},
}
@inproceedings{fawzi_manitest_2015,
	title        = {Manitest: {Are} classifiers really invariant?},
	author       = {Fawzi, Alhussein and Frossard, Pascal},
	year         = 2015,
	booktitle    = {Procedings of the {British} {Machine} {Vision} {Conference} 2015},
	publisher    = {British Machine Vision Association},
	address      = {Swansea},
	doi          = {10.5244/C.29.106},
	isbn         = {978-1-901725-53-7},
	url          = {http://www.bmva.org/bmvc/2015/papers/paper106/index.html},
	urldate      = {2021-05-25},
	language     = {en},
}
@article{Franz15,
	title        = {Universal spectrum of normal modes in low-temperature glasses},
	author       = {Franz, Silvio and Parisi, Giorgio and Urbani, Pierfrancesco and Zamponi, Francesco},
	year         = 2015,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 112,
	number       = 47,
}
@book{gikhman2015theory,
	title        = {The theory of stochastic processes I},
	author       = {Gikhman, Iosif I and Skorokhod, Anatoli V},
	year         = 2015,
	publisher    = {Springer},
}
@inproceedings{han15a,
	title        = {Learning both Weights and Connections for Efficient Neural Network},
	author       = {Han, Song and Pool, Jeff and Tran, John and Dally, William},
	year         = 2015,
	booktitle    = {Advances in Neural Information Processing Systems (NIPS)},
}
@article{hardt2015train,
	title        = {Train faster, generalize better: Stability of stochastic gradient descent},
	author       = {Hardt, Moritz and Recht, Benjamin and Singer, Yoram},
	year         = 2015,
	journal      = {arXiv preprint arXiv:1509.01240},
}
@article{huval2015empirical,
	title        = {An empirical evaluation of deep learning on highway driving},
	author       = {Huval, Brody and Wang, Tao and Tandon, Sameep and Kiske, Jeff and Song, Will and Pazhayampallil, Joel and Andriluka, Mykhaylo and Rajpurkar, Pranav and Migimatsu, Toki and Cheng-Yue, Royce and others},
	year         = 2015,
	journal      = {arXiv preprint arXiv:1504.01716},
}
@inproceedings{Ioffe15,
	title        = {Batch normalization: Accelerating deep network training by reducing internal covariate shift},
	author       = {Ioffe, Sergey and Szegedy, Christian},
	year         = 2015,
	booktitle    = {International conference on machine learning},
}
@article{Kingma14,
	title        = {Adam: A method for stochastic optimization},
	author       = {Kingma, Diederik P and Ba, Jimmy},
	year         = 2015,
	journal      = {International Conference on Learning Representations},
}
@article{Lecun15,
	title        = {Deep learning},
	author       = {LeCun, Yann and Bengio, Yoshua and Hinton, Geoffrey},
	year         = 2015,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 521,
	number       = 7553,
}
@article{Muller14,
	title        = {Marginal Stability in Structural, Spin, and Electron Glasses},
	author       = {M{\"u}ller, Markus and Wyart, Matthieu},
	year         = 2015,
	journal      = {Annual Review of Condensed Matter Physics},
	volume       = 6,
	number       = 1,
	doi          = {10.1146/annurev-conmatphys-031214-014614},
	bdsk-url-1   = {http://www.annualreviews.org/doi/abs/10.1146/annurev-conmatphys-031214-014614},
	bdsk-url-2   = {http://dx.doi.org/10.1146/annurev-conmatphys-031214-014614},
}
@inproceedings{mobahi2015theoretical,
	title        = {A Theoretical Analysis of Optimization by Gaussian Continuation.},
	author       = {Mobahi, Hossein and Fisher III, John W},
	year         = 2015,
	booktitle    = {AAAI},
	organization = {Citeseer},
}
@inproceedings{mobahi2015link,
	title        = {On the link between gaussian homotopy continuation and convex envelopes},
	author       = {Mobahi, Hossein and Fisher III, John W},
	year         = 2015,
	booktitle    = {International Workshop on Energy Minimization Methods in Computer Vision and Pattern Recognition},
	organization = {Springer},
}
@inproceedings{neyshabur_norm-based_2015,
	title        = {Norm-based capacity control in neural networks},
	author       = {Neyshabur, Behnam and Tomioka, Ryota and Srebro, Nathan},
	year         = 2015,
	booktitle    = {Conference on {Learning} {Theory}},
	publisher    = {PMLR},
}
@article{sagun2014explorations,
	title        = {Explorations on high dimensional landscapes},
	author       = {Sagun, Levent and {G\"uney}, V. {U\u{g}ur} and {G{\'{e}}rard} {Ben Arous} and LeCun, Yann},
	year         = 2015,
	journal      = {International Conference on Learning Representations Workshop Contribution, arXiv:1412.6615},
}
@article{simonyan_very_2015,
	title        = {Very {Deep} {Convolutional} {Networks} for {Large}-{Scale} {Image} {Recognition}},
	author       = {Simonyan, K. and Zisserman, Andrew},
	year         = 2015,
	journal      = {ICLR},
	abstract     = {In this work we investigate the effect of the convolutional network depth on its accuracy in the large-scale image recognition setting. Our main contribution is a thorough evaluation of networks of increasing depth using an architecture with very small (3x3) convolution filters, which shows that a significant improvement on the prior-art configurations can be achieved by pushing the depth to 16-19 weight layers. These findings were the basis of our ImageNet Challenge 2014 submission, where our team secured the first and the second places in the localisation and classification tracks respectively. We also show that our representations generalise well to other datasets, where they achieve state-of-the-art results. We have made our two best-performing ConvNet models publicly available to facilitate further research on the use of deep visual representations in computer vision.},
}
@article{zdeborovareview,
	title        = {Statistical physics of inference: Thresholds and algorithms},
	author       = {{Zdeborov{\'a}}, Lenka and Krzakala, Florent},
	year         = 2016,
	journal      = {Advances in Physics},
	publisher    = {Taylor \& Francis},
	volume       = 65,
	number       = 5,
}
@inproceedings{amodei2016deep,
	title        = {Deep speech 2: End-to-end speech recognition in english and mandarin},
	author       = {Amodei, Dario and Ananthanarayanan, Sundaram and Anubhai, Rishita and Bai, Jingliang and Battenberg, Eric and Case, Carl and Casper, Jared and Catanzaro, Bryan and Cheng, Qiang and Chen, Guoliang and others},
	year         = 2016,
	booktitle    = {International conference on machine learning},
}
@article{anselmi_unsupervised_2016,
	title        = {Unsupervised learning of invariant representations},
	author       = {Anselmi, Fabio and Leibo, Joel Z. and Rosasco, Lorenzo and Mutch, Jim and Tacchetti, Andrea and Poggio, Tomaso},
	year         = 2016,
	month        = jun,
	journal      = {Theoretical Computer Science},
	series       = {Biologically {Inspired} {Processes} in {Neural} {Computation}},
	volume       = 633,
	doi          = {10.1016/j.tcs.2015.06.048},
	issn         = {0304-3975},
	url          = {https://www.sciencedirect.com/science/article/pii/S0304397515005587},
	urldate      = {2022-09-21},
	abstract     = {The present phase of Machine Learning is characterized by supervised learning algorithms relying on large sets of labeled examples (n→∞). The next phase is likely to focus on algorithms capable of learning from very few labeled examples (n→1), like humans seem able to do. We propose an approach to this problem and describe the underlying theory, based on the unsupervised, automatic learning of a “good” representation for supervised learning, characterized by small sample complexity. We consider the case of visual object recognition, though the theory also applies to other domains like speech. The starting point is the conjecture, proved in specific cases, that image representations which are invariant to translation, scaling and other transformations can considerably reduce the sample complexity of learning. We prove that an invariant and selective signature can be computed for each image or image patch: the invariance can be exact in the case of group transformations and approximate under non-group transformations. A module performing filtering and pooling, like the simple and complex cells described by Hubel and Wiesel, can compute such signature. The theory offers novel unsupervised learning algorithms for “deep” architectures for image and speech recognition. We conjecture that the main computational goal of the ventral stream of visual cortex is to provide a hierarchical representation of new objects/images which is invariant to transformations, stable, and selective for recognition—and show how this representation may be continuously learned in an unsupervised way during development and visual experience.},
	language     = {en},
	keywords     = {Convolutional networks, Cortex, Hierarchy, Invariance},
}
@article{baldassi2016unreasonable,
	title        = {Unreasonable effectiveness of learning neural networks: {From} accessible states and robust ensembles to basic algorithmic schemes},
	author       = {Baldassi, Carlo and Borgs, Christian and Chayes, Jennifer T. and Ingrosso, Alessandro and Lucibello, Carlo and Saglietti, Luca and Zecchina, Riccardo},
	year         = 2016,
	month        = nov,
	journal      = {Proceedings of the National Academy of Sciences},
	volume       = 113,
	number       = 48,
	doi          = {10.1073/pnas.1608103113},
	issn         = {0027-8424, 1091-6490},
	urldate      = {2016-11-30},
	language     = {en},
}
@article{Balduzzi16,
	title        = {Deep online convex optimization with gated games},
	author       = {Balduzzi, David},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1604.01952},
}
@incollection{birolileshouches,
	title        = {Slow relaxations and non-equilibrium dynamics in classical and quantum systems},
	author       = {Biroli, Giulio},
	year         = 2016,
	booktitle    = {Strongly Interacting Quantum Systems Out of Equilibrium},
	publisher    = {Oxford University Press},
	address      = {Oxford},
	editor       = {Thierry Giamarchi, Andrew J. Millis, Olivier Parcollet},
}
@article{bloemendal2016principal,
	title        = {On the principal components of sample covariance matrices},
	author       = {Bloemendal, Alex and Knowles, Antti and Yau, Horng-Tzer and Yin, Jun},
	year         = 2016,
	journal      = {Probability Theory and Related Fields},
	publisher    = {Springer},
	volume       = 164,
	number       = {1-2},
}
@article{chaudhari2016entropy,
	title        = {Entropy-SGD: Biasing Gradient Descent Into Wide Valleys},
	author       = {Chaudhari, Pratik and Choromanska, Anna and Soatto, Stefano and LeCun, Yann and Baldassi, Carlo and Borgs, Christian and Chayes, Jennifer and Sagun, Levent and Zecchina, Riccardo},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1611.01838},
}
@article{Chaudhari16,
	title        = {Entropy-sgd: Biasing gradient descent into wide valleys},
	author       = {Chaudhari, Pratik and Choromanska, Anna and Soatto, Stefano and LeCun, Yann and Baldassi, Carlo and Borgs, Christian and Chayes, Jennifer and Sagun, Levent and Zecchina, Riccardo},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1611.01838},
}
@inproceedings{daniely2016toward,
	title        = {Toward deeper understanding of neural networks: The power of initialization and a dual view on expressivity},
	author       = {Daniely, Amit and Frostig, Roy and Singer, Yoram},
	year         = 2016,
	booktitle    = {Advances In Neural Information Processing Systems},
}
@article{de2016comparing,
	title        = {Comparing molecules and solids across structural and alchemical space},
	author       = {De, Sandip and Bart{\'o}k, Albert P and Cs{\'a}nyi, G{\'a}bor and Ceriotti, Michele},
	year         = 2016,
	journal      = {Physical Chemistry Chemical Physics},
	publisher    = {Royal Society of Chemistry},
	volume       = 18,
	number       = 20,
}
@article{dieleman2016exploiting,
	title        = {Exploiting cyclic symmetry in convolutional neural networks},
	author       = {Dieleman, Sander and De Fauw, Jeffrey and Kavukcuoglu, Koray},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1602.02660},
}
@inproceedings{eldan2016power,
	title        = {The power of depth for feedforward neural networks},
	author       = {Eldan, Ronen and Shamir, Ohad},
	year         = 2016,
	booktitle    = {Conference on Learning Theory},
}
@article{Franz16,
	title        = {The simplest model of jamming},
	author       = {Franz, Silvio and Parisi, Giorgio},
	year         = 2016,
	journal      = {Journal of Physics A: Mathematical and Theoretical},
	publisher    = {IOP Publishing},
	volume       = 49,
	number       = 14,
}
@book{goodfellow_deep_2016,
	title        = {Deep {Learning}},
	author       = {Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron},
	year         = 2016,
	month        = nov,
	publisher    = {The MIT Press},
	address      = {Cambridge, Massachusetts},
	isbn         = {978-0-262-03561-3},
	language     = {English},
}
@article{gulcehre2016mollifying,
	title        = {Mollifying Networks},
	author       = {Gulcehre, Caglar and Moczulski, Marcin and Visin, Francesco and Bengio, Yoshua},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1608.04980},
}
@article{han15b,
	title        = {Deep Compression: Compressing Deep Neural Networks with Pruning, Trained Quantization and Huffman Coding},
	author       = {Han, Song and Mao, Huizi and Dally, William J},
	year         = 2016,
	journal      = {International Conference on Learning Representations (ICLR)},
}
@article{hauberg_dreaming_2016,
	title        = {Dreaming {More} {Data}: {Class}-dependent {Distributions} over {Diffeomorphisms} for {Learned} {Data} {Augmentation}},
	author       = {Hauberg, Søren and Freifeld, Oren and Larsen, Anders Boesen Lindbo and Fisher III, John W. and Hansen, Lars Kai},
	year         = 2016,
	month        = jun,
	journal      = {arXiv:1510.02795 [cs]},
	url          = {http://arxiv.org/abs/1510.02795},
	urldate      = {2021-10-20},
	note         = {arXiv: 1510.02795},
	abstract     = {Data augmentation is a key element in training high-dimensional models. In this approach, one synthesizes new observations by applying pre-specified transformations to the original training data; e.g.{\textasciitilde}new images are formed by rotating old ones. Current augmentation schemes, however, rely on manual specification of the applied transformations, making data augmentation an implicit form of feature engineering. With an eye towards true end-to-end learning, we suggest learning the applied transformations on a per-class basis. Particularly, we align image pairs within each class under the assumption that the spatial transformation between images belongs to a large class of diffeomorphisms. We then learn a class-specific probabilistic generative models of the transformations in a Riemannian submanifold of the Lie group of diffeomorphisms. We demonstrate significant performance improvements in training deep neural nets over manually-specified augmentation schemes. Our code and augmented datasets are available online.},
	keywords     = {Computer Science - Computer Vision and Pattern Recognition},
}
@inproceedings{hazan2016graduated,
	title        = {On graduated optimization for stochastic non-convex problems},
	author       = {Hazan, Elad and Levy, Kfir Yehuda and Shalev-Shwartz, Shai},
	year         = 2016,
	booktitle    = {International Conference on Machine Learning},
}
@inproceedings{he_deep_2016,
	title        = {Deep {Residual} {Learning} for {Image} {Recognition}},
	author       = {He, K. and Zhang, X. and Ren, S. and Sun, J.},
	year         = 2016,
	month        = jun,
	booktitle    = {2016 {IEEE} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} ({CVPR})},
	doi          = {10.1109/CVPR.2016.90},
	note         = {ISSN: 1063-6919},
	abstract     = {Deeper neural networks are more difficult to train. We present a residual learning framework to ease the training of networks that are substantially deeper than those used previously. We explicitly reformulate the layers as learning residual functions with reference to the layer inputs, instead of learning unreferenced functions. We provide comprehensive empirical evidence showing that these residual networks are easier to optimize, and can gain accuracy from considerably increased depth. On the ImageNet dataset we evaluate residual nets with a depth of up to 152 layers - 8× deeper than VGG nets [40] but still having lower complexity. An ensemble of these residual nets achieves 3.57\% error on the ImageNet test set. This result won the 1st place on the ILSVRC 2015 classification task. We also present analysis on CIFAR-10 with 100 and 1000 layers. The depth of representations is of central importance for many visual recognition tasks. Solely due to our extremely deep representations, we obtain a 28\% relative improvement on the COCO object detection dataset. Deep residual nets are foundations of our submissions to ILSVRC \& COCO 2015 competitions1, where we also won the 1st places on the tasks of ImageNet detection, ImageNet localization, COCO detection, and COCO segmentation.},
	keywords     = {CIFAR-10, COCO object detection dataset, COCO segmentation, Complexity theory, deep residual learning, deep residual nets, deeper neural network training, Degradation, ILSVRC \& COCO 2015 competitions, ILSVRC 2015 classification task, image classification, image recognition, Image recognition, Image segmentation, ImageNet dataset, ImageNet localization, ImageNet test set, learning (artificial intelligence), neural nets, Neural networks, object detection, residual function learning, residual nets, Training, VGG nets, visual recognition tasks, Visualization},
}
@inproceedings{He16,
	title        = {Deep residual learning for image recognition},
	author       = {He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
	year         = 2016,
	booktitle    = {Proceedings of the IEEE conference on computer vision and pattern recognition},
}
@inproceedings{kawaguchi2016deep,
	title        = {Deep learning without poor local minima},
	author       = {Kawaguchi, Kenji},
	year         = 2016,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{keskar2016large,
	title        = {On Large-Batch Training for Deep Learning: Generalization Gap and Sharp Minima},
	author       = {Keskar, Nitish Shirish and Mudigere, Dheevatsa and Nocedal, Jorge and Smelyanskiy, Mikhail and Tang, Ping Tak Peter},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1609.04836},
}
@article{lee2016gradient,
	title        = {Gradient descent converges to minimizers},
	author       = {Lee, Jason D and Simchowitz, Max and Jordan, Michael I and Recht, Benjamin},
	year         = 2016,
	journal      = {University of California, Berkeley},
	volume       = 1050,
}
@article{Lipton16,
	title        = {Stuck in a what? adventures in weight space},
	author       = {Lipton, Zachary C},
	year         = 2016,
	journal      = {International Conference on Learning Representations},
}
@article{loshchilov_sgdr_2016,
	title        = {{SGDR}: {Stochastic} {Gradient} {Descent} with {Warm} {Restarts}},
	author       = {Loshchilov, Ilya and Hutter, Frank},
	year         = 2016,
	month        = nov,
	url          = {https://openreview.net/forum?id=Skq89Scxx},
	urldate      = {2021-02-07},
	language     = {en},
	abstract     = {We propose a simple warm restart technique for stochastic gradient descent to improve its anytime performance.},
}
@article{mallat2016understanding,
	title        = {Understanding deep convolutional networks},
	author       = {Mallat, St{\'e}phane},
	year         = 2016,
	journal      = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
	publisher    = {The Royal Society Publishing},
	volume       = 374,
	number       = 2065,
}
@article{mei2016landscape,
	title        = {The landscape of empirical risk for non-convex losses},
	author       = {Mei, Song and Bai, Yu and Montanari, Andrea},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1607.06534},
}
@article{mobahi2016training,
	title        = {Training Recurrent Neural Networks by Diffusion},
	author       = {Mobahi, Hossein},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1601.04114},
}
@article{panageas2016gradient,
	title        = {Gradient descent only converges to minimizers: Non-isolated critical points and invariant regions},
	author       = {Panageas, Ioannis and Piliouras, Georgios},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1605.00405},
}
@book{poggio_visual_2016,
	title        = {Visual {Cortex} and {Deep} {Networks}: {Learning} {Invariant} {Representations}},
	author       = {Poggio, Tomaso A. and Anselmi, Fabio},
	year         = 2016,
	month        = sep,
	doi          = {10.7551/mitpress/10177.001.0001},
	url          = {https://direct.mit.edu/books/book/4088/Visual-Cortex-and-Deep-NetworksLearning-Invariant},
	urldate      = {2022-09-21},
	abstract     = {A mathematical framework that describes learning of invariant representations in the ventral stream, offering both theoretical development and applications.The},
	language     = {en},
}
@inproceedings{Zagoruyko2016WRN,
	title        = {Wide Residual Networks},
	author       = {Sergey Zagoruyko and Nikos Komodakis},
	year         = 2016,
	booktitle    = {BMVC},
}
@article{shi2016end,
	title        = {An end-to-end trainable neural network for image-based sequence recognition and its application to scene text recognition},
	author       = {Shi, Baoguang and Bai, Xiang and Yao, Cong},
	year         = 2016,
	journal      = {IEEE transactions on pattern analysis and machine intelligence},
	publisher    = {IEEE},
	volume       = 39,
	number       = 11,
}
@article{Silver16,
	title        = {Mastering the game of Go with deep neural networks and tree search},
	author       = {Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
	year         = 2016,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 529,
	number       = 7587,
}
@article{Soudry2016,
	title        = {No bad local minima: Data independent training error guarantees for multilayer neural networks},
	author       = {Soudry, Daniel and Carmon, Yair},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1605.08361},
}
@article{urban2016deep,
	title        = {Do deep convolutional nets really need to be deep and convolutional?},
	author       = {Urban, Gregor and Geras, Krzysztof J and Kahou, Samira Ebrahimi and Aslan, Ozlem and Wang, Shengjie and Caruana, Rich and Mohamed, Abdelrahman and Philipose, Matthai and Richardson, Matt},
	year         = 2016,
	journal      = {arXiv preprint arXiv:1603.05691},
}
@article{wen16,
	title        = {Learning Structured Sparsity in Deep Neural Networks},
	author       = {Wei Wen and Chunpeng Wu and Yandan Wang and Yiran Chen and Hai Li},
	year         = 2016,
	journal      = {CoRR},
	volume       = {abs/1608.03665},
	url          = {http://arxiv.org/abs/1608.03665},
	timestamp    = {Mon, 30 Jan 2017 17:08:13 +0100},
	biburl       = {http://dblp.uni-trier.de/rec/bib/journals/corr/WenWWCL16},
	bibsource    = {dblp computer science bibliography, http://dblp.org},
}
@article{Yan16,
	title        = {On variational arguments for vibrational modes near jamming},
	author       = {Yan, Le and DeGiuli, Eric and Wyart, Matthieu},
	year         = 2016,
	journal      = {EPL (Europhysics Letters)},
	publisher    = {IOP Publishing},
	volume       = 114,
	number       = 2,
	date         = 2016,
}
@article{Achille17,
	title        = {Emergence of invariance and disentangling in deep representations},
	author       = {Achille, Alessandro and Soatto, Stefano},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1706.01350},
}
@article{arpit2017closer,
	title        = {A closer look at memorization in deep networks},
	author       = {Arpit, Devansh and {Jastrz{\k{e}}bski}, {Stanis{\l}aw} and Ballas, Nicolas and Krueger, David and Bengio, Emmanuel and Kanwal, Maxinder S and Maharaj, Tegan and Fischer, Asja and Courville, Aaron and Bengio, Yoshua and others},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1706.05394},
}
@article{bach2017breaking,
	title        = {Breaking the curse of dimensionality with convex neural networks},
	author       = {Bach, Francis},
	year         = 2017,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 18,
	number       = 1,
}
@article{Balduzzi17,
	title        = {The Shattered Gradients Problem: If resnets are the answer, then what is the question?},
	author       = {Balduzzi, David and Frean, Marcus and Leary, Lennox and Lewis, JP and Ma, Kurt Wan-Duo and McWilliams, Brian},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1702.08591},
}
@article{Ballard17,
	title        = {Energy landscapes for machine learning},
	author       = {Ballard, Andrew J and Das, Ritankar and Martiniani, Stefano and Mehta, Dhagash and Sagun, Levent and Stevenson, Jacob D and Wales, David J},
	year         = 2017,
	journal      = {Physical Chemistry Chemical Physics},
	publisher    = {Royal Society of Chemistry},
}
@article{ballard2017perspective,
	title        = {Perspective: Energy Landscapes for Machine Learning},
	author       = {Ballard, Andrew J and Das, Ritankar and Martiniani, Stefano and Mehta, Dhagash and Sagun, Levent and Stevenson, Jacob D and Wales, David J},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1703.07915},
}
@article{barbier2017phase,
	title        = {Phase transitions, optimal errors and optimality of message-passing in generalized linear models},
	author       = {Barbier, Jean and Krzakala, Florent and Macris, Nicolas and Miolane, L{\'e}o and Zdeborov{\'a}, Lenka},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1708.03395},
}
@article{benarousj,
	title        = {Spectral gap estimates in mean field spin glasses},
	author       = {Ben Arous, {G{\'e}rard} and Jagannath, Aukosh},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1705.04243},
}
@article{carleo2017solving,
	title        = {Solving the quantum many-body problem with artificial neural networks},
	author       = {Carleo, Giuseppe and Troyer, Matthias},
	year         = 2017,
	journal      = {Science},
	publisher    = {American Association for the Advancement of Science},
	volume       = 355,
	number       = 6325,
}
@article{dinh2017sharp,
	title        = {Sharp Minima Can Generalize For Deep Nets},
	author       = {Dinh, Laurent and Pascanu, Razvan and Bengio, Samy and Bengio, Yoshua},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1703.04933},
}
@article{facco_estimating_2017,
	title        = {Estimating the intrinsic dimension of datasets by a minimal neighborhood information},
	author       = {Facco, Elena and d’Errico, Maria and Rodriguez, Alex and Laio, Alessandro},
	year         = 2017,
	month        = 9,
	journal      = {Scientific Reports},
	volume       = 7,
	number       = 1,
	doi          = {10.1038/s41598-017-11873-y},
	issn         = {2045-2322},
	url          = {https://www.nature.com/articles/s41598-017-11873-y},
	copyright    = {2017 The Author(s)},
	note         = {Number: 1 Publisher: Nature Publishing Group},
	language     = {en},
	abstract     = {Analyzing large volumes of high-dimensional data is an issue of fundamental importance in data science, molecular simulations and beyond. Several approaches work on the assumption that the important content of a dataset belongs to a manifold whose Intrinsic Dimension (ID) is much lower than the crude large number of coordinates. Such manifold is generally twisted and curved; in addition points on it will be non-uniformly distributed: two factors that make the identification of the ID and its exploitation really hard. Here we propose a new ID estimator using only the distance of the first and the second nearest neighbor of each point in the sample. This extreme minimality enables us to reduce the effects of curvature, of density variation, and the resulting computational cost. The ID estimator is theoretically exact in uniformly distributed datasets, and provides consistent measures in general. When used in combination with block analysis, it allows discriminating the relevant dimensions as a function of the block size. This allows estimating the ID even when the data lie on a manifold perturbed by a high-dimensional noise, a situation often encountered in real world data sets. We demonstrate the usefulness of the approach on molecular simulations and image analysis.},
}
@article{fawzi2017robustness,
	title        = {The robustness of deep networks: A geometrical perspective},
	author       = {Fawzi, Alhussein and Moosavi-Dezfooli, Seyed-Mohsen and Frossard, Pascal},
	year         = 2017,
	journal      = {IEEE Signal Processing Magazine},
	publisher    = {IEEE},
	volume       = 34,
	number       = 6,
}
@article{Franz17,
	title        = {Universality of the SAT-UNSAT (jamming) threshold in non-convex continuous constraint satisfaction problems},
	author       = {Franz, Silvio and Parisi, Giorgio and Sevelev, Maxime and Urbani, Pierfrancesco and Zamponi, Francesco},
	year         = 2017,
	journal      = {SciPost Physics},
	volume       = 2,
	number       = 3,
}
@article{Franz17b,
	title        = {Mean-field avalanches in jammed spheres},
	author       = {Franz, Silvio and Spigler, Stefano},
	year         = 2017,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 95,
	number       = 2,
}
@article{Freeman16,
	title        = {Topology and Geometry of Deep Rectified Network Optimization Landscapes},
	author       = {Freeman, C Daniel and Bruna, Joan},
	year         = 2017,
	journal      = {International Conference on Learning Representations},
}
@article{Gastaldi17,
	title        = {Shake-Shake regularization of 3-branch residual networks},
	author       = {Gastaldi, Xavier},
	year         = 2017,
	journal      = {International Conference on Learning Representations},
}
@article{goyal2017accurate,
	title        = {Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour},
	author       = {Goyal, Priya and {Doll{\'a}r}, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1706.02677},
}
@inproceedings{haeffele2017global,
	title        = {Global optimality in neural network training},
	author       = {Haeffele, Benjamin D and Vidal, {Ren{\'e}}},
	year         = 2017,
	booktitle    = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},
}
@online{xiao2017/online,
	title        = {Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms},
	author       = {Han Xiao and Kashif Rasul and Roland Vollgraf},
	year         = 2017,
	date         = {2017-08-28},
	eprintclass  = {cs.LG},
	eprinttype   = {arXiv},
	eprint       = {cs.LG/1708.07747},
}
@article{hestness2017deep,
	title        = {Deep learning scaling is predictable, empirically},
	author       = {Hestness, Joel and Narang, Sharan and Ardalani, Newsha and Diamos, Gregory and Jun, Heewoo and Kianinejad, Hassan and Patwary, Md and Ali, Mostofa and Yang, Yang and Zhou, Yanqi},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1712.00409},
}
@inproceedings{Hoffer17,
	title        = {Train longer, generalize better: closing the generalization gap in large batch training of neural networks},
	author       = {Hoffer, Elad and Hubara, Itay and Soudry, Daniel},
	year         = 2017,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@inproceedings{lee2017ability,
	title        = {On the Ability of Neural Nets to Express Distributions},
	author       = {Holden Lee and Rong Ge and Tengyu Ma and Andrej Risteski and Sanjeev Arora},
	year         = 2017,
	month        = {07--10 Jul},
	booktitle    = {Proceedings of the 2017 Conference on Learning Theory},
	publisher    = {PMLR},
	address      = {Amsterdam, Netherlands},
	series       = {Proceedings of Machine Learning Research},
	volume       = 65,
	url          = {http://proceedings.mlr.press/v65/lee17a.html},
	editor       = {Satyen Kale and Ohad Shamir},
	abstract     = {Deep neural nets have caused a revolution in many classification tasks. A related ongoing revolution?also theoretically not understood?concerns their ability to serve as generative models for complicated types of data such as images and texts. These models are trained using ideas like variational autoencoders and Generative Adversarial Networks. We take a first cut at explaining the expressivity of multilayer nets by giving a sufficient criterion for a function to be approximable by a neural network with $n$ hidden layers. A key ingredient is Barron?s Theorem (Barron, 1993), which gives a Fourier criterion for approximability of a function by a neural network with 1 hidden layer. We show that a composition of $n$ functions which satisfy certain Fourier conditions (?Barron functions?) can be approximated by a $n+1$-layer neural network. For probability distributions, this translates into a criterion for a probability distribution to be approximable in Wasserstein distance?a natural metric on probability distributions?by a neural network applied to a fixed base distribution (e.g., multivariate gaussian). Building up recent lower bound work, we also give an example function that shows that composition of Barron functions is more expressive than Barron functions alone.},
}
@article{jastrzkebski2017three,
	title        = {Three Factors Influencing Minima in SGD},
	author       = {Jastrzebski, {Stanis{\l}aw} and Kenton, Zachary and Arpit, Devansh and Ballas, Nicolas and Fischer, Asja and Bengio, Yoshua and Storkey, Amos},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1711.04623},
}
@article{krishnan2017neumann,
	title        = {Neumann Optimizer: A Practical Optimization Algorithm for Deep Neural Networks},
	author       = {Krishnan, Shankar and Xiao, Ying and Saurous, Rif A},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1712.03298},
}
@article{laurent2017multilinear,
	title        = {The Multilinear Structure of ReLU Networks},
	author       = {Laurent, Thomas and von Brecht, James},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1712.10132},
}
@article{mezard2017mean,
	title        = {Mean-field message-passing equations in the Hopfield model and its generalizations},
	author       = {M{\'e}zard, Marc},
	year         = 2017,
	journal      = {Physical Review E},
	publisher    = {APS},
	volume       = 95,
	number       = 2,
}
@inproceedings{Raghu16,
	title        = {On the Expressive Power of Deep Neural Networks},
	author       = {Maithra Raghu and Ben Poole and Jon Kleinberg and Surya Ganguli and Jascha Sohl-Dickstein},
	year         = 2017,
	month        = {06--11 Aug},
	booktitle    = {Proceedings of the 34th International Conference on Machine Learning},
	publisher    = {PMLR},
	address      = {International Convention Centre, Sydney, Australia},
	series       = {Proceedings of Machine Learning Research},
	volume       = 70,
	url          = {http://proceedings.mlr.press/v70/raghu17a.html},
	editor       = {Doina Precup and Yee Whye Teh},
	abstract     = {We propose a new approach to the problem of neural network expressivity, which seeks to characterize how structural properties of a neural network family affect the functions it is able to compute. Our approach is based on an interrelated set of measures of expressivity, unified by the novel notion of trajectory length, which measures how the output of a network changes as the input sweeps along a one-dimensional path. Our findings show that: (1) The complexity of the computed function grows exponentially with depth (2) All weights are not equal: trained networks are more sensitive to their lower (initial) layer weights (3) Trajectory regularization is a simpler alternative to batch normalization, with the same performance.},
}
@article{neyshabur2017geometry,
	title        = {Geometry of optimization and implicit regularization in deep learning},
	author       = {Neyshabur, Behnam and Tomioka, Ryota and Salakhutdinov, Ruslan and Srebro, Nathan},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1705.03071},
}
@article{ninarello2017models,
	title        = {Models and algorithms for the next generation of glass transition studies},
	author       = {Ninarello, Andrea and Berthier, Ludovic and Coslovich, Daniele},
	year         = 2017,
	journal      = {Physical Review X},
	publisher    = {APS},
	volume       = 7,
	number       = 2,
}
@inproceedings{rudi2017generalization,
	title        = {Generalization properties of learning with random features},
	author       = {Rudi, Alessandro and Rosasco, Lorenzo},
	year         = 2017,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{Sagun16,
	title        = {Singularity of the Hessian in Deep Learning},
	author       = {Sagun, Levent and Bottou, {L{\'e}on} and LeCun, Yann},
	year         = 2017,
	journal      = {International Conference on Learning Representations},
}
@article{sagun2017empirical,
	title        = {Empirical Analysis of the Hessian of Over-Parametrized Neural Networks},
	author       = {Sagun, Levent and Evci, Utku and {G\"uney}, V. {U\u{g}ur} and Dauphin, Yann and Bottou, {L\'eon}},
	year         = 2017,
	journal      = {ICLR 2018 Workshop Contribution, arXiv:1706.04454},
}
@article{shwartz2017opening,
	title        = {Opening the Black Box of Deep Neural Networks via Information},
	author       = {Shwartz-Ziv, Ravid and Tishby, Naftali},
	year         = 2017,
	journal      = {arXiv preprint arXiv:1703.00810},
}
@article{silver2017mastering,
	title        = {Mastering the game of go without human knowledge},
	author       = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
	year         = 2017,
	journal      = {nature},
	publisher    = {Nature Publishing Group},
	volume       = 550,
	number       = 7676,
}
@article{Silver17,
	title        = {Mastering the game of Go without human knowledge},
	author       = {Silver, David and Schrittwieser, Julian and Simonyan, Karen and Antonoglou, Ioannis and Huang, Aja and Guez, Arthur and Hubert, Thomas and Baker, Lucas and Lai, Matthew and Bolton, Adrian and others},
	year         = 2017,
	journal      = {Nature},
	publisher    = {Nature Publishing Group},
	volume       = 550,
	number       = 7676,
}
@article{van2017learning,
	title        = {Learning phase transitions by confusion},
	author       = {Van Nieuwenburg, Evert PL and Liu, Ye-Hua and Huber, Sebastian D},
	year         = 2017,
	journal      = {Nature Physics},
	publisher    = {Nature Publishing Group},
	volume       = 13,
	number       = 5,
}
@article{xiao_fashion-mnist_2017,
	title        = {Fashion-{MNIST}: a {Novel} {Image} {Dataset} for {Benchmarking} {Machine} {Learning} {Algorithms}},
	author       = {Xiao, Han and Rasul, Kashif and Vollgraf, Roland},
	year         = 2017,
	month        = sep,
	journal      = {arXiv:1708.07747 [cs, stat]},
	url          = {http://arxiv.org/abs/1708.07747},
	urldate      = {2021-02-06},
	note         = {arXiv: 1708.07747},
	annote       = {Comment: Dataset is freely available at https://github.com/zalandoresearch/fashion-mnist Benchmark is available at http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/},
	abstract     = {We present Fashion-MNIST, a new dataset comprising of 28x28 grayscale images of 70,000 fashion products from 10 categories, with 7,000 images per category. The training set has 60,000 images and the test set has 10,000 images. Fashion-MNIST is intended to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms, as it shares the same image size, data format and the structure of training and testing splits. The dataset is freely available at https://github.com/zalandoresearch/fashion-mnist},
	keywords     = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
}
@article{zhang2016understanding,
	title        = {Understanding deep learning requires rethinking generalization},
	author       = {Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
	year         = 2017,
	journal      = {5th International Conference on Learning Representations, {ICLR} 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings},
}
@article{Zhang16,
	title        = {Understanding deep learning requires rethinking generalization},
	author       = {Zhang, Chiyuan and Bengio, Samy and Hardt, Moritz and Recht, Benjamin and Vinyals, Oriol},
	year         = 2017,
	journal      = {International Conference on Learning Representations},
}
@article{vstefko2018autonomous,
	title        = {Autonomous illumination control for localization microscopy},
	author       = {{\v{S}}tefko, Marcel and Ottino, Baptiste and Douglass, Kyle M and Manley, Suliana},
	year         = 2018,
	journal      = {Optics express},
	publisher    = {Optical Society of America},
	volume       = 26,
	number       = 23,
}
@inproceedings{alaifari_adef_2018,
	title        = {{ADef}: an {Iterative} {Algorithm} to {Construct} {Adversarial} {Deformations}},
	author       = {Alaifari, Rima and Alberti, Giovanni S. and Gauksson, Tandri},
	year         = 2018,
	month        = sep,
	url          = {https://openreview.net/forum?id=Hk4dFjR5K7},
	urldate      = {2021-05-25},
	language     = {en},
	abstract     = {We propose a new, efficient algorithm to construct adversarial examples by means of deformations, rather than additive perturbations.},
}
@inproceedings{matthews2018gaussian,
	title        = {Gaussian Process Behaviour in Wide Deep Neural Networks},
	author       = {Alexander G. de G. Matthews and Jiri Hron and Mark Rowland and Richard E. Turner and Zoubin Ghahramani},
	year         = 2018,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=H1-nGgWC-},
}
@inproceedings{athalye_synthesizing_2018,
	title        = {Synthesizing {Robust} {Adversarial} {Examples}},
	author       = {Athalye, Anish and Engstrom, Logan and Ilyas, Andrew and Kwok, Kevin},
	year         = 2018,
	month        = jul,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v80/athalye18b.html},
	urldate      = {2021-05-25},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Standard methods for generating adversarial examples for neural networks do not consistently fool neural network classifiers in the physical world due to a combination of viewpoint shifts, camera n...},
}
@inproceedings{aubin2018committee,
	title        = {The committee machine: Computational to statistical gaps in learning a two-layers neural network},
	author       = {Aubin, Benjamin and Maillard, Antoine and Krzakala, Florent and Macris, Nicolas and Zdeborov{\'a}, Lenka and others},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{azulay2018deep,
	title        = {Why do deep convolutional networks generalize so poorly to small image transformations?},
	author       = {Azulay, Aharon and Weiss, Yair},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1805.12177},
}
@article{BJREM,
	title        = {Activated aging dynamics and effective trap model description in the random energy model},
	author       = {Baity-Jesi, Marco and Biroli, Giulio and Cammarota, Chiara},
	year         = 2018,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	publisher    = {IOP Publishing},
	volume       = 2018,
	number       = 1,
}
@inproceedings{Baity18,
	title        = {Comparing Dynamics: Deep Neural Networks versus Glassy Systems},
	author       = {Baity-Jesi, Marco and Sagun, Levent and Geiger, Mario and Spigler, Stefano and Arous, Gerard Ben and Cammarota, Chiara and LeCun, Yann and Wyart, Matthieu and Biroli, Giulio},
	year         = 2018,
	month        = {10--15 Jul},
	booktitle    = {Proceedings of the 35th International Conference on Machine Learning},
	publisher    = {PMLR},
	address      = {Stockholmsmässan, Stockholm Sweden},
	series       = {Proceedings of Machine Learning Research},
	volume       = 80,
	url          = {http://proceedings.mlr.press/v80/baity-jesi18a.html},
	editor       = {Dy, Jennifer and Krause, Andreas},
	abstract     = {We analyze numerically the training dynamics of deep neural networks (DNN) by using methods developed in statistical physics of glassy systems. The two main issues we address are the complexity of the loss-landscape and of the dynamics within it, and to what extent DNNs share similarities with glassy systems. Our findings, obtained for different architectures and data-sets, suggest that during the training process the dynamics slows down because of an increasingly large number of flat directions. At large times, when the loss is approaching zero, the system diffuses at the bottom of the landscape. Despite some similarities with the dynamics of mean-field glassy systems, in particular, the absence of barrier crossing, we find distinctive dynamical behaviors in the two cases, thus showing that the statistical properties of the corresponding loss and energy landscapes are different. In contrast, when the network is under-parametrized we observe a typical glassy behavior, thus suggesting the existence of different phases depending on whether the network is under-parametrized or over-parametrized.},
}
@article{bansal2018minnorm,
	title        = {Minnorm training: an algorithm for training overcomplete deep neural networks},
	author       = {Bansal, Yamini and Advani, Madhu and Cox, David D and Saxe, Andrew M},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1806.00730},
}
@article{Brito18a,
	title        = {Universality of jamming of nonspherical particles},
	author       = {Brito, Carolina and Ikeda, Harukuni and Urbani, Pierfrancesco and Wyart, Matthieu and Zamponi, Francesco},
	year         = 2018,
	month        = nov,
	journal      = {Proceedings of the National Academy of Sciences},
	volume       = 115,
	number       = 46,
	doi          = {10.1073/pnas.1812457115},
	issn         = {0027-8424, 1091-6490},
	url          = {https://www.pnas.org/content/115/46/11736},
	urldate      = {2020-09-29},
	copyright    = {© 2018 . http://www.pnas.org/site/aboutpnas/licenses.xhtmlPublished under the PNAS license.},
	note         = {Publisher: National Academy of Sciences Section: Physical Sciences},
	language     = {en},
	pmid         = 30381457,
	abstract     = {Amorphous packings of nonspherical particles such as ellipsoids and spherocylinders are known to be hypostatic: The number of mechanical contacts between particles is smaller than the number of degrees of freedom, thus violating Maxwell’s mechanical stability criterion. In this work, we propose a general theory of hypostatic amorphous packings and the associated jamming transition. First, we show that many systems fall into a same universality class. As an example, we explicitly map ellipsoids into a system of “breathing” particles. We show by using a marginal stability argument that in both cases jammed packings are hypostatic and that the critical exponents related to the contact number and the vibrational density of states are the same. Furthermore, we introduce a generalized perceptron model which can be solved analytically by the replica method. The analytical solution predicts critical exponents in the same hypostatic jamming universality class. Our analysis further reveals that the force and gap distributions of hypostatic jamming do not show power-law behavior, in marked contrast to the isostatic jamming of spherical particles. Finally, we confirm our theoretical predictions by numerical simulations.},
	keywords     = {glass, jamming, marginal stability, nonspherical particles},
}
@article{brito2018theory,
	title        = {Theory for Swap Acceleration near the Glass and Jamming Transitions},
	author       = {Brito, Carolina and Lerner, Edan and Wyart, Matthieu},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1801.03796},
}
@incollection{Chizat2018,
	title        = {{On the Global Convergence of Gradient Descent for Over-parameterized Models using Optimal Transport}},
	author       = {Chizat, L\'{e}na\"{\i}c and Bach, Francis},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems 31},
	publisher    = {Curran Associates, Inc.},
}
@article{coja2018information,
	title        = {Information-theoretic thresholds from the cavity method},
	author       = {Coja-Oghlan, Amin and Krzakala, Florent and Perkins, Will and Zdeborova, Lenka},
	year         = 2018,
	journal      = {Advances in Mathematics},
	publisher    = {Elsevier},
	volume       = 333,
}
@article{Cooper18,
	title        = {The loss landscape of overparameterized neural networks},
	author       = {Cooper, Yaim},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1804.10200},
}
@inproceedings{gabrie2018entropy,
	title        = {Entropy and mutual information in models of deep neural networks},
	author       = {Gabri{\'e}, Marylou and Manoel, Andre and Luneau, Cl{\'e}ment and Macris, Nicolas and Krzakala, Florent and Zdeborov{\'a}, Lenka and others},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@string{epje = {Eur.\ Phys.\ J E}}
@string{epl = {Europhys.\ Lett.}}
@string{pnas = {Proc.\ Natl.\ Acad.\ Sci.\ U.S.A.}}
@string{prb = {Phys.\ Rev.\ B}}
@string{pre = {Phys.\ Rev.\ E}}
@string{prl = {Phys.\ Rev.\ Lett.}}
@string{rmp = {Rev.\ Mod.\ Phys.}}
@inproceedings{jacot2018neural,
	title        = {Neural Tangent Kernel: Convergence and Generalization in Neural Networks},
	author       = {Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
	year         = 2018,
	booktitle    = {Proceedings of the 32Nd International Conference on Neural Information Processing Systems},
	location     = {Montr\&\#233;al, Canada},
	publisher    = {Curran Associates Inc.},
	address      = {USA},
	series       = {NIPS'18},
	url          = {http://dl.acm.org/citation.cfm?id=3327757.3327948},
	numpages     = 10,
	acmid        = 3327948,
}
@article{Lee2017,
	title        = {Deep Neural Networks as Gaussian Processes},
	author       = {Jae Hoon Lee and Yasaman Bahri and Roman Novak and Samuel S. Schoenholz and Jeffrey Pennington and Jascha Sohl-Dickstein},
	year         = 2018,
	journal      = {ICLR},
}
@inproceedings{kanbak_geometric_2018,
	title        = {Geometric {Robustness} of {Deep} {Networks}: {Analysis} and {Improvement}},
	author       = {Kanbak, Can and Moosavi-Dezfooli, Seyed-Mohsen and Frossard, Pascal},
	year         = 2018,
	month        = jun,
	booktitle    = {2018 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition}},
	publisher    = {IEEE},
	address      = {Salt Lake City, UT},
	doi          = {10.1109/CVPR.2018.00467},
	isbn         = {978-1-5386-6420-9},
	url          = {https://ieeexplore.ieee.org/document/8578565/},
	urldate      = {2021-05-25},
	language     = {en},
}
@article{Li18,
	title        = {Measuring the intrinsic dimension of objective landscapes},
	author       = {Li, Chunyuan and Farkhoor, Heerad and Liu, Rosanne and Yosinski, Jason},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1804.08838},
}
@article{liang2018just,
	title        = {Just Interpolate: Kernel" Ridgeless" Regression Can Generalize},
	author       = {Liang, Tengyuan and Rakhlin, Alexander},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1808.00387},
}
@article{liao2018dynamics,
	title        = {The Dynamics of Learning: A Random Matrix Approach},
	author       = {Liao, Zhenyu and Couillet, Romain},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1805.11917},
}
@article{maennel2018gradient,
	title        = {Gradient descent quantizes relu network features},
	author       = {Maennel, Hartmut and Bousquet, Olivier and Gelly, Sylvain},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1803.08367},
}
@inproceedings{mahajan2018exploring,
	title        = {Exploring the limits of weakly supervised pretraining},
	author       = {Mahajan, Dhruv and Girshick, Ross and Ramanathan, Vignesh and He, Kaiming and Paluri, Manohar and Li, Yixuan and Bharambe, Ashwin and van der Maaten, Laurens},
	year         = 2018,
	booktitle    = {Proceedings of the European Conference on Computer Vision (ECCV)},
}
@article{mei2018mean,
	title        = {A mean field view of the landscape of two-layer neural networks},
	author       = {Mei, Song and Montanari, Andrea and Nguyen, Phan-Minh},
	year         = 2018,
	month        = aug,
	journal      = {Proceedings of the National Academy of Sciences},
	volume       = 115,
	number       = 33,
	doi          = {10.1073/pnas.1806579115},
	issn         = {0027-8424, 1091-6490},
	url          = {https://www.pnas.org/content/115/33/E7665},
	urldate      = {2020-09-29},
	copyright    = {Copyright © 2018 the Author(s). Published by PNAS.. https://creativecommons.org/licenses/by-nc-nd/4.0/This open access article is distributed under Creative Commons Attribution-NonCommercial-NoDerivatives License 4.0 (CC BY-NC-ND).},
	note         = {Publisher: National Academy of Sciences Section: PNAS Plus},
	language     = {en},
	pmid         = 30054315,
	abstract     = {Multilayer neural networks are among the most powerful models in machine learning, yet the fundamental reasons for this success defy mathematical understanding. Learning a neural network requires optimizing a nonconvex high-dimensional objective (risk function), a problem that is usually attacked using stochastic gradient descent (SGD). Does SGD converge to a global optimum of the risk or only to a local optimum? In the former case, does this happen because local minima are absent or because SGD somehow avoids them? In the latter, why do local minima reached by SGD have good generalization properties? In this paper, we consider a simple case, namely two-layer neural networks, and prove that—in a suitable scaling limit—SGD dynamics is captured by a certain nonlinear partial differential equation (PDE) that we call distributional dynamics (DD). We then consider several specific examples and show how DD can be used to prove convergence of SGD to networks with nearly ideal generalization error. This description allows for “averaging out” some of the complexities of the landscape of neural networks and can be used to prove a general convergence result for noisy SGD.},
	keywords     = {gradient flow, neural networks, partial differential equations, stochastic gradient descent, Wasserstein space},
}
@article{neal2018modern,
	title        = {A modern take on the bias-variance tradeoff in neural networks},
	author       = {Neal, Brady and Mittal, Sarthak and Baratin, Aristide and Tantia, Vinayak and Scicluna, Matthew and Lacoste-Julien, Simon and Mitliagkas, Ioannis},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1810.08591},
}
@article{neyshabur2018towards,
	title        = {Towards Understanding the Role of Over-Parametrization in Generalization of Neural Networks},
	author       = {Neyshabur, Behnam and Li, Zhiyuan and Bhojanapalli, Srinadh and LeCun, Yann and Srebro, Nathan},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1805.12076},
}
@article{novak2018sensitivity,
	title        = {Sensitivity and generalization in neural networks: an empirical study},
	author       = {Novak, Roman and Bahri, Yasaman and Abolafia, Daniel A and Pennington, Jeffrey and Sohl-Dickstein, Jascha},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1802.08760},
}
@article{rotskoff2018neural,
	title        = {Neural networks as Interacting Particle Systems: Asymptotic convexity of the Loss Landscape and Universal Scaling of the Approximation Error},
	author       = {Rotskoff, Grant M and Vanden-Eijnden, Eric},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1805.00915},
}
@article{ruderman_pooling_2018,
	title        = {Pooling is neither necessary nor sufficient for appropriate deformation stability in {CNNs}},
	author       = {Ruderman, Avraham and Rabinowitz, Neil C. and Morcos, Ari S. and Zoran, Daniel},
	year         = 2018,
	month        = may,
	journal      = {arXiv:1804.04438 [cs, stat]},
	url          = {http://arxiv.org/abs/1804.04438},
	urldate      = {2021-05-17},
	note         = {arXiv: 1804.04438},
	annote       = {Comment: NIPS 2018 submission},
	abstract     = {Many of our core assumptions about how neural networks operate remain empirically untested. One common assumption is that convolutional neural networks need to be stable to small translations and deformations to solve image recognition tasks. For many years, this stability was baked into CNN architectures by incorporating interleaved pooling layers. Recently, however, interleaved pooling has largely been abandoned. This raises a number of questions: Are our intuitions about deformation stability right at all? Is it important? Is pooling necessary for deformation invariance? If not, how is deformation invariance achieved in its absence? In this work, we rigorously test these questions, and find that deformation stability in convolutional networks is more nuanced than it first appears: (1) Deformation invariance is not a binary property, but rather that different tasks require different degrees of deformation stability at different layers. (2) Deformation stability is not a fixed property of a network and is heavily adjusted over the course of training, largely through the smoothness of the convolutional filters. (3) Interleaved pooling layers are neither necessary nor sufficient for achieving the optimal form of deformation stability for natural image classification. (4) Pooling confers too much deformation stability for image classification at initialization, and during training, networks have to learn to counteract this inductive bias. Together, these findings provide new insights into the role of interleaved pooling and deformation invariance in CNNs, and demonstrate the importance of rigorous empirical testing of even our most basic assumptions about the working of neural networks.},
	keywords     = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
}
@inproceedings{sandler_mobilenetv2_2018,
	title        = {{MobileNetV2}: {Inverted} {Residuals} and {Linear} {Bottlenecks}},
	author       = {Sandler, Mark and Howard, Andrew and Zhu, Menglong and Zhmoginov, Andrey and Chen, Liang-Chieh},
	year         = 2018,
	month        = jun,
	booktitle    = {2018 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition}},
	publisher    = {IEEE},
	address      = {Salt Lake City, UT},
	doi          = {10.1109/CVPR.2018.00474},
	isbn         = {978-1-5386-6420-9},
	url          = {https://ieeexplore.ieee.org/document/8578572/},
	urldate      = {2021-04-13},
	language     = {en},
	abstract     = {In this paper we describe a new mobile architecture, MobileNetV2, that improves the state of the art performance of mobile models on multiple tasks and benchmarks as well as across a spectrum of different model sizes. We also describe efficient ways of applying these mobile models to object detection in a novel framework we call SSDLite. Additionally, we demonstrate how to build mobile semantic segmentation models through a reduced form of DeepLabv3 which we call Mobile DeepLabv3.},
}
@article{soudry2018implicit,
	title        = {The implicit bias of gradient descent on separable data},
	author       = {Soudry, Daniel and Hoffer, Elad and Nacson, Mor Shpigel and Gunasekar, Suriya and Srebro, Nathan},
	year         = 2018,
	journal      = {Journal of Machine Learning Research},
	volume       = 19,
	number       = 70,
}
@inproceedings{stich2018sparsified,
	title        = {Sparsified SGD with memory},
	author       = {Stich, Sebastian U and Cordonnier, Jean-Baptiste and Jaggi, Martin},
	year         = 2018,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{venturi2018neural,
	title        = {Neural Networks with Finite Intrinsic Dimension have no Spurious Valleys},
	author       = {Venturi, Luca and Bandeira, Afonso and Bruna, Joan},
	year         = 2018,
	journal      = {arXiv preprint arXiv:1802.06384},
}
@inproceedings{xiao_spatially_2018,
	title        = {Spatially {Transformed} {Adversarial} {Examples}},
	author       = {Xiao, Chaowei and Zhu, Jun-Yan and Li, Bo and He, Warren and Liu, Mingyan and Song, Dawn},
	year         = 2018,
	month        = feb,
	url          = {https://openreview.net/forum?id=HyydRMZC-},
	urldate      = {2021-05-25},
	language     = {en},
	abstract     = {We propose a new approach for generating adversarial examples based on spatial transformation, which produces perceptually realistic examples compared to existing attacks.},
}
@inproceedings{alcorn_strike_2019,
	title        = {Strike ({With}) a {Pose}: {Neural} {Networks} {Are} {Easily} {Fooled} by {Strange} {Poses} of {Familiar} {Objects}},
	author       = {Alcorn, Michael A. and Li, Qi and Gong, Zhitao and Wang, Chengfei and Mai, Long and Ku, Wei-Shinn and Nguyen, Anh},
	year         = 2019,
	month        = jun,
	booktitle    = {2019 {IEEE}/{CVF} {Conference} on {Computer} {Vision} and {Pattern} {Recognition} ({CVPR})},
	publisher    = {IEEE},
	address      = {Long Beach, CA, USA},
	doi          = {10.1109/CVPR.2019.00498},
	isbn         = {978-1-72813-293-8},
	url          = {https://ieeexplore.ieee.org/document/8954212/},
	urldate      = {2021-05-25},
	language     = {en},
	abstract     = {Despite excellent performance on stationary test sets, deep neural networks (DNNs) can fail to generalize to out-of-distribution (OoD) inputs, including natural, nonadversarial ones, which are common in real-world settings. In this paper, we present a framework for discovering DNN failures that harnesses 3D renderers and 3D models. That is, we estimate the parameters of a 3D renderer that cause a target DNN to misbehave in response to the rendered image. Using our framework and a self-assembled dataset of 3D objects, we investigate the vulnerability of DNNs to OoD poses of well-known objects in ImageNet. For objects that are readily recognized by DNNs in their canonical poses, DNNs incorrectly classify 97\% of their pose space. In addition, DNNs are highly sensitive to slight pose perturbations. Importantly, adversarial poses transfer across models and datasets. We find that 99.9\% and 99.4\% of the poses misclassified by Inception-v3 also transfer to the AlexNet and ResNet-50 image classifiers trained on the same ImageNet dataset, respectively, and 75.5\% transfer to the YOLOv3 object detector trained on MS COCO.},
}
@inproceedings{Allen-Zhu2018,
	title        = {A {Convergence} {Theory} for {Deep} {Learning} via {Over}-{Parameterization}},
	author       = {Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
	year         = 2019,
	month        = may,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v97/allen-zhu19a.html},
	urldate      = {2020-09-29},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Deep neural networks (DNNs) have demonstrated dominating performance in many fields; since AlexNet, networks used in practice are going wider and deeper. On the theoretical side, a long line of wor...},
}
@inproceedings{ansuini2019intrinsic,
	title        = {Intrinsic dimension of data representations in deep neural networks},
	author       = {Ansuini, Alessio and Laio, Alessandro and Macke, Jakob H and Zoccolan, Davide},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@incollection{arora2019exact,
	title        = {On {Exact} {Computation} with an {Infinitely} {Wide} {Neural} {Net}},
	author       = {Arora, Sanjeev and Du, Simon S and Hu, Wei and Li, Zhiyuan and Salakhutdinov, Russ R and Wang, Ruosong},
	year         = 2019,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems} 32},
	publisher    = {Curran Associates, Inc.},
	url          = {http://papers.nips.cc/paper/9025-on-exact-computation-with-an-infinitely-wide-neural-net.pdf},
	urldate      = {2020-09-29},
	editor       = {Wallach, H. and Larochelle, H. and Beygelzimer, A. and Alché-Buc, F. d{\textbackslash}textquotesingle and Fox, E. and Garnett, R.},
}
@inproceedings{arora2019harnessing,
	title        = {Harnessing the {Power} of {Infinitely} {Wide} {Deep} {Nets} on {Small}-data {Tasks}},
	author       = {Arora, Sanjeev and Du, Simon S. and Li, Zhiyuan and Salakhutdinov, Ruslan and Wang, Ruosong and Yu, Dingli},
	year         = 2019,
	month        = sep,
	url          = {https://openreview.net/forum?id=rkl8sJBYvH},
	urldate      = {2020-12-30},
	language     = {en},
	abstract     = {We verify neural tangent kernel is powerful on small data via experiments on UCI datasets, small CIFAR 10 and low-shot learning on VOC07.},
}
@article{barbier2019adaptive,
	title        = {The adaptive interpolation method: a simple scheme to prove replica formulas in Bayesian inference},
	author       = {Barbier, Jean and Macris, Nicolas},
	year         = 2019,
	journal      = {Probability theory and related fields},
	publisher    = {Springer},
	volume       = 174,
	number       = {3-4},
}
@article{basri_convergence_2019,
	title        = {The {Convergence} {Rate} of {Neural} {Networks} for {Learned} {Functions} of {Different} {Frequencies}},
	author       = {Basri, Ronen and Jacobs, David and Kasten, Yoni and Kritchman, Shira},
	year         = 2019,
	month        = dec,
	journal      = {arXiv:1906.00425 [cs, eess, stat]},
	url          = {http://arxiv.org/abs/1906.00425},
	urldate      = {2022-03-03},
	note         = {arXiv: 1906.00425},
	annote       = {relu has only odd components on spherical harmonics, when adding a bias all components are restored.},
}
@article{belkin2019reconciling,
	title        = {Reconciling modern machine-learning practice and the classical bias--variance trade-off},
	author       = {Belkin, Mikhail and Hsu, Daniel and Ma, Siyuan and Mandal, Soumik},
	year         = 2019,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 116,
	number       = 32,
}
@article{bietti2019group,
	title        = {Group invariance, stability to deformations, and complexity of deep convolutional representations},
	author       = {Bietti, Alberto and Mairal, Julien},
	year         = 2019,
	journal      = {The Journal of Machine Learning Research},
	publisher    = {JMLR. org},
	volume       = 20,
	number       = 1,
}
@article{bietti2019inductive,
	title        = {On the inductive bias of neural tangent kernels},
	author       = {Bietti, Alberto and Mairal, Julien},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1905.12173},
}
@article{boyer2019representer,
	title        = {{On Representer Theorems and Convex Regularization}},
	author       = {Boyer, Claire and Chambolle, Antonin and De Castro, Yohann and Duval, Vincent and De Gournay, Fr{\'e}d{\'e}ric and Weiss, Pierre},
	year         = 2019,
	month        = May,
	journal      = {{SIAM Journal on Optimization}},
	publisher    = {{Society for Industrial and Applied Mathematics}},
	volume       = 29,
	number       = 2,
	doi          = {10.1137/18M1200750},
	url          = {https://hal.archives-ouvertes.fr/hal-01823135},
	keywords     = {Inverse problems ; Representer theorem ; Convex regularization ; Vector space ; Total variation},
	hal_id       = {hal-01823135},
	hal_version  = {v3},
}
@inproceedings{chizat2019lazy,
	title        = {On lazy training in differentiable programming},
	author       = {Chizat, Lenaic and Oyallon, Edouard and Bach, Francis},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{dyer2019asymptotics,
	title        = {Asymptotics of wide networks from feynman diagrams},
	author       = {Dyer, Ethan and Gur-Ari, Guy},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1909.11304},
}
@article{Dyer19,
	title        = {Asymptotics of wide networks from feynman diagrams},
	author       = {Dyer, Ethan and Gur-Ari, Guy},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1909.11304},
}
@inproceedings{engstrom_exploring_2019,
	title        = {Exploring the {Landscape} of {Spatial} {Robustness}},
	author       = {Engstrom, Logan and Tran, Brandon and Tsipras, Dimitris and Schmidt, Ludwig and Madry, Aleksander},
	year         = 2019,
	month        = may,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v97/engstrom19a.html},
	urldate      = {2021-05-25},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {The study of adversarial robustness has so far largely focused on perturbations bound in \${\textbackslash}ell\_p\$-norms. However, state-of-the-art models turn out to be also vulnerable to other, more natural class...},
}
@article{franz2018jamming,
	title        = {Jamming in {Multilayer} {Supervised} {Learning} {Models}},
	author       = {Franz, Silvio and Hwang, Sungmin and Urbani, Pierfrancesco},
	year         = 2019,
	month        = oct,
	journal      = {Physical Review Letters},
	volume       = 123,
	number       = 16,
	doi          = {10.1103/PhysRevLett.123.160602},
	url          = {https://link.aps.org/doi/10.1103/PhysRevLett.123.160602},
	urldate      = {2020-09-29},
	note         = {Publisher: American Physical Society},
	abstract     = {Critical jamming transitions are characterized by an astonishing degree of universality. Analytic and numerical evidence points to the existence of a large universality class that encompasses finite and infinite dimensional spheres and continuous constraint satisfaction problems (CCSP) such as the nonconvex perceptron and related models. In this Letter we investigate multilayer neural networks (MLNN) learning random associations as models for CCSP that could potentially define different jamming universality classes. As opposed to simple perceptrons and infinite dimensional spheres, which are described by a single effective field in terms of which the constraints appear to be one dimensional, the description of MLNN involves multiple fields, and the constraints acquire a multidimensional character. We first study the models numerically and show that similarly to the perceptron, whenever jamming is isostatic, the sphere universality class is recovered, we then write the exact mean-field equations for the models and identify a dimensional reduction mechanism that leads to a scaling regime identical to the one of spheres.},
}
@article{franz2019critical,
	title        = {Critical jammed phase of the linear perceptron},
	author       = {Franz, Silvio and Sclocchi, Antonio and Urbani, Pierfrancesco},
	year         = 2019,
	journal      = {Physical Review Letters},
	publisher    = {APS},
	volume       = 123,
	number       = 11,
}
@article{Geiger18,
	title        = {Jamming transition as a paradigm to understand the loss landscape of deep neural networks},
	author       = {Geiger, Mario and Spigler, Stefano and d'Ascoli, Stéphane and Sagun, Levent and Baity-Jesi, Marco and Biroli, Giulio and Wyart, Matthieu},
	year         = 2019,
	month        = jul,
	journal      = {Physical Review E},
	volume       = 100,
	number       = 1,
	doi          = {10.1103/PhysRevE.100.012115},
	url          = {https://link.aps.org/doi/10.1103/PhysRevE.100.012115},
	urldate      = {2020-09-29},
	note         = {Publisher: American Physical Society},
	abstract     = {Deep learning has been immensely successful at a variety of tasks, ranging from classification to artificial intelligence. Learning corresponds to fitting training data, which is implemented by descending a very high-dimensional loss function. Understanding under which conditions neural networks do not get stuck in poor minima of the loss, and how the landscape of that loss evolves as depth is increased, remains a challenge. Here we predict, and test empirically, an analogy between this landscape and the energy landscape of repulsive ellipses. We argue that in fully connected deep networks a phase transition delimits the over- and underparametrized regimes where fitting can or cannot be achieved. In the vicinity of this transition, properties of the curvature of the minima of the loss (the spectrum of the Hessian) are critical. This transition shares direct similarities with the jamming transition by which particles form a disordered solid as the density is increased, which also occurs in certain classes of computational optimization and learning problems such as the perceptron. Our analysis gives a simple explanation as to why poor minima of the loss cannot be encountered in the overparametrized regime. Interestingly, we observe that the ability of fully connected networks to fit random data is independent of their depth, an independence that appears to also hold for real data. We also study a quantity Δ which characterizes how well (Δ{\textless}0) or badly (Δ{\textgreater}0) a datum is learned. At the critical point it is power-law distributed on several decades, P+(Δ)∼Δθ for Δ{\textgreater}0 and P−(Δ)∼(−Δ)−γ for Δ{\textless}0, with exponents that depend on the choice of activation function. This observation suggests that near the transition the loss landscape has a hierarchical structure and that the learning dynamics is prone to avalanche-like dynamics, with abrupt changes in the set of patterns that are learned.},
}
@inproceedings{ghorbani2019limitations,
	title        = {Limitations of lazy training of two-layers neural network},
	author       = {Ghorbani, Behrooz and Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{ghorbani2019linearized,
	title        = {Linearized two-layers neural networks in high dimension},
	author       = {Ghorbani, Behrooz and Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1904.12191},
}
@inproceedings{goldt2019dynamics,
	title        = {Dynamics of stochastic gradient descent for two-layer neural networks in the teacher-student setup},
	author       = {Goldt, Sebastian and Advani, Madhu and Saxe, Andrew M and Krzakala, Florent and Zdeborov{\'a}, Lenka},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{hanin2019finite,
	title        = {Finite depth and width corrections to the neural tangent kernel},
	author       = {Hanin, Boris and Nica, Mihai},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1909.05989},
}
@article{jacot2019hessian,
	title        = {The asymptotic spectrum of the Hessian of DNN throughout training},
	author       = {Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1910.02875},
}
@article{jacot2019hessian2,
	title        = {The Neural Tangent Kernel describes the Hessian of Overparametrized DNNs},
	author       = {Jacot, Arthur and Gabriel, Franck and Hongler, Clement},
	year         = 2019,
	journal      = {},
}
@incollection{lee2019wide,
	title        = {Wide {Neural} {Networks} of {Any} {Depth} {Evolve} as {Linear} {Models} {Under} {Gradient} {Descent}},
	author       = {Lee, Jaehoon and Xiao, Lechao and Schoenholz, Samuel and Bahri, Yasaman and Novak, Roman and Sohl-Dickstein, Jascha and Pennington, Jeffrey},
	year         = 2019,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems} 32},
	publisher    = {Curran Associates, Inc.},
	url          = {http://papers.nips.cc/paper/9063-wide-neural-networks-of-any-depth-evolve-as-linear-models-under-gradient-descent.pdf},
	urldate      = {2020-09-29},
	editor       = {Wallach, H. and Larochelle, H. and Beygelzimer, A. and Alché-Buc, F. d{\textbackslash}textquotesingle and Fox, E. and Garnett, R.},
}
@inproceedings{li2019towards,
	title        = {Towards explaining the regularization effect of initial large learning rate in training neural networks},
	author       = {Li, Yuanzhi and Wei, Colin and Ma, Tengyu},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@inproceedings{mei2019mean,
	title        = {Mean-field theory of two-layers neural networks: dimension-free bounds and kernel limit},
	author       = {Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
	year         = 2019,
	month        = jun,
	booktitle    = {Conference on {Learning} {Theory}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v99/mei19a.html},
	urldate      = {2020-12-30},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {We consider learning two layer neural networks using stochastic gradient descent. The mean-field description of this learning dynamics approximates the evolution of the network weights by an evolut...},
}
@article{nguyen2019mean,
	title        = {Mean Field Limit of the Learning Dynamics of Multilayer Neural Networks},
	author       = {Nguyen, Phan-Minh},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1902.02880},
}
@article{oymak2019generalization,
	title        = {Generalization guarantees for neural networks via harnessing the low-rank structure of the jacobian},
	author       = {Oymak, Samet and Fabian, Zalan and Li, Mingchen and Soltanolkotabi, Mahdi},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1906.05392},
}
@inproceedings{park2019effect,
	title        = {The {Effect} of {Network} {Width} on {Stochastic} {Gradient} {Descent} and {Generalization}: an {Empirical} {Study}},
	author       = {Park, Daniel and Sohl-Dickstein, Jascha and Le, Quoc and Smith, Samuel},
	year         = 2019,
	month        = may,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v97/park19b.html},
	urldate      = {2020-12-30},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {We investigate how the final parameters found by stochastic gradient descent are influenced by over-parameterization. We generate families of models by increasing the number of channels in a base n...},
}
@inproceedings{paszke_pytorch_2019,
	title        = {{PyTorch}: {An} {Imperative} {Style}, {High}-{Performance} {Deep} {Learning} {Library}},
	author       = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and Desmaison, Alban and Kopf, Andreas and Yang, Edward and DeVito, Zachary and Raison, Martin and Tejani, Alykhan and Chilamkurthy, Sasank and Steiner, Benoit and Fang, Lu and Bai, Junjie and Chintala, Soumith},
	year         = 2019,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher    = {Curran Associates, Inc.},
	volume       = 32,
	url          = {https://papers.nips.cc/paper/2019/hash/bdbca288fee7f92f2bfa9f7012727740-Abstract.html},
	urldate      = {2022-05-16},
}
@article{recanatesi2019dimensionality,
	title        = {Dimensionality compression and expansion in Deep Neural Networks},
	author       = {Recanatesi, Stefano and Farrell, Matthew and Advani, Madhu and Moore, Timothy and Lajoie, Guillaume and Shea-Brown, Eric},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1906.00443},
}
@inproceedings{novak2018bayesian,
	title        = {Bayesian Deep Convolutional Networks with Many Channels are Gaussian Processes},
	author       = {Roman Novak and Lechao Xiao and Yasaman Bahri and Jaehoon Lee and Greg Yang and Daniel A. Abolafia and Jeffrey Pennington and Jascha Sohl-dickstein},
	year         = 2019,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=B1g30j0qF7},
}
@article{saxe2019information,
	title        = {On the information bottleneck theory of deep learning},
	author       = {Saxe, Andrew M and Bansal, Yamini and Dapello, Joel and Advani, Madhu and Kolchinsky, Artemy and Tracey, Brendan D and Cox, David D},
	year         = 2019,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	publisher    = {IOP Publishing},
	volume       = 2019,
	number       = 12,
}
@inproceedings{yaida2018fluctuationdissipation,
	title        = {Fluctuation-dissipation relations for stochastic gradient descent},
	author       = {Sho Yaida},
	year         = 2019,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=SkNksoRctQ},
}
@inproceedings{Du2019,
	title        = {Gradient Descent Provably Optimizes Over-parameterized Neural Networks},
	author       = {Simon S. Du and Xiyu Zhai and Barnabas Poczos and Aarti Singh},
	year         = 2019,
	booktitle    = {International Conference on Learning Representations},
	url          = {https://openreview.net/forum?id=S1eK3i09YQ},
}
@inproceedings{pmlr-v97-simsekli19a,
	title        = {A Tail-Index Analysis of Stochastic Gradient Noise in Deep Neural Networks},
	author       = {Simsekli, Umut and Sagun, Levent and Gurbuzbalaban, Mert},
	year         = 2019,
	month        = {09--15 Jun},
	booktitle    = {Proceedings of the 36th International Conference on Machine Learning},
	publisher    = {PMLR},
	address      = {Long Beach, California, USA},
	series       = {Proceedings of Machine Learning Research},
	volume       = 97,
	url          = {http://proceedings.mlr.press/v97/simsekli19a.html},
	editor       = {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},
	abstract     = {The gradient noise (GN) in the stochastic gradient descent (SGD) algorithm is often considered to be Gaussian in the large data regime by assuming that the classical central limit theorem (CLT) kicks in. This assumption is often made for mathematical convenience, since it enables SGD to be analyzed as a stochastic differential equation (SDE) driven by a Brownian motion. We argue that the Gaussianity assumption might fail to hold in deep learning settings and hence render the Brownian motion-based analyses inappropriate. Inspired by non-Gaussian natural phenomena, we consider the GN in a more general context and invoke the generalized CLT (GCLT), which suggests that the GN converges to a heavy-tailed $\alpha$-stable random variable. Accordingly, we propose to analyze SGD as an SDE driven by a Lévy motion. Such SDEs can incur ?jumps?, which force the SDE transition from narrow minima to wider minima, as proven by existing metastability theory. To validate the $\alpha$-stable assumption, we conduct experiments on common deep learning scenarios and show that in all settings, the GN is highly non-Gaussian and admits heavy-tails. We investigate the tail behavior in varying network architectures and sizes, loss functions, and datasets. Our results open up a different perspective and shed more light on the belief that SGD prefers wide minima.},
}
@article{Spigler18,
	title        = {A jamming transition from under- to over-parametrization affects generalization in deep learning},
	author       = {Spigler, S. and Geiger, M. and d’Ascoli, S. and Sagun, L. and Biroli, G. and Wyart, M.},
	year         = 2019,
	month        = oct,
	journal      = {Journal of Physics A: Mathematical and Theoretical},
	volume       = 52,
	number       = 47,
	doi          = {10.1088/1751-8121/ab4c8b},
	issn         = {1751-8121},
	url          = {https://iopscience.iop.org/article/10.1088/1751-8121/ab4c8b/meta},
	urldate      = {2020-09-29},
	note         = {Publisher: IOP Publishing},
	language     = {en},
}
@inproceedings{tan_efficientnet_2019,
	title        = {{EfficientNet}: {Rethinking} {Model} {Scaling} for {Convolutional} {Neural} {Networks}},
	author       = {Tan, Mingxing and Le, Quoc},
	year         = 2019,
	month        = may,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v97/tan19a.html},
	urldate      = {2021-02-06},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Convolutional Neural Networks (ConvNets) are commonly developed at a fixed resource budget, and then scaled up for better accuracy if more resources are given. In this paper, we systematically stud...},
}
@inproceedings{tsuzuku2019structural,
	title        = {On the structural sensitivity of deep convolutional networks to the directions of fourier basis functions},
	author       = {Tsuzuku, Yusuke and Sato, Issei},
	year         = 2019,
	booktitle    = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
}
@article{yang2019scaling,
	title        = {Scaling limits of wide neural networks with weight sharing: Gaussian process behavior, gradient independence, and neural tangent kernel derivation},
	author       = {Yang, Greg},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1902.04760},
}
@inproceedings{yehudai2019power,
	title        = {On the power and limitations of random features for understanding neural networks},
	author       = {Yehudai, Gilad and Shamir, Ohad},
	year         = 2019,
	booktitle    = {Advances in Neural Information Processing Systems},
}
@article{yin2019fourier,
	title        = {A fourier perspective on model robustness in computer vision},
	author       = {Yin, Dong and Lopes, Raphael Gontijo and Shlens, Jonathon and Cubuk, Ekin D and Gilmer, Justin},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1906.08988},
}
@article{zhang2019making,
	title        = {Making convolutional networks shift-invariant again},
	author       = {Zhang, Richard},
	year         = 2019,
	journal      = {arXiv preprint arXiv:1904.11486},
}
@article{abbe2020poly,
	title        = {Poly-time universality and limitations of deep learning},
	author       = {Abbe, Emmanuel and Sandon, Colin},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2001.02992},
}
@article{advani2017high,
	title        = {High-dimensional dynamics of generalization error in neural networks},
	author       = {Advani, Madhu S. and Saxe, Andrew M. and Sompolinsky, Haim},
	year         = 2020,
	month        = sep,
	journal      = {Neural Networks},
	doi          = {10.1016/j.neunet.2020.08.022},
	issn         = {0893-6080},
	url          = {http://www.sciencedirect.com/science/article/pii/S0893608020303117},
	urldate      = {2020-09-29},
	language     = {en},
	abstract     = {We perform an analysis of the average generalization dynamics of large neural networks trained using gradient descent. We study the practically-relevant “high-dimensional” regime where the number of free parameters in the network is on the order of or even larger than the number of examples in the dataset. Using random matrix theory and exact solutions in linear models, we derive the generalization error and training error dynamics of learning and analyze how they depend on the dimensionality of data and signal to noise ratio of the learning problem. We find that the dynamics of gradient descent learning naturally protect against overtraining and overfitting in large networks. Overtraining is worst at intermediate network sizes, when the effective number of free parameters equals the number of samples, and thus can be reduced by making a network smaller or larger. Additionally, in the high-dimensional regime, low generalization error requires starting with small initial weights. We then turn to non-linear neural networks, and show that making networks very large does not harm their generalization performance. On the contrary, it can in fact reduce overtraining, even without early stopping or regularization of any sort. We identify two novel phenomena underlying this behavior in overcomplete models: first, there is a frozen subspace of the weights in which no learning occurs under gradient descent; and second, the statistical properties of the high-dimensional regime yield better-conditioned input correlations which protect against overtraining. We demonstrate that standard application of theories such as Rademacher complexity are inaccurate in predicting the generalization performance of deep neural networks, and derive an alternative bound which incorporates the frozen subspace and conditioning effects and qualitatively matches the behavior observed in simulation.},
	keywords     = {Generalization error, Neural networks, Random matrix theory},
}
@inproceedings{bordelon2020spectrum,
	title        = {Spectrum {Dependent} {Learning} {Curves} in {Kernel} {Regression} and {Wide} {Neural} {Networks}},
	author       = {Bordelon, Blake and Canatar, Abdulkadir and Pehlevan, Cengiz},
	year         = 2020,
	month        = nov,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v119/bordelon20a.html},
	urldate      = {2020-12-30},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {We derive analytical expressions for the generalization performance of kernel regression as a function of the number of training samples using theoretical methods from Gaussian processes and statis...},
}
@inproceedings{brown_language_2020,
	title        = {Language {Models} are {Few}-{Shot} {Learners}},
	author       = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
	year         = 2020,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher    = {Curran Associates, Inc.},
	volume       = 33,
	url          = {https://papers.nips.cc/paper/2020/hash/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html},
	urldate      = {2022-04-25},
}
@article{chen2020dynamical,
	title        = {A {Dynamical} {Central} {Limit} {Theorem} for {Shallow} {Neural} {Networks}},
	author       = {Chen, Zhengdao and Rotskoff, Grant and Bruna, Joan and Vanden-Eijnden, Eric},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	url          = {https://papers.nips.cc/paper/2020/hash/fc5b3186f1cf0daece964f78259b7ba0-Abstract.html},
	urldate      = {2020-12-30},
	language     = {en},
}
@inproceedings{chizat2020implicit,
	title        = {Implicit {Bias} of {Gradient} {Descent} for {Wide} {Two}-layer {Neural} {Networks} {Trained} with the {Logistic} {Loss}},
	author       = {Chizat, Lénaïc and Bach, Francis},
	year         = 2020,
	month        = jul,
	booktitle    = {Conference on {Learning} {Theory}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v125/chizat20a.html},
	urldate      = {2020-12-30},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Neural networks trained to minimize the logistic (a.k.a. cross-entropy) loss with gradient-based methods are observed to perform well in many supervised classification tasks. Towards understanding...},
}
@inproceedings{d2020double,
	title        = {Double {Trouble} in {Double} {Descent}: {Bias} and {Variance}(s) in the {Lazy} {Regime}},
	author       = {D’Ascoli, Stéphane and Refinetti, Maria and Biroli, Giulio and Krzakala, Florent},
	year         = 2020,
	month        = nov,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v119/d-ascoli20a.html},
	urldate      = {2020-12-29},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Deep neural networks can achieve remarkable generalization performances while interpolating the training data. Rather than the U-curve emblematic of the bias-variance trade-off, their test error of...},
}
@article{de2020sparsity,
	title        = {On Sparsity in Overparametrised Shallow ReLU Networks},
	author       = {de Dios, Jaume and Bruna, Joan},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2006.10225},
}
@article{dietler2020yeaz,
	title        = {YeaZ: A convolutional neural network for highly accurate, label-free segmentation of yeast microscopy images},
	author       = {Dietler, Nicola and Minder, Matthias and Gligorovski, Vojislav and Economou, Augoustina Maria and Joly, Denis Alain Henri Lucien and Sadeghi, Ahmad and Chan, Chun Hei Michael and Kozi{\'n}ski, Mateusz and Weigert, Martin and Bitbol, Anne-Florence and others},
	year         = 2020,
	journal      = {bioRxiv},
	publisher    = {Cold Spring Harbor Laboratory},
}
@article{geiger2019scaling,
	title        = {Scaling description of generalization with number of parameters in deep learning},
	author       = {Geiger, Mario and Jacot, Arthur and Spigler, Stefano and Gabriel, Franck and Sagun, Levent and d'Ascoli, Stéphane and Biroli, Giulio and Hongler, Clément and Wyart, Matthieu},
	year         = 2020,
	month        = feb,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	volume       = 2020,
	number       = 2,
	doi          = {10.1088/1742-5468/ab633c},
	issn         = {1742-5468},
	url          = {https://doi.org/10.1088%2F1742-5468%2Fab633c},
	urldate      = {2020-09-29},
	note         = {Publisher: IOP Publishing},
	language     = {en},
	abstract     = {Supervised deep learning involves the training of neural networks with a large number N of parameters. For large enough N, in the so-called over-parametrized regime, one can essentially fit the training data points. Sparsity-based arguments would suggest that the generalization error increases as N grows past a certain threshold N *. Instead, empirical studies have shown that in the over-parametrized regime, generalization error keeps decreasing with N. We resolve this paradox through a new framework. We rely on the so-called Neural Tangent Kernel, which connects large neural nets to kernel methods, to show that the initialization causes finite-size random fluctuations of the neural net output function f N around its expectation . These affect the generalization error for classification: under natural assumptions, it decays to a plateau value in a power-law fashion ∼N −1/2. This description breaks down at a so-called jamming transition N = N *. At this threshold, we argue that diverges. This result leads to a plausible explanation for the cusp in test error known to occur at N *. Our results are confirmed by extensive empirical observations on the MNIST and CIFAR image datasets. Our analysis finally suggests that, given a computational envelope, the smallest generalization error is obtained using several networks of intermediate sizes, just beyond N *, and averaging their outputs.},
}
@article{geiger2019disentangling,
	title        = {Disentangling feature and lazy training in deep neural networks},
	author       = {Geiger, Mario and Spigler, Stefano and Jacot, Arthur and Wyart, Matthieu},
	year         = 2020,
	month        = nov,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	volume       = 2020,
	number       = 11,
	doi          = {10.1088/1742-5468/abc4de},
	issn         = {1742-5468},
	url          = {https://doi.org/10.1088/1742-5468/abc4de},
	urldate      = {2020-12-30},
	note         = {Publisher: IOP Publishing},
	language     = {en},
	abstract     = {Two distinct limits for deep learning have been derived as the network width h → ∞, depending on how the weights of the last layer scale with h. In the neural tangent Kernel (NTK) limit, the dynamics becomes linear in the weights and is described by a frozen kernel Θ (the NTK). By contrast, in the mean-field limit, the dynamics can be expressed in terms of the distribution of the parameters associated with a neuron, that follows a partial differential equation. In this work we consider deep networks where the weights in the last layer scale as αh −1/2 at initialization. By varying α and h, we probe the crossover between the two limits. We observe two the previously identified regimes of ‘lazy training’ and ‘feature training’. In the lazy-training regime, the dynamics is almost linear and the NTK barely changes after initialization. The feature-training regime includes the mean-field formulation as a limiting case and is characterized by a kernel that evolves in time, and thus learns some features. We perform numerical experiments on MNIST, Fashion-MNIST, EMNIST and CIFAR10 and consider various architectures. We find that: (i) the two regimes are separated by an α* that scales as . (ii) Network architecture and data structure play an important role in determining which regime is better: in our tests, fully-connected networks perform generally better in the lazy-training regime, unlike convolutional networks. (iii) In both regimes, the fluctuations δF induced on the learned function by initial conditions decay as , leading to a performance that increases with h. The same improvement can also be obtained at an intermediate width by ensemble-averaging several networks that are trained independently. (iv) In the feature-training regime we identify a time scale , such that for t ≪ t 1 the dynamics is linear. At t ∼ t 1, the output has grown by a magnitude and the changes of the tangent kernel {\textbar} {\textbar}ΔΘ{\textbar} {\textbar} become significant. Ultimately, it follows for ReLU and Softplus activation functions, with a {\textless} 2 and a → 2 as depth grows. We provide scaling arguments supporting these findings.},
}
@article{ghorbani2020neural,
	title        = {When {Do} {Neural} {Networks} {Outperform} {Kernel} {Methods}?},
	author       = {Ghorbani, Behrooz and Mei, Song and Misiakiewicz, Theodor and Montanari, Andrea},
	year         = 2020,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 33,
	url          = {https://proceedings.neurips.cc/paper/2020/hash/a9df2255ad642b923d95503b9a7958d8-Abstract.html},
	urldate      = {2020-12-30},
	language     = {en},
}
@inproceedings{gluch2020constructing,
	title        = {Constructing a provably adversarially-robust classifier from a high accuracy one},
	author       = {Gluch, Grzegorz and Urbanke, R{\"u}diger},
	year         = 2020,
	booktitle    = {International Conference on Artificial Intelligence and Statistics},
}
@article{goldt2019modelling,
	title        = {Modeling the {Influence} of {Data} {Structure} on {Learning} in {Neural} {Networks}: {The} {Hidden} {Manifold} {Model}},
	author       = {Goldt, Sebastian and Mézard, Marc and Krzakala, Florent and Zdeborová, Lenka},
	year         = 2020,
	month        = dec,
	journal      = {Physical Review X},
	volume       = 10,
	number       = 4,
	doi          = {10.1103/PhysRevX.10.041044},
	url          = {https://link.aps.org/doi/10.1103/PhysRevX.10.041044},
	urldate      = {2020-12-30},
	note         = {Publisher: American Physical Society},
	abstract     = {Understanding the reasons for the success of deep neural networks trained using stochastic gradient-based methods is a key open problem for the nascent theory of deep learning. The types of data where these networks are most successful, such as images or sequences of speech, are characterized by intricate correlations. Yet, most theoretical work on neural networks does not explicitly model training data or assumes that elements of each data sample are drawn independently from some factorized probability distribution. These approaches are, thus, by construction blind to the correlation structure of real-world datasets and their impact on learning in neural networks. Here, we introduce a generative model for structured datasets that we call the hidden manifold model. The idea is to construct high-dimensional inputs that lie on a lower-dimensional manifold, with labels that depend only on their position within this manifold, akin to a single-layer decoder or generator in a generative adversarial network. We demonstrate that learning of the hidden manifold model is amenable to an analytical treatment by proving a “Gaussian equivalence property” (GEP), and we use the GEP to show how the dynamics of two-layer neural networks trained using one-pass stochastic gradient descent is captured by a set of integro-differential equations that track the performance of the network at all times. This approach permits us to analyze in detail how a neural network learns functions of increasing complexity during training, how its performance depends on its size, and how it is impacted by parameters such as the learning rate or the dimension of the hidden manifold.},
}
@article{grigorescu_survey_2020,
	title        = {A survey of deep learning techniques for autonomous driving},
	author       = {Grigorescu, Sorin and Trasnea, Bogdan and Cocias, Tiberiu and Macesanu, Gigel},
	year         = 2020,
	journal      = {Journal of Field Robotics},
	volume       = 37,
	number       = 3,
	doi          = {10.1002/rob.21918},
	issn         = {1556-4967},
	url          = {https://onlinelibrary.wiley.com/doi/abs/10.1002/rob.21918},
	urldate      = {2022-04-25},
	note         = {\_eprint: https://onlinelibrary.wiley.com/doi/pdf/10.1002/rob.21918},
	language     = {en},
}
@inproceedings{jacot2020implicit,
	title        = {Implicit {Regularization} of {Random} {Feature} {Models}},
	author       = {Jacot, Arthur and Simsek, Berfin and Spadaro, Francesco and Hongler, Clement and Gabriel, Franck},
	year         = 2020,
	month        = nov,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v119/jacot20a.html},
	urldate      = {2020-12-29},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {Random Features (RF) models are used as efficient parametric approximations of kernel methods. We investigate, by means of random matrix theory, the connection between Gaussian RF models and Kernel...},
}
@inproceedings{kayhan2020translation,
	title        = {On translation invariance in cnns: Convolutional layers can exploit absolute spatial location},
	author       = {Kayhan, Osman Semih and Gemert, Jan C van},
	year         = 2020,
	booktitle    = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
}
@article{kopitkov2019neural,
	title        = {Neural {Spectrum} {Alignment}: {Empirical} {Study}},
	author       = {Kopitkov, Dmitry and Indelman, Vadim},
	year         = 2020,
	journal      = {Artificial {Neural} {Networks} and {Machine} {Learning} – {ICANN} 2020},
	publisher    = {Springer International Publishing},
}
@article{lee2020finite,
	title        = {Finite Versus Infinite Neural Networks: an Empirical Study},
	author       = {Lee, Jaehoon and Schoenholz, Samuel S and Pennington, Jeffrey and Adlam, Ben and Xiao, Lechao and Novak, Roman and Sohl-Dickstein, Jascha},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2007.15801},
}
@article{neyshabur_towards_2020,
	title        = {Towards {Learning} {Convolutions} from {Scratch}},
	author       = {Neyshabur, Behnam},
	year         = 2020,
	month        = jul,
	journal      = {arXiv:2007.13657 [cs, stat]},
	url          = {http://arxiv.org/abs/2007.13657},
	urldate      = {2022-04-08},
	note         = {arXiv: 2007.13657},
	abstract     = {Convolution is one of the most essential components of architectures used in computer vision. As machine learning moves towards reducing the expert bias and learning it from data, a natural next step seems to be learning convolution-like structures from scratch. This, however, has proven elusive. For example, current state-of-the-art architecture search algorithms use convolution as one of the existing modules rather than learning it from data. In an attempt to understand the inductive bias that gives rise to convolutions, we investigate minimum description length as a guiding principle and show that in some settings, it can indeed be indicative of the performance of architectures. To find architectures with small description length, we propose \${\textbackslash}beta\$-LASSO, a simple variant of LASSO algorithm that, when applied on fully-connected networks for image classification tasks, learns architectures with local connections and achieves state-of-the-art accuracies for training fully-connected nets on CIFAR-10 (85.19\%), CIFAR-100 (59.56\%) and SVHN (94.07\%) bridging the gap between fully-connected and convolutional nets.},
	keywords     = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning, Statistics - Machine Learning},
	annote       = {Comment: 18 pages, 9 figures, 4 tables},
}
@article{nguyen2020rigorous,
	title        = {A rigorous framework for the mean field limit of multilayer neural networks},
	author       = {Nguyen, Phan-Minh and Pham, Huy Tuan},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2001.11443},
}
@article{ongie2019function,
	title        = {A {Function} {Space} {View} of {Bounded} {Norm} {Infinite} {Width} {ReLU} {Nets}: {The} {Multivariate} {Case}},
	author       = {Ongie, Greg and Willett, Rebecca and Soudry, Daniel and Srebro, Nathan},
	year         = 2020,
	month        = apr,
	journal      = {8th International Conference on Learning Representations, {ICLR} 2020, Addis Ababa, Ethiopia, April 26-30, 2020},
	url          = {https://iclr.cc/virtual_2020/poster_H1lNPxHKDH.html},
	urldate      = {2020-12-30},
	language     = {en},
	abstract     = {We give a tight characterization of the (vectorized Euclidean) norm of weights required to realize a function \$f:{\textbackslash}mathbb\{R\}{\textbackslash}rightarrow {\textbackslash}mathbb\{R\}{\textasciicircum}d\$ as a single hidden-layer ReLU network with an unbounded number of units (infinite width), extending the univariate characterization of Savarese et al. (2019) to the multivariate case.},
}
@article{papyan2020prevalence,
	title        = {Prevalence of neural collapse during the terminal phase of deep learning training},
	author       = {Papyan, Vardan and Han, XY and Donoho, David L},
	year         = 2020,
	journal      = {Proceedings of the National Academy of Sciences},
	publisher    = {National Acad Sciences},
	volume       = 117,
	number       = 40,
}
@inproceedings{shankar2020neural,
	title        = {Neural {Kernels} {Without} {Tangents}},
	author       = {Shankar, Vaishaal and Fang, Alex and Guo, Wenshuo and Fridovich-Keil, Sara and Ragan-Kelley, Jonathan and Schmidt, Ludwig and Recht, Benjamin},
	year         = 2020,
	month        = nov,
	booktitle    = {International {Conference} on {Machine} {Learning}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v119/shankar20a.html},
	urldate      = {2020-12-29},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {We investigate the connections between neural networks and simple building blocks in kernel space. In particular, using well established feature space tools such as direct sum, averaging, and momen...},
}
@inproceedings{shen_anatomical_2020,
	title        = {Anatomical {Data} {Augmentation} via {Fluid}-{Based} {Image} {Registration}},
	author       = {Shen, Zhengyang and Xu, Zhenlin and Olut, Sahin and Niethammer, Marc},
	year         = 2020,
	booktitle    = {Medical {Image} {Computing} and {Computer} {Assisted} {Intervention} – {MICCAI} 2020},
	publisher    = {Springer International Publishing},
	address      = {Cham},
	series       = {Lecture {Notes} in {Computer} {Science}},
	doi          = {10.1007/978-3-030-59716-0_31},
	isbn         = {978-3-030-59716-0},
	language     = {en},
	editor       = {Martel, Anne L. and Abolmaesumi, Purang and Stoyanov, Danail and Mateus, Diana and Zuluaga, Maria A. and Zhou, S. Kevin and Racoceanu, Daniel and Joskowicz, Leo},
	abstract     = {We introduce a fluid-based image augmentation method for medical image analysis. In contrast to existing methods, our framework generates anatomically meaningful images via interpolation from the geodesic subspace underlying given samples. Our approach consists of three steps: 1) given a source image and a set of target images, we construct a geodesic subspace using the Large Deformation Diffeomorphic Metric Mapping (LDDMM) model; 2) we sample transformations from the resulting geodesic subspace; 3) we obtain deformed images and segmentations via interpolation. Experiments on brain (LPBA) and knee (OAI) data illustrate the performance of our approach on two tasks: 1) data augmentation during training and testing for image segmentation; 2) one-shot learning for single atlas image segmentation. We demonstrate that our approach generates anatomically meaningful data and improves performance on these tasks over competing approaches. Code is available at https://github.com/uncbiag/easyreg.},
}
@article{sirignano2018mean,
	title        = {Mean {Field} {Analysis} of {Neural} {Networks}: {A} {Law} of {Large} {Numbers}},
	author       = {Sirignano, Justin and Spiliopoulos, Konstantinos},
	year         = 2020,
	month        = jan,
	journal      = {SIAM Journal on Applied Mathematics},
	volume       = 80,
	number       = 2,
	doi          = {10.1137/18M1192184},
	issn         = {0036-1399},
	url          = {https://epubs.siam.org/doi/abs/10.1137/18M1192184},
	urldate      = {2020-12-30},
	note         = {Publisher: Society for Industrial and Applied Mathematics},
	abstract     = {Machine learning, and in particular neural network models, have revolutionized fields such as image, text, and speech recognition. Today, many important real-world applications in these areas are driven by neural networks. There are also growing applications in engineering, robotics, medicine, and finance. Despite their immense success in practice, there is limited mathematical understanding of neural networks. This paper illustrates how neural networks can be studied via stochastic analysis and develops approaches for addressing some of the technical challenges which arise. We analyze one-layer neural networks in the asymptotic regime of simultaneously (a) large network sizes and (b) large numbers of stochastic gradient descent training iterations. We rigorously prove that the empirical distribution of the neural network parameters converges to the solution of a nonlinear partial differential equation. This result can be considered a law of large numbers for neural networks. In addition, a consequence of our analysis is that the trained parameters of the neural network asymptotically become independent, a property which is commonly called “propagation of chaos.”},
}
@article{sirignano2020mean,
	title        = {Mean field analysis of neural networks: A central limit theorem},
	author       = {Sirignano, Justin and Spiliopoulos, Konstantinos},
	year         = 2020,
	journal      = {Stochastic Processes and their Applications},
	publisher    = {Elsevier},
	volume       = 130,
	number       = 3,
}
@article{spigler2019asymptotic,
	title        = {Asymptotic learning curves of kernel methods: empirical data versus teacher–student paradigm},
	author       = {Spigler, Stefano and Geiger, Mario and Wyart, Matthieu},
	year         = 2020,
	month        = dec,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	volume       = 2020,
	number       = 12,
	doi          = {10.1088/1742-5468/abc61d},
	issn         = {1742-5468},
	url          = {https://doi.org/10.1088/1742-5468/abc61d},
	urldate      = {2020-12-30},
	note         = {Publisher: IOP Publishing},
	language     = {en},
	abstract     = {How many training data are needed to learn a supervised task? It is often observed that the generalization error decreases as n −β where n is the number of training examples and β is an exponent that depends on both data and algorithm. In this work we measure β when applying kernel methods to real datasets. For MNIST we find β ≈ 0.4 and for CIFAR10 β ≈ 0.1, for both regression and classification tasks, and for Gaussian or Laplace kernels. To rationalize the existence of non-trivial exponents that can be independent of the specific kernel used, we study the teacher–student framework for kernels. In this scheme, a teacher generates data according to a Gaussian random field, and a student learns them via kernel regression. With a simplifying assumption—namely that the data are sampled from a regular lattice—we derive analytically β for translation invariant kernels, using previous results from the kriging literature. Provided that the student is not too sensitive to high frequencies, β depends only on the smoothness and dimension of the training data. We confirm numerically that these predictions hold when the training points are sampled at random on a hypersphere. Overall, the test error is found to be controlled by the magnitude of the projection of the true function on the kernel eigenvectors whose rank is larger than n. Using this idea we predict the exponent β from real data by performing kernel PCA, leading to β ≈ 0.36 for MNIST and β ≈ 0.07 for CIFAR10, in good agreement with observations. We argue that these rather large exponents are possible due to the small effective dimension of the data.},
}
@inproceedings{woodworth2020kernel,
	title        = {Kernel and {Rich} {Regimes} in {Overparametrized} {Models}},
	author       = {Woodworth, Blake and Gunasekar, Suriya and Lee, Jason D. and Moroshko, Edward and Savarese, Pedro and Golan, Itay and Soudry, Daniel and Srebro, Nathan},
	year         = 2020,
	month        = jul,
	booktitle    = {Conference on {Learning} {Theory}},
	publisher    = {PMLR},
	url          = {http://proceedings.mlr.press/v125/woodworth20a.html},
	urldate      = {2020-12-29},
	note         = {ISSN: 2640-3498},
	language     = {en},
	abstract     = {A recent line of work studies overparametrized neural networks in the “kernel regime,” i.e. when  during training the network behaves as a kernelized linear predictor, and thus, training with grad...},
}
@article{yang2020feature,
	title        = {Feature learning in infinite-width neural networks},
	author       = {Yang, Greg and Hu, Edward J},
	year         = 2020,
	journal      = {arXiv preprint arXiv:2011.14522},
}
@article{bietti_deep_2021,
	title        = {Deep {Equals} {Shallow} for {ReLU} {Networks} in {Kernel} {Regimes}},
	author       = {Bietti, Alberto and Bach, Francis},
	year         = 2021,
	month        = aug,
	journal      = {arXiv:2009.14397 [cs, stat]},
	url          = {http://arxiv.org/abs/2009.14397},
	urldate      = {2022-03-03},
	note         = {arXiv: 2009.14397},
}
@article{bronstein_geometric_2021,
	title        = {Geometric deep learning: {Grids}, groups, graphs, geodesics, and gauges},
	author       = {Bronstein, Michael M. and Bruna, Joan and Cohen, Taco and Veličković, Petar},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2104.13478},
}
@article{chen_equivalence_2021,
	title        = {On the {Equivalence} between {Neural} {Network} and {Support} {Vector} {Machine}},
	author       = {Chen, Yilan and Huang, Wei and Nguyen, Lam M. and Weng, Tsui-Wei},
	year         = 2021,
	month        = nov,
	journal      = {arXiv:2111.06063 [cs, math, stat]},
	url          = {http://arxiv.org/abs/2111.06063},
	urldate      = {2022-04-08},
	note         = {arXiv: 2111.06063},
	annote       = {Comment: 35th Conference on Neural Information Processing Systems (NeurIPS 2021)},
}
@article{chizat2021sparse,
	title        = {Sparse optimization on measures with over-parameterized gradient descent},
	author       = {Chizat, Lenaic},
	year         = 2021,
	journal      = {Mathematical Programming},
	publisher    = {Springer},
}
@article{cui_generalization_2021,
	title        = {Generalization error rates in kernel regression: {The} crossover from the noiseless to noisy regime},
	author       = {Cui, Hugo and Loureiro, Bruno and Krzakala, Florent and Zdeborová, Lenka},
	year         = 2021,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 34,
}
@inproceedings{favero_locality_2021,
	title        = {Locality defeats the curse of dimensionality in convolutional teacher-student scenarios},
	author       = {Favero, Alessandro and Cagnetta, Francesco and Wyart, Matthieu},
	year         = 2021,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher    = {Curran Associates, Inc.},
	volume       = 34,
	url          = {https://proceedings.neurips.cc/paper/2021/hash/4e8eaf897c638d519710b1691121f8cb-Abstract.html},
	urldate      = {2022-04-25},
}
@article{geiger_landscape_2021,
	title        = {Landscape and training regimes in deep learning},
	author       = {Geiger, Mario and Petrini, Leonardo and Wyart, Matthieu},
	year         = 2021,
	month        = aug,
	journal      = {Physics Reports},
	series       = {Landscape and training regimes in deep learning},
	volume       = 924,
	doi          = {10.1016/j.physrep.2021.04.001},
	issn         = {0370-1573},
	url          = {https://www.sciencedirect.com/science/article/pii/S0370157321001290},
	urldate      = {2022-04-08},
	language     = {en},
}
@article{geiger2020perspective,
	title        = {Landscape and training regimes in deep learning},
	author       = {Geiger, Mario and Petrini, Leonardo and Wyart, Matthieu},
	year         = 2021,
	month        = apr,
	journal      = {Physics Reports},
	doi          = {10.1016/j.physrep.2021.04.001},
	issn         = {0370-1573},
	url          = {https://www.sciencedirect.com/science/article/pii/S0370157321001290},
	urldate      = {2021-05-17},
	abstract     = {Deep learning algorithms are responsible for a technological revolution in a variety of tasks including image recognition or Go playing. Yet, why they work is not understood. Ultimately, they manage to classify data lying in high dimension – a feat generically impossible due to the geometry of high dimensional space and the associated curse of dimensionality. Understanding what kind of structure, symmetry or invariance makes data such as images learnable is a fundamental challenge. Other puzzles include that (i) learning corresponds to minimizing a loss in high dimension, which is in general not convex and could well get stuck bad minima. (ii) Deep learning predicting power increases with the number of fitting parameters, even in a regime where data are perfectly fitted. In this manuscript, we review recent results elucidating (i, ii) and the perspective they offer on the (still unexplained) curse of dimensionality paradox. We base our theoretical discussion on the (h,α) plane where h controls the number of parameters and α the scale of the output of the network at initialization, and provide new systematic measures of performance in that plane for two common image classification datasets. We argue that different learning regimes can be organized into a phase diagram. A line of critical points sharply delimits an under-parametrized phase from an over-parametrized one. In over-parametrized nets, learning can operate in two regimes separated by a smooth cross-over. At large initialization, it corresponds to a kernel method, whereas for small initializations features can be learnt, together with invariants in the data. We review the properties of these different phases, of the transition separating them and some open questions. Our treatment emphasizes analogies with physical systems, scaling arguments and the development of numerical observables to quantitatively test these results empirically. Practical implications are also discussed, including the benefit of averaging nets with distinct initial weights, or the choice of parameters (h,α) optimizing performance.},
	language     = {en},
	keywords     = {Curse of dimensionality, Deep learning, Feature learning, Jamming, Lazy training, Loss landscape, Neural networks, Neural tangent kernel},
}
@article{ortiz2021can,
	title        = {What can linearized neural networks actually say about generalization?},
	author       = {Ortiz-Jim{\'e}nez, Guillermo and Moosavi-Dezfooli, Seyed-Mohsen and Frossard, Pascal},
	year         = 2021,
	journal      = {Advances in Neural Information Processing Systems},
	volume       = 34,
}
@article{paccolat2020compressing,
	title        = {Geometric compression of invariant manifolds in neural networks},
	author       = {Paccolat, Jonas and Petrini, Leonardo and Geiger, Mario and Tyloo, Kevin and Wyart, Matthieu},
	year         = 2021,
	month        = apr,
	journal      = {Journal of Statistical Mechanics: Theory and Experiment},
	volume       = 2021,
	number       = 4,
	doi          = {10.1088/1742-5468/abf1f3},
	issn         = {1742-5468},
	url          = {https://iopscience.iop.org/article/10.1088/1742-5468/abf1f3/meta},
	urldate      = {2021-04-26},
	note         = {Publisher: IOP Publishing},
	language     = {en},
}
@article{paccolat2020isotropic,
	title        = {How isotropic kernels perform on simple invariants},
	author       = {Paccolat, Jonas and Spigler, Stefano and Wyart, Matthieu},
	year         = 2021,
	month        = mar,
	journal      = {Machine Learning: Science and Technology},
	volume       = 2,
	number       = 2,
	doi          = {10.1088/2632-2153/abd485},
	issn         = {2632-2153},
	url          = {https://doi.org/10.1088/2632-2153/abd485},
	urldate      = {2021-03-11},
	note         = {Publisher: IOP Publishing},
	language     = {en},
	abstract     = {We investigate how the training curve of isotropic kernel methods depends on the symmetry of the task to be learned, in several settings. (i) We consider a regression task, where the target function is a Gaussian random field that depends only on variables, fewer than the input dimension d. We compute the expected test error ϵ that follows where p is the size of the training set. We find that β ∼ 1/d independently of , supporting previous findings that the presence of invariants does not resolve the curse of dimensionality for kernel regression. (ii) Next we consider support-vector binary classification and introduce the stripe model, where the data label depends on a single coordinate , corresponding to parallel decision boundaries separating labels of different signs, and consider that there is no margin at these interfaces. We argue and confirm numerically that, for large bandwidth, , where ξ ∈ (0, 2) is the exponent characterizing the singularity of the kernel at the origin. This estimation improves classical bounds obtainable from Rademacher complexity. In this setting there is no curse of dimensionality since as . (iii) We confirm these findings for the spherical model, for which . (iv) In the stripe model, we show that, if the data are compressed along their invariants by some factor λ (an operation believed to take place in deep networks), the test error is reduced by a factor .},
}
@inproceedings{petrini_relative_2021,
	title        = {Relative stability toward diffeomorphisms indicates performance in deep nets},
	author       = {Petrini, Leonardo and Favero, Alessandro and Geiger, Mario and Wyart, Matthieu},
	year         = 2021,
	booktitle    = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher    = {Curran Associates, Inc.},
	volume       = 34,
	url          = {https://proceedings.neurips.cc/paper/2021/hash/497476fe61816251905e8baafdf54c23-Abstract.html},
	urldate      = {2022-04-08},
}
@article{refinetti2021classifying,
	title        = {Classifying high-dimensional Gaussian mixtures: Where kernel methods fail and neural networks succeed},
	author       = {Refinetti, Maria and Goldt, Sebastian and Krzakala, Florent and Zdeborov{\'a}, Lenka},
	year         = 2021,
	journal      = {arXiv preprint arXiv:2102.11742},
}
@misc{sulam_adversarial_2021,
	title        = {Adversarial {Robustness} of {Supervised} {Sparse} {Coding}},
	author       = {Sulam, Jeremias and Muthukumar, Ramchandran and Arora, Raman},
	year         = 2021,
	month        = jan,
	publisher    = {arXiv},
	url          = {http://arxiv.org/abs/2010.12088},
	urldate      = {2022-09-29},
	note         = {arXiv:2010.12088 [cs, stat]},
	abstract     = {Several recent results provide theoretical insights into the phenomena of adversarial examples. Existing results, however, are often limited due to a gap between the simplicity of the models studied and the complexity of those deployed in practice. In this work, we strike a better balance by considering a model that involves learning a representation while at the same time giving a precise generalization bound and a robustness certificate. We focus on the hypothesis class obtained by combining a sparsity-promoting encoder coupled with a linear classifier, and show an interesting interplay between the expressivity and stability of the (supervised) representation map and a notion of margin in the feature space. We bound the robust risk (to \${\textbackslash}ell\_2\$-bounded perturbations) of hypotheses parameterized by dictionaries that achieve a mild encoder gap on training data. Furthermore, we provide a robustness certificate for end-to-end classification. We demonstrate the applicability of our analysis by computing certified accuracy on real data, and compare with other alternatives for certified robustness.},
	keywords     = {Computer Science - Machine Learning, Statistics - Machine Learning},
}
@article{bach_learning_2022,
	title        = {Learning {Theory} from {First} {Principles}},
	author       = {Bach, Francis},
	year         = 2022,
	journal      = {\href{https://francisbach.com/i-am-writing-a-book/}{In preparation}},
	url          = {https://francisbach.com/i-am-writing-a-book/},
	language     = {en},
}
@article{tomasini2022failure,
	title        = {Failure and success of the spectral bias prediction for Kernel Ridge Regression: the case of low-dimensional data},
	author       = {Tomasini, Umberto M and Sclocchi, Antonio and Wyart, Matthieu},
	year         = 2022,
	journal      = {arXiv preprint arXiv:2202.03348},
}