{
  "id": "2107.09370",
  "version": "v1",
  "published": "2021-07-20T09:43:31.000Z",
  "updated": "2021-07-20T09:43:31.000Z",
  "title": "An Embedding of ReLU Networks and an Analysis of their Identifiability",
  "authors": [
    "Pierre Stock",
    "Rémi Gribonval"
  ],
  "categories": [
    "cs.LG"
  ],
  "abstract": "Neural networks with the Rectified Linear Unit (ReLU) nonlinearity are described by a vector of parameters $\\theta$, and realized as a piecewise linear continuous function $R_{\\theta}: x \\in \\mathbb R^{d} \\mapsto R_{\\theta}(x) \\in \\mathbb R^{k}$. Natural scalings and permutations operations on the parameters $\\theta$ leave the realization unchanged, leading to equivalence classes of parameters that yield the same realization. These considerations in turn lead to the notion of identifiability -- the ability to recover (the equivalence class of) $\\theta$ from the sole knowledge of its realization $R_{\\theta}$. The overall objective of this paper is to introduce an embedding for ReLU neural networks of any depth, $\\Phi(\\theta)$, that is invariant to scalings and that provides a locally linear parameterization of the realization of the network. Leveraging these two key properties, we derive some conditions under which a deep ReLU network is indeed locally identifiable from the knowledge of the realization on a finite set of samples $x_{i} \\in \\mathbb R^{d}$. We study the shallow case in more depth, establishing necessary and sufficient conditions for the network to be identifiable from a bounded subset $\\mathcal X \\subseteq \\mathbb R^{d}$.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2021-07-20T09:43:31.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "identifiability",
      "realization",
      "equivalence class",
      "relu neural networks",
      "deep relu network"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}