{
  "id": "2210.06143",
  "version": "v1",
  "published": "2022-10-12T12:49:20.000Z",
  "updated": "2022-10-12T12:49:20.000Z",
  "title": "On the Importance of Gradient Norm in PAC-Bayesian Bounds",
  "authors": [
    "Itai Gat",
    "Yossi Adi",
    "Alexander Schwing",
    "Tamir Hazan"
  ],
  "comment": "NeurIPS 22. arXiv admin note: text overlap with arXiv:2002.09866",
  "categories": [
    "cs.LG",
    "stat.ML"
  ],
  "abstract": "Generalization bounds which assess the difference between the true risk and the empirical risk, have been studied extensively. However, to obtain bounds, current techniques use strict assumptions such as a uniformly bounded or a Lipschitz loss function. To avoid these assumptions, in this paper, we follow an alternative approach: we relax uniform bounds assumptions by using on-average bounded loss and on-average bounded gradient norm assumptions. Following this relaxation, we propose a new generalization bound that exploits the contractivity of the log-Sobolev inequalities. These inequalities add an additional loss-gradient norm term to the generalization bound, which is intuitively a surrogate of the model complexity. We apply the proposed bound on Bayesian deep nets and empirically analyze the effect of this new loss-gradient norm term on different neural architectures.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-10-12T12:49:20.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "pac-bayesian bounds",
      "generalization bound",
      "additional loss-gradient norm term",
      "importance",
      "on-average bounded gradient norm assumptions"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}