{
  "id": "2311.09632",
  "version": "v1",
  "published": "2023-11-16T07:31:03.000Z",
  "updated": "2023-11-16T07:31:03.000Z",
  "title": "Online Continual Knowledge Learning for Language Models",
  "authors": [
    "Yuhao Wu",
    "Tongjun Shi",
    "Karthick Sharma",
    "Chun Wei Seah",
    "Shuhao Zhang"
  ],
  "categories": [
    "cs.CL",
    "cs.AI"
  ],
  "abstract": "Large Language Models (LLMs) serve as repositories of extensive world knowledge, enabling them to perform tasks such as question-answering and fact-checking. However, this knowledge can become obsolete as global contexts change. In this paper, we introduce a novel problem in the realm of continual learning: Online Continual Knowledge Learning (OCKL). This problem formulation aims to manage the dynamic nature of world knowledge in LMs under real-time constraints. We propose a new benchmark and evaluation metric designed to measure both the rate of new knowledge acquisition and the retention of previously learned knowledge. Our empirical evaluation, conducted using a variety of state-of-the-art methods, establishes robust base-lines for OCKL. Our results reveal that existing continual learning approaches are unfortunately insufficient for tackling the unique challenges posed by OCKL. We identify key factors that influence the trade-off between knowledge acquisition and retention, thereby advancing our understanding of how to train LMs in a continually evolving environment.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-11-16T07:31:03.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "online continual knowledge learning",
      "knowledge acquisition",
      "large language models",
      "establishes robust base-lines",
      "global contexts change"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}