{
  "id": "2106.09685",
  "version": "v1",
  "published": "2021-06-17T17:37:18.000Z",
  "updated": "2021-06-17T17:37:18.000Z",
  "title": "LoRA: Low-Rank Adaptation of Large Language Models",
  "authors": [
    "Edward J. Hu",
    "Yelong Shen",
    "Phillip Wallis",
    "Zeyuan Allen-Zhu",
    "Yuanzhi Li",
    "Shean Wang",
    "Weizhu Chen"
  ],
  "categories": [
    "cs.CL",
    "cs.AI",
    "cs.LG"
  ],
  "abstract": "The dominant paradigm of natural language processing consists of large-scale pre-training on general domain data and adaptation to particular tasks or domains. As we pre-train larger models, conventional fine-tuning, which retrains all model parameters, becomes less feasible. Using GPT-3 175B as an example, deploying many independent instances of fine-tuned models, each with 175B parameters, is extremely expensive. We propose Low-Rank Adaptation, or LoRA, which freezes the pre-trained model weights and injects trainable rank decomposition matrices into each layer of the Transformer architecture, greatly reducing the number of trainable parameters for downstream tasks. For GPT-3, LoRA can reduce the number of trainable parameters by 10,000 times and the computation hardware requirement by 3 times compared to full fine-tuning. LoRA performs on-par or better than fine-tuning in model quality on both GPT-3 and GPT-2, despite having fewer trainable parameters, a higher training throughput, and no additional inference latency. We also provide an empirical investigation into rank-deficiency in language model adaptations, which sheds light on the efficacy of LoRA. We release our implementation in GPT-2 at https://github.com/microsoft/LoRA .",
  "revisions": [
    {
      "version": "v1",
      "updated": "2021-06-17T17:37:18.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "large language models",
      "low-rank adaptation",
      "trainable parameters",
      "injects trainable rank decomposition matrices",
      "natural language processing consists"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}