{
  "id": "2205.06262",
  "version": "v1",
  "published": "2022-05-12T17:59:00.000Z",
  "updated": "2022-05-12T17:59:00.000Z",
  "title": "FETA: A Benchmark for Few-Sample Task Transfer in Open-Domain Dialogue",
  "authors": [
    "Alon Albalak",
    "Yi-Lin Tuan",
    "Pegah Jandaghi",
    "Connor Pryor",
    "Luke Yoffe",
    "Deepak Ramachandran",
    "Lise Getoor",
    "Jay Pujara",
    "William Yang Wang"
  ],
  "comment": "code available at https://github.com/alon-albalak/TLiDB",
  "categories": [
    "cs.CL"
  ],
  "abstract": "Task transfer, transferring knowledge contained in related tasks, holds the promise of reducing the quantity of labeled data required to fine-tune language models. Dialogue understanding encompasses many diverse tasks, yet task transfer has not been thoroughly studied in conversational AI. This work explores conversational task transfer by introducing FETA: a benchmark for few-sample task transfer in open-domain dialogue. FETA contains two underlying sets of conversations upon which there are 10 and 7 tasks annotated, enabling the study of intra-dataset task transfer; task transfer without domain adaptation. We utilize three popular language models and three learning algorithms to analyze the transferability between 132 source-target task pairs and create a baseline for future work. We run experiments in the single- and multi-source settings and report valuable findings, e.g., most performance trends are model-specific, and span extraction and multiple-choice tasks benefit the most from task transfer. In addition to task transfer, FETA can be a valuable resource for future research into the efficiency and generalizability of pre-training datasets and model architectures, as well as for learning settings such as continual and multitask learning.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-05-12T17:59:00.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "few-sample task transfer",
      "open-domain dialogue",
      "multiple-choice tasks benefit",
      "source-target task pairs",
      "popular language models"
    ],
    "tags": [
      "github project"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}