{
  "id": "2009.10717",
  "version": "v1",
  "published": "2020-09-22T17:59:06.000Z",
  "updated": "2020-09-22T17:59:06.000Z",
  "title": "Asynchronous Distributed Optimization with Randomized Delays",
  "authors": [
    "Margalit Glasgow",
    "Mary Wootters"
  ],
  "categories": [
    "cs.LG",
    "cs.DC",
    "stat.ML"
  ],
  "abstract": "In this work, we study asynchronous finite sum minimization in a distributed-data setting with a central parameter server. While asynchrony is well understood in parallel settings where the data is accessible by all machines, little is known for the distributed-data setting. We introduce a variant of SAGA called ADSAGA for the distributed-data setting where each machine stores a partition of the data. We show that with independent exponential work times -- a common assumption in distributed optimization -- ADSAGA converges in $\\tilde{O}\\left(\\left(n + \\sqrt{m}\\kappa\\right)\\log(1/\\epsilon)\\right)$ iterations, where $n$ is the number of component functions, $m$ is the number of machines, and $\\kappa$ is a condition number. We empirically compare the iteration complexity of ADSAGA to existing parallel and distributed algorithms, including synchronous mini-batch algorithms.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2020-09-22T17:59:06.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "asynchronous distributed optimization",
      "randomized delays",
      "study asynchronous finite sum minimization",
      "distributed-data setting",
      "independent exponential work times"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}