{
  "id": "2308.06780",
  "version": "v1",
  "published": "2023-08-13T14:25:54.000Z",
  "updated": "2023-08-13T14:25:54.000Z",
  "title": "Neural Networks at a Fraction with Pruned Quaternions",
  "authors": [
    "Sahel Mohammad Iqbal",
    "Subhankar Mishra"
  ],
  "journal": "In Proceedings of the 6th Joint International Conference on Data Science & Management of Data (CODS-COMAD '23). ACM. 19-27 (2023)",
  "doi": "10.1145/3570991.3570997",
  "categories": [
    "cs.LG",
    "cs.CV"
  ],
  "abstract": "Contemporary state-of-the-art neural networks have increasingly large numbers of parameters, which prevents their deployment on devices with limited computational power. Pruning is one technique to remove unnecessary weights and reduce resource requirements for training and inference. In addition, for ML tasks where the input data is multi-dimensional, using higher-dimensional data embeddings such as complex numbers or quaternions has been shown to reduce the parameter count while maintaining accuracy. In this work, we conduct pruning on real and quaternion-valued implementations of different architectures on classification tasks. We find that for some architectures, at very high sparsity levels, quaternion models provide higher accuracies than their real counterparts. For example, at the task of image classification on CIFAR-10 using Conv-4, at $3\\%$ of the number of parameters as the original model, the pruned quaternion version outperforms the pruned real by more than $10\\%$. Experiments on various network architectures and datasets show that for deployment in extremely resource-constrained environments, a sparse quaternion network might be a better candidate than a real sparse model of similar architecture.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-08-13T14:25:54.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "architecture",
      "contemporary state-of-the-art neural networks",
      "sparse quaternion network",
      "remove unnecessary weights",
      "reduce resource requirements"
    ],
    "tags": [
      "journal article"
    ],
    "publication": {
      "publisher": "ACM",
      "journal": "Commun. ACM"
    },
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}