{
  "id": "2304.03486",
  "version": "v1",
  "published": "2023-04-07T05:45:26.000Z",
  "updated": "2023-04-07T05:45:26.000Z",
  "title": "Can we learn better with hard samples?",
  "authors": [
    "Subin Sahayam",
    "John Zakkam",
    "Umarani Jayaraman"
  ],
  "categories": [
    "cs.CV",
    "cs.AI"
  ],
  "abstract": "In deep learning, mini-batch training is commonly used to optimize network parameters. However, the traditional mini-batch method may not learn the under-represented samples and complex patterns in the data, leading to a longer time for generalization. To address this problem, a variant of the traditional algorithm has been proposed, which trains the network focusing on mini-batches with high loss. The study evaluates the effectiveness of the proposed training using various deep neural networks trained on three benchmark datasets (CIFAR-10, CIFAR-100, and STL-10). The deep neural networks used in the study are ResNet-18, ResNet-50, Efficient Net B4, EfficientNetV2-S, and MobilenetV3-S. The experimental results showed that the proposed method can significantly improve the test accuracy and speed up the convergence compared to the traditional mini-batch training method. Furthermore, we introduce a hyper-parameter delta ({\\delta}) that decides how many mini-batches are considered for training. Experiments on various values of {\\delta} found that the performance of the proposed method for smaller {\\delta} values generally results in similar test accuracy and faster generalization. We show that the proposed method generalizes in 26.47% less number of epochs than the traditional mini-batch method in EfficientNet-B4 on STL-10. The proposed method also improves the test top-1 accuracy by 7.26% in ResNet-18 on CIFAR-100.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-04-07T05:45:26.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "learn better",
      "hard samples",
      "deep neural networks",
      "traditional mini-batch method",
      "traditional mini-batch training method"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}