{
  "id": "2007.15745",
  "version": "v1",
  "published": "2020-07-30T21:11:01.000Z",
  "updated": "2020-07-30T21:11:01.000Z",
  "title": "On Hyperparameter Optimization of Machine Learning Algorithms: Theory and Practice",
  "authors": [
    "Li Yang",
    "Abdallah Shami"
  ],
  "comment": "69 Pages, 10 tables, accepted in Neurocomputing, Elsevier",
  "doi": "10.1016/j.neucom.2020.07.061",
  "categories": [
    "cs.LG",
    "stat.ML"
  ],
  "abstract": "Machine learning algorithms have been used widely in various applications and areas. To fit a machine learning model into different problems, its hyper-parameters must be tuned. Selecting the best hyper-parameter configuration for machine learning models has a direct impact on the model's performance. It often requires deep knowledge of machine learning algorithms and appropriate hyper-parameter optimization techniques. Although several automatic optimization techniques exist, they have different strengths and drawbacks when applied to different types of problems. In this paper, optimizing the hyper-parameters of common machine learning models is studied. We introduce several state-of-the-art optimization techniques and discuss how to apply them to machine learning algorithms. Many available libraries and frameworks developed for hyper-parameter optimization problems are provided, and some open challenges of hyper-parameter optimization research are also discussed in this paper. Moreover, experiments are conducted on benchmark datasets to compare the performance of different optimization methods and provide practical examples of hyper-parameter optimization. This survey paper will help industrial users, data analysts, and researchers to better develop machine learning models by identifying the proper hyper-parameter configurations effectively.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2020-07-30T21:11:01.000Z"
    }
  ],
  "analyses": {
    "subjects": [
      "68T01",
      "90C31"
    ],
    "keywords": [
      "machine learning algorithms",
      "hyperparameter optimization",
      "appropriate hyper-parameter optimization techniques",
      "common machine learning models",
      "best hyper-parameter configuration"
    ],
    "tags": [
      "journal article"
    ],
    "publication": {
      "publisher": "Elsevier"
    },
    "note": {
      "typesetting": "TeX",
      "pages": 69,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}