{
  "id": "1809.09095",
  "version": "v1",
  "published": "2018-09-23T15:48:28.000Z",
  "updated": "2018-09-23T15:48:28.000Z",
  "title": "On Reinforcement Learning for Full-length Game of StarCraft",
  "authors": [
    "Zhen-Jia Pang",
    "Ruo-Ze Liu",
    "Zhou-Yu Meng",
    "Yi Zhang",
    "Yang Yu",
    "Tong Lu"
  ],
  "categories": [
    "cs.LG",
    "cs.AI",
    "stat.ML"
  ],
  "abstract": "StarCraft II poses a grand challenge for reinforcement learning. The main difficulties of it include huge state and action space and a long-time horizon. In this paper, we investigate a hierarchical reinforcement learning approach for StarCraft II. The hierarchy involves two levels of abstraction. One is the macro-action automatically extracted from expert's trajectories, which reduces the action space in an order of magnitude yet remains effective. The other is a two-layer hierarchical architecture which is modular and easy to scale, enabling a curriculum transferring from simpler tasks to more complex tasks. The reinforcement training algorithm for this architecture is also investigated. On a 64x64 map and using restrictive units, we achieve a winning rate of more than 99\\% against the difficulty level-1 built-in AI. Through the curriculum transfer learning algorithm and a mixture of combat model, we can achieve over 93\\% winning rate of Protoss against the most difficult non-cheating built-in AI (level-7) of Terran, training within two days using a single machine with only 48 CPU cores and 8 K40 GPUs. It also shows strong generalization performance, when tested against never seen opponents including cheating levels built-in AI and all levels of Zerg and Protoss built-in AI. We hope this study could shed some light on the future research of large-scale reinforcement learning.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2018-09-23T15:48:28.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "reinforcement learning",
      "full-length game",
      "action space",
      "protoss built-in ai",
      "curriculum transfer learning algorithm"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}