{
  "id": "2010.10711",
  "version": "v1",
  "published": "2020-10-21T02:09:33.000Z",
  "updated": "2020-10-21T02:09:33.000Z",
  "title": "On the Global Self-attention Mechanism for Graph Convolutional Networks",
  "authors": [
    "Chen Wang",
    "Chengyuan Deng"
  ],
  "categories": [
    "cs.LG",
    "stat.ML"
  ],
  "abstract": "Applying Global Self-attention (GSA) mechanism over features has achieved remarkable success on Convolutional Neural Networks (CNNs). However, it is not clear if Graph Convolutional Networks (GCNs) can similarly benefit from such a technique. In this paper, inspired by the similarity between CNNs and GCNs, we study the impact of the Global Self-attention mechanism on GCNs. We find that consistent with the intuition, the GSA mechanism allows GCNs to capture feature-based vertex relations regardless of edge connections; As a result, the GSA mechanism can introduce extra expressive power to the GCNs. Furthermore, we analyze the impacts of the GSA mechanism on the issues of overfitting and over-smoothing. We prove that the GSA mechanism can alleviate both the overfitting and the over-smoothing issues based on some recent technical developments. Experiments on multiple benchmark datasets illustrate both superior expressive power and less significant overfitting and over-smoothing problems for the GSA-augmented GCNs, which corroborate the intuitions and the theoretical results.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2020-10-21T02:09:33.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "graph convolutional networks",
      "global self-attention mechanism",
      "gsa mechanism",
      "feature-based vertex relations regardless",
      "multiple benchmark datasets illustrate"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}