use github api to find github releases

vulnerabilities/package_managers.py

@@ -19,6 +19,7 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+import os
 import asyncio
 import dataclasses
 import xml.etree.ElementTree as ET
@@ -30,6 +31,7 @@
 from typing import Set
 
 from aiohttp import ClientSession
+import aiohttp
 from aiohttp.client_exceptions import ClientResponseError
 from aiohttp.client_exceptions import ServerDisconnectedError
 from bs4 import BeautifulSoup
@@ -48,6 +50,10 @@ class VersionResponse:
     newer_versions: Set[str] = dataclasses.field(default_factory=set)
 
 
+class GraphQLError(Exception):
+    pass
+
+
 class VersionAPI:
     def __init__(self, cache: Mapping[str, Set[Version]] = None):
         self.cache = cache or {}
@@ -377,21 +383,92 @@ def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
 class GitHubTagsAPI(VersionAPI):
 
     package_type = "github"
+    GQL_QUERY = """
+    query getTags($name: String!, $owner: String!, $after: String)
+    {
+        repository(name: $name, owner: $owner) {
+            refs(refPrefix: "refs/tags/", first: 100, after: $after) {
+                totalCount
+                pageInfo {
+                    endCursor
+                    hasNextPage
+                }
+                nodes {
+                    name
+                    target {
+                        ... on Commit {
+                            committedDate
+                        }
+                        ... on Tag {
+                                target {
+                                ... on Commit {
+                                    committedDate
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }"""
 
-    async def fetch(self, owner_repo: str, session) -> None:
+    def __init__(self, cache: Mapping[str, Set[Version]] = None):
+        self.gh_token = os.getenv("GH_TOKEN")
+        super().__init__(cache=cache)
+
+    async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
         """
         owner_repo is a string of format "{repo_owner}/{repo_name}"
         Example value of owner_repo = "nexB/scancode-toolkit"
         """
         self.cache[owner_repo] = set()
-        endpoint = f"https://github.com/{owner_repo}"
-
-        tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
-        elements = ET.fromstring(tags_xml)
-        for entry in elements.iter("entry"):
-            name = entry.find("name").text
-            release_date = dateparser.parse(entry.find("commit/date").text)
-            self.cache[owner_repo].add(Version(value=name, release_date=release_date))
+        if self.gh_token:
+            # graphql api cannot work without api token
+            session.headers["Authorization"] = "token " + self.gh_token
+            endpoint = f"https://api.github.com/graphql"
+            owner, name = owner_repo.split("/")
+            query = {"query": self.GQL_QUERY, "variables": {"name": name, "owner": owner}}
+
+            while True:
+                response = await session.post(endpoint, json=query)
+                resp_json = await response.json()
+
+                if "errors" in resp_json:
+                    raise GraphQLError(resp_json["errors"])
+
+                refs = resp_json["data"]["repository"]["refs"]
+
+                for entry in refs["nodes"]:
+                    name = entry["name"]
+                    target = entry["target"]
+                    # in case the tag is a signed tag, then the commit info is in target['target']
+                    if "committedDate" not in target:
+                        target = target["target"]
+                    if "committedDate" in target:
+                        release_date = dateparser.parse(target["committedDate"])
+                    else:
+                        # but tags can actually point to tree and not commit, so there is no date
+                        # probably this only happened for linux. Github cannot even properly display it.
+                        # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/refs/tags/v2.6.11
+                        release_date = None
+                    self.cache[owner_repo].add(Version(value=name, release_date=release_date))
+
+                if not refs["pageInfo"]["hasNextPage"]:
+                    break
+                # to fetch next page, we just set the after variable to endCursor
+                query["variables"]["after"] = refs["pageInfo"]["endCursor"]
+
+        else:
+            # In case we don't have GH_TOKEN, we use the svn ls method to get the tags
+            # It allows to get all the information needed in one request without any rate limiting
+            # this method is however not scalable for larger repo and the api is unresponsive
+            # for repo with > 50 tags
+            endpoint = f"https://github.com/{owner_repo}"
+            tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
+            elements = ET.fromstring(tags_xml)
+            for entry in elements.iter("entry"):
+                name = entry.find("name").text
+                release_date = dateparser.parse(entry.find("commit/date").text)
 
 
 class HexVersionAPI(VersionAPI):

vulnerabilities/tests/test_data/github_api/release_response.json

@@ -0,0 +1,76 @@
+[
+  {
+    "url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782",
+    "assets_url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782/assets",
+    "upload_url": "https://uploads.github.com/repos/nexB/vulnerablecode/releases/32748782/assets{?name,label}",
+    "html_url": "https://github.com/nexB/vulnerablecode/releases/tag/v20.10",
+    "id": 32748782,
+    "author": {
+      "login": "pombredanne",
+      "id": 675997,
+      "node_id": "MDQ6VXNlcjY3NTk5Nw==",
+      "avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/pombredanne",
+      "html_url": "https://github.com/pombredanne",
+      "followers_url": "https://api.github.com/users/pombredanne/followers",
+      "following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
+      "gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
+      "organizations_url": "https://api.github.com/users/pombredanne/orgs",
+      "repos_url": "https://api.github.com/users/pombredanne/repos",
+      "events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/pombredanne/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "node_id": "MDc6UmVsZWFzZTMyNzQ4Nzgy",
+    "tag_name": "v20.10",
+    "target_commitish": "main",
+    "name": "v20.10",
+    "draft": false,
+    "prerelease": false,
+    "created_at": "2020-09-28T12:31:16Z",
+    "published_at": "2020-10-19T10:46:17Z",
+    "assets": [
+      {
+        "url": "https://api.github.com/repos/nexB/vulnerablecode/releases/assets/27230021",
+        "id": 27230021,
+        "node_id": "MDEyOlJlbGVhc2VBc3NldDI3MjMwMDIx",
+        "name": "vulnerablecode-2020-10-19.json.xz",
+        "label": null,
+        "uploader": {
+          "login": "pombredanne",
+          "id": 675997,
+          "node_id": "MDQ6VXNlcjY3NTk5Nw==",
+          "avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/pombredanne",
+          "html_url": "https://github.com/pombredanne",
+          "followers_url": "https://api.github.com/users/pombredanne/followers",
+          "following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
+          "gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
+          "organizations_url": "https://api.github.com/users/pombredanne/orgs",
+          "repos_url": "https://api.github.com/users/pombredanne/repos",
+          "events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/pombredanne/received_events",
+          "type": "User",
+          "site_admin": false
+        },
+        "content_type": "application/x-xz",
+        "state": "uploaded",
+        "size": 13603356,
+        "download_count": 20,
+        "created_at": "2020-10-20T09:40:08Z",
+        "updated_at": "2020-10-20T09:40:25Z",
+        "browser_download_url": "https://github.com/nexB/vulnerablecode/releases/download/v20.10/vulnerablecode-2020-10-19.json.xz"
+      }
+    ],
+    "tarball_url": "https://api.github.com/repos/nexB/vulnerablecode/tarball/v20.10",
+    "zipball_url": "https://api.github.com/repos/nexB/vulnerablecode/zipball/v20.10",
+    "body": "This release comes with the new calver versioning scheme and an initial data dump.\r\n\r\nTo load the JSON data attached here:\r\n- extract it with `unxz vulnerablecode-2020-10-19.json.xz`\r\n- run `DJANGO_DEV=1 python manage.py loaddata vulnerablecode-2020-10-19.json`\r\n\r\nThe data import is not optimized yet and takes a long time."
+  }
+]

vulnerabilities/tests/test_data/records/nexB_vulnerablecode_1.json

@@ -0,0 +1 @@
+{"data":{"repository":{"refs":{"totalCount":2,"pageInfo":{"endCursor":"Mg","hasNextPage":false},"nodes":[{"name":"v0.1","target":{"committedDate":"2019-12-03T13:48:53Z"}},{"name":"v20.10","target":{"committedDate":"2020-09-28T12:31:16Z"}}]}}}}

vulnerabilities/tests/test_data/records/torvalds_linux_1.json

@@ -0,0 +1 @@
+{"data":{"repository":{"refs":{"totalCount":713,"pageInfo":{"endCursor":"MTAw","hasNextPage":true},"nodes":[{"name":"v2.6.11-tree","target":{"target":{}}},{"name":"v2.6.11","target":{"target":{}}},{"name":"v2.6.12-rc2","target":{"target":{"committedDate":"2005-04-16T22:20:36Z"}}},{"name":"v2.6.12-rc3","target":{"target":{"committedDate":"2005-04-20T23:24:21Z"}}},{"name":"v2.6.12-rc4","target":{"target":{"committedDate":"2005-05-07T05:20:31Z"}}},{"name":"v2.6.12-rc5","target":{"target":{"committedDate":"2005-05-25T03:31:20Z"}}},{"name":"v2.6.12-rc6","target":{"target":{"committedDate":"2005-06-06T15:22:29Z"}}},{"name":"v2.6.12","target":{"target":{"committedDate":"2005-06-17T19:48:29Z"}}},{"name":"v2.6.13-rc1","target":{"target":{"committedDate":"2005-06-29T05:57:29Z"}}},{"name":"v2.6.13-rc2","target":{"target":{"committedDate":"2005-07-06T03:46:33Z"}}},{"name":"v2.6.13-rc3","target":{"target":{"committedDate":"2005-07-13T04:46:46Z"}}},{"name":"v2.6.13-rc4","target":{"target":{"committedDate":"2005-07-28T22:44:44Z"}}},{"name":"v2.6.13-rc5","target":{"target":{"committedDate":"2005-08-02T04:45:48Z"}}},{"name":"v2.6.13-rc6","target":{"target":{"committedDate":"2005-08-07T18:18:56Z"}}},{"name":"v2.6.13-rc7","target":{"target":{"committedDate":"2005-08-24T03:39:14Z"}}},{"name":"v2.6.13","target":{"target":{"committedDate":"2005-08-28T23:41:01Z"}}},{"name":"v2.6.14-rc1","target":{"target":{"committedDate":"2005-09-13T03:12:09Z"}}},{"name":"v2.6.14-rc2","target":{"target":{"committedDate":"2005-09-20T03:00:41Z"}}},{"name":"v2.6.14-rc3","target":{"target":{"committedDate":"2005-09-30T21:17:35Z"}}},{"name":"v2.6.14-rc4","target":{"target":{"committedDate":"2005-10-11T01:19:19Z"}}},{"name":"v2.6.14-rc5","target":{"target":{"committedDate":"2005-10-20T06:23:05Z"}}},{"name":"v2.6.14","target":{"target":{"committedDate":"2005-10-28T00:02:08Z"}}},{"name":"v2.6.15-rc1","target":{"target":{"committedDate":"2005-11-12T01:43:36Z"}}},{"name":"v2.6.15-rc2","target":{"target":{"committedDate":"2005-11-20T03:25:03Z"}}},{"name":"v2.6.15-rc3","target":{"target":{"committedDate":"2005-11-29T03:51:27Z"}}},{"name":"v2.6.15-rc4","target":{"target":{"committedDate":"2005-12-01T06:25:15Z"}}},{"name":"v2.6.15-rc5","target":{"target":{"committedDate":"2005-12-04T05:10:42Z"}}},{"name":"v2.6.15-rc6","target":{"target":{"committedDate":"2005-12-19T00:36:54Z"}}},{"name":"v2.6.15-rc7","target":{"target":{"committedDate":"2005-12-24T23:47:48Z"}}},{"name":"v2.6.15","target":{"target":{"committedDate":"2006-01-03T03:21:10Z"}}},{"name":"v2.6.16-rc1","target":{"target":{"committedDate":"2006-01-17T07:44:47Z"}}},{"name":"v2.6.16-rc2","target":{"target":{"committedDate":"2006-02-03T06:03:08Z"}}},{"name":"v2.6.16-rc3","target":{"target":{"committedDate":"2006-02-13T00:27:25Z"}}},{"name":"v2.6.16-rc4","target":{"target":{"committedDate":"2006-02-17T22:23:45Z"}}},{"name":"v2.6.16-rc5","target":{"target":{"committedDate":"2006-02-27T05:09:35Z"}}},{"name":"v2.6.16-rc6","target":{"target":{"committedDate":"2006-03-11T22:12:55Z"}}},{"name":"v2.6.16","target":{"target":{"committedDate":"2006-03-20T05:53:29Z"}}},{"name":"v2.6.17-rc1","target":{"target":{"committedDate":"2006-04-03T03:22:10Z"}}},{"name":"v2.6.17-rc2","target":{"target":{"committedDate":"2006-04-19T03:00:49Z"}}},{"name":"v2.6.17-rc3","target":{"target":{"committedDate":"2006-04-27T02:19:25Z"}}},{"name":"v2.6.17-rc4","target":{"target":{"committedDate":"2006-05-11T23:31:53Z"}}},{"name":"v2.6.17-rc5","target":{"target":{"committedDate":"2006-05-25T01:50:17Z"}}},{"name":"v2.6.17-rc6","target":{"target":{"committedDate":"2006-06-06T00:57:02Z"}}},{"name":"v2.6.17","target":{"target":{"committedDate":"2006-06-18T01:49:35Z"}}},{"name":"v2.6.18-rc1","target":{"target":{"committedDate":"2006-07-06T04:09:49Z"}}},{"name":"v2.6.18-rc2","target":{"target":{"committedDate":"2006-07-15T21:53:08Z"}}},{"name":"v2.6.18-rc3","target":{"target":{"committedDate":"2006-07-30T06:15:36Z"}}},{"name":"v2.6.18-rc4","target":{"target":{"committedDate":"2006-08-06T18:20:11Z"}}},{"name":"v2.6.18-rc5","target":{"target":{"committedDate":"2006-08-28T03:41:48Z"}}},{"name":"v2.6.18-rc6","target":{"target":{"committedDate":"2006-09-04T02:19:48Z"}}},{"name":"v2.6.18-rc7","target":{"target":{"committedDate":"2006-09-13T01:41:36Z"}}},{"name":"v2.6.18","target":{"target":{"committedDate":"2006-09-20T03:42:06Z"}}},{"name":"v2.6.19-rc1","target":{"target":{"committedDate":"2006-10-05T02:57:05Z"}}},{"name":"v2.6.19-rc2","target":{"target":{"committedDate":"2006-10-13T16:25:04Z"}}},{"name":"v2.6.19-rc3","target":{"target":{"committedDate":"2006-10-23T23:02:02Z"}}},{"name":"v2.6.19-rc4","target":{"target":{"committedDate":"2006-10-31T03:37:36Z"}}},{"name":"v2.6.19-rc5","target":{"target":{"committedDate":"2006-11-08T02:24:20Z"}}},{"name":"v2.6.19-rc6","target":{"target":{"committedDate":"2006-11-16T04:03:40Z"}}},{"name":"v2.6.19","target":{"target":{"committedDate":"2006-11-29T21:57:37Z"}}},{"name":"v2.6.20-rc1","target":{"target":{"committedDate":"2006-12-14T01:14:23Z"}}},{"name":"v2.6.20-rc2","target":{"target":{"committedDate":"2006-12-24T04:00:32Z"}}},{"name":"v2.6.20-rc3","target":{"target":{"committedDate":"2007-01-01T00:53:20Z"}}},{"name":"v2.6.20-rc4","target":{"target":{"committedDate":"2007-01-07T05:45:51Z"}}},{"name":"v2.6.20-rc5","target":{"target":{"committedDate":"2007-01-12T18:54:26Z"}}},{"name":"v2.6.20-rc6","target":{"target":{"committedDate":"2007-01-25T02:19:28Z"}}},{"name":"v2.6.20-rc7","target":{"target":{"committedDate":"2007-01-31T03:42:57Z"}}},{"name":"v2.6.20","target":{"target":{"committedDate":"2007-02-04T18:44:54Z"}}},{"name":"v2.6.21-rc1","target":{"target":{"committedDate":"2007-02-21T04:32:30Z"}}},{"name":"v2.6.21-rc2","target":{"target":{"committedDate":"2007-02-28T04:59:12Z"}}},{"name":"v2.6.21-rc3","target":{"target":{"committedDate":"2007-03-07T04:41:20Z"}}},{"name":"v2.6.21-rc4","target":{"target":{"committedDate":"2007-03-16T00:20:01Z"}}},{"name":"v2.6.21-rc5","target":{"target":{"committedDate":"2007-03-25T22:56:23Z"}}},{"name":"v2.6.21-rc6","target":{"target":{"committedDate":"2007-04-06T02:36:56Z"}}},{"name":"v2.6.21-rc7","target":{"target":{"committedDate":"2007-04-15T23:50:57Z"}}},{"name":"v2.6.21","target":{"target":{"committedDate":"2007-04-26T03:08:32Z"}}},{"name":"v2.6.22-rc1","target":{"target":{"committedDate":"2007-05-13T01:45:56Z"}}},{"name":"v2.6.22-rc2","target":{"target":{"committedDate":"2007-05-19T04:06:17Z"}}},{"name":"v2.6.22-rc3","target":{"target":{"committedDate":"2007-05-26T02:55:14Z"}}},{"name":"v2.6.22-rc4","target":{"target":{"committedDate":"2007-06-05T00:57:25Z"}}},{"name":"v2.6.22-rc5","target":{"target":{"committedDate":"2007-06-17T02:09:12Z"}}},{"name":"v2.6.22-rc6","target":{"target":{"committedDate":"2007-06-24T23:21:48Z"}}},{"name":"v2.6.22-rc7","target":{"target":{"committedDate":"2007-07-01T19:54:24Z"}}},{"name":"v2.6.22","target":{"target":{"committedDate":"2007-07-08T23:32:17Z"}}},{"name":"v2.6.23-rc1","target":{"target":{"committedDate":"2007-07-22T20:41:00Z"}}},{"name":"v2.6.23-rc2","target":{"target":{"committedDate":"2007-08-04T02:49:55Z"}}},{"name":"v2.6.23-rc3","target":{"target":{"committedDate":"2007-08-13T04:25:24Z"}}},{"name":"v2.6.23-rc4","target":{"target":{"committedDate":"2007-08-28T01:32:35Z"}}},{"name":"v2.6.23-rc5","target":{"target":{"committedDate":"2007-09-01T06:08:24Z"}}},{"name":"v2.6.23-rc6","target":{"target":{"committedDate":"2007-09-11T02:50:29Z"}}},{"name":"v2.6.23-rc7","target":{"target":{"committedDate":"2007-09-19T23:01:13Z"}}},{"name":"v2.6.23-rc8","target":{"target":{"committedDate":"2007-09-25T00:33:10Z"}}},{"name":"v2.6.23-rc9","target":{"target":{"committedDate":"2007-10-02T03:24:52Z"}}},{"name":"v2.6.23","target":{"target":{"committedDate":"2007-10-09T20:31:38Z"}}},{"name":"v2.6.24-rc1","target":{"target":{"committedDate":"2007-10-24T03:50:57Z"}}},{"name":"v2.6.24-rc2","target":{"target":{"committedDate":"2007-11-06T21:57:46Z"}}},{"name":"v2.6.24-rc3","target":{"target":{"committedDate":"2007-11-17T05:16:36Z"}}},{"name":"v2.6.24-rc4","target":{"target":{"committedDate":"2007-12-04T04:26:10Z"}}},{"name":"v2.6.24-rc5","target":{"target":{"committedDate":"2007-12-11T03:48:43Z"}}},{"name":"v2.6.24-rc6","target":{"target":{"committedDate":"2007-12-21T01:25:48Z"}}},{"name":"v2.6.24-rc7","target":{"target":{"committedDate":"2008-01-06T21:45:38Z"}}}]}}}}