Prepare new version to fix duplicate issues for yanked crates

Issues for yanked crates start with "Crate ... v..." not with
"RUSTSEC-". So the filter on the existing issues list was too strict,
not detecting the issue for yanked crates. This causes duplicate issues.

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 22.12.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -14,13 +14,13 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: v5.11.3
     # https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md
     hooks:
       - id: isort
         args: ["--profile=black"]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.0
+    rev: v3.3.1
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.5] - 2022-12-22
+
+* Fix duplicate issues for yanked crates.
+
+    The previous version introduced a bug where existing issues were not properly detected.
+    This only affected issues for yanked crates.
+    Now duplicate issues will no longer be created.
+
+## [1.1.4] - 2022-12-22
+
+* Handle warnings without any associated advisory.
+
+    This occurs for yanked crates, where the `advisory` field is `null` in the JSON output.
+    Now a message is shown that the crate and version is yanked.
+
 ## [1.1.3] - 2022-12-05
 
 * Fix the path to the cargo installation directory to fix caching.

audit.py

@@ -11,6 +11,15 @@ import requests
 # https://github.com/alstr/todo-to-issue-action/blob/25c80e9c4999d107bec208af49974d329da26370/main.py
 # Originally licensed under MIT license
 
+# Timeout in seconds for requests methods
+TIMEOUT = 30
+
+
+def debug(message: str) -> None:
+    """Print a debug message to the GitHub Action log"""
+    newline = "\n"
+    print(f"""::debug::{message.replace(newline, " ")}""")
+
 
 class Issue:
     """Basic Issue model for collecting the necessary info to send to GitHub."""
@@ -21,7 +30,7 @@ class Issue:
         labels: List[str],
         assignees: List[str],
         body: str,
-        rustsec_id: str,
+        rustsec_id: str,  # Should be the start of the title
     ) -> None:
         self.title = title
         self.labels = labels
@@ -59,62 +68,85 @@ class Entry:
         self.entry_type = entry_type
         self.warning_type = warning_type
 
+    def id(self) -> str:
+        """
+        Return the ID of the entry.
+        """
+
+        # IMPORTANT: Coordinate this value with the `_get_existing_issues` method below.
+        # Any value returned here must also be present in the filtering there, since the id will be used in the issue title.
+
+        advisory = self.entry.get("advisory", None)
+        if advisory:
+            return advisory["id"]
+        else:
+            return f"Crate {self.entry['package']['name']} {self.entry['package']['version']}"
+
     def _entry_table(self) -> str:
-        advisory = self.entry["advisory"]
+        advisory = self.entry.get("advisory", None)
 
-        table = []
-        table.append(("Details", ""))
-        table.append(("---", "---"))
-        table.append(("Package", f"`{advisory['package']}`"))
-        table.append(("Version", f"`{self.entry['package']['version']}`"))
-        if self.warning_type is not None:
-            table.append(("Warning", str(self.warning_type)))
-        table.append(("URL", advisory["url"]))
-        table.append(
-            (
-                "Patched Versions",
-                " OR ".join(self.entry["versions"]["patched"])
-                if len(self.entry["versions"]["patched"]) > 0
-                else "n/a",
-            )
-        )
-        if len(self.entry["versions"]["unaffected"]) > 0:
+        if advisory:
+            table = []
+            table.append(("Details", ""))
+            table.append(("---", "---"))
+            table.append(("Package", f"`{advisory['package']}`"))
+            table.append(("Version", f"`{self.entry['package']['version']}`"))
+            if self.warning_type is not None:
+                table.append(("Warning", str(self.warning_type)))
+            table.append(("URL", advisory["url"]))
             table.append(
                 (
-                    "Unaffected Versions",
-                    " OR ".join(self.entry["versions"]["unaffected"]),
+                    "Patched Versions",
+                    " OR ".join(self.entry["versions"]["patched"])
+                    if len(self.entry["versions"]["patched"]) > 0
+                    else "n/a",
                 )
             )
-        if len(advisory["aliases"]) > 0:
-            table.append(
-                (
-                    "Aliases",
-                    ", ".join(
-                        Entry._md_autolink_advisory_id(advisory_id)
-                        for advisory_id in advisory["aliases"]
-                    ),
+            if len(self.entry["versions"]["unaffected"]) > 0:
+                table.append(
+                    (
+                        "Unaffected Versions",
+                        " OR ".join(self.entry["versions"]["unaffected"]),
+                    )
                 )
-            )
-        if len(advisory["related"]) > 0:
-            table.append(
-                (
-                    "Related Advisories",
-                    ", ".join(
-                        Entry._md_autolink_advisory_id(advisory_id)
-                        for advisory_id in advisory["related"]
-                    ),
+            if len(advisory["aliases"]) > 0:
+                table.append(
+                    (
+                        "Aliases",
+                        ", ".join(
+                            Entry._md_autolink_advisory_id(advisory_id)
+                            for advisory_id in advisory["aliases"]
+                        ),
+                    )
+                )
+            if len(advisory["related"]) > 0:
+                table.append(
+                    (
+                        "Related Advisories",
+                        ", ".join(
+                            Entry._md_autolink_advisory_id(advisory_id)
+                            for advisory_id in advisory["related"]
+                        ),
+                    )
                 )
-            )
 
-        table_parts = []
-        for row in table:
-            table_parts.append("| ")
-            table_parts.append(row[0])
-            table_parts.append(" | ")
-            table_parts.append(row[1])
-            table_parts.append(" |\n")
+            table_parts = []
+            for row in table:
+                table_parts.append("| ")
+                table_parts.append(row[0])
+                table_parts.append(" | ")
+                table_parts.append(row[1])
+                table_parts.append(" |\n")
 
-        return "".join(table_parts)
+            return "".join(table_parts)
+        else:
+            # There is no advisory.
+            # This occurs when a yanked version is detected.
+
+            name = self.entry["package"]["name"]
+            return f"""{self.id()} is yanked.
+Switch to a different version of `{name}` to resolve this issue.
+"""
 
     @classmethod
     def _md_autolink_advisory_id(cls, advisory_id: str) -> str:
@@ -132,37 +164,64 @@ class Entry:
         return advisory_id
 
     def format_as_markdown(self) -> str:
-        advisory = self.entry["advisory"]
+        advisory = self.entry.get("advisory", None)
 
-        entry_table = self._entry_table()
-        # Replace the @ with a ZWJ to avoid triggering markdown autolinks
-        # Otherwise GitHub will interpret the @ as a mention
-        description = advisory["description"].replace("@", "@\u200d")
+        if advisory:
+            entry_table = self._entry_table()
+            # Replace the @ with a ZWJ to avoid triggering markdown autolinks
+            # Otherwise GitHub will interpret the @ as a mention
+            description = advisory["description"].replace("@", "@\u200d")
 
-        md = f"""## {self.entry_type.icon()} {advisory['id']}: {advisory['title']}
+            md = f"""## {self.entry_type.icon()} {advisory['id']}: {advisory['title']}
 
 {entry_table}
 
 {description}
 """
-        return md
+            return md
+        else:
+            # There is no advisory.
+            # This occurs when a yanked version is detected.
+
+            name = self.entry["package"]["name"]
+            return f"""## {self.entry_type.icon()} {self.id()} is yanked.
+
+Switch to a different version of `{name}` to resolve this issue.
+"""
 
     def format_as_issue(self, labels: List[str], assignees: List[str]) -> Issue:
-        advisory = self.entry["advisory"]
+        advisory = self.entry.get("advisory", None)
 
-        entry_table = self._entry_table()
+        if advisory:
+            entry_table = self._entry_table()
 
-        title = f"{advisory['id']}: {advisory['title']}"
-        body = f"""{entry_table}
+            title = f"{self.id()}: {advisory['title']}"
+            body = f"""{entry_table}
 
 {advisory['description']}"""
-        return Issue(
-            title=title,
-            labels=labels,
-            assignees=assignees,
-            body=body,
-            rustsec_id=advisory["id"],
-        )
+            return Issue(
+                title=title,
+                labels=labels,
+                assignees=assignees,
+                body=body,
+                rustsec_id=self.id(),
+            )
+        else:
+            # There is no advisory.
+            # This occurs when a yanked version is detected.
+
+            name = self.entry["package"]["name"]
+            title = f"{self.id()} is yanked"
+            body = (
+                f"""Switch to a different version of `{name}` to resolve this issue."""
+            )
+            return Issue(
+                title=title,
+                labels=labels,
+                assignees=assignees,
+                body=body,
+                rustsec_id=self.id(),
+            )
 
 
 class GitHubClient:
@@ -183,6 +242,10 @@ class GitHubClient:
         # Retrieve the existing repo issues now so we can easily check them later.
         self._get_existing_issues()
 
+        debug("Existing issues:")
+        for issue in self.existing_issues:
+            debug(f"* {issue['title']}")
+
     def _get_existing_issues(self, page: int = 1) -> None:
         """Populate the existing issues list."""
         params: Dict[str, Union[str, int]] = {
@@ -190,8 +253,9 @@ class GitHubClient:
             "page": page,
             "state": "open",
         }
+        debug(f"Fetching existing issues from GitHub: {page=}")
         list_issues_request = requests.get(
-            self.issues_url, headers=self.issue_headers, params=params
+            self.issues_url, headers=self.issue_headers, params=params, timeout=TIMEOUT
         )
         if list_issues_request.status_code == 200:
             self.existing_issues.extend(
@@ -199,6 +263,7 @@ class GitHubClient:
                     issue
                     for issue in list_issues_request.json()
                     if issue["title"].startswith("RUSTSEC-")
+                    or issue["title"].startswith("Crate ")
                 ]
             )
             links = list_issues_request.links
@@ -208,6 +273,7 @@ class GitHubClient:
     def create_issue(self, issue: Issue) -> Optional[int]:
         """Create a dict containing the issue details and send it to GitHub."""
         title = issue.title
+        debug(f"Creating issue: {title=}")
 
         # Check if the current issue already exists - if so, skip it.
         # The below is a simple and imperfect check based on the issue title.
@@ -226,9 +292,14 @@ class GitHubClient:
                         existing_issue["url"],
                         headers=self.issue_headers,
                         data=json.dumps(body),
+                        timeout=TIMEOUT,
                     )
                     return update_request.status_code
 
+        debug(
+            f"""No existing issue found for "{issue.rustsec_id}". Creating new issue."""
+        )
+
         new_issue_body = {"title": title, "body": issue.body, "labels": issue.labels}
 
         # We need to check if any assignees/milestone specified exist, otherwise issue creation will fail.
@@ -236,7 +307,9 @@ class GitHubClient:
         for assignee in issue.assignees:
             assignee_url = f"{self.repos_url}{self.repo}/assignees/{assignee}"
             assignee_request = requests.get(
-                url=assignee_url, headers=self.issue_headers
+                url=assignee_url,
+                headers=self.issue_headers,
+                timeout=TIMEOUT,
             )
             if assignee_request.status_code == 204:
                 valid_assignees.append(assignee)
@@ -248,6 +321,7 @@ class GitHubClient:
             url=self.issues_url,
             headers=self.issue_headers,
             data=json.dumps(new_issue_body),
+            timeout=TIMEOUT,
         )
 
         return new_issue_request.status_code
@@ -255,7 +329,10 @@ class GitHubClient:
     def close_issue(self, issue: Dict[str, Any]) -> int:
         body = {"state": "closed"}
         close_request = requests.patch(
-            issue["url"], headers=self.issue_headers, data=json.dumps(body)
+            issue["url"],
+            headers=self.issue_headers,
+            data=json.dumps(body),
+            timeout=TIMEOUT,
         )
         return close_request.status_code
 
@@ -317,12 +394,15 @@ def run() -> None:
         extra_args.append("--deny")
         extra_args.append("warnings")
 
+    audit_cmd = ["cargo", "audit", "--json"] + extra_args + ignore_args
+    debug(f"Running command: {audit_cmd}")
     completed = subprocess.run(
-        ["cargo", "audit", "--json"] + extra_args + ignore_args,
+        audit_cmd,
         capture_output=True,
         text=True,
         check=False,
     )
+    debug(f"Command output: {completed.stdout}")
     data = json.loads(completed.stdout)
 
     summary = create_summary(data)
@@ -356,7 +436,7 @@ def run() -> None:
         num_existing_issues = len(gh_client.existing_issues)
         for entry in entries:
             for ex_issue in gh_client.existing_issues:
-                if ex_issue["title"].startswith(entry.entry["advisory"]["id"]):
+                if ex_issue["title"].startswith(entry.id()):
                     gh_client.existing_issues.remove(ex_issue)
         num_old_issues = len(gh_client.existing_issues)
         print(