DXVK Nightly: Simplify by Inheriting from DXVK Ctmod (#509)

* DXVK: Refactor to use __get_data method

Same idea to what we do in GE-Proton ctmod,
allows us to override the data and extract
path in child classes

* DXVK: Refactor extraction to `self.__extract` method

* DXVK: Type hinting

* DXVK Nightly: Simplify by Inheriting DXVK

* DXVK Nightly: Fix fetching Windows MSVC Builds

* DXVK Nightly: Refactor to use commit hash again instead of workflow ID

This allows us to display the commit hash in the releases dropdown
which helps with readibility, and it also allows us to preserve
the functionality of the `get_info_url` method which relies on
using the commit hash.

* DXVK Nightly: Only include workflow runs from `master`

* DXVK Nightly: Refactor extraction

The logic specifically for extracting DXVK Nightly (ZIP)
now lives inside of the DXVK Nightly ctmod.

The main DXVK Ctmod now only knows how to extract itself.

Author: sonic2kk

pupgui2/resources/ctmods/ctmod_z0dxvk.py

@@ -21,20 +21,20 @@
 
 class CtInstaller(QObject):
 
-    BUFFER_SIZE = 65536
-    CT_URL = 'https://api.github.com/repos/doitsujin/dxvk/releases'
-    CT_INFO_URL = 'https://github.com/doitsujin/dxvk/releases/tag/'
+    BUFFER_SIZE: int = 65536
+    CT_URL: str = 'https://api.github.com/repos/doitsujin/dxvk/releases'
+    CT_INFO_URL: str = 'https://github.com/doitsujin/dxvk/releases/tag/'
 
-    p_download_progress_percent = 0
-    download_progress_percent = Signal(int)
-    message_box_message = Signal((str, str, QMessageBox.Icon))
+    p_download_progress_percent: int = 0
+    download_progress_percent: Signal = Signal(int)
+    message_box_message: Signal = Signal((str, str, QMessageBox.Icon))
 
     def __init__(self, main_window = None):
         super(CtInstaller, self).__init__()
-        self.p_download_canceled = False
-        self.release_format = 'tar.gz'
+        self.p_download_canceled: bool = False
+        self.release_format: str = 'tar.gz'
 
-        self.rs = requests.Session()
+        self.rs: requests.Session = requests.Session()
         rs_headers = build_headers_with_authorization({}, main_window.web_access_tokens, 'github')
         self.rs.headers.update(rs_headers)
 
@@ -95,37 +95,66 @@ def is_system_compatible(self):
         """
         return True
 
-    def fetch_releases(self, count=100, page=1):
+    def fetch_releases(self, count: int = 100, page: int = 1):
         """
         List available releases
         Return Type: list[str]
         """
         return fetch_project_releases(self.CT_URL, self.rs, count=count, page=page)
 
-    def get_tool(self, version, install_dir, temp_dir):
+    def __get_data(self, version: str, install_dir: str) -> tuple[dict | None, str | None]:
+
         """
-        Download and install the compatibility tool
-        Return Type: bool
+        Get needed download data and path to extract directory.
+        Return Type: diple[dict | None, str | None]
         """
 
         data = self.__fetch_data(version)
         if not data or 'download' not in data:
+            return (None, None)
+
+        dxvk_dir = self.get_extract_dir(install_dir)
+
+        return (data, dxvk_dir)
+
+    def __extract(self, archive_path: str, extract_dir: str) -> bool:
+
+        """
+        Extract the tool archive at the given path.
+        Return Type: bool
+        """
+
+        if not archive_path or not extract_dir:
+            return False
+        
+        # DXVK and DXVK Async are 'tar.gz'
+        tar_type = self.release_format.split('.')[-1]
+
+        return extract_tar(archive_path, extract_dir, mode=tar_type)
+
+    def get_tool(self, version: str, install_dir: str, temp_dir: str) -> bool:
+        """
+        Download and install the compatibility tool
+        Return Type: bool
+        """
+
+        data, dxvk_dir = self.__get_data(version, install_dir)
+        if not data:
             return False
 
         # Should be updated to support Heroic, like ctmod_d8vk
-        dxvk_tar = os.path.join(temp_dir, data['download'].split('/')[-1])
-        if not self.__download(url=data['download'], destination=dxvk_tar, known_size=data.get('size', 0)):
+        dxvk_archive: str = os.path.join(temp_dir, data['download'].split('/')[-1])
+        if not self.__download(url=data['download'], destination=dxvk_archive, known_size=data.get('size', 0)):
             return False
 
-        dxvk_dir = self.get_extract_dir(install_dir)
-        if not extract_tar(dxvk_tar, dxvk_dir, mode='gz'):
+        if not dxvk_dir or not self.__extract(dxvk_archive, dxvk_dir):
             return False
 
         self.__set_download_progress_percent(100)
 
         return True
 
-    def get_info_url(self, version):
+    def get_info_url(self, version: str) -> str:
         """
         Get link with info about version (eg. GitHub release page)
         Return Type: str

pupgui2/resources/ctmods/ctmod_z2dxvknightly.py

@@ -3,99 +3,115 @@
 # Copyright (C) 2022 DavidoTek, partially based on AUNaseef's protonup
 
 import os
-import requests
 
-from PySide6.QtCore import QObject, QCoreApplication, Signal, Property
+from PySide6.QtCore import QCoreApplication
 
-from pupgui2.util import ghapi_rlcheck, extract_zip
-from pupgui2.util import build_headers_with_authorization
+from pupgui2.util import extract_zip, ghapi_rlcheck
+
+from pupgui2.resources.ctmods.ctmod_z0dxvk import CtInstaller as DXVKInstaller
 
 
 CT_NAME = 'DXVK (nightly)'
 CT_LAUNCHERS = ['lutris', 'advmode']
 CT_DESCRIPTION = {'en': QCoreApplication.instance().translate('ctmod_z2dxvknightly', '''Nightly version of DXVK (master branch), a Vulkan based implementation of Direct3D 8, 9, 10 and 11 for Linux/Wine.<br/><br/><b>Warning: Nightly version is unstable, use with caution!</b>''')}
 
 
-class CtInstaller(QObject):
+class CtInstaller(DXVKInstaller):
 
-    BUFFER_SIZE = 65536
-    CT_URL = 'https://api.github.com/repos/doitsujin/dxvk/actions/artifacts'
-    CT_INFO_URL = 'https://github.com/doitsujin/dxvk/commit/'
+    BUFFER_SIZE: int = 65536
+    CT_WORKFLOW_URL: str = 'https://api.github.com/repos/doitsujin/dxvk/actions/workflows'
+    CT_ARTIFACT_URL: str = 'https://api.github.com/repos/doitsujin/dxvk/actions/runs/{}/artifacts'
+    CT_ALL_ARTIFACTS_URL: str = 'https://api.github.com/repos/doitsujin/dxvk/actions/artifacts'
+    CT_INFO_URL: str = 'https://github.com/doitsujin/dxvk/commit/'
 
-    p_download_progress_percent = 0
-    download_progress_percent = Signal(int)
+    DXVK_WORKFLOW_NAME: str = 'artifacts'
 
     def __init__(self, main_window = None):
-        super(CtInstaller, self).__init__()
-        self.p_download_canceled = False
 
-        self.rs = requests.Session()
-        rs_headers = build_headers_with_authorization({}, main_window.web_access_tokens, 'github')
-        self.rs.headers.update(rs_headers)
+        super().__init__(main_window)
+
+        self.release_format: str = 'zip'
 
-    def get_download_canceled(self):
-        return self.p_download_canceled
+    def __fetch_workflows(self, count: int = 30) -> list[str]:
 
-    def set_download_canceled(self, val):
-        self.p_download_canceled = val
+        """
+        Get all active, successful runs in the DXVK Linux-compatible workflow.
+        Return Type: list
+        """
 
-    download_canceled = Property(bool, get_download_canceled, set_download_canceled)
+        workflow_request_url: str = f'{self.CT_WORKFLOW_URL}?per_page={str(count)}'
+        workflow_response_json: dict = self.rs.get(workflow_request_url).json()
 
-    def __set_download_progress_percent(self, value : int):
-        if self.p_download_progress_percent == value:
-            return
-        self.p_download_progress_percent = value
-        self.download_progress_percent.emit(value)
+        tags: list[str] = []
+        for workflow in workflow_response_json.get('workflows', {}):
+            if workflow['state'] != "active" or self.DXVK_WORKFLOW_NAME not in workflow['path']:
+                continue
+
+            page = 1
+            while page != -1 and page <= 5:  # fetch more (up to 5 pages) if first releases all failed
+                at_least_one_failed = False  # ensure the reason that len(tags)=0 is that releases failed
+
+                workflow_runs_request_url: str = f'{workflow["url"]}/runs?per_page={count}&page={page}'
+                workflow_runs_response_json: dict = self.rs.get(workflow_runs_request_url).json()
+
+                for run in workflow_runs_response_json.get('workflow_runs', {}):
+                    if run['head_branch'] != 'master':
+                        continue
+
+                    if run['conclusion'] == "failure":
+                        at_least_one_failed = True
+
+                        continue
+
+                    # TODO can make this generic so that i.e. this DXVK Ctmod can use commmit SHAs but Proton-tkg can use workflow IDs?
+                    #      then this could be a generic function shared between ctmods and could be in a util file, unit tested, etc
+                    commit_hash: str = str(run['head_commit']['id'][:7])
+                    tags.append(commit_hash)
+
+                if len(tags) == 0 and at_least_one_failed:
+                    page += 1
+
+                    continue
+
+                page = -1
+
+        return tags
+
+    def fetch_releases(self, count: int = 30, page: int = 1) -> list[str]:
 
-    def __download(self, url, destination, f_size):
-        # f_size in argumentbecause artifacts don't have Content-Length.
         """
-        Download files from url to destination
-        Return Type: bool
+        List available releases.
+        Return Type: str[]
         """
-        try:
-            file = requests.get(url, stream=True)
-        except OSError:
-            return False
 
-        self.__set_download_progress_percent(1) # 1 download started
-        c_count = int(f_size / self.BUFFER_SIZE)
-        c_current = 1
-        destination = os.path.expanduser(destination)
-        os.makedirs(os.path.dirname(destination), exist_ok=True)
-        with open(destination, 'wb') as dest:
-            for chunk in file.iter_content(chunk_size=self.BUFFER_SIZE):
-                if self.download_canceled:
-                    self.download_canceled = False
-                    self.__set_download_progress_percent(-2) # -2 download canceled
-                    return False
-                if chunk:
-                    dest.write(chunk)
-                    dest.flush()
-                self.__set_download_progress_percent(int(min(c_current / c_count * 98.0, 98.0))) # 1-98, 100 after extract
-                c_current += 1
-        self.__set_download_progress_percent(99) # 99 download complete
-        return True
+        return self.__fetch_workflows(count=count)
 
     def __get_artifact_from_commit(self, commit):
+
         """
         Get artifact from commit
         Return Type: str
         """
-        for artifact in self.rs.get(f'{self.CT_URL}?per_page=100').json()["artifacts"]:
-            if artifact['workflow_run']['head_sha'][:len(commit)] == commit:
+
+        for artifact in self.rs.get(f'{self.CT_ALL_ARTIFACTS_URL}?per_page=100').json()["artifacts"]:
+            # DXVK appends '-msvc-output' to Windows builds
+            # See: https://github.com/doitsujin/dxvk/blob/20a6fae8a7f60e7719724b229552eba1ae6c3427/.github/workflows/test-build-windows.yml#L80
+            if artifact['workflow_run']['head_sha'][:len(commit)] == commit and not artifact['name'].endswith('-msvc-output'):
                 artifact['workflow_run']['head_sha'] = commit
                 return artifact
+        
         return None
 
-    def __fetch_github_data(self, tag):
+    def __fetch_github_data(self, tag: str):
+
         """
         Fetch GitHub release information
         Return Type: dict
         Content(s):
-            'version', 'date', 'download', 'size', 'checksum'
+            'version', 'date', 'download', 'size'
         """
-        # Tag in this case is the commit hash.
+
+        # Tag in this case is the commit hash
         data = self.__get_artifact_from_commit(tag)
         if not data:
             return
@@ -105,50 +121,39 @@ def __fetch_github_data(self, tag):
         values['size'] = data['size_in_bytes']
         return values
 
-    def is_system_compatible(self):
-        """
-        Are the system requirements met?
-        Return Type: bool
-        """
-        return True
+    def __get_data(self, version: str, install_dir: str) -> tuple[dict | None, str | None]:
 
-    def fetch_releases(self, count=100, page=1):
         """
-        List available releases
-        Return Type: str[]
+        Get needed download data and path to extract directory.
+        Return Type: diple[dict | None, str | None]
         """
-        tags = []
-        for artifact in ghapi_rlcheck(self.rs.get(f'{self.CT_URL}?per_page={count}&page={page}').json()).get("artifacts", {}):
-            workflow = artifact['workflow_run']
-            if workflow["head_branch"] != "master" or artifact["expired"]:
-                continue
-            tags.append(workflow['head_sha'][:7])
-        return tags
 
-    def get_tool(self, version, install_dir, temp_dir):
+        data = self.__fetch_github_data(version)
+        if not data or not 'download' in data:
+            return (None, None)
+
+        # TODO This is hardcoded to Lutris as DXVK Nightly currently doesn't support any other launchers -- Could possibly add support for Heroic in future
+        dxvk_dir = os.path.join(install_dir, '../../runtime/dxvk', 'dxvk-git-' + data['version'])
+
+        return (data, dxvk_dir)
+
+    def __extract(self, archive_path: str, extract_dir: str) -> bool:
+
         """
-    Download and install the compatibility tool
+        Extract the tool archive at the given path.
         Return Type: bool
         """
-        data = self.__fetch_github_data(version)
-        if not data or 'download' not in data:
-            return False
 
-        dxvk_zip = os.path.join(temp_dir, data['download'].split('/')[-1])
-        if not self.__download(url=data['download'], destination=dxvk_zip, f_size=data['size']):
+        if not archive_path or not extract_dir:
             return False
 
-        dxvk_dir = os.path.join(install_dir, '../../runtime/dxvk', 'dxvk-git-' + data['version'])
-        if not extract_zip(dxvk_zip, dxvk_dir):
-            return False
+        return extract_zip(archive_path, extract_dir)
 
-        self.__set_download_progress_percent(100)
+    def get_info_url(self, version: str) -> str:
 
-        return True
-
-    def get_info_url(self, version):
         """
         Get link with info about version (eg. GitHub release page)
         Return Type: str
         """
+
         return self.CT_INFO_URL + version