Merge pull request #33 from matyalatte/dev

v0.6.0 update

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on:
     tags:
       - "v*"
   workflow_dispatch:
-  
+
 env:
   ZIP_NAME: UE4-DDS-Tools
   GUI_VERSION: v0.6.3
@@ -14,7 +14,7 @@ jobs:
   build:
     runs-on: windows-2022
     steps:
-    
+
       - name: Check tag
         run: |
           if [[ ${{ github.ref }} == refs/tags/v* ]]; then
@@ -24,10 +24,10 @@ jobs:
           fi
           echo "TAG=$TAG" >> "$GITHUB_ENV"
         shell: bash
-        
-      - uses: actions/checkout@v3
-      - run: git submodule update --init --recursive external/Texconv-Custom-DLL
-      
+
+      - uses: actions/checkout@v4
+      - run: git submodule update --init --recursive
+
       - name: Download embeddable python
         run: |
           mkdir -p release
@@ -37,7 +37,7 @@ jobs:
 
       - name: build dll
         run: |
-          external/Texconv-Custom-DLL/batch_files/build_wic_support.bat
+          external/Texconv-Custom-DLL/batch_files/build_without_vcruntime.bat
           mkdir -p release/src/directx
           cp external/Texconv-Custom-DLL/texconv.dll release/src/directx
 
@@ -51,13 +51,24 @@ jobs:
           cp bat/* release
         shell: bash
 
-      - name: Download Tuw
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install Meson
+        run: pip install meson
+
+      - name: Prepare MSVC for Windows
+        if: runner.os == 'Windows'
+        uses: bus1/cabuild/action/msdevshell@v1
+        with:
+          architecture: x64
+
+      - name: Build Tuw
         run: |
-          curl -OL https://github.com/matyalatte/tuw/releases/download/${{env.GUI_VERSION}}/Tuw-${{env.GUI_VERSION}}-${{ runner.os }}-x64.zip
-          unzip -d Tuw Tuw-${{env.GUI_VERSION}}-${{ runner.os }}-x64.zip
-          cp Tuw/Tuw.exe release_gui/GUI.exe
+          external/tuw/batch_files/build_with_ucrt.bat
+          cp external/tuw/build/ReleaseUCRT/Tuw.exe release_gui/GUI.exe
           cp gui_definition.json release_gui
-        shell: bash
 
       - name: Archive Release
         uses: thedoctor0/zip-release@master
@@ -77,7 +88,7 @@ jobs:
 
       - name: Create Release Draft
         id: create-release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           tag_name: ${{ env.TAG }}
           name: ${{ env.TAG }}

.github/workflows/download_portable_python.bat

@@ -25,13 +25,19 @@ del pyexpat.pyd unicodedata.pyd winsound.pyd
 REM Remove unnecessary files from pythonXXX.zip
 powershell Expand-Archive -Force -Path python%PYTHON_VER_SHORT%.zip
 cd python%PYTHON_VER_SHORT%
-rmdir /s /q curses dbm distutils email html http lib2to3 msilib
+rmdir /s /q asyncio curses dbm distutils email html http lib2to3 msilib
 rmdir /s /q pydoc_data site-packages sqlite3 tomllib unittest urllib
 rmdir /s /q wsgiref xml xmlrpc zoneinfo
-del ast.pyc bz2.pyc calendar.pyc csv.pyc doctest.pyc ftplib.pyc gzip.pyc
-del imaplib.pyc ipaddress.pyc mailbox.pyc nntplib.pyc nturl2path.pyc
-del optparse.pyc pdb.pyc pickletools.pyc pydoc.pyc
-del smtpd.pyc smtplib.pyc ssl.pyc tarfile.pyc
+del _aix_support.pyc _compression.pyc _markupbase.pyc _osx_support.pyc _pydecimal.pyc _strptime.pyc
+del aifc.pyc ast.pyc asynchat.pyc asyncore.pyc base64.pyc bz2.pyc
+del calendar.pyc cgi.pyc cgitb.pyc configparser.pyc cProfile.pyc csv.pyc
+del datetime.pyc difflib.pyc dis.pyc doctest.pyc fileinput.pyc fractions.pyc ftplib.pyc gzip.pyc
+del getopt.pyc graphlib.pyc hashlib.pyc hmac.pyc imaplib.pyc imghdr.pyc ipaddress.pyc lzma.pyc
+del mailbox.pyc mailcap.pyc netrc.pyc nntplib.pyc nturl2path.pyc
+del opcode.pyc optparse.pyc pdb.pyc pickletools.pyc plistlib.pyc poplib.pyc pprint.pyc profile.pyc pstats.pyc pydoc.pyc quopri.pyc
+del rlcompleter.pyc sched.pyc shelve.pyc shlex.pyc smtpd.pyc smtplib.pyc sndhdr.pyc socketserver.pyc ssl.pyc statistics.pyc sunau.pyc symtable.pyc
+del tarfile.pyc telnetlib.pyc timeit.pyc tty.pyc
+del uuid.pyc wave.pyc webbrowser.pyc xdrlib.pyc zipapp.pyc zipfile.pyc zipimport.pyc
 powershell Compress-Archive -Force -Path * -Destination ../python%PYTHON_VER_SHORT%.zip
 cd ..
 rmdir /s /q python%PYTHON_VER_SHORT%

.github/workflows/test.yml

@@ -1,11 +1,11 @@
 name: test
 
 on:
-  push:
-    tags:
-      - "v*"
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
-  
+
 env:
   PYTHON_VERSION: 3.10.11
 
@@ -14,25 +14,27 @@ jobs:
     runs-on: windows-2022
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: build dll
         run: |
-          external/Texconv-Custom-DLL/batch_files/build_wic_support.bat
+          external/Texconv-Custom-DLL/batch_files/build_without_vcruntime.bat
           cp external/Texconv-Custom-DLL/texconv.dll src/directx
 
       - name: Set up Python v${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install pytest
-        run: pip install flake8 pytest pytest-cov
+        run: pip install flake8 codespell pytest pytest-cov
 
       - name: Linting
-        run: flake8 --max-line-length 119
+        run: |
+          flake8 --max-line-length 119 --exclude ".git,.pytest_cache,external,htmlcov,python,tmp"
+          codespell -L "splitted,ue" -S ".git,.pytest_cache,external,htmlcov,python,tmp"
 
       - name: Test
         env:

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "external/Matya-Uasset-Samples"]
 	path = external/Matya-Uasset-Samples
 	url = https://github.com/matyalatte/Matya-Uasset-Samples.git
+[submodule "external/tuw"]
+	path = external/tuw
+	url = https://github.com/matyalatte/tuw.git

docs/README.md

@@ -2,7 +2,7 @@
 ![test](https://github.com/matyalatte/UE4-DDS-tools/actions/workflows/test.yml/badge.svg)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-# UE4-DDS-Tools ver0.5.6
+# UE4-DDS-Tools ver0.6.0
 
 Texture modding tools for UE games.  
 You can inject texture files (.dds, .tga, .hdr, etc.) into UE assets.  
@@ -16,7 +16,7 @@ You can inject texture files (.dds, .tga, .hdr, etc.) into UE assets.
 
 ## Supported UE versions
 
-- UE5.0 ~ 5.3
+- UE5.0 ~ 5.4
 - UE4.0 ~ 4.27
 - FF7R
 - Borderlands3
@@ -97,6 +97,14 @@ Download one of the zip files from [here](https://github.com/matyalatte/UE4-DDS-
 - `UE4-DDS-tools-*-Batch.zip` is for batch file users.
 - `UE4-DDS-tools-*-GUI.zip` is for GUI users.
 
+> `GUI.exe` only works on Windows10 or later.  
+> If you use windows 8 or older, you can copy an executable from [Tuw-*-Windows-x64.zip](https://github.com/matyalatte/tuw/releases).
+
+> Released packages only work on Windows.
+> See here if you use macOS or Linux.  
+> [For Unix Systems · matyalatte/UE4-DDS-Tools Wiki](https://github.com/matyalatte/UE4-DDS-Tools/wiki/For-Unix-Systems)
+
+
 ## Getting Started
 
 - [For GUI users](https://github.com/matyalatte/UE4-DDS-Tools/wiki/How-to-Use)

docs/changelog.txt

@@ -1,4 +1,11 @@
+ver 0.6.0
+- Supported ucas/utoc assets.
+- Added support for UE5.4.
+- Updated DirectXTex to the March 2024 release.
+- Updated GUI to v0.6.4
 - Fixed typos.
+- Removed unused python libraries.
+- Fixed an error when reading non-texture assets for UE5.2 or later.
 
 ver 0.5.6
 - Added support for ETC formats. (ETC1, ETC2_RGB, and ETC2_RGBA)

gui_definition.json

@@ -1,5 +1,5 @@
 {
-    "recommended": "0.6.3",
+    "recommended": "0.6.4",
     "minimum_required": "0.5.0",
     "gui": [
         {
@@ -35,6 +35,7 @@
                     "type": "choice",
                     "label": "UE version",
                     "items": [
+                        { "label": "5.4" },
                         { "label": "5.3" },
                         { "label": "5.2" },
                         { "label": "5.1" },
@@ -139,6 +140,7 @@
                     "type": "choice",
                     "label": "UE version",
                     "items": [
+                        { "label": "5.4" },
                         { "label": "5.3" },
                         { "label": "5.2" },
                         { "label": "5.1" },

src/config.json

@@ -1,4 +1,4 @@
 {
-    "$desc": "UE version. 5.3 ~ 4.0, ff7r, and borderlands3 are available.",
+    "$desc": "UE version. 5.4 ~ 4.0, ff7r, and borderlands3 are available.",
     "version": "4.26"
 }

src/directx/texconv.py

@@ -193,8 +193,7 @@ def __texconv(self, file: str, args: list[str],
         if out not in [".", ""] and not os.path.exists(out):
             mkdir(out)
 
-        args += ["-y"]
-        args += [os.path.normpath(file)]
+        args += ["-y", "--", os.path.normpath(file)]
 
         args_p = [ctypes.c_wchar_p(arg) for arg in args]
         args_p = (ctypes.c_wchar_p*len(args_p))(*args_p)
@@ -227,7 +226,7 @@ def __texassemble(self, file: str, new_file: str, args: list[str], verbose=True)
         out = os.path.dirname(new_file)
         if out not in [".", ""] and not os.path.exists(out):
             mkdir(out)
-        args += ["-y", "-o", new_file, file]
+        args += ["-y", "-o", new_file, "--", file]
 
         args_p = [ctypes.c_wchar_p(arg) for arg in args]
         args_p = (ctypes.c_wchar_p*len(args_p))(*args_p)

src/main.py

@@ -10,16 +10,17 @@
 
 # my scripts
 from util import (compare, get_ext, get_temp_dir,
-                  get_file_list, get_base_folder, remove_quotes)
+                  get_file_list, get_base_folder, remove_quotes,
+                  check_python_version)
 from unreal.uasset import Uasset, UASSET_EXT
 from directx.dds import DDS
 from directx.dxgi_format import DXGI_FORMAT
 from directx.texconv import Texconv, is_windows
 
-TOOL_VERSION = "0.5.6"
+TOOL_VERSION = "0.6.0"
 
-# UE version: 4.0 ~ 5.3, ff7r, borderlands3
-UE_VERSIONS = ["4." + str(i) for i in range(28)] + ["5." + str(i) for i in range(4)] + ["ff7r", "borderlands3"]
+# UE version: 4.0 ~ 5.4, ff7r, borderlands3
+UE_VERSIONS = ["4." + str(i) for i in range(28)] + ["5." + str(i) for i in range(5)] + ["ff7r", "borderlands3"]
 
 # Supported file extensions.
 TEXTURES = ["dds", "tga", "hdr"]
@@ -346,7 +347,7 @@ def copy(folder, file, args, texture_file=None):
 
 # UE version for textures
 UTEX_VERSIONS = [
-    "5.3", "5.2", "5.1", "5.0",
+    "5.4", "5.3", "5.2", "5.1", "5.0",
     "4.26 ~ 4.27", "4.24 ~ 4.25", "4.23", "4.20 ~ 4.22",
     "4.16 ~ 4.19", "4.15", "4.14", "4.12 ~ 4.13", "4.11", "4.10",
     "4.9", "4.8", "4.7", "4.4 ~ 4.6", "4.3", "4.0 ~ 4.2",
@@ -575,6 +576,9 @@ def main(args, config={}):
 
     print(f"UE4 DDS Tools ver{TOOL_VERSION} by Matyalatte")
 
+    # ensure that the python version is 3.10 or later
+    check_python_version(3, 10)
+
     args = get_args()
     config = get_config()
     main(args, config=config)

src/unreal/archive.py

@@ -22,6 +22,7 @@ class ArchiveBase:
     io: IOBase
     is_reading = False
     is_writing = False
+    is_ucas = False
 
     def __init__(self, io: IOBase, endian="little", context: dict = {}):
         self.io = io
@@ -65,13 +66,39 @@ def write(self, obj):
     def close(self):
         self.io.close()
 
+    def align(self, base: int):
+        pad = (base - self.tell() % base) % base
+        self == (Bytes, b"\x00" * pad, "pad", pad)
+
+    def raise_error(self, msg="Parse failed. Make sure you specified UE version correctly."):
+        if (hasattr(self, "uasset")):
+            msg += " (" + self.uasset.file_name + ")"
+        raise RuntimeError(msg)
+
     def check(self, actual, expected, msg="Parse failed. Make sure you specified UE version correctly."):
         if actual == expected:
             return
         print(f"offset: {self.tell()}")
         print(f"actual: {actual}")
         print(f"expected: {expected}")
-        raise RuntimeError(msg)
+        self.raise_error(msg)
+
+    def check_buffer_size(self, size):
+        if self.tell() + size > self.size:
+            raise RuntimeError(
+                "There is no buffer that has specified size."
+                f" (Offset: {self.tell()}, Size: {size})"
+            )
+
+    def update_with_current_offset(self, obj, attr_name, base=0):
+        if self.is_reading:
+            # Checks obj.attr_name is the same as the current offset
+            current_offs = self.tell() - base
+            serialized_offs = getattr(obj, attr_name)
+            self.check(serialized_offs, current_offs)
+        else:
+            # Update obj.attr_name with the current offset
+            setattr(obj, attr_name, self.tell() - base)
 
 
 class ArchiveRead(ArchiveBase):
@@ -121,11 +148,7 @@ class Buffer(Bytes):
     @staticmethod
     def read(ar: ArchiveBase) -> bytes:
         size = ar.args[0]
-        if ar.tell() + size > ar.size:
-            raise RuntimeError(
-                "There is no buffer that has specified size."
-                f" (Offset: {ar.tell()}, Size: {size})"
-            )
+        ar.check_buffer_size(size)
         return ar.read(size)
 
 
@@ -234,6 +257,26 @@ def write(ar: ArchiveBase, val: str):
         ar.write(str_byte + b"\x00" * (1 + utf16))
 
 
+class StringWithLen:
+    @staticmethod
+    def get_args(ar: ArchiveBase):
+        num = ar.args[0]
+        utf16 = ar.args[1]
+        encode = "utf-16-le" if utf16 else "ascii"
+        return num, utf16, encode
+
+    @staticmethod
+    def read(ar: ArchiveBase) -> str:
+        num, utf16, encode = StringWithLen.get_args(ar)
+        string = ar.read(num * (1 + utf16)).decode(encode)
+        return string
+
+    @staticmethod
+    def write(ar: ArchiveBase, val: str):
+        _, utf16, encode = StringWithLen.get_args(ar)
+        ar.write(val.encode(encode))
+
+
 class SerializableBase:
     def serialize(self, ar: ArchiveBase):  # pragma: no cover
         pass