Merge pull request #39 from matyalatte/dev

v0.6.1 update

.github/workflows/build.yml

@@ -70,6 +70,18 @@ jobs:
           cp external/tuw/build/ReleaseUCRT/Tuw.exe release_gui/GUI.exe
           cp gui_definition.json release_gui
 
+      - name: Upload as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ZIP_NAME }}-${{ env.TAG }}-Batch.zip
+          path: release
+
+      - name: Upload as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ZIP_NAME }}-${{ env.TAG }}-GUI.zip
+          path: release_gui
+
       - name: Archive Release
         uses: thedoctor0/zip-release@master
         with:

.github/workflows/test.yml

@@ -33,8 +33,8 @@ jobs:
 
       - name: Linting
         run: |
-          flake8 --max-line-length 119 --exclude ".git,.pytest_cache,external,htmlcov,python,tmp"
-          codespell -L "splitted,ue" -S ".git,.pytest_cache,external,htmlcov,python,tmp"
+          flake8
+          codespell
 
       - name: Test
         env:

docs/README.md

@@ -2,7 +2,7 @@
 ![test](https://github.com/matyalatte/UE4-DDS-tools/actions/workflows/test.yml/badge.svg)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-# UE4-DDS-Tools ver0.6.0
+# UE4-DDS-Tools ver0.6.1
 
 Texture modding tools for UE games.  
 You can inject texture files (.dds, .tga, .hdr, etc.) into UE assets.  

docs/changelog.txt

@@ -1,3 +1,9 @@
+ver 0.6.1
+- Added support for non-2D textures with mipmaps.
+- Fixed errors when reading some ucas/utoc assets from UE5.4 games
+- Fixed some error messages about input paths.
+- Fixed errors when reading some broken dds files.
+
 ver 0.6.0
 - Supported ucas/utoc assets.
 - Added support for UE5.4.

gui_definition.json

@@ -21,7 +21,16 @@
                     "label": "Texture file (dds, tga, hdr, png, jpg, or bmp)",
                     "extension": "any files | *",
                     "add_quotes": true,
-                    "empty_message": "Drop an image here!"
+                    "empty_message": "Drop an image here!",
+                    "platforms": [ "win" ]
+                },
+                {
+                    "type": "file",
+                    "label": "Texture file (dds, tga, or hdr)",
+                    "extension": "any files | *",
+                    "add_quotes": true,
+                    "empty_message": "Drop an image here!",
+                    "platforms": [ "mac", "linux" ]
                 },
                 {
                     "type": "folder",

setup.cfg

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 119
+exclude = .git,.pytest_cache,external,htmlcov,python,tmp
+
+[codespell]
+ignore-words-list = splitted,ue
+skip = .git,.pytest_cache,external,htmlcov,python,tmp

src/directx/dds.py

@@ -316,6 +316,7 @@ def read(f: IOBase) -> "DDSHeader":
         head = DDSHeader()
         f.readinto(head)
         head.mipmap_num += head.mipmap_num == 0
+        head.depth += head.depth == 0
 
         # DXT10 header
         if head.pixel_format.is_dx10():
@@ -331,8 +332,6 @@ def read(f: IOBase) -> "DDSHeader":
             raise RuntimeError("Not DDS file.")
         if head.dx10_header.resource_dimension == 2:
             raise RuntimeError("1D textures are unsupported.")
-        if (head.is_array() or head.is_3d()) and head.has_mips():
-            raise RuntimeError(f"Loaded {head.get_texture_type()} texture has mipmaps. This is unexpected.")
 
         return head
 
@@ -520,7 +519,7 @@ def load(file: str, verbose=False):
             dds.print(verbose)
 
             if f.tell() != end_offset:
-                raise RuntimeError("Parse failed. (Not the end of the file.)")
+                raise RuntimeError(f"Parse failed. (Not the end of the file. Offset: {f.tell()})")
 
         return dds
 

src/directx/texconv.py

@@ -6,29 +6,12 @@
 """
 import ctypes
 import os
-import platform
 import shutil
 import tempfile
 
 from .dds import DDS, DDSHeader, is_hdr
 from .dxgi_format import DXGI_FORMAT
-from util import mkdir
-
-
-def get_os_name():
-    return platform.system()
-
-
-def is_windows():
-    return get_os_name() == "Windows"
-
-
-def is_linux():
-    return get_os_name() == "Linux"
-
-
-def is_mac():
-    return get_os_name() == "Darwin"
+from util import mkdir, get_os_name, is_windows, is_mac, is_linux
 
 
 class Texconv:

src/main.py

@@ -11,17 +11,26 @@
 # my scripts
 from util import (compare, get_ext, get_temp_dir,
                   get_file_list, get_base_folder, remove_quotes,
-                  check_python_version)
+                  check_python_version, is_windows)
 from unreal.uasset import Uasset, UASSET_EXT
 from directx.dds import DDS
 from directx.dxgi_format import DXGI_FORMAT
-from directx.texconv import Texconv, is_windows
+from directx.texconv import Texconv
 
-TOOL_VERSION = "0.6.0"
+TOOL_VERSION = "0.6.1"
 
 # UE version: 4.0 ~ 5.4, ff7r, borderlands3
 UE_VERSIONS = ["4." + str(i) for i in range(28)] + ["5." + str(i) for i in range(5)] + ["ff7r", "borderlands3"]
 
+# UE version for textures
+UTEX_VERSIONS = [
+    "5.4", "5.3", "5.2", "5.1", "5.0",
+    "4.26 ~ 4.27", "4.24 ~ 4.25", "4.23", "4.20 ~ 4.22",
+    "4.16 ~ 4.19", "4.15", "4.14", "4.12 ~ 4.13", "4.11", "4.10",
+    "4.9", "4.8", "4.7", "4.4 ~ 4.6", "4.3", "4.0 ~ 4.2",
+    "ff7r", "borderlands3"
+]
+
 # Supported file extensions.
 TEXTURES = ["dds", "tga", "hdr"]
 if is_windows():
@@ -345,16 +354,6 @@ def copy(folder, file, args, texture_file=None):
     asset.save(new_file)
 
 
-# UE version for textures
-UTEX_VERSIONS = [
-    "5.4", "5.3", "5.2", "5.1", "5.0",
-    "4.26 ~ 4.27", "4.24 ~ 4.25", "4.23", "4.20 ~ 4.22",
-    "4.16 ~ 4.19", "4.15", "4.14", "4.12 ~ 4.13", "4.11", "4.10",
-    "4.9", "4.8", "4.7", "4.4 ~ 4.6", "4.3", "4.0 ~ 4.2",
-    "ff7r", "borderlands3"
-]
-
-
 @stdout_wrapper
 def check_version(folder, file, args, texture_file=None):
     """Check mode (check file version)"""
@@ -457,6 +456,9 @@ def fix_args(args, config):
     if args.max_workers is not None and args.max_workers <= 0:
         args.max_workers = None
 
+    if args.export_as == "hdr":
+        args.export_as = "tga"
+
 
 def print_args(args):
     mode = args.mode
@@ -488,11 +490,13 @@ def check_args(args):
     mode = args.mode
     if os.path.isfile(args.save_folder):
         raise RuntimeError(f"Output path is not a folder. ({args.save_folder})")
+    if args.file == "":
+        raise RuntimeError("Specify a uasset file.")
     if not os.path.exists(args.file):
         raise RuntimeError(f"Path not found. ({args.file})")
     if mode == "inject":
         if args.texture is None or args.texture == "":
-            raise RuntimeError("Specify texture file.")
+            raise RuntimeError("Specify a texture file.")
         if os.path.isdir(args.file):
             if not os.path.isdir(args.texture):
                 raise RuntimeError(
@@ -506,7 +510,7 @@ def check_args(args):
         raise RuntimeError(f"Unsupported mode. ({mode})")
     if mode != "check" and args.version not in UE_VERSIONS:
         raise RuntimeError(f"Unsupported version. ({args.version})")
-    if args.export_as not in ["tga", "png", "dds", "jpg", "bmp"]:
+    if args.export_as not in TEXTURES:
         raise RuntimeError(f"Unsupported format to export. ({args.export_as})")
     if args.image_filter.lower() not in IMAGE_FILTERS:
         raise RuntimeError(f"Unsupported image filter. ({args.image_filter})")

src/unreal/archive.py

@@ -28,15 +28,19 @@ def __init__(self, io: IOBase, endian="little", context: dict = {}):
         self.io = io
         self.size = get_size(io)
         self.endian = endian
-        for key, val in context.items():
-            setattr(self, key, val)
+
+        self.update_context(context)
 
         if isinstance(io, BytesIO):
             self.name = "BytesIO"
         else:
             self.name = io.name
         self.args = None
 
+    def update_context(self, context: dict = {}):
+        for key, val in context.items():
+            setattr(self, key, val)
+
     def __lshift__(self, val: tuple):  # pragma: no cover
         """Read or write attributes.
         Notes:
@@ -100,6 +104,9 @@ def update_with_current_offset(self, obj, attr_name, base=0):
             # Update obj.attr_name with the current offset
             setattr(obj, attr_name, self.tell() - base)
 
+    def is_eof(self):
+        return self.tell() == self.size
+
 
 class ArchiveRead(ArchiveBase):
     is_reading = True

src/unreal/city_hash.py

@@ -71,7 +71,7 @@ def hash_len_0to16(binary: bytes) -> int:
     if length > 0:
         a = binary[0]
         b = binary[length >> 1]
-        c = binary[:-1]
+        c = binary[-1]
         y = (a + (b << 8)) & MASK_32
         z = (length + (c << 2)) & MASK_32
         return (shift_mix(y * k2 ^ z * k0) * k2) & MASK_64

src/unreal/file_summary.py

@@ -341,11 +341,15 @@ def serialize_name_map(self, ar: ArchiveBase, name_list: list[ZenName]) -> list[
         list(map(lambda x: x.serialize_hash(ar), name_list))
         list(map(lambda x: x.serialize_head(ar), name_list))
         list(map(lambda x: x.serialize_string(ar), name_list))
-        if ar.is_writing:
-            self.pad_size = (8 - (ar.tell() % 8)) % 8
         if ar.version >= "5.4":
+            if ar.is_writing and self.align_name_map:
+                self.pad_size = (8 - (ar.tell() % 8)) % 8
             ar << (Uint64, self, "pad_size")
             ar == (Buffer, b"\x00" * self.pad_size, "pad", self.pad_size)
+            if ar.is_reading:
+                # Some assets don't use alignment somehow.
+                self.align_name_map = ar.tell() % 8 == 0
+
         return name_list
 
     def serialize_data_resources(self, ar: ArchiveBase,

src/unreal/uasset.py

@@ -56,12 +56,25 @@ def __init__(self, file_path: str, version: str = "ff7r", verbose=False):
         print("load: " + uasset_file)
 
         self.version = VersionInfo(version)
-        self.context = {"version": self.version, "verbose": verbose, "valid": False, "uasset": self}
-        ar = ArchiveRead(open(uasset_file, "rb"), context=self.context)
+        self.context_verbose = verbose
+        self.context_valid = False
+        self.is_ucas = False
+        self.has_end_tag = True
+        ar = ArchiveRead(open(uasset_file, "rb"), context=self.get_ar_context())
         self.serialize(ar)
         ar.close()
         self.read_export_objects(verbose=verbose)
 
+    def get_ar_context(self):
+        context = {
+            "version": self.version,
+            "verbose": self.context_verbose,
+            "valid": self.context_valid,
+            "is_ucas": self.is_ucas,
+            "uasset": self
+        }
+        return context
+
     def print_name_map(self):
         print("Names")
         for i, name in zip(range(len(self.name_list)), self.name_list):
@@ -181,13 +194,9 @@ def check_tag(self, ar: ArchiveBase):
             ar.seek(0)
         if self.tag == Uasset.TAG:
             ar.endian = "little"
-            ar.is_ucas = False
-            self.context["is_ucas"] = False
             self.is_ucas = False
         elif self.tag == Uasset.TAG_SWAPPED:
             ar.endian = "big"
-            ar.is_ucas = False
-            self.context["is_ucas"] = False
             self.is_ucas = False
         else:
             if ar.version <= "4.24":
@@ -196,9 +205,8 @@ def check_tag(self, ar: ArchiveBase):
                 if self.tag != b"\x00\x00\x00\x00" and self.tag != b"\x01\x00\x00\x00":
                     raise ar.raise_error(f"Invalid tag detected. ({self.tag})")
             ar.endian = "little"
-            ar.is_ucas = True
-            self.context["is_ucas"] = True
             self.is_ucas = True
+        ar.update_context(self.get_ar_context())
 
     def read_export_objects(self, verbose=False):
         uexp_io = self.get_io(ext="uexp", rb=True)
@@ -239,11 +247,11 @@ def save(self, file: str, valid=False):
         uasset_file = self.file_name + ".uasset"
         print("save :" + uasset_file)
 
-        self.context["verbose"] = False
-        self.context["valid"] = valid
+        self.context_verbose = False
+        self.context_valid = valid
         self.write_export_objects()
 
-        ar = ArchiveWrite(open(uasset_file, "wb"), context=self.context)
+        ar = ArchiveWrite(open(uasset_file, "wb"), context=self.get_ar_context())
 
         self.serialize(ar)
 
@@ -307,9 +315,9 @@ def __get_io_base(self, file: str, bin: bytes, rb: bool) -> IOBase:
             opened_io = open(file, "rb" if rb else "wb")
 
         if rb:
-            return ArchiveRead(opened_io, context=self.context)
+            return ArchiveRead(opened_io, context=self.get_ar_context())
         else:
-            return ArchiveWrite(opened_io, context=self.context)
+            return ArchiveWrite(opened_io, context=self.get_ar_context())
 
     def get_io(self, ext="uexp", rb=True) -> IOBase:
         if ext not in self.io_dict or self.io_dict[ext] is None:
@@ -322,9 +330,12 @@ def __close_io(self, ext="uexp", rb=True):
         ar = self.io_dict[ext]
         if ext == "uexp":
             self.uexp_size = ar.tell()
-            if (self.has_uexp() and not ar.is_ucas) or ar.version == "5.3":
+            if self.has_end_tag and ((self.has_uexp() and not ar.is_ucas) or ar.version >= "5.3"):
                 if rb:
-                    ar.check(ar.read(4), Uasset.TAG)
+                    if ar.is_eof():
+                        self.has_end_tag = False
+                    else:
+                        ar.check(ar.read(4), Uasset.TAG)
                 else:
                     ar.write(Uasset.TAG)
         else: