Format 25C capability

DAT_Image_File.txt

@@ -2,7 +2,7 @@ DAT Image File
 Author: Michael Gaisser (mjgaisser@gmail.com)
 Site: https://github.com/MikeG621
 Released: 2010.02.14
-Updated: 2019.09.21
+Updated: 2022.02.27
 =====
 
 The .DAT file used in X-wing Alliance is an archive file that can contain
@@ -20,7 +20,7 @@ NAME	LENGTH	DESC
 ----	------	----
 BYTE	1	unsigned 8-bit
 SHORT	2	signed Int16
-INT	4	signed Int32
+INT 	4	signed Int32
 LONG	8	signed Int64
 
 --
@@ -61,10 +61,15 @@ struct Sub
 	0x00	SubHeader
 	0x12	ImageHeader
 	0x3E	Colors[ImageHeader.NumberOfColors]
+#if (Type!=25C)
 		Row[Height]
-		#if (Type==7 || Type==23)
+#else
+	0x3E	BYTE[5]	CompressionParameters
+	0x43	BYTE[] LzmaData
+#endif
+#if (Type==7 || Type==23)
 		BYTE	Reserved (0)
-		#endif
+#endif
 }
 
 struct SubHeader (size 0x12)
@@ -105,49 +110,49 @@ struct Color (size 0x3)
 
 struct Row
 {
-	#if (Type==7 || Type==23)
+#if (Type==7 || Type==23)
 	BYTE	NumberOfOperations
 	OpCode[NumberOfOperations]
-	#elseif (Type==24 || Type==25)
+#elseif (Type==24 || Type==25)
 	Pixel[Width]
-	#endif
+#endif
 }
 
 struct OpCode
 {
 	BYTE	Value
-	#if (Type==7 && ~Value & 0x80) (Read)
+#if (Type==7 && ~Value & 0x80) (Read)
 	BYTE[Value &0x7F]	ColorIndexes
-	#elseif (Type==23 && Value & 0xC0) (Read with Alpha)
+#elseif (Type==23 && Value & 0xC0) (Read with Alpha)
 	Pixel[Value & 0x3F]
-	#elseif (Type==23 && Value < 0x40) (Read Opaque)
+#elseif (Type==23 && Value < 0x40) (Read Opaque)
 	BYTE[Value]
-	#else (Short Transparent)
-	#endif
+#else (Short Transparent)
+#endif
 }
 
 struct Pixel
 {
-	#if (Type==23)
+#if (Type==23)
 	BYTE	Alpha
-	#endif
-	#if (Type==25)
+#endif
+#if (Type==25)
 	BYTE	Blue
 	BYTE	Green
 	BYTE	Red
-	#else
+#else
 	BYTE	ColorIndex
-	#endif
-	#if (Type==24 || Type==25)
+#endif
+#if (Type==24 || Type==25)
 	BYTE	Alpha
-	#endif
+#endif
 }
 
 =====
 DAT Structure Details
 
 Yet another image format for the X-wing series, although this one is XWA only.
-Four different raw formats are bundled under this one type, the difference is
+Five different raw formats are bundled under this one type, the difference is
 the type of bitmaps that are displayed. More on that in a sec.
 
 As mentioned at the beginning, the images are categorized as Groups and Subs.
@@ -207,6 +212,12 @@ type. The Type determines the type (gasp) of image and how it needs to be read.
 0x17	Alpha blended bitmap (32bpp), compressed
 0x18	Alpha blended bitmap (32bpp), uncompressed
 0x19	Full ARGB image (32bpp), uncompressed
+0x19	Full ARGB image (32bpp), LZMA compressed (aka "25C")
+
+Yes, those are both 0x19. The way that compression gets flagged is that the
+ImageHeader.NumberOfColors (couple paragraphs down) is set to 1. There really
+aren't any colors defined so that array is empty, but that's the way that 25C
+is defined.
 
 Width and Height are easy enough, they're one-indexed values. The GroupID and
 SubID values should also be self-explanatory. SubHeader.Length could also be
@@ -233,7 +244,8 @@ Okay, here's where most of the magic happens. The only common similarity
 between the first three Types are the length of the Rows array itself, and that
 they are all indexed color images. Once we get to Type 25 (which again requires
 a hook to be installed) then there's little to no processing that needs to be
-done and the size of Row doubles.
+done and the size of Row doubles. For Type 25C, the rows are omitted entirely
+since the data is LZMA compressed and is processed separately.
 
 First off, Type 7; Transparent. Mostly used as the backdrop type, it also
 has the tightest compression rate for filesize. The Row starts off with the
@@ -295,4 +307,9 @@ zero (which of course means all Group.NumberOfColors values are zero) and
 Sub.ImageHeaderOffset is 0x2C since there's no Color array to skip over.
 
 For the image data itself, it's simple raw pixel data at this point, four bytes
-in the BGRA order with no extra termination data, they go from one to the next.
+in the BGRA order with no extra termination data, they go from one to the next.
+
+Type "25C"; Full ARGB, LZMA compressed, requires 32-bit hook. The
+data is the same as 25, however it's stored as LZMA compressed data. Per the
+LZMA documentation, the first 5 bytes are the compression parameters that
+defines exactly how it's stored.

DatFile.cs

@@ -1,6 +1,6 @@
 /*
  * Idmr.ImageFormat.Dat, Allows editing capability of LA *.DAT Image files
- * Copyright (C) 2009-2021 Michael Gaisser (mjgaisser@gmail.com)
+ * Copyright (C) 2009-2022 Michael Gaisser (mjgaisser@gmail.com)
  * 
  * This library is free software; you can redistribute it and/or modify it
  * under the terms of the Mozilla Public License; either version 2.0 of the
@@ -10,13 +10,17 @@
  * library is free of defects, merchantable, fit for a particular purpose or
  * non-infringing. See the full license text for more details.
  *
- * If a copy of the MPL (MPL.txt) was not distributed with this file,
+ * If a copy of the MPL (License.txt) was not distributed with this file,
  * you can obtain one at http://mozilla.org/MPL/2.0/.
  *
- * VERSION: 2.3
+ * VERSION: 2.4
  */
 
 /* CHANGE LOG
+ * v2.4, 220227
+ * [NEW] Format 25C capability
+ * [UPD] Forced garbage collection after load. I know that's taboo, shush.
+ * [UPD] during Save, write the raw data from EncodeImage for Full32bpp images since that's no longer
  * v2.3, 210606
  * [UPD] Ignores empty Groups
  * v2.2, 190922
@@ -62,6 +66,7 @@ public DatFile(string file)
 				fs = File.OpenRead(file);
 				DecodeFile(new BinaryReader(fs).ReadBytes((int)fs.Length));
 				fs.Close();
+				GC.Collect();	// for files like Planet2 that are 100% Full32, this'll immediately release half the memory
 			}
 			catch (Exception x)
 			{
@@ -86,7 +91,7 @@ public void Save()
 				for (int g = 0; g < NumberOfGroups; g++) if (Groups[g].ID < 0) throw new InvalidDataException("Not all Groups have been initialized");
 				updateGroupHeaders();
 				for (int g = 0; g < NumberOfGroups; g++)
-					for (int s = 0; s < Groups[g].NumberOfSubs; s++) Groups[g].Subs[s]._headerUpdate();
+					for (int s = 0; s < Groups[g].NumberOfSubs; s++) Groups[g].Subs[s].UpdateHeader();
 				if (File.Exists(_filePath)) File.Copy(_filePath, tempFile);	// create backup
 				File.Delete(_filePath);
 				fs = File.OpenWrite(_filePath);
@@ -96,10 +101,10 @@ public void Save()
 				bw.Write((short)1);
 				bw.Write(NumberOfGroups);
 				bw.Write(Groups.NumberOfSubs);
-				bw.Write(_length);
+				bw.Write(length);
 				bw.Write(Groups.NumberOfColors);
 				fs.Position += 8;	// long 0
-				bw.Write(_dataOffset);
+				bw.Write(dataOffset);
 				for (int g = 0; g < NumberOfGroups; g++) bw.Write(Groups[g]._header);
 				// Groups
 				for (int g = 0; g < NumberOfGroups; g++)
@@ -114,7 +119,12 @@ public void Save()
 							bw.Write(Groups[g].Subs[s].Colors[c].G);
 							bw.Write(Groups[g].Subs[s].Colors[c].B);
 						}
-						bw.Write(Groups[g].Subs[s]._rows);
+						if (Groups[g].Subs[s].Type == Sub.ImageType.Full32bppArgb)
+						{
+							// _rows is null to save RAM, so execute the copy to bytes again.
+							bw.Write(Sub.EncodeImage(Groups[g].Subs[s].Image, Sub.ImageType.Full32bppArgb, null, out _, out _));
+						}
+						else bw.Write(Groups[g].Subs[s]._rows);
 					}
 				}
 				fs.SetLength(fs.Position);
@@ -166,8 +176,8 @@ public void DecodeFile(byte[] rawData)
 				{
 					for (int j = 0; j < Groups[i].NumberOfSubs; j++)
 					{
-						int length = BitConverter.ToInt32(rawData, offset + 0xE);
-						byte[] sub = new byte[length + Sub._subHeaderLength];
+						int dataLength = BitConverter.ToInt32(rawData, offset + 0xE);
+						byte[] sub = new byte[dataLength + Sub._subHeaderLength];
 						ArrayFunctions.TrimArray(rawData, offset, sub);
 						Groups[i].Subs[j] = new Sub(sub);
 						offset += sub.Length;
@@ -226,25 +236,25 @@ void updateGroupHeaders()
 			{
 				// GroupID is always up-to-date
 				ArrayFunctions.WriteToArray(Groups[i].NumberOfSubs, Groups[i]._header, 2);
-				ArrayFunctions.WriteToArray(Groups[i]._length, Groups[i]._header, 4);
+				ArrayFunctions.WriteToArray(Groups[i].length, Groups[i]._header, 4);
 				ArrayFunctions.WriteToArray(Groups[i].Subs.NumberOfColors, Groups[i]._header, 8);
 				// Reserved(0) is always up-to-date
-				if (i == 0) Groups[i]._dataOffset = 0;
-				else Groups[i]._dataOffset = Groups[i - 1]._dataOffset + Groups[i - 1]._length;
+				if (i == 0) Groups[i].dataOffset = 0;
+				else Groups[i].dataOffset = Groups[i - 1].dataOffset + Groups[i - 1].length;
 			}
 		}
 
 		/// <summary>Gets sum of Groups.Length values</summary>
-		int _length
+		int length
 		{
 			get
 			{
 				int l = 0;
-				for (int i = 0; i < Groups.Count; i++) l += Groups[i]._length;
+				for (int i = 0; i < Groups.Count; i++) l += Groups[i].length;
 				return l;
 			}
 		}
 
-		int _dataOffset { get { return NumberOfGroups * Group._headerLength; } }
+		int dataOffset { get { return NumberOfGroups * Group._headerLength; } }
 	}
 }

Group.cs

@@ -1,6 +1,6 @@
 /*
  * Idmr.ImageFormat.Dat, Allows editing capability of LucasArts *.DAT Image files
- * Copyright (C) 2009-2019 Michael Gaisser (mjgaisser@gmail.com)
+ * Copyright (C) 2009-2022 Michael Gaisser (mjgaisser@gmail.com)
  * Licensed under the MPL v2.0 or later
  * 
  * Full notice in DatFile.cs
@@ -112,7 +112,7 @@ public short ID
 		#endregion public properties
 
 		/// <summary>Sum of Subs.Length values</summary>
-		internal int _length
+		internal int length
 		{
 			get
 			{
@@ -122,7 +122,7 @@ internal int _length
 			}
 		}
 		/// <summary>***Must be updated at the Dat level***</summary>
-		internal int _dataOffset
+		internal int dataOffset
 		{
 			get { return BitConverter.ToInt32(_header, 0x14); }
 			set { ArrayFunctions.WriteToArray(value, _header, 0x14); }

ImageFormat.Dat.csproj

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>9.0.21022</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{CB13D130-495C-4F73-9B23-FE688973EF0D}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Idmr.ImageFormat</RootNamespace>
+    <AssemblyName>Idmr.ImageFormat.Dat</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <FileUpgradeFlags>
+    </FileUpgradeFlags>
+    <UpgradeBackupLocation>
+    </UpgradeBackupLocation>
+    <OldToolsVersion>3.5</OldToolsVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <DocumentationFile>bin\Release\Idmr.ImageFormat.Dat.XML</DocumentationFile>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Idmr.Common, Version=1.0.2.18456, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\Libraries\Idmr.Common.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Data" />
+    <Reference Include="System.Drawing" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="SevenZip\Compression\LZMA\Encoder.LiteralEncoder.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Encoder.LenPriceTableEncoder.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Encoder.Optimal.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Decoder.LiteralDecoder.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Decoder.LenDecoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\BitDecoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\BitTreeDecoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\Decoder.cs" />
+    <Compile Include="DatFile.cs" />
+    <Compile Include="Group.cs" />
+    <Compile Include="GroupCollection.cs" />
+    <Compile Include="Sub.cs" />
+    <Compile Include="SubCollection.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="SevenZip\ICoder.cs" />
+    <Compile Include="SevenZip\CRC.cs" />
+    <Compile Include="SevenZip\Compression\LZ\IMatchFinder.cs" />
+    <Compile Include="SevenZip\Compression\LZ\BinTree.cs" />
+    <Compile Include="SevenZip\Compression\LZ\InWindow.cs" />
+    <Compile Include="SevenZip\Compression\LZ\OutWindow.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Base.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Decoder.cs" />
+    <Compile Include="SevenZip\Compression\LZMA\Encoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\Encoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\BitEncoder.cs" />
+    <Compile Include="SevenZip\Compression\RangeCoder\BitTreeEncoder.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="SevenZip\Compression\LZMA\Encoder.LenEncoder.cs" />
+    <None Include="DAT_Image_File.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="License.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="Readme.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+  <PropertyGroup>
+    <PostBuildEvent>copy /y "$(TargetPath)" "C:\Users\Me\Documents\Visual Studio 2008\Libraries\"</PostBuildEvent>
+  </PropertyGroup>
+</Project>