Fixed some bugs for Pack compression, pack file service and shares Content folder generation. Also tidied up game info.

AssetEditor/AssetEditor.csproj

@@ -28,6 +28,14 @@
         <ProjectReference Include="..\Shared\SharedUI\Shared.Ui.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+        <Content Include="..\GameWorld\ContentProject\Content\bin\Windows\Content\**\*.*">
+            <Link>Content\%(RecursiveDir)%(Filename)%(Extension)</Link>
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+            <CopyToPublishDirectory>Always</CopyToPublishDirectory>
+        </Content>
+    </ItemGroup>
+
     <PropertyGroup>
         <DebugType>embedded</DebugType>
         <Authors>AssetEdCommunity</Authors>
@@ -43,22 +51,19 @@
         <None Include="..\.editorconfig" />
     </ItemGroup>
 
-
-
     <ItemGroup>
       <Resource Include="AssetEditorIcon.png" />
     </ItemGroup>
 
     <PropertyGroup>
         <!--
-    Make sure any documentation comments which are included in code get checked for syntax during the build, but do
-    not report warnings for missing comments.
-    CS1573: Parameter 'parameter' has no matching param tag in the XML comment for 'parameter' (but other parameters do)
-    CS1591: Missing XML comment for publicly visible type or member 'Type_or_Member'
-    CS1712: Type parameter 'type_parameter' has no matching typeparam tag in the XML comment on 'type_or_member' (but other type parameters do)
--->
+        Make sure any documentation comments which are included in code get checked for syntax during the build, but do
+        not report warnings for missing comments.
+        CS1573: Parameter 'parameter' has no matching param tag in the XML comment for 'parameter' (but other parameters do)
+        CS1591: Missing XML comment for publicly visible type or member 'Type_or_Member'
+        CS1712: Type parameter 'type_parameter' has no matching typeparam tag in the XML comment on 'type_or_member' (but other type parameters do)
+        -->
         <GenerateDocumentationFile>False</GenerateDocumentationFile>
         <NoWarn>$(NoWarn),1573,1591,1712</NoWarn>
     </PropertyGroup>
-
 </Project>

AssetEditor/ViewModels/SettingsViewModel.cs

@@ -37,12 +37,12 @@ public SettingsViewModel(ApplicationSettingsService settingsService)
             RenderEngineBackgroundColours = new ObservableCollection<BackgroundColour>((BackgroundColour[])Enum.GetValues(typeof(BackgroundColour)));
             CurrentRenderEngineBackgroundColour = _settingsService.CurrentSettings.RenderEngineBackgroundColour;
             StartMaximised = _settingsService.CurrentSettings.StartMaximised;
-            Games = new ObservableCollection<GameTypeEnum>(GameInformationDatabase.Games.OrderBy(g => g.DisplayName).Select(g => g.Type));
+            Games = new ObservableCollection<GameTypeEnum>(GameInformationDatabase.Games.Values.OrderBy(game => game.DisplayName).Select(game => game.Type));
             CurrentGame = _settingsService.CurrentSettings.CurrentGame;
             LoadCaPacksByDefault = _settingsService.CurrentSettings.LoadCaPacksByDefault;
             ShowCAWemFiles = _settingsService.CurrentSettings.ShowCAWemFiles;
             OnlyLoadLod0ForReferenceMeshes = _settingsService.CurrentSettings.OnlyLoadLod0ForReferenceMeshes;
-            foreach (var game in GameInformationDatabase.Games.OrderBy(g => g.DisplayName))
+            foreach (var game in GameInformationDatabase.Games.Values.OrderBy(game => game.DisplayName))
             {
                 GameDirectores.Add(
                     new GamePathItem()

GameWorld/ContentProject/ContentProject.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
       <TargetFramework>net9.0-windows</TargetFramework>
   </PropertyGroup>
@@ -24,7 +24,6 @@
       <ProjectReference Include="..\..\Shared\SharedCore\Shared.Core.csproj" />
     </ItemGroup>
 
-
     <Target Name="RestoreDotnetTools" BeforeTargets="Restore">
         <Message Text="Restoring dotnet tools" Importance="High" />
         <Exec Command="dotnet tool restore" />

GameWorld/View3D/Services/SkeletonAnimationLookUpHelper.cs

@@ -1,5 +1,4 @@
 using System.Collections.ObjectModel;
-using System.Diagnostics;
 using System.IO;
 using Serilog;
 using Shared.Core.ErrorHandling;
@@ -129,23 +128,19 @@ void LoadFromPackFileContainer(PackFileContainer packFileContainer)
 
         void FileDiscovered(byte[] byteChunk, PackFileContainer container, string fullPath, ref List<string> skeletonFileNameList, ref Dictionary<string, List<AnimationReference>> animationList)
         {
-            // Skip broken animations, as the errors are annoying when the debuger is attached.
-            if (Debugger.IsAttached)
+            var brokenFiles = new string[]
             {
-                var brokenAnims = new string[]
-                {
-                     "rigidmodels\\buildings\\roman_aqueduct_straight\\roman_aqueduct_straight_piece01_destruct01_anim.anim",
-                     "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim",
-                     "animations\\battle\\humanoid13b\\golgfag\\docking\\hu13b_golgfag_docking_armed_02.anim",
-                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_shoot_back_crossbow_01.anim",
-                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_reload_crossbow_01.anim",
-                     "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim"
-                };
-                if (brokenAnims.Contains(fullPath))
-                {
-                    _logger.Here().Warning("Skipping loading of known broken file - " + fullPath);
-                    return;
-                }
+                "rigidmodels\\buildings\\roman_aqueduct_straight\\roman_aqueduct_straight_piece01_destruct01_anim.anim",
+                "animations\\battle\\raptor02\\subset\\colossal_squig\\deaths\\rp2_colossalsquig_death_01.anim",
+                "animations\\battle\\humanoid13b\\golgfag\\docking\\hu13b_golgfag_docking_armed_02.anim",
+                "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_shoot_back_crossbow_01.anim",
+                "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_swc_rider1_reload_crossbow_01.anim",
+                "animations\\battle\\humanoid13\\ogre\\rider\\hq3b_stonehorn_wb\\sword_and_crossbow\\missile_action\\crossbow\\hu13_hq3b_sp_rider1_shoot_ready_crossbow_01.anim"
+            };
+            if (brokenFiles.Contains(fullPath))
+            {
+                _logger.Here().Warning("Skipping loading of known broken file - " + fullPath);
+                return;
             }
 
             try

Shared/SharedCore/PackFiles/Models/DataSource.cs

@@ -1,5 +1,4 @@
 using Shared.Core.ByteParsing;
-using Shared.Core.Settings;
 
 namespace Shared.Core.PackFiles.Models
 {
@@ -159,62 +158,23 @@ public byte[] ReadData(Stream knownStream)
             return data;
         }
 
-        public ByteChunk ReadDataAsChunk()
-        {
-            return new ByteChunk(ReadData());
-        }
-
-        public void SetCompressionInfo(GameInformation gameInformation, string rootFolder, string extension)
+        public byte[] ReadDataWithoutDecompressing()
         {
-            // Check if the game supports any compression at all
-            if (gameInformation.CompressionFormats.All(compressionFormat => compressionFormat == CompressionFormat.None))
-                return;
-
-            // We use isTable because non-loc tables don't have an extension
-            var isTable = rootFolder == "db" || extension == ".loc";
-            var hasExtension = !string.IsNullOrEmpty(extension);
-
-            // Don't compress files that aren't tables and don't have extensions
-            if (!isTable && !hasExtension)
-            {
-                CompressionFormat = CompressionFormat.None;
-                IsCompressed = false;
-                return;
-            }
-
-            // Only in WH3 (and newer games?) is the table compression bug fixed
-            if (isTable && gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd) && gameInformation.Type == GameTypeEnum.Warhammer3)
+            var data = new byte[Size];
+            using (Stream stream = File.Open(_parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
             {
-                CompressionFormat = CompressionFormat.Zstd;
-                IsCompressed = true;
-                return;
+                stream.Seek(Offset, SeekOrigin.Begin);
+                stream.Read(data, 0, data.Length);
             }
 
-            // Games that support the other formats won't use Lzma1 as it's legacy so if it's set then it's for a game that only uses it so keep it
-            if (CompressionFormat == CompressionFormat.Lzma1 && gameInformation.CompressionFormats.Contains(CompressionFormat.Lzma1))
-                return;
+            if (IsEncrypted)
+                data = PackFileEncryption.Decrypt(data);
+            return data;
+        }
 
-            // Anything that shouldn't be None or Lz4 is set to Zstd unless the game doesn't support that in which case use None
-            if (PackFileCompression.NoneFileTypes.Contains(extension))
-            {
-                CompressionFormat = CompressionFormat.None;
-                IsCompressed = false;
-            }
-            else if (PackFileCompression.Lz4FileTypes.Contains(extension) && gameInformation.CompressionFormats.Contains(CompressionFormat.Lz4))
-            {
-                CompressionFormat = CompressionFormat.Lz4;
-                IsCompressed = true;
-            }
-            else if (gameInformation.CompressionFormats.Contains(CompressionFormat.Zstd))
-            {
-                CompressionFormat = CompressionFormat.Zstd;
-                IsCompressed = true;
-            }
-            else
-            {
-                CompressionFormat = CompressionFormat.None;
-                IsCompressed = false;
-            }
+        public ByteChunk ReadDataAsChunk()
+        {
+            return new ByteChunk(ReadData());
         }
     }
 

Shared/SharedCore/PackFiles/Models/PackFileContainer.cs

@@ -3,6 +3,12 @@
 
 namespace Shared.Core.PackFiles.Models
 {
+    public record PackFileWriteInfo(
+        PackFile PackFile,
+        long FileSizeMetadataPosition,
+        CompressionFormat CurrentCompressionFormat,
+        CompressionFormat IntendedCompressionFormat);
+
     public class PackFileContainer
     {
         public string Name { get; set; }
@@ -55,70 +61,105 @@ public void SaveToByteArray(BinaryWriter writer, GameInformation gameInformation
             Header.FileCount = (uint)FileList.Count;
             PackFileSerializer.WriteHeader(Header, (uint)fileNamesOffset, writer);
 
-            // Save all the files
+            var filesToWrite = new List<PackFileWriteInfo>();
+
+            // Write file metadata
             foreach (var file in sortedFiles)
             {
                 var packFile = file.Value;
-                var packedFileSource = (PackedFileSource)file.Value.DataSource;
-                var data = packedFileSource.ReadData();
+                var fileSize = (int)packFile.DataSource.Size;
 
-                var fileExtension = packFile.Extension;
+                // Determine compression info
+                var currentCompressionFormat = CompressionFormat.None;
+                if (packFile.DataSource is PackedFileSource packedFileSource)
+                    currentCompressionFormat = packedFileSource.CompressionFormat;
+                var firstFilePathPart = file.Key.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries).First();
+                var intendedCompressionFormat = PackFileCompression.GetCompressionFormat(gameInformation, firstFilePathPart, packFile.Extension);
+                var shouldCompress = intendedCompressionFormat != CompressionFormat.None;
 
-                var segments = file.Key.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries);
-                var rootFolder = segments.First();
-
-                packedFileSource.SetCompressionInfo(gameInformation, rootFolder, fileExtension);
-
-                var fileSize = data.Length;
-                if (packedFileSource.IsCompressed)
-                {
-                    var compressedData = PackFileCompression.Compress(data, packedFileSource.CompressionFormat);
-                    fileSize = compressedData.Length;
-                }
-                writer.Write(fileSize);
+                // File size placeholder (rewritten later)
+                var fileSizePosition = writer.BaseStream.Position;
+                writer.Write(0);
 
                 // Timestamp
                 if (Header.HasIndexWithTimeStamp)
                     writer.Write(0);
 
                 // Compression
                 if (Header.Version == PackFileVersion.PFH5)
-                    writer.Write(packedFileSource.IsCompressed);
+                    writer.Write(shouldCompress);
 
                 // Filename
                 var fileNameBytes = Encoding.UTF8.GetBytes(file.Key);
                 writer.Write(fileNameBytes);
 
                 // Zero terminator
                 writer.Write((byte)0);
+
+                filesToWrite.Add(new PackFileWriteInfo(
+                    packFile,
+                    fileSizePosition,
+                    currentCompressionFormat,
+                    intendedCompressionFormat));
             }
 
-            var packedFileSourceParent = new PackedFileSourceParent()
-            {
-                FilePath = SystemFilePath,
-            };
+            var packedFileSourceParent = new PackedFileSourceParent { FilePath = SystemFilePath };
 
             // Write the files
-            foreach (var file in sortedFiles)
+            foreach (var file in filesToWrite)
             {
-                var packFile = file.Value;
-                var packedFileSource = (PackedFileSource)packFile.DataSource;
+                var packFile = file.PackFile;
+                byte[] data;
+                var fileSize = 0;
+                uint uncompressedFileSize = 0;
+
+                // Read the data
+                var shouldCompress = file.IntendedCompressionFormat != CompressionFormat.None;
+                var isCorrectCompressionFormat = file.CurrentCompressionFormat == file.IntendedCompressionFormat;
+                if (shouldCompress && !isCorrectCompressionFormat)
+                {
+                    // Decompress the data 
+                    var uncompressedData = packFile.DataSource.ReadData();
+                    uncompressedFileSize = (uint)uncompressedData.Length;
 
+                    // Compress the data into the right format
+                    var compressedData = PackFileCompression.Compress(uncompressedData, file.IntendedCompressionFormat);
+                    data = compressedData;
+                    fileSize = compressedData.Length;
+                }
+                else if (packFile.DataSource is PackedFileSource packedFileSource && isCorrectCompressionFormat)
+                {
+                    // The data is already in the right format so just get the compressed data
+                    uncompressedFileSize = packedFileSource.UncompressedSize;
+                    var compressedData = packedFileSource.ReadDataWithoutDecompressing();
+                    data = compressedData;
+                    fileSize = data.Length;
+                }
+                else
+                {
+                    data = packFile.DataSource.ReadData();
+                    fileSize = data.Length;
+                }
+
+                // Write the data
                 var offset = writer.BaseStream.Position;
-                var data = packedFileSource.ReadData();
-                if (packedFileSource.IsCompressed)
-                    data = PackFileCompression.Compress(data, packedFileSource.CompressionFormat);
+                writer.Write(data);
+
+                // Patch the file size metadata placeholder 
+                var currentPosition = writer.BaseStream.Position;
+                writer.BaseStream.Position = file.FileSizeMetadataPosition;
+                writer.Write(fileSize);
+                writer.BaseStream.Position = currentPosition;
 
+                // Update DataSource
                 packFile.DataSource = new PackedFileSource(
                     packedFileSourceParent,
                     offset,
-                    data.Length,
-                    packedFileSource.IsEncrypted,
-                    packedFileSource.IsCompressed,
-                    packedFileSource.CompressionFormat,
-                    packedFileSource.UncompressedSize);
-
-                writer.Write(data);
+                    fileSize,
+                    Header.HasEncryptedData,
+                    shouldCompress,
+                    file.IntendedCompressionFormat,
+                    uncompressedFileSize);
             }
         }
     }