[VFS] STFS/XContent structure improvements

Now makes use of xe::be<T> and removes need for read() method
2021-04-17 13:44:54 +01:00
parent 010e14b4cd
commit 84f566d92b
2 changed files with 419 additions and 308 deletions
--- a/src/xenia/vfs/devices/stfs_container_device.h
+++ b/src/xenia/vfs/devices/stfs_container_device.h
@@ -15,6 +15,8 @@
 #include <string>

 #include "xenia/base/mapped_memory.h"
+#include "xenia/base/string_util.h"
+#include "xenia/kernel/util/xex2_info.h"
 #include "xenia/vfs/device.h"

 namespace xe {
@@ -24,144 +26,377 @@ namespace vfs {

 class StfsContainerEntry;

-enum class StfsPackageType {
-  kCon,
-  kPirs,
-  kLive,
+enum XContentPackageType : uint32_t {
+  kPackageTypeCon = 0x434F4E20,
+  kPackageTypePirs = 0x50495253,
+  kPackageTypeLive = 0x4C495645,
 };

-enum class StfsContentType : uint32_t {
-  kArcadeTitle = 0x000D0000,
-  kAvatarItem = 0x00009000,
-  kCacheFile = 0x00040000,
-  kCommunityGame = 0x02000000,
-  kGamesOnDemand = 0x00007000,
-  kGameDemo = 0x00080000,
-  kGamerPicture = 0x00020000,
-  kGameTitle = 0x000A0000,
-  kGameTrailer = 0x000C0000,
-  kGameVideo = 0x00400000,
-  kInstalledGame = 0x00004000,
-  kInstaller = 0x000B0000,
-  kIptvPauseBuffer = 0x00002000,
-  kLicenseStore = 0x000F0000,
-  kMarketplaceContent = 0x00000002,
-  kMovie = 0x00100000,
-  kMusicVideo = 0x00300000,
-  kPodcastVideo = 0x00500000,
-  kProfile = 0x00010000,
-  kPublisher = 0x00000003,
+enum XContentType : uint32_t {
  kSavedGame = 0x00000001,
-  kStorageDownload = 0x00050000,
-  kTheme = 0x00030000,
-  kTV = 0x00200000,
-  kVideo = 0x00090000,
-  kViralVideo = 0x00600000,
-  kXboxDownload = 0x00070000,
-  kXboxOriginalGame = 0x00005000,
-  kXboxSavedGame = 0x00060000,
+  kMarketplaceContent = 0x00000002,
+  kPublisher = 0x00000003,
  kXbox360Title = 0x00001000,
+  kIptvPauseBuffer = 0x00002000,
+  kXNACommunity = 0x00003000,
+  kInstalledGame = 0x00004000,
  kXboxTitle = 0x00005000,
+  kSocialTitle = 0x00006000,
+  kGamesOnDemand = 0x00007000,
+  kSUStoragePack = 0x00008000,
+  kAvatarItem = 0x00009000,
+  kProfile = 0x00010000,
+  kGamerPicture = 0x00020000,
+  kTheme = 0x00030000,
+  kCacheFile = 0x00040000,
+  kStorageDownload = 0x00050000,
+  kXboxSavedGame = 0x00060000,
+  kXboxDownload = 0x00070000,
+  kGameDemo = 0x00080000,
+  kVideo = 0x00090000,
+  kGameTitle = 0x000A0000,
+  kInstaller = 0x000B0000,
+  kGameTrailer = 0x000C0000,
+  kArcadeTitle = 0x000D0000,
  kXNA = 0x000E0000,
+  kLicenseStore = 0x000F0000,
+  kMovie = 0x00100000,
+  kTV = 0x00200000,
+  kMusicVideo = 0x00300000,
+  kGameVideo = 0x00400000,
+  kPodcastVideo = 0x00500000,
+  kViralVideo = 0x00600000,
+  kCommunityGame = 0x02000000,
 };

-enum class StfsPlatform : uint8_t {
-  kXbox360 = 0x02,
-  kPc = 0x04,
-};
-
-enum class StfsDescriptorType : uint32_t {
+enum class XContentVolumeType : uint32_t {
  kStfs = 0,
  kSvod = 1,
 };

-struct StfsVolumeDescriptor {
-  bool Read(const uint8_t* p);
-
-  uint8_t descriptor_size;
+/* STFS structures */
+XEPACKEDSTRUCT(StfsVolumeDescriptor, {
+  uint8_t descriptor_length;
  uint8_t version;
-  uint8_t flags;
+  union {
+    struct {
+      uint8_t read_only_format : 1;  // if set, only uses a single backing-block
+                                     // per hash table (no resiliency),
+                                     // otherwise uses two
+      uint8_t root_active_index : 1;  // if set, uses secondary backing-block
+                                      // for the highest-level hash table
+      uint8_t directory_overallocated : 1;
+      uint8_t directory_index_bounds_valid : 1;
+    };
+    uint8_t as_byte;
+  } flags;
  uint16_t file_table_block_count;
-  uint32_t file_table_block_number;
+  uint8_t file_table_block_number_0;
+  uint8_t file_table_block_number_1;
+  uint8_t file_table_block_number_2;
  uint8_t top_hash_table_hash[0x14];
-  uint32_t total_allocated_block_count;
-  uint32_t total_unallocated_block_count;
+  be<uint32_t> allocated_block_count;
+  be<uint32_t> free_block_count;
+
+  uint32_t file_table_block_number() {
+    return uint32_t(file_table_block_number_0) |
+           (uint32_t(file_table_block_number_1) << 8) |
+           (uint32_t(file_table_block_number_2) << 16);
+  }
+});
+static_assert_size(StfsVolumeDescriptor, 0x24);
+
+struct StfsHashEntry {
+  uint8_t sha1[0x14];
+
+  uint8_t info0;  // usually contains flags
+
+  uint8_t info1;
+  uint8_t info2;
+  uint8_t info3;
+
+  // If this is a level0 entry, this points to the next block in the chain
+  uint32_t level0_next_block() {
+    return uint32_t(info3) | (uint32_t(info2) << 8) | (uint32_t(info1) << 16);
+  }
+
+  void level0_next_block(uint32_t value) {
+    info3 = uint8_t(value & 0xFF);
+    info2 = uint8_t((value >> 8) & 0xFF);
+    info1 = uint8_t((value >> 16) & 0xFF);
+  }
+
+  // If this is level 1 or 2, this says whether the hash table this entry refers
+  // to is using the secondary block or not
+  bool levelN_activeindex() { return info0 & 0x40; }
+
+  bool levelN_writeable() { return info0 & 0x80; }
 };
+static_assert_size(StfsHashEntry, 0x18);

-enum SvodDeviceFeatures {
-  kFeatureHasEnhancedGDFLayout = 0x40,
-};
-
-enum SvodLayoutType {
-  kUnknownLayout = 0x0,
-  kEnhancedGDFLayout = 0x1,
-  kXSFLayout = 0x2,
-  kSingleFileLayout = 0x4,
-};
-
-struct SvodVolumeDescriptor {
-  bool Read(const uint8_t* p);
-
-  uint8_t descriptor_size;
+/* SVOD structures */
+struct SvodDeviceDescriptor {
+  uint8_t descriptor_length;
  uint8_t block_cache_element_count;
  uint8_t worker_thread_processor;
  uint8_t worker_thread_priority;
-  uint8_t hash[0x14];
-  uint8_t device_features;
-  uint32_t data_block_count;
-  uint32_t data_block_offset;
-  // 0x5 padding bytes...
-
-  SvodLayoutType layout_type;
-};
-
-class StfsHeader {
- public:
-  bool Read(const uint8_t* p);
-
-  uint8_t license_entries[0x100];
-  uint8_t header_hash[0x14];
-  uint32_t header_size;
-  StfsContentType content_type;
-  uint32_t metadata_version;
-  uint64_t content_size;
-  uint32_t media_id;
-  uint32_t version;
-  uint32_t base_version;
-  uint32_t title_id;
-  StfsPlatform platform;
-  uint8_t executable_type;
-  uint8_t disc_number;
-  uint8_t disc_in_set;
-  uint32_t save_game_id;
-  uint8_t console_id[0x5];
-  uint8_t profile_id[0x8];
+  uint8_t first_fragment_hash_entry[0x14];
  union {
-    StfsVolumeDescriptor stfs_volume_descriptor;
-    SvodVolumeDescriptor svod_volume_descriptor;
-  };
-  uint32_t data_file_count;
-  uint64_t data_file_combined_size;
-  StfsDescriptorType descriptor_type;
-  uint8_t device_id[0x14];
-  char16_t display_names[0x900 / 2];
-  char16_t display_descs[0x900 / 2];
-  char16_t publisher_name[0x80 / 2];
-  char16_t title_name[0x80 / 2];
-  uint8_t transfer_flags;
-  uint32_t thumbnail_image_size;
-  uint32_t title_thumbnail_image_size;
-  uint8_t thumbnail_image[0x4000];
-  uint8_t title_thumbnail_image[0x4000];
+    struct {
+      uint8_t must_be_zero_for_future_usage : 6;
+      uint8_t enhanced_gdf_layout : 1;
+      uint8_t zero_for_downlevel_clients : 1;
+    };
+    uint8_t as_byte;
+  } features;
+  uint8_t num_data_blocks2;
+  uint8_t num_data_blocks1;
+  uint8_t num_data_blocks0;
+  uint8_t start_data_block0;
+  uint8_t start_data_block1;
+  uint8_t start_data_block2;
+  uint8_t reserved[5];

-  // Metadata v2 Fields
+  uint32_t num_data_blocks() {
+    return uint32_t(num_data_blocks0) | (uint32_t(num_data_blocks1) << 8) |
+           (uint32_t(num_data_blocks2) << 16);
+  }
+
+  uint32_t start_data_block() {
+    return uint32_t(start_data_block0) | (uint32_t(start_data_block1) << 8) |
+           (uint32_t(start_data_block2) << 16);
+  }
+};
+static_assert_size(SvodDeviceDescriptor, 0x24);
+
+/* XContent structures */
+struct XContentMediaData {
  uint8_t series_id[0x10];
  uint8_t season_id[0x10];
-  int16_t season_number;
-  int16_t episode_number;
-  char16_t additonal_display_names[0x300 / 2];
-  char16_t additional_display_descriptions[0x300 / 2];
+  be<uint16_t> season_number;
+  be<uint16_t> episode_number;
 };
+static_assert_size(XContentMediaData, 0x24);
+
+struct XContentAvatarAssetData {
+  be<uint32_t> sub_category;
+  be<uint32_t> colorizable;
+  uint8_t asset_id[0x10];
+  uint8_t skeleton_version_mask;
+  uint8_t reserved[0xB];
+};
+static_assert_size(XContentAvatarAssetData, 0x24);
+
+struct XContentAttributes {
+  uint8_t profile_transfer : 1;
+  uint8_t device_transfer : 1;
+  uint8_t move_only_transfer : 1;
+  uint8_t kinect_enabled : 1;
+  uint8_t disable_network_storage : 1;
+  uint8_t deep_link_supported : 1;
+  uint8_t reserved : 2;
+};
+static_assert_size(XContentAttributes, 1);
+
+XEPACKEDSTRUCT(XContentMetadata, {
+  static const uint32_t kThumbLengthV1 = 0x4000;
+  static const uint32_t kThumbLengthV2 = 0x3D00;
+
+  static const uint32_t kNumLanguagesV1 = 9;
+  // metadata_version 2 adds 3 languages inside thumbnail/title_thumbnail space
+  static const uint32_t kNumLanguagesV2 = 12;
+
+  be<XContentType> content_type;
+  be<uint32_t> metadata_version;
+  be<uint64_t> content_size;
+  xex2_opt_execution_info execution_info;
+  uint8_t console_id[5];
+  be<uint64_t> profile_id;
+  union {
+    StfsVolumeDescriptor stfs_volume_descriptor;
+    SvodDeviceDescriptor svod_volume_descriptor;
+  };
+  be<uint32_t> data_file_count;
+  be<uint64_t> data_file_size;
+  be<XContentVolumeType> volume_type;
+  be<uint64_t> online_creator;
+  be<uint32_t> category;
+  uint8_t reserved2[0x20];
+  union {
+    XContentMediaData media_data;
+    XContentAvatarAssetData avatar_asset_data;
+  };
+  uint8_t device_id[0x14];
+  union {
+    be<uint16_t> display_name_raw[kNumLanguagesV1][128];
+    char16_t display_name_chars[kNumLanguagesV1][128];
+  };
+  union {
+    be<uint16_t> description_raw[kNumLanguagesV1][128];
+    char16_t description_chars[kNumLanguagesV1][128];
+  };
+  union {
+    be<uint16_t> publisher_raw[64];
+    char16_t publisher_chars[64];
+  };
+  union {
+    be<uint16_t> title_name_raw[64];
+    char16_t title_name_chars[64];
+  };
+  union {
+    XContentAttributes bits;
+    uint8_t as_byte;
+  } flags;
+  be<uint32_t> thumbnail_size;
+  be<uint32_t> title_thumbnail_size;
+  uint8_t thumbnail[kThumbLengthV2];
+  union {
+    be<uint16_t> display_name_ex_raw[kNumLanguagesV2 - kNumLanguagesV1][128];
+    char16_t display_name_ex_chars[kNumLanguagesV2 - kNumLanguagesV1][128];
+  };
+  uint8_t title_thumbnail[kThumbLengthV2];
+  union {
+    be<uint16_t> description_ex_raw[kNumLanguagesV2 - kNumLanguagesV1][128];
+    char16_t description_ex_chars[kNumLanguagesV2 - kNumLanguagesV1][128];
+  };
+
+  std::u16string display_name(uint32_t lang_id) const {
+    lang_id--;
+    if (lang_id >= kNumLanguagesV2) {
+      assert_always();
+      lang_id = 0;  // no room for this lang, read from english slot..
+    }
+
+    const be<uint16_t>* str = 0;
+    if (lang_id >= 0 && lang_id < kNumLanguagesV1) {
+      str = display_name_raw[lang_id];
+    } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 &&
+               metadata_version >= 2) {
+      str = display_name_ex_raw[lang_id - kNumLanguagesV1];
+    }
+
+    if (!str) {
+      return u"";
+    }
+
+    return load_and_swap<std::u16string>(str);
+  }
+
+  std::u16string description(uint32_t lang_id) const {
+    lang_id--;
+    if (lang_id >= kNumLanguagesV2) {
+      assert_always();
+      lang_id = 0;  // no room for this lang, read from english slot..
+    }
+
+    const be<uint16_t>* str = 0;
+    if (lang_id >= 0 && lang_id < kNumLanguagesV1) {
+      str = description_raw[lang_id];
+    } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 &&
+               metadata_version >= 2) {
+      str = description_ex_raw[lang_id - kNumLanguagesV1];
+    }
+
+    if (!str) {
+      return u"";
+    }
+
+    return load_and_swap<std::u16string>(str);
+  }
+
+  std::u16string publisher() const {
+    return load_and_swap<std::u16string>(publisher_raw);
+  }
+
+  std::u16string title_name() const {
+    return load_and_swap<std::u16string>(title_name_raw);
+  }
+
+  bool set_display_name(uint32_t lang_id, const std::u16string_view value) {
+    lang_id--;
+    if (lang_id >= kNumLanguagesV2) {
+      assert_always();
+      lang_id = 0;  // no room for this lang, store in english slot..
+    }
+
+    char16_t* str = 0;
+    if (lang_id >= 0 && lang_id < kNumLanguagesV1) {
+      str = display_name_chars[lang_id];
+    } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 &&
+               metadata_version >= 2) {
+      str = display_name_ex_chars[lang_id - kNumLanguagesV1];
+    }
+
+    if (!str) {
+      return false;
+    }
+
+    string_util::copy_and_swap_truncating(str, value,
+                                          countof(display_name_chars[0]));
+    return true;
+  }
+
+  bool set_description(uint32_t lang_id, const std::u16string_view value) {
+    lang_id--;
+    if (lang_id >= kNumLanguagesV2) {
+      assert_always();
+      lang_id = 0;  // no room for this lang, store in english slot..
+    }
+
+    char16_t* str = 0;
+    if (lang_id >= 0 && lang_id < kNumLanguagesV1) {
+      str = description_chars[lang_id];
+    } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 &&
+               metadata_version >= 2) {
+      str = description_ex_chars[lang_id - kNumLanguagesV1];
+    }
+
+    if (!str) {
+      return false;
+    }
+
+    string_util::copy_and_swap_truncating(str, value,
+                                          countof(description_chars[0]));
+    return true;
+  }
+
+  bool set_publisher(const std::u16string_view value) {
+    string_util::copy_and_swap_truncating(publisher_chars, value,
+                                          countof(publisher_chars));
+    return true;
+  }
+
+  bool set_title_name(const std::u16string_view value) {
+    string_util::copy_and_swap_truncating(title_name_chars, value,
+                                          countof(title_name_chars));
+    return true;
+  }
+});
+static_assert_size(XContentMetadata, 0x93D6);
+
+struct XContentLicense {
+  be<uint64_t> licensee_id;
+  be<uint32_t> license_bits;
+  be<uint32_t> license_flags;
+};
+static_assert_size(XContentLicense, 0x10);
+
+XEPACKEDSTRUCT(XContentHeader, {
+  be<XContentPackageType> magic;
+  uint8_t signature[0x228];
+  XContentLicense licenses[0x10];
+  uint8_t content_id[0x14];
+  be<uint32_t> header_size;
+});
+static_assert_size(XContentHeader, 0x344);
+
+struct StfsHeader {
+  XContentHeader header;
+  XContentMetadata metadata;
+  // TODO: title/system updates contain more data after XContentMetadata, seems
+  // to affect header.header_size
+};
+static_assert_size(StfsHeader, 0x971A);

 class StfsContainerDevice : public Device {
 public:
@@ -187,6 +422,7 @@ class StfsContainerDevice : public Device {

 private:
  const uint32_t kSectorSize = 0x1000;
+  const uint32_t kBlocksPerHashLevel[3] = {170, 28900, 4913000};

  enum class Error {
    kSuccess = 0,
@@ -196,18 +432,17 @@ class StfsContainerDevice : public Device {
    kErrorDamagedFile = -31,
  };

-  struct BlockHash {
-    uint32_t next_block_index;
-    uint32_t info;
+  enum class SvodLayoutType {
+    kUnknown = 0x0,
+    kEnhancedGDF = 0x1,
+    kXSF = 0x2,
+    kSingleFile = 0x4,
  };

-  const uint32_t kSTFSHashSpacing = 170;
-
+  uint32_t ReadMagic(const std::filesystem::path& path);
  bool ResolveFromFolder(const std::filesystem::path& path);

  Error MapFiles();
-  static Error ReadPackageType(const uint8_t* map_ptr, size_t map_size,
-                               StfsPackageType* package_type_out);
  Error ReadHeaderAndVerify(const uint8_t* map_ptr, size_t map_size);

  Error ReadSVOD();
@@ -218,8 +453,8 @@ class StfsContainerDevice : public Device {
  Error ReadSTFS();
  size_t BlockToOffsetSTFS(uint64_t block);

-  BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index,
-                         uint32_t table_offset);
+  StfsHashEntry GetBlockHash(const uint8_t* map_ptr, uint32_t block_index,
+                             uint32_t table_offset);

  std::string name_;
  std::filesystem::path host_path_;
@@ -229,8 +464,8 @@ class StfsContainerDevice : public Device {
  size_t base_offset_;
  size_t magic_offset_;
  std::unique_ptr<Entry> root_entry_;
-  StfsPackageType package_type_;
  StfsHeader header_;
+  SvodLayoutType svod_layout_;
  uint32_t table_size_shift_;
 };