Skip to content

Commit

Permalink
File Downloader Submodule Major Refactor (Hi3Helper.Http) (#563)
Browse files Browse the repository at this point in the history 
# Main Goal
As per title, this PR contains some huge changes to our own File
Downloader Submodule implementation that we use for Collapse called:
Hi3Helper.Http.
The changes brings some new features to improve the reliability of the
downloading process for certain parts, like Game Installation/Update,
Game Repair, Cache Updates and Internal APIs. The changes that worth to
mention are including:
- **[New]** Bandwidth/Speed Limiter
  With this new feature, the user can now set the download speed limit.
- **[New]** Download File Size Pre-allocation.
Thanks to this feature, the download file would not need to have merged
before the file can be used.
- **[Imp][Performance]** Reduce unnecessary ``async`` thread overhead.
- **[Imp]** Use delegates in replacement for event-based
``DownloadEvent`` to notify the download progress.
This has benefits if you're performing download multiple files within a
single Downloader instance for other purposes. By using delegates, you
can subscribe the download progress to a specific file you wanted to
track.
- **[Imp]** Changing the language specification from C# 8.0 to C# 9.0.

However, this PR requires some changes in some parts, including:
- [x] Game Repair
  - [x] Genshin Impact
  - [x] Honkai Impact 3rd
  - [x] Honkai: Star Rail
- [x] Cache Update
  - [x] Honkai Impact 3rd
  - [x] Honkai: Star Rail
- [x] Launcher API/Sprites Loader
  - [x] HoYoPlay
  - [x] Sophon
- [x] ``FallbackCDNUtil``
- [x] Game Conversion Utils
- [x] ``InstallManagerBase``
  - [x] Genshin Impact
  - [x] Honkai Impact 3rd
  - [x] Honkai: Star Rail
  - [x] Zenless Zone Zero
- [x] Launcher Updater

## PR Status :
- Overall Status : Completed
- Commits : Completed
- Synced to base (Collapse:main) : Not yet
- Build status : OK
- Crashing : No
- Bug found caused by PR : -

### Templates

<details>
  <summary>Changelog Prefixes</summary>
  
  ```
    **[New]**
    **[Imp]**
    **[Fix]**
    **[Loc]**
    **[Doc]**
  ```

</details>
  • Loading branch information
@neon-nyan
neon-nyan authored Aug 31, 2024
2 parents 622a3b9 + e210969 commit 71699b0
Show file tree
Hide file tree
Showing 43 changed files with 2,049 additions and 1,113 deletions.
94 changes: 42 additions & 52 deletions CollapseLauncher/Classes/CachesManagement/Honkai/Fetch.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,66 +25,56 @@ internal partial class HonkaiCache
private async Task<List<CacheAsset>> Fetch(CancellationToken token)
{
// Initialize asset index for the return
List<CacheAsset> returnAsset = new();
List<CacheAsset> returnAsset = [];

// Initialize new proxy-aware HttpClient
using HttpClient httpClientNew = new HttpClientBuilder()
.UseLauncherConfig(_downloadThreadCount + 16)
.UseLauncherConfig(_downloadThreadCount + _downloadThreadCountReserved)
.SetUserAgent(_userAgent)
.SetAllowedDecompression(DecompressionMethods.None)
.Create();

// Use HttpClient instance on fetching
using Http httpClient = new Http(true, 5, 1000, _userAgent, httpClientNew);
try
{
// Subscribe the event listener
httpClient.DownloadProgress += _httpClient_FetchAssetProgress;
// Use a new DownloadClient for fetching
DownloadClient downloadClient = DownloadClient.CreateInstance(httpClientNew);

// Build _gameRepoURL from loading Dispatcher and Gateway
await BuildGameRepoURL(token);
// Build _gameRepoURL from loading Dispatcher and Gateway
await BuildGameRepoURL(downloadClient, token);

// Iterate type and do fetch
foreach (CacheAssetType type in Enum.GetValues<CacheAssetType>())
// Iterate type and do fetch
foreach (CacheAssetType type in Enum.GetValues<CacheAssetType>())
{
// Skip for unused type
switch (type)
{
// Skip for unused type
switch (type)
{
case CacheAssetType.Unused:
case CacheAssetType.Dispatcher:
case CacheAssetType.Gateway:
case CacheAssetType.General:
case CacheAssetType.IFix:
case CacheAssetType.DesignData:
case CacheAssetType.Lua:
continue;
}
case CacheAssetType.Unused:
case CacheAssetType.Dispatcher:
case CacheAssetType.Gateway:
case CacheAssetType.General:
case CacheAssetType.IFix:
case CacheAssetType.DesignData:
case CacheAssetType.Lua:
continue;
}

// uint = Count of the assets available
// long = Total size of the assets available
(int, long) count = await FetchByType(type, httpClient, returnAsset, token);
// uint = Count of the assets available
// long = Total size of the assets available
(int, long) count = await FetchByType(type, downloadClient, returnAsset, token);

// Write a log about the metadata
LogWriteLine($"Cache Metadata [T: {type}]:", LogType.Default, true);
LogWriteLine($" Cache Count = {count.Item1}", LogType.NoTag, true);
LogWriteLine($" Cache Size = {SummarizeSizeSimple(count.Item2)}", LogType.NoTag, true);
// Write a log about the metadata
LogWriteLine($"Cache Metadata [T: {type}]:", LogType.Default, true);
LogWriteLine($" Cache Count = {count.Item1}", LogType.NoTag, true);
LogWriteLine($" Cache Size = {SummarizeSizeSimple(count.Item2)}", LogType.NoTag, true);

// Increment the Total Size and Count
_progressAllCountTotal += count.Item1;
_progressAllSizeTotal += count.Item2;
}
}
finally
{
// Unsubscribe the event listener and dispose Http client
httpClient.DownloadProgress -= _httpClient_FetchAssetProgress;
// Increment the Total Size and Count
_progressAllCountTotal += count.Item1;
_progressAllSizeTotal += count.Item2;
}

// Return asset index
return returnAsset;
}

private async Task BuildGameRepoURL(CancellationToken token)
private async Task BuildGameRepoURL(DownloadClient downloadClient, CancellationToken token)
{
KianaDispatch dispatch = null;
Exception lastException = null;
Expand All @@ -93,7 +83,7 @@ private async Task BuildGameRepoURL(CancellationToken token)
{
try
{
// Init the key and decrypt it if exist.
// Init the key and decrypt it if existed.
if (string.IsNullOrEmpty(_gameVersionManager.GamePreset.DispatcherKey))
{
throw new NullReferenceException("Dispatcher key is null or empty!");
Expand All @@ -102,7 +92,7 @@ private async Task BuildGameRepoURL(CancellationToken token)
string key = _gameVersionManager.GamePreset.DispatcherKey;

// Try assign dispatcher
dispatch = await KianaDispatch.GetDispatch(baseURL,
dispatch = await KianaDispatch.GetDispatch(downloadClient, baseURL,
_gameVersionManager.GamePreset.GameDispatchURLTemplate,
_gameVersionManager.GamePreset.GameDispatchChannelName,
key, _gameVersion.VersionArray, token);
Expand All @@ -120,13 +110,13 @@ private async Task BuildGameRepoURL(CancellationToken token)

// Get gatewayURl and fetch the gateway
_gameGateway =
await KianaDispatch.GetGameserver(dispatch!, _gameVersionManager.GamePreset.GameGatewayDefault!, token);
await KianaDispatch.GetGameserver(downloadClient, dispatch!, _gameVersionManager.GamePreset.GameGatewayDefault!, token);
_gameRepoURL = BuildAssetBundleURL(_gameGateway);
}

private string BuildAssetBundleURL(KianaDispatch gateway) => CombineURLFromString(gateway!.AssetBundleUrls![0], "/{0}/editor_compressed/");
private static string BuildAssetBundleURL(KianaDispatch gateway) => CombineURLFromString(gateway!.AssetBundleUrls![0], "/{0}/editor_compressed/");

private async Task<(int, long)> FetchByType(CacheAssetType type, Http httpClient, List<CacheAsset> assetIndex, CancellationToken token)
private async Task<(int, long)> FetchByType(CacheAssetType type, DownloadClient downloadClient, List<CacheAsset> assetIndex, CancellationToken token)
{
// Set total activity string as "Fetching Caches Type: <type>"
_status!.ActivityStatus = string.Format(Lang!._CachesPage!.CachesStatusFetchingType!, type);
Expand All @@ -145,9 +135,9 @@ private async Task BuildGameRepoURL(CancellationToken token)

// Get a direct HTTP Stream
await using HttpResponseInputStream remoteStream = await HttpResponseInputStream.CreateStreamAsync(
httpClient.GetHttpClient(), assetIndexURL, null, null, token);
downloadClient.GetHttpClient(), assetIndexURL, null, null, null, null, null, token);

using XORStream stream = new XORStream(remoteStream);
await using XORStream stream = new XORStream(remoteStream);

// Build the asset index and return the count and size of each type
(int, long) returnValue = await BuildAssetIndex(type, baseURL, stream, assetIndex, token);
Expand Down Expand Up @@ -263,7 +253,7 @@ private IEnumerable<CacheAsset> EnumerateCacheTextAsset(CacheAssetType type, IEn

// Initialize local HTTP client
using HttpClient client = new HttpClientBuilder()
.UseLauncherConfig(_downloadThreadCount + 16)
.UseLauncherConfig(_downloadThreadCount + _downloadThreadCountReserved)
.SetUserAgent(_userAgent)
.SetAllowedDecompression(DecompressionMethods.None)
.Create();
Expand Down Expand Up @@ -339,16 +329,16 @@ private bool IsValidRegionFile(string input, string lang)
public KianaDispatch GetCurrentGateway() => _gameGateway;

public async Task<(List<CacheAsset>, string, string, int)> GetCacheAssetList(
Http httpClient, CacheAssetType type, CancellationToken token)
DownloadClient downloadClient, CacheAssetType type, CancellationToken token)
{
// Initialize asset index for the return
List<CacheAsset> returnAsset = new();

// Build _gameRepoURL from loading Dispatcher and Gateway
await BuildGameRepoURL(token);
await BuildGameRepoURL(downloadClient, token);

// Fetch the progress
_ = await FetchByType(type, httpClient, returnAsset, token);
_ = await FetchByType(type, downloadClient, returnAsset, token);

// Return the list and base asset bundle repo URL
return (returnAsset, _gameGateway!.ExternalAssetUrls!.FirstOrDefault(), BuildAssetBundleURL(_gameGateway),
Expand Down
66 changes: 7 additions & 59 deletions CollapseLauncher/Classes/CachesManagement/Honkai/Update.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using CollapseLauncher.Helper;
using CollapseLauncher.Interfaces;
using Hi3Helper;
using Hi3Helper.Data;
using Hi3Helper.Http;
using System;
using System.Collections.Generic;
Expand All @@ -22,21 +21,19 @@ private async Task<bool> Update(List<CacheAsset> updateAssetIndex, List<CacheAss
{
// Initialize new proxy-aware HttpClient
using HttpClient client = new HttpClientBuilder<SocketsHttpHandler>()
.UseLauncherConfig(_downloadThreadCount + 16)
.UseLauncherConfig(_downloadThreadCount + _downloadThreadCountReserved)
.SetUserAgent(_userAgent)
.SetAllowedDecompression(DecompressionMethods.None)
.Create();

// Assign Http client
Http httpClient = new Http(true, 5, 1000, _userAgent, client);
// Use the new DownloadClient instance
DownloadClient downloadClient = DownloadClient.CreateInstance(client);
try
{
// Set IsProgressAllIndetermined as false and update the status
_status!.IsProgressAllIndetermined = true;
UpdateStatus();

// Subscribe the event listener
httpClient.DownloadProgress += _httpClient_UpdateAssetProgress;
// Iterate the asset index and do update operation
ObservableCollection<IAssetProperty> assetProperty = new ObservableCollection<IAssetProperty>(AssetEntry);
if (_isBurstDownloadEnabled)
Expand All @@ -52,7 +49,7 @@ await Parallel.ForEachAsync(
new ParallelOptions { CancellationToken = token, MaxDegreeOfParallelism = _downloadThreadCount },
async (asset, innerToken) =>
{
await UpdateCacheAsset(asset, httpClient, innerToken);
await UpdateCacheAsset(asset, downloadClient, _httpClient_UpdateAssetProgress, innerToken);
});
}
else
Expand All @@ -66,7 +63,7 @@ await Parallel.ForEachAsync(
#endif
, assetProperty))
{
await UpdateCacheAsset(asset, httpClient, token);
await UpdateCacheAsset(asset, downloadClient, _httpClient_UpdateAssetProgress, token);
}
}

Expand All @@ -82,11 +79,6 @@ await Parallel.ForEachAsync(
LogWriteLine($"An error occured while updating cache file!\r\n{ex}", LogType.Error, true);
throw;
}
finally
{
// Unsubscribe the event listener and dispose Http client
httpClient.DownloadProgress -= _httpClient_UpdateAssetProgress;
}
}

private void UpdateCacheVerifyList(List<CacheAsset> assetIndex)
Expand All @@ -110,7 +102,7 @@ private void UpdateCacheVerifyList(List<CacheAsset> assetIndex)
}
}

private async Task UpdateCacheAsset((CacheAsset AssetIndex, IAssetProperty AssetProperty) asset, Http httpClient, CancellationToken token)
private async Task UpdateCacheAsset((CacheAsset AssetIndex, IAssetProperty AssetProperty) asset, DownloadClient downloadClient, DownloadProgressDelegate downloadProgress, CancellationToken token)
{
// Increment total count and update the status
_progressAllCountCurrent++;
Expand All @@ -132,28 +124,11 @@ private async Task UpdateCacheAsset((CacheAsset AssetIndex, IAssetProperty Asset
// Other than unused file, do this action
else
{
// Assign and check the path of the asset directory
string assetDir = Path.GetDirectoryName(asset.AssetIndex.ConcatPath);
if (!Directory.Exists(assetDir))
{
Directory.CreateDirectory(assetDir!);
}

#if DEBUG
LogWriteLine($"Downloading cache [T: {asset.AssetIndex.DataType}]: {asset.AssetIndex.N} at URL: {asset.AssetIndex.ConcatURL}", LogType.Debug, true);
#endif

// Do multi-session download for asset that has applicable size
if (asset.AssetIndex.CS >= _sizeForMultiDownload && !_isBurstDownloadEnabled)
{
await httpClient!.Download(asset.AssetIndex.ConcatURL, asset.AssetIndex.ConcatPath, _downloadThreadCount, true, token);
await httpClient.Merge(token);
}
// Do single-session download for others
else
{
await httpClient!.Download(asset.AssetIndex.ConcatURL, asset.AssetIndex.ConcatPath, true, null, null, token);
}
await RunDownloadTask(asset.AssetIndex.CS, asset.AssetIndex.ConcatPath, asset.AssetIndex.ConcatURL, downloadClient, downloadProgress, token);

#if !DEBUG
LogWriteLine($"Downloaded cache [T: {asset.AssetIndex.DataType}]: {asset.AssetIndex.N}", LogType.Default, true);
Expand All @@ -163,32 +138,5 @@ private async Task UpdateCacheAsset((CacheAsset AssetIndex, IAssetProperty Asset
// Remove Asset Entry display
PopRepairAssetEntry(asset.AssetProperty);
}

private async void _httpClient_UpdateAssetProgress(object sender, DownloadEvent e)
{
// Update current progress percentages and speed
_progress!.ProgressAllPercentage = _progressAllSizeCurrent != 0 ?
ConverterTool.GetPercentageNumber(_progressAllSizeCurrent, _progressAllSizeTotal) :
0;

if (e!.State != DownloadState.Merging)
{
_progressAllSizeCurrent += e.Read;
}
long speed = (long)(_progressAllSizeCurrent / _stopwatch!.Elapsed.TotalSeconds);

if (await CheckIfNeedRefreshStopwatch())
{
// Update current activity status
_status!.IsProgressAllIndetermined = false;
string timeLeftString = string.Format(Lang!._Misc!.TimeRemainHMSFormat!, ((_progressAllSizeCurrent - _progressAllSizeTotal) / ConverterTool.Unzeroed(speed)).ToTimeSpanNormalized());
_status.ActivityAll = string.Format(Lang!._Misc!.Downloading + ": {0}/{1} ", _progressAllCountCurrent, _progressAllCountTotal)
+ string.Format($"({Lang._Misc.SpeedPerSec})", ConverterTool.SummarizeSizeSimple(speed))
+ $" | {timeLeftString}";

// Trigger update
UpdateAll();
}
}
}
}
Loading

0 comments on commit 71699b0

Please sign in to comment.