using System; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Http; using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; using TwitchDownloaderCore.Interfaces; using TwitchDownloaderCore.Models; namespace TwitchDownloaderCore.Tools { public static class DownloadTools { /// /// Downloads the requested to the without storing it in memory. /// /// The to perform the download operation. /// The url of the file to download. /// The path to the file where download will be saved. /// Path to a file whose contents will be written to the start of the destination file. /// The maximum download speed in kibibytes per second, or -1 for no maximum. /// Logger. /// A containing a to cancel the operation. /// The expected length of the downloaded file, or -1 if the content length header is not present. /// The may be canceled by this method. public static async Task DownloadFileAsync(HttpClient httpClient, Uri url, string destinationFile, [AllowNull] string headerFile, int throttleKib, ITaskLogger logger, CancellationTokenSource cancellationTokenSource = null) { using var request = new HttpRequestMessage(HttpMethod.Get, url); // For some reason tryUnmute fails when we don't explicitly define Accept-Encoding request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("deflate")); request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("br")); var cancellationToken = cancellationTokenSource?.Token ?? CancellationToken.None; using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var fileMode = FileMode.Create; if (!string.IsNullOrWhiteSpace(headerFile)) { await using var headerFs = new FileStream(headerFile, FileMode.Open, FileAccess.Read, FileShare.Read); await using var destinationFs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read); await headerFs.CopyToAsync(destinationFs, cancellationToken); fileMode = FileMode.Append; } // Why are we setting a CTS CancelAfter timer? See lay295#265 const int SIXTY_SECONDS = 60; if (throttleKib == -1 || !response.Content.Headers.ContentLength.HasValue) { cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(SIXTY_SECONDS)); } else { const double ONE_KIBIBYTE = 1024d; cancellationTokenSource?.CancelAfter(TimeSpan.FromSeconds(Math.Max( SIXTY_SECONDS, response.Content.Headers.ContentLength!.Value / ONE_KIBIBYTE / throttleKib * 8 // Allow up to 8x the shortest download time given the thread bandwidth ))); } switch (throttleKib) { case -1: { await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read); await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); break; } default: { try { await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken); await using var throttledStream = new ThrottledStream(contentStream, throttleKib); await using var fs = new FileStream(destinationFile, fileMode, FileAccess.Write, FileShare.Read); await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); } catch (IOException ex) when (ex.Message.Contains("EOF")) { // If we get an exception for EOF, it may be related to the throttler. Try again without it. logger.LogVerbose($"Unexpected EOF, retrying without bandwidth throttle. Message: {ex.Message}."); await Task.Delay(2_000, cancellationToken); goto case -1; } break; } } // Reset the cts timer so it can be reused for the next download on this thread. // Is there a friendlier way to do this? Yes. Does it involve creating and destroying 4,000 CancellationTokenSources that are almost never cancelled? Also Yes. cancellationTokenSource?.CancelAfter(TimeSpan.FromMilliseconds(uint.MaxValue - 1)); return response.Content.Headers.ContentLength ?? -1; } /// /// Some old twitch VODs have files with a query string at the end such as 1.ts?offset=blah which isn't a valid filename /// public static string RemoveQueryString(string inputString) { var queryIndex = inputString.IndexOf('?'); if (queryIndex == -1) { return inputString; } return inputString[..queryIndex]; } } }