Add a setting to control whether to respect advisory rate limits (#1342)

DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs

@@ -27,8 +27,19 @@ public abstract class DiscordCommandBase : ICommand
     )]
     public bool IsBotToken { get; init; } = false;
 
+    [CommandOption(
+        "respect-rate-limits",
+        Description = "Whether to respect advisory rate limits. "
+            + "If disabled, only hard rate limits (i.e. 429 responses) will be respected."
+    )]
+    public bool ShouldRespectRateLimits { get; init; } = true;
+
     private DiscordClient? _discordClient;
-    protected DiscordClient Discord => _discordClient ??= new DiscordClient(Token);
+    protected DiscordClient Discord =>
+        _discordClient ??= new DiscordClient(
+            Token,
+            ShouldRespectRateLimits ? RateLimitPreference.RespectAll : RateLimitPreference.IgnoreAll
+        );
 
     public virtual ValueTask ExecuteAsync(IConsole console)
     {

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -18,7 +18,10 @@ using JsonExtensions.Reading;
 
 namespace DiscordChatExporter.Core.Discord;
 
-public class DiscordClient(string token)
+public class DiscordClient(
+    string token,
+    RateLimitPreference rateLimitPreference = RateLimitPreference.RespectAll
+)
 {
     private readonly Uri _baseUri = new("https://discord.com/api/v10/", UriKind.Absolute);
     private TokenKind? _resolvedTokenKind;
@@ -47,33 +50,41 @@ public class DiscordClient(string token)
                     innerCancellationToken
                 );
 
-                // If this was the last request available before hitting the rate limit,
+                // Discord has advisory rate limits (communicated via response headers), but they are typically
-                // wait out the reset time so that future requests can succeed.
+                // way stricter than the actual rate limits enforced by the server.
-                // This may add an unnecessary delay in case the user doesn't intend to
+                // The user may choose to ignore the advisory rate limits and only retry on hard rate limits,
-                // make any more requests, but implementing a smarter solution would
+                // if they want to prioritize speed over compliance (and safety of their account/bot).
-                // require properly keeping track of Discord's global/per-route/per-resource
+                // https://github.com/Tyrrrz/DiscordChatExporter/issues/1021
-                // rate limits and that's just way too much effort.
+                if (rateLimitPreference.IsRespectedFor(tokenKind))
-                // https://discord.com/developers/docs/topics/rate-limits
-                var remainingRequestCount = response
-                    .Headers.TryGetValue("X-RateLimit-Remaining")
-                    ?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
-
-                var resetAfterDelay = response
-                    .Headers.TryGetValue("X-RateLimit-Reset-After")
-                    ?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
-                    .Pipe(TimeSpan.FromSeconds);
-
-                if (remainingRequestCount <= 0 && resetAfterDelay is not null)
                 {
-                    var delay =
+                    var remainingRequestCount = response
-                        // Adding a small buffer to the reset time reduces the chance of getting
+                        .Headers.TryGetValue("X-RateLimit-Remaining")
-                        // rate limited again, because it allows for more requests to be released.
+                        ?.Pipe(s => int.Parse(s, CultureInfo.InvariantCulture));
-                        (resetAfterDelay.Value + TimeSpan.FromSeconds(1))
-                        // Sometimes Discord returns an absurdly high value for the reset time, which
-                        // is not actually enforced by the server. So we cap it at a reasonable value.
-                        .Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60));
 
-                    await Task.Delay(delay, innerCancellationToken);
+                    var resetAfterDelay = response
+                        .Headers.TryGetValue("X-RateLimit-Reset-After")
+                        ?.Pipe(s => double.Parse(s, CultureInfo.InvariantCulture))
+                        .Pipe(TimeSpan.FromSeconds);
+
+                    // If this was the last request available before hitting the rate limit,
+                    // wait out the reset time so that future requests can succeed.
+                    // This may add an unnecessary delay in case the user doesn't intend to
+                    // make any more requests, but implementing a smarter solution would
+                    // require properly keeping track of Discord's global/per-route/per-resource
+                    // rate limits and that's just way too much effort.
+                    // https://discord.com/developers/docs/topics/rate-limits
+                    if (remainingRequestCount <= 0 && resetAfterDelay is not null)
+                    {
+                        var delay =
+                            // Adding a small buffer to the reset time reduces the chance of getting
+                            // rate limited again, because it allows for more requests to be released.
+                            (resetAfterDelay.Value + TimeSpan.FromSeconds(1))
+                            // Sometimes Discord returns an absurdly high value for the reset time, which
+                            // is not actually enforced by the server. So we cap it at a reasonable value.
+                            .Clamp(TimeSpan.Zero, TimeSpan.FromSeconds(60));
+
+                        await Task.Delay(delay, innerCancellationToken);
+                    }
                 }
 
                 return response;

DiscordChatExporter.Core/Discord/RateLimitPreference.cs

@@ -0,0 +1,36 @@
+using System;
+
+namespace DiscordChatExporter.Core.Discord;
+
+[Flags]
+public enum RateLimitPreference
+{
+    IgnoreAll = 0,
+    RespectForUserTokens = 0b1,
+    RespectForBotTokens = 0b10,
+    RespectAll = RespectForUserTokens | RespectForBotTokens,
+}
+
+public static class RateLimitPreferenceExtensions
+{
+    internal static bool IsRespectedFor(
+        this RateLimitPreference rateLimitPreference,
+        TokenKind tokenKind
+    ) =>
+        tokenKind switch
+        {
+            TokenKind.User => (rateLimitPreference & RateLimitPreference.RespectForUserTokens) != 0,
+            TokenKind.Bot => (rateLimitPreference & RateLimitPreference.RespectForBotTokens) != 0,
+            _ => throw new ArgumentOutOfRangeException(nameof(tokenKind)),
+        };
+
+    public static string GetDisplayName(this RateLimitPreference rateLimitPreference) =>
+        rateLimitPreference switch
+        {
+            RateLimitPreference.IgnoreAll => "Always ignore",
+            RateLimitPreference.RespectForUserTokens => "Respect for user tokens",
+            RateLimitPreference.RespectForBotTokens => "Respect for bot tokens",
+            RateLimitPreference.RespectAll => "Always respect",
+            _ => throw new ArgumentOutOfRangeException(nameof(rateLimitPreference)),
+        };
+}

DiscordChatExporter.Gui/Converters/RateLimitPreferenceToStringConverter.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Globalization;
+using Avalonia.Data.Converters;
+using DiscordChatExporter.Core.Discord;
+
+namespace DiscordChatExporter.Gui.Converters;
+
+public class RateLimitPreferenceToStringConverter : IValueConverter
+{
+    public static RateLimitPreferenceToStringConverter Instance { get; } = new();
+
+    public object? Convert(
+        object? value,
+        Type targetType,
+        object? parameter,
+        CultureInfo culture
+    ) =>
+        value is RateLimitPreference rateLimitPreference
+            ? rateLimitPreference.GetDisplayName()
+            : default;
+
+    public object ConvertBack(
+        object? value,
+        Type targetType,
+        object? parameter,
+        CultureInfo culture
+    ) => throw new NotSupportedException();
+}

DiscordChatExporter.Gui/Services/SettingsService.cs

@@ -3,6 +3,7 @@ using System.IO;
 using System.Text.Json.Serialization;
 using Cogwheel;
 using CommunityToolkit.Mvvm.ComponentModel;
+using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Exporting;
 using DiscordChatExporter.Gui.Framework;
 using DiscordChatExporter.Gui.Models;
@@ -28,6 +29,10 @@ public partial class SettingsService()
     [ObservableProperty]
     public partial bool IsTokenPersisted { get; set; } = true;
 
+    [ObservableProperty]
+    public partial RateLimitPreference RateLimitPreference { get; set; } =
+        RateLimitPreference.RespectAll;
+
     [ObservableProperty]
     public partial ThreadInclusionMode ThreadInclusionMode { get; set; }
 

DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs

@@ -123,7 +123,7 @@ public partial class DashboardViewModel : ViewModelBase
             AvailableChannels = null;
             SelectedChannels.Clear();
 
-            _discord = new DiscordClient(token);
+            _discord = new DiscordClient(token, _settingsService.RateLimitPreference);
             _settingsService.LastToken = token;
 
             var guilds = await _discord.GetUserGuildsAsync();

DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Utils.Extensions;
 using DiscordChatExporter.Gui.Framework;
 using DiscordChatExporter.Gui.Models;
@@ -42,6 +43,15 @@ public class SettingsViewModel : DialogViewModelBase
         set => _settingsService.IsTokenPersisted = value;
     }
 
+    public IReadOnlyList<RateLimitPreference> AvailableRateLimitPreferences { get; } =
+        Enum.GetValues<RateLimitPreference>();
+
+    public RateLimitPreference RateLimitPreference
+    {
+        get => _settingsService.RateLimitPreference;
+        set => _settingsService.RateLimitPreference = value;
+    }
+
     public IReadOnlyList<ThreadInclusionMode> AvailableThreadInclusionModes { get; } =
         Enum.GetValues<ThreadInclusionMode>();
 

DiscordChatExporter.Gui/Views/Dialogs/SettingsView.axaml

@@ -54,6 +54,25 @@
                         <ToggleSwitch DockPanel.Dock="Right" IsChecked="{Binding IsTokenPersisted}" />
                     </DockPanel>
 
+                    <!--  Rate limit preference  -->
+                    <DockPanel
+                        Margin="16,8"
+                        LastChildFill="False"
+                        ToolTip.Tip="Whether to respect advisory rate limits. If disabled, only hard rate limits (i.e. 429 responses) will be respected.">
+                        <TextBlock DockPanel.Dock="Left" Text="Rate limit preference" />
+                        <ComboBox
+                            Width="150"
+                            DockPanel.Dock="Right"
+                            ItemsSource="{Binding AvailableRateLimitPreferences}"
+                            SelectedItem="{Binding RateLimitPreference}">
+                            <ComboBox.ItemTemplate>
+                                <DataTemplate>
+                                    <TextBlock Text="{Binding Converter={x:Static converters:RateLimitPreferenceToStringConverter.Instance}}" />
+                                </DataTemplate>
+                            </ComboBox.ItemTemplate>
+                        </ComboBox>
+                    </DockPanel>
+
                     <!--  Thread inclusion mode  -->
                     <DockPanel
                         Margin="16,8"