From a317fa2e322a0afa24ef71c570928325d68ef696 Mon Sep 17 00:00:00 2001
From: Rik De Peuter <rikdepeuter@gmail.com>
Date: Thu, 24 Oct 2024 08:55:24 +0200
Subject: [PATCH] fix: deal with 429 and extra logging

---
 src/Ops.Tickets.Console/Program.cs | 111 ++++++++++++++++++-----------
 1 file changed, 71 insertions(+), 40 deletions(-)

diff --git a/src/Ops.Tickets.Console/Program.cs b/src/Ops.Tickets.Console/Program.cs
index d141c6f..e91257c 100644
--- a/src/Ops.Tickets.Console/Program.cs
+++ b/src/Ops.Tickets.Console/Program.cs
@@ -6,6 +6,7 @@
     using System.Globalization;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Net.Http.Json;
     using System.Text;
@@ -60,54 +61,81 @@ await Parallel.ForEachAsync(
                     tickets,
                     new ParallelOptions { MaxDegreeOfParallelism = 5, CancellationToken = cts.Token },
                     async (ticket, token) =>
-                {
-                    if(token.IsCancellationRequested)
-                        return;
-
-                    Interlocked.Increment(ref countProcessed);
-                    Console.WriteLine($"Processing ticket: {ticket.TicketUrl} ({Math.Round((double)countProcessed / tickets.Count * 100, 2)}%)");
-
-                    try
                     {
-                        var done = false;
-                        while (!done)
-                        {
-                            var response = await client.GetFromJsonAsync<TicketResponse>(ticket.TicketUrl, token);
-
-                            done = response!.Status.Equals("complete") || response.Status.Equals("error");
-                            if (!done)
-                            {
-                                Console.WriteLine($"Ticket has status {response.Status}. Waiting 2 seconds.");
-                                await Task.Delay(TimeSpan.FromSeconds(2), token);
-                                continue;
-                            }
+                        if (token.IsCancellationRequested)
+                            return;
 
-                            if (response.Status.Equals("complete"))
-                            {
-                                completedTickets.Add(ticket);
-                            }
+                        Interlocked.Increment(ref countProcessed);
+                        Console.WriteLine($"Processing ticket: {ticket.TicketUrl} ({Math.Round((double)countProcessed / tickets.Count * 100, 2)}%)");
 
-                            if (response.Status.Equals("error"))
+                        try
+                        {
+                            var done = false;
+                            while (!done)
                             {
-                                errorTickets.Add(new TicketErrorRecord(ticket.Id, ticket.TicketUrl, response.Result.Json));
+                                try
+                                {
+                                    var response = await client.GetFromJsonAsync<TicketResponse>(ticket.TicketUrl, token);
+
+                                    done = response!.Status.Equals("complete") || response.Status.Equals("error");
+                                    if (!done)
+                                    {
+                                        Console.WriteLine($"Ticket has status {response.Status}. Waiting 2 seconds.");
+                                        await Task.Delay(TimeSpan.FromSeconds(2), token);
+                                        continue;
+                                    }
+
+                                    if (response.Status.Equals("complete"))
+                                    {
+                                        completedTickets.Add(ticket);
+                                    }
+
+                                    if (response.Status.Equals("error"))
+                                    {
+                                        errorTickets.Add(new TicketErrorRecord(ticket.Id, ticket.TicketUrl, response.Result.Json));
+                                    }
+                                }
+                                catch (WebException ex)
+                                {
+                                    if (ex.Response is HttpWebResponse response
+                                        && response.StatusCode == HttpStatusCode.TooManyRequests)
+                                    {
+                                        await Task.Delay(50);
+                                        continue;
+                                    }
+
+                                    throw;
+                                }
+
+                                done = true;
                             }
-
-                            done = true;
                         }
-
-                    }
-                    catch (Exception e)
-                    {
-                        Console.WriteLine($"Failed to process id: {ticket}");
-                        Console.WriteLine(e);
-                        await cts.CancelAsync();
-                        throw;
-                    }
-                });
+                        catch (Exception e)
+                        {
+                            Console.WriteLine($"Failed to process id: {ticket.TicketUrl}");
+                            Console.WriteLine(e);
+                            await cts.CancelAsync();
+                            throw;
+                        }
+                    });
             }
             finally
             {
-                var completedTicketFileStream = new FileStream(Path.Join(outputCsvFolder, "completed.csv"), FileMode.CreateNew);
+                Directory.CreateDirectory(outputCsvFolder);
+
+                var completedPath = Path.Join(outputCsvFolder, "completed.csv");
+                if (File.Exists(completedPath))
+                {
+                    File.Delete(completedPath);
+                }
+
+                var errorPath = Path.Join(outputCsvFolder, "error.csv");
+                if (File.Exists(errorPath))
+                {
+                    File.Delete(errorPath);
+                }
+
+                var completedTicketFileStream = new FileStream(completedPath, FileMode.CreateNew);
                 await using var completedTicketStreamWriter = new StreamWriter(completedTicketFileStream);
                 await using var completedTicketCsvWriter = new CsvWriter(completedTicketStreamWriter, CsvConfiguration);
 
@@ -115,9 +143,12 @@ await Parallel.ForEachAsync(
                 await completedTicketCsvWriter.WriteRecordsAsync(completedTickets, CancellationToken.None);
                 await completedTicketCsvWriter.FlushAsync();
 
+                Console.WriteLine($"Completed tickets: {completedTickets.Count}");
+                Console.WriteLine($"Error tickets: {errorTickets.Count}");
+
                 if (errorTickets.Any())
                 {
-                    var errorTicketFileStream = new FileStream(Path.Join(outputCsvFolder, "error.csv"), FileMode.CreateNew);
+                    var errorTicketFileStream = new FileStream(errorPath, FileMode.CreateNew);
                     await using var errorTicketStreamWriter = new StreamWriter(errorTicketFileStream);
                     await using var errorTicketCsvWriter = new CsvWriter(errorTicketStreamWriter, CsvConfiguration);