Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/CurseForgeCommunity/CFLookup
Browse files Browse the repository at this point in the history
  • Loading branch information
@itssimple
itssimple committed Dec 26, 2023
2 parents 0651c7e + 1f8a6b7 commit 89124c3
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 61 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/CI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ on:
push:
branches: [master]

concurrency:
group: environment-${{ github.ref }}-ci
cancel-in-progress: true

jobs:
generate:
runs-on: ubuntu-latest
Expand Down
146 changes: 104 additions & 42 deletions CFLookup/Jobs/GetLatestUpdatedModPerGame.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
using Hangfire;
using Hangfire.Server;
using Microsoft.Data.SqlClient;
using Newtonsoft.Json;
using StackExchange.Redis;
using System.Text;
using System.Text.Json;

namespace CFLookup.Jobs
{
Expand All @@ -16,8 +16,8 @@ public class GetLatestUpdatedModPerGame
{
public static async Task RunAsync(PerformContext context)
{
using(var scope = Program.ServiceProvider.CreateScope())
{
using (var scope = Program.ServiceProvider.CreateScope())
{
var cfClient = scope.ServiceProvider.GetRequiredService<ApiClient>();
var db = scope.ServiceProvider.GetRequiredService<MSSQLDB>();
var _redis = scope.ServiceProvider.GetRequiredService<ConnectionMultiplexer>();
Expand All @@ -28,11 +28,11 @@ public static async Task RunAsync(PerformContext context)

var allGames = new List<Game>();
var games = await cfClient.GetGamesAsync();
if(games != null && games.Pagination.ResultCount > 0)
if (games != null && games.Pagination.ResultCount > 0)
{
allGames.AddRange(games.Data);
var index = 0;
while(games.Pagination.ResultCount > 0)
while (games.Pagination.ResultCount > 0)
{
index += games.Pagination.PageSize;
games = await cfClient.GetGamesAsync(index);
Expand All @@ -46,51 +46,113 @@ public static async Task RunAsync(PerformContext context)
Mod? latestUpdatedModData = null;
CurseForge.APIClient.Models.Files.File? latestUpdatedFileData = null;

foreach(var game in allGames)
var privateGames = new List<int>
{
449, // Skyrim
540, // 7 Days To Die
4482, // Subnautica
4593, // Final Fantasy XV
4619, // The Last of Us
4819, // American Truck Simulator
6351, // Mario Party 3
18237, // Staxel
66004, // Starfield
73492, // Kerbal Space Program 2
78022, // Minecraft Bedrock
78023, // Timber and Stone
78072, // Hometopia
78101, // Tiny Life
78103, // art of rally
78163, // Astro Colony
78225, // The Anacrusis
78251, // Kingshunt
78496, // Hero's Tour
79630, // GTA-SA
79805, // CurseForge Demo
80345, // Dwerve
81975, // LEAP
82010, // KSP QA Test Game
82047, // Oaken
82164, // Unity SDK Tester
83357, // River City Girls 2
83374, // ARK
83375, // Far Cry 6
83444, // WorldBox - God Simulator
83453, // Minecraft Legends
83981, // Unreal Test Game
84529, // NighspadeTest001
84530, // OWITestGame
84610, // Test01
84658, // AI M3
84749, // Minecraft
84801, // stopdeletingmystuffiamtesting
84810, // Oaken_Testing
};

foreach (var privateGame in privateGames)
{
var game = await cfClient.GetGameAsync(privateGame);
if (game != null && game.Data != null)
{
allGames.Add(game.Data);
}
}

foreach (var game in allGames)
{
Console.WriteLine($"Starting to check for latest updated mod for {game.Name}");
await _db.StringSetAsync($"cf-game-{game.Id}", JsonSerializer.Serialize(game), TimeSpan.FromDays(1));

var latestUpdatedMod = await cfClient.SearchModsAsync(game.Id, sortField: ModsSearchSortField.LastUpdated, sortOrder: ModsSearchSortOrder.Descending, pageSize: 1);
if(latestUpdatedMod != null && latestUpdatedMod.Pagination.ResultCount > 0)
if (latestUpdatedMod != null && latestUpdatedMod.Pagination.ResultCount > 0)
{
var mod = latestUpdatedMod.Data.First();
var latestUpdatedFile = mod.LatestFiles.OrderByDescending(f => f.FileDate).First();
Console.WriteLine($"Latest updated mod for {game.Name} is {mod.Name} with {mod.DownloadCount} downloads and the latest file was updated {latestUpdatedFile.FileDate}");
if (lastUpdatedMod < latestUpdatedFile.FileDate)
var latestUpdatedFile = mod.LatestFiles.OrderByDescending(f => f.FileDate).FirstOrDefault();
if (latestUpdatedFile != null)
{
lastUpdatedMod = latestUpdatedFile.FileDate;
latestUpdatedModData = mod;
latestUpdatedFileData = latestUpdatedFile;
}

var existingGame = await db.ExecuteSingleRowAsync<FileProcessingStatus>(
"SELECT * FROM fileProcessingStatus WHERE gameId = @gameId",
new SqlParameter("@gameId", game.Id)
);

if(existingGame == null)
{
// New game, insert it
await db.ExecuteNonQueryAsync(
"INSERT INTO fileProcessingStatus (last_updated_utc, gameId, modId, fileId) VALUES (@last_updated_utc, @gameId, @modId, @fileId)",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@gameId", game.Id),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id)
Console.WriteLine($"Latest updated mod for {game.Name} is {mod.Name} with {mod.DownloadCount} downloads and the latest file was updated {latestUpdatedFile.FileDate}");
if (lastUpdatedMod < latestUpdatedFile.FileDate)
{
lastUpdatedMod = latestUpdatedFile.FileDate;
latestUpdatedModData = mod;
latestUpdatedFileData = latestUpdatedFile;
}

await _db.StringSetAsync($"cf-mod-{mod.Id}", JsonSerializer.Serialize(mod), TimeSpan.FromDays(1));
await _db.StringSetAsync($"cf-file-{latestUpdatedFile.Id}", JsonSerializer.Serialize(latestUpdatedFile), TimeSpan.FromDays(1));

var existingGame = await db.ExecuteSingleRowAsync<FileProcessingStatus>(
"SELECT * FROM fileProcessingStatus WHERE gameId = @gameId",
new SqlParameter("@gameId", game.Id)
);

if (existingGame == null)
{
// New game, insert it
await db.ExecuteNonQueryAsync(
"INSERT INTO fileProcessingStatus (last_updated_utc, gameId, modId, fileId) VALUES (@last_updated_utc, @gameId, @modId, @fileId)",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@gameId", game.Id),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id)
);
}
else
{
// Existing game, update it
await db.ExecuteNonQueryAsync(
"UPDATE fileProcessingStatus SET last_updated_utc = @last_updated_utc, modId = @modId, fileId = @fileId WHERE gameId = @gameId",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id),
new SqlParameter("@gameId", game.Id)
);
}
}
else
{
// Existing game, update it
await db.ExecuteNonQueryAsync(
"UPDATE fileProcessingStatus SET last_updated_utc = @last_updated_utc, modId = @modId, fileId = @fileId WHERE gameId = @gameId",
new SqlParameter("@last_updated_utc", latestUpdatedFile.FileDate),
new SqlParameter("@modId", mod.Id),
new SqlParameter("@fileId", latestUpdatedFile.Id),
new SqlParameter("@gameId", game.Id)
);
Console.WriteLine($"No updated files found for {game.Name} and mod {mod.Name}");
}

}
else
{
Expand All @@ -100,13 +162,13 @@ await db.ExecuteNonQueryAsync(

Console.WriteLine($"Last updated mod was updated {lastUpdatedMod}");

if(lastUpdatedMod < DateTimeOffset.UtcNow.AddHours(-3) && latestUpdatedModData != null && latestUpdatedFileData != null)
if (lastUpdatedMod < DateTimeOffset.UtcNow.AddHours(-3) && latestUpdatedModData != null && latestUpdatedFileData != null)
{
Console.WriteLine("No mods were updated in the last 3 hours, file processing might be down.");

var warned = await _db.StringGetAsync("cf-file-processing-warning");

if(warned.HasValue && warned == "true")
if (warned.HasValue && warned == "true")
{
Console.WriteLine("Already warned about this, skipping.");
return;
Expand All @@ -118,7 +180,7 @@ await db.ExecuteNonQueryAsync(
Environment.GetEnvironmentVariable("DISCORD_WEBHOOK", EnvironmentVariableTarget.Process) ??
string.Empty;

if(!string.IsNullOrWhiteSpace(discordWebhook))
if (!string.IsNullOrWhiteSpace(discordWebhook))
{
var message = @$"No mods were updated in the last 3 hours, file processing might be down.
Last updated mod was updated {lastUpdatedMod}, and it was {latestUpdatedModData.Name}
Expand All @@ -130,7 +192,7 @@ await db.ExecuteNonQueryAsync(
flags = 4
};

var json = JsonConvert.SerializeObject(payload);
var json = JsonSerializer.Serialize(payload);
var content = new StringContent(json, Encoding.UTF8, "application/json");
await httpClient.PostAsync(discordWebhook, content);
}
Expand Down
16 changes: 8 additions & 8 deletions CFLookup/Pages/FileProcessingInfo.cshtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@
<th scope="col" class="text-end">Latest update</th>
</tr>
</thead>
<tbody>
@foreach (var row in Model.ModFiles)
{
<tbody>
<tr>
<td scope="row"><a href="https://www.curseforge.com/@row.Game.Slug" target="_blank">@row.Game.Name</a></td>
<td><a href="@row.Mod.Links.WebsiteUrl" target="_blank">@row.Mod.Name</a></td>
<td><a href="@row.Mod.Links.WebsiteUrl/files/@row.File.Id" target="_blank">@row.File.FileName</a></td>
<td class="text-end">@row.LatestUpdatedUtc</td>
</tr>
</tbody>
<tr>
<td scope="row"><a href="https://www.curseforge.com/@row.Game.Slug" target="_blank">@row.Game.Name</a></td>
<td><a href="@row.Mod.Links.WebsiteUrl" target="_blank">@row.Mod.Name</a></td>
<td><a href="@row.Mod.Links.WebsiteUrl/files/@row.File.Id" target="_blank">@row.File.FileName</a></td>
<td class="text-end" title="@row.LatestUpdatedUtc">@row.SinceLatestUpdate.ToHumanReadableFormat(true) ago</td>
</tr>
}
</tbody>
</table>
10 changes: 5 additions & 5 deletions CFLookup/Pages/FileProcessingInfo.cshtml.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using CFLookup.Models;
using CurseForge.APIClient;
using CurseForge.APIClient.Models.Files;
using CurseForge.APIClient.Models.Games;
using CurseForge.APIClient.Models.Mods;
using Microsoft.AspNetCore.Mvc;
Expand Down Expand Up @@ -29,11 +28,11 @@ public async Task<IActionResult> OnGetAsync()
{
var gameProcessingInfo = await _db.ExecuteListAsync<FileProcessingStatus>("SELECT * FROM fileProcessingStatus ORDER BY last_updated_utc DESC");

foreach(var info in gameProcessingInfo)
foreach (var info in gameProcessingInfo)
{
Game game;
var gameCache = await _redis.StringGetAsync($"cf-game-{info.GameId}");
if(!gameCache.IsNullOrEmpty)
if (!gameCache.IsNullOrEmpty)
{
game = JsonSerializer.Deserialize<Game>(gameCache)!;
}
Expand All @@ -45,7 +44,7 @@ public async Task<IActionResult> OnGetAsync()

Mod mod;
var modCache = await _redis.StringGetAsync($"cf-mod-{info.ModId}");
if(!modCache.IsNullOrEmpty)
if (!modCache.IsNullOrEmpty)
{
mod = JsonSerializer.Deserialize<Mod>(modCache)!;
}
Expand All @@ -57,7 +56,7 @@ public async Task<IActionResult> OnGetAsync()

CurseForge.APIClient.Models.Files.File file;
var fileCache = await _redis.StringGetAsync($"cf-file-{info.FileId}");
if(!fileCache.IsNullOrEmpty)
if (!fileCache.IsNullOrEmpty)
{
file = JsonSerializer.Deserialize<CurseForge.APIClient.Models.Files.File>(fileCache)!;
}
Expand Down Expand Up @@ -88,5 +87,6 @@ public class GameModFileProcessingInfo
public Mod Mod { get; set; }
public CurseForge.APIClient.Models.Files.File File { get; set; }
public DateTimeOffset LatestUpdatedUtc { get; set; }
public TimeSpan SinceLatestUpdate => DateTimeOffset.UtcNow - LatestUpdatedUtc;
}
}
22 changes: 16 additions & 6 deletions CFLookup/SharedMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public static async Task<List<Category>> GetCategoryInfo(IDatabaseAsync _redis,
FileIds = new List<int> { fileId }
});

if(file.Data.Count == 0)
if (file.Data.Count == 0)
return (null, null, null);

var mod = await _cfApiClient.GetModAsync(file.Data[0].ModId);
Expand Down Expand Up @@ -358,12 +358,22 @@ public static string GetProjectNameFromFile(string url)
return Path.GetFileName(url);
}

public static string ToHumanReadableFormat(this TimeSpan timeSpan)
public static string ToHumanReadableFormat(this TimeSpan timeSpan, bool shortText = false, double skipMinutesAfterHours = 24.0f, double skipSecondsAfterHours = 1.0f)
{
return timeSpan.TotalSeconds <= 0 ? "0 seconds" : string.Format("{0}{1}{2}",
timeSpan.Hours > 0 ? string.Format($"{timeSpan.Hours:n0} hour{{0}}, ", timeSpan.Hours != 1 ? "s" : string.Empty) : string.Empty,
timeSpan.Minutes > 0 ? string.Format($"{timeSpan.Minutes:n0} minute{{0}}, ", timeSpan.Minutes != 1 ? "s" : string.Empty) : string.Empty,
timeSpan.Seconds > 0 ? string.Format($"{timeSpan.Seconds:n0} second{{0}}", timeSpan.Seconds != 1 ? "s" : string.Empty) : string.Empty
var secondText = shortText ? "s" : " second" + (timeSpan.Seconds != 1 ? "s" : string.Empty);
var minuteText = shortText ? "m" : " minute" + (timeSpan.Minutes != 1 ? "s" : string.Empty);
var hourText = shortText ? "h" : " hour" + (timeSpan.Hours != 1 ? "s" : string.Empty);
var dayText = shortText ? "d" : " day" + (timeSpan.Days != 1 ? "s" : string.Empty);

return timeSpan.TotalSeconds <= 0 ? "0 seconds" : string.Format("{0}{1}{2}{3}",
timeSpan.Days > 0 ? $"{timeSpan.Days:n0}{dayText}, " : string.Empty,
timeSpan.Hours > 0 ? $"{timeSpan.Hours:n0}{hourText}, " : string.Empty,
timeSpan.TotalHours <= skipMinutesAfterHours ?
timeSpan.Minutes > 0 ? $"{timeSpan.Minutes:n0}{minuteText}, " : string.Empty :
string.Empty,
timeSpan.TotalHours <= skipSecondsAfterHours ?
timeSpan.Seconds > 0 ? $"{timeSpan.Seconds:n0}{secondText}" : string.Empty :
string.Empty
).Trim(new[] { ' ', ',' });
}
}
Expand Down

0 comments on commit 89124c3

Please sign in to comment.