Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
mythz committed Mar 21, 2024
1 parent c37b20c commit e33f199
Show file tree
Hide file tree
Showing 26 changed files with 552 additions and 192 deletions.
18 changes: 17 additions & 1 deletion MyApp.ServiceInterface/BackgroundMqServices.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using MyApp.ServiceModel;
using ServiceStack;
using ServiceStack.IO;
using ServiceStack.OrmLite;

namespace MyApp.ServiceInterface;

Expand All @@ -18,4 +19,19 @@ public async Task Any(DiskTasks request)
r2.DeleteFiles(request.CdnDeleteFiles);
}
}
}

public async Task Any(AnalyticsTasks request)
{
if (request.RecordPostStat != null && !Stats.IsAdminOrModerator(request.RecordPostStat.UserName))
{
using var analyticsDb = HostContext.AppHost.GetDbConnection(Databases.Analytics);
await analyticsDb.InsertAsync(request.RecordPostStat);
}

if (request.RecordSearchStat != null && !Stats.IsAdminOrModerator(request.RecordSearchStat.UserName))
{
using var analyticsDb = HostContext.AppHost.GetDbConnection(Databases.Analytics);
await analyticsDb.InsertAsync(request.RecordSearchStat);
}
}
}
33 changes: 12 additions & 21 deletions MyApp.ServiceInterface/Data/DbExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,34 +12,25 @@ public static SqlExpression<Post> WhereContainsTag(this SqlExpression<Post> q, s
if (tag != null)
{
tag = tag.UrlDecode().Replace("'","").Replace("\\","").SqlVerifyFragment();
q.UnsafeWhere("',' || tags || ',' like '%," + tag + ",%'");
q.UnsafeWhere("',' || Tags || ',' LIKE '%," + tag + ",%'");
}
return q;
}

public static SqlExpression<Post> WhereSearch(this SqlExpression<Post> q, string? search)
public static SqlExpression<PostFts> WhereContainsTag(this SqlExpression<PostFts> q, string? tag)
{
if (tag != null)
{
tag = tag.UrlDecode().Replace("'","").Replace("\\","").SqlVerifyFragment();
q.UnsafeWhere($"Tags match '\"{tag}\"'");
}
return q;
}

public static SqlExpression<Post> WhereSearch(this SqlExpression<Post> q, string? search, int? skip, int take)
{
if (!string.IsNullOrEmpty(search))
{
search = search.Trim();
if (search.StartsWith('[') && search.EndsWith(']'))
{
q.WhereContainsTag(search.TrimStart('[').TrimEnd(']'));
}
else
{
var sb = StringBuilderCache.Allocate();
var words = search.Split(' ');
for (var i = 0; i < words.Length; i++)
{
if (sb.Length > 0)
sb.Append(" AND ");
sb.AppendLine("(title like '%' || {" + i + "} || '%' or summary like '%' || {" + i + "} || '%' or tags like '%' || {" + i + "} || '%')");
}

var sql = StringBuilderCache.ReturnAndFree(sb);
q.UnsafeWhere(sql, words.Cast<object>().ToArray());
}
}
return q;
}
Expand Down
24 changes: 0 additions & 24 deletions MyApp.ServiceInterface/Data/IdFiles.cs

This file was deleted.

101 changes: 101 additions & 0 deletions MyApp.ServiceInterface/Data/QuestionFiles.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
using System.Collections.Concurrent;
using MyApp.ServiceModel;
using ServiceStack;
using ServiceStack.IO;

namespace MyApp.Data;

public class QuestionFiles(int id, string dir1, string dir2, string fileId, List<IVirtualFile> files, bool remote=false)
{
public const int MostVotedScore = 10;
public const int AcceptedScore = 9;
public static Dictionary<string,int> ModelScores = new()
{
["phi"] = 1, //2.7B
["gemma:2b"] = 2,
["qwen:4b"] = 3, //4B
["codellama"] = 4, //7B
["gemma"] = 5, //7B
["deepseek-coder:6.7b"] = 5, //6.7B
["mistral"] = 7, //7B
["mixtral"] = 8, //47B
["accepted"] = 9,
["most-voted"] = 10,
};

public int Id { get; init; } = id;
public string Dir1 { get; init; } = dir1;
public string Dir2 { get; init; } = dir2;
public string DirPath = "/{Dir1}/{Dir2}";
public string FileId { get; init; } = fileId;
public List<IVirtualFile> Files { get; init; } = files;
public bool LoadedRemotely { get; set; } = remote;
public ConcurrentDictionary<string, string> FileContents { get; } = [];
public QuestionAndAnswers? Question { get; set; }

public async Task<QuestionAndAnswers?> GetQuestionAsync()
{
if (Question == null)
{
await LoadQuestionAndAnswersAsync();
}
return Question;
}

public async Task LoadContentsAsync()
{
if (FileContents.Count > 0) return;
var tasks = new List<Task>();
tasks.AddRange(Files.Select(async file => {
FileContents[file.VirtualPath] = await file.ReadAllTextAsync();
}));
await Task.WhenAll(tasks);
}

public async Task LoadQuestionAndAnswersAsync()
{
var questionFileName = FileId + ".json";
await LoadContentsAsync();

var to = new QuestionAndAnswers();
foreach (var entry in FileContents)
{
var fileName = entry.Key.LastRightPart('/');
if (fileName == questionFileName)
{
to.Post = entry.Value.FromJson<Post>();
}
else if (fileName.StartsWith(FileId + ".a."))
{
to.Answers.Add(entry.Value.FromJson<Answer>());
}
else if (fileName.StartsWith(FileId + ".h."))
{
var post = entry.Value.FromJson<Post>();
var userName = fileName.Substring((FileId + ".h.").Length).LeftPart('.');
var answer = new Answer
{
Id = $"{post.Id}",
Model = userName,
UpVotes = userName == "most-voted" ? MostVotedScore : AcceptedScore,
Choices = [
new()
{
Index = 1,
Message = new() { Role = userName, Content = post.Body ?? "" }
}
]
};
if (to.Answers.All(x => x.Id != answer.Id))
to.Answers.Add(answer);
}
}

if (to.Post == null)
return;

to.Answers.Each(x => x.UpVotes = x.UpVotes == 0 ? ModelScores.GetValueOrDefault(x.Model, 1) : x.UpVotes);
to.Answers.Sort((a, b) => b.Votes - a.Votes);
Question = to;
}
}
64 changes: 64 additions & 0 deletions MyApp.ServiceInterface/Data/QuestionsProvider.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
using Microsoft.Extensions.Logging;
using ServiceStack;
using ServiceStack.IO;
using ServiceStack.Messaging;

namespace MyApp.Data;

public class QuestionsProvider(ILogger<QuestionsProvider> log, IMessageProducer mqClient, IVirtualFiles fs, R2VirtualFiles r2)
{
public QuestionFiles GetLocalQuestionFiles(int id)
{
var (dir1, dir2, fileId) = id.ToFileParts();

var files = fs.GetDirectory($"{dir1}/{dir2}").GetAllMatchingFiles($"{fileId}.*")
.OrderByDescending(x => x.LastModified)
.ToList();

return new QuestionFiles(id: id, dir1: dir1, dir2: dir2, fileId: fileId, files: files);
}

public async Task<QuestionFiles> GetRemoteQuestionFilesAsync(int id)
{
var (dir1, dir2, fileId) = id.ToFileParts();

var files = (await r2.EnumerateFilesAsync($"{dir1}/{dir2}").ToListAsync())
.Where(x => x.Name.Glob($"{fileId}.*"))
.OrderByDescending(x => x.LastModified)
.Cast<IVirtualFile>()
.ToList();

return new QuestionFiles(id: id, dir1: dir1, dir2: dir2, fileId: fileId, files: files, remote:true);
}

public async Task<QuestionFiles> GetQuestionFilesAsync(int id)
{
var localFiles = GetLocalQuestionFiles(id);
if (localFiles.Files.Count > 0)
return localFiles;

log.LogInformation("No local cached files for question {Id}, fetching from R2...", id);
var r = await GetRemoteQuestionFilesAsync(id);
if (r.Files.Count > 0)
{
var lastModified = r.Files.Max(x => x.LastModified);
log.LogInformation("Fetched {Count} files from R2 for question {Id}, last modified: '{LastModified}'", r.Files.Count, id, lastModified);
}
return r;
}

public async Task<QuestionFiles> GetQuestionAsync(int id)
{
var questionFiles = await GetQuestionFilesAsync(id);
await questionFiles.GetQuestionAsync();
if (questionFiles.LoadedRemotely)
{
log.LogInformation("Caching question {Id}'s {Count} remote files locally...", id, questionFiles.FileContents.Count);
foreach (var entry in questionFiles.FileContents)
{
await fs.WriteFileAsync(entry.Key, entry.Value);
}
}
return questionFiles;
}
}
78 changes: 1 addition & 77 deletions MyApp.ServiceInterface/Data/R2Extensions.cs
Original file line number Diff line number Diff line change
@@ -1,25 +1,7 @@
using MyApp.ServiceModel;
using ServiceStack;
using ServiceStack.IO;

namespace MyApp.Data;
namespace MyApp.Data;

public static class R2Extensions
{
public const int MostVotedScore = 10;
public const int AcceptedScore = 9;
public static Dictionary<string,int> ModelScores = new()
{
["starcoder2:3b"] = 1, //3B
["phi"] = 2, //2.7B
["gemma:2b"] = 3,
["gemma"] = 4, //7B
["codellama"] = 5, //7B
["mistral"] = 6, //7B
["starcoder2:15b"] = 7, //15B
["mixtral"] = 8, //47B
};

public static (string dir1, string dir2, string fileId) ToFileParts(this int id)
{
var idStr = $"{id}".PadLeft(9, '0');
Expand All @@ -28,62 +10,4 @@ public static (string dir1, string dir2, string fileId) ToFileParts(this int id)
var fileId = idStr[6..];
return (dir1, dir2, fileId);
}

public static async Task<IdFiles> GetQuestionFilesAsync(this R2VirtualFiles r2, int id)
{
var (dir1, dir2, fileId) = ToFileParts(id);

var files = (await r2.EnumerateFilesAsync($"{dir1}/{dir2}").ToListAsync())
.Where(x => x.Name.Glob($"{fileId}.*"))
.OrderByDescending(x => x.LastModified)
.Cast<IVirtualFile>()
.ToList();

return new IdFiles(id: id, dir1: dir1, dir2: dir2, fileId: fileId, files: files);
}

public static async Task<QuestionAndAnswers?> ToQuestionAndAnswers(this IdFiles idFiles)
{
var fileName = idFiles.FileId + ".json";
await idFiles.LoadContentsAsync();

var to = new QuestionAndAnswers();
foreach (var entry in idFiles.FileContents)
{
if (entry.Key == fileName)
{
to.Post = entry.Value.FromJson<Post>();
}
else if (entry.Key.StartsWith(idFiles.FileId + ".a."))
{
to.Answers.Add(entry.Value.FromJson<Answer>());
}
else if (entry.Key.StartsWith(idFiles.FileId + ".h."))
{
var post = entry.Value.FromJson<Post>();
var answer = new Answer
{
Id = $"{post.Id}",
Model = "human",
UpVotes = entry.Key.Contains("h.most-voted") ? MostVotedScore : AcceptedScore,
Choices = [
new()
{
Index = 1,
Message = new() { Role = "human", Content = post.Body ?? "" }
}
]
};
if (to.Answers.All(x => x.Id != answer.Id))
to.Answers.Add(answer);
}
}

if (to.Post == null)
return null;

to.Answers.Each(x => x.UpVotes = x.UpVotes == 0 ? ModelScores.GetValueOrDefault(x.Model, 1) : x.UpVotes);
to.Answers.Sort((a, b) => b.Votes - a.Votes);
return to;
}
}
16 changes: 16 additions & 0 deletions MyApp.ServiceInterface/Data/StatUtils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
using Microsoft.AspNetCore.Http;
using MyApp.ServiceModel;

namespace MyApp.Data;

public static class StatUtils
{
public static T WithRequest<T>(this T stat, HttpContext? ctx) where T : StatBase
{
var user = ctx?.User;
stat.UserName = user?.Identity?.Name;
stat.RemoteIp = ctx?.Connection.RemoteIpAddress?.ToString();
stat.CreatedDate = DateTime.UtcNow;
return stat;
}
}
4 changes: 2 additions & 2 deletions MyApp.ServiceModel/Icons.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ namespace MyApp.ServiceModel;
public static class Icons
{
public const string Dashboard = "<svg fill='none' viewBox='0 0 24 24' stroke-width='1.5' stroke='currentColor' aria-hidden='true'><path stroke-linecap='round' stroke-linejoin='round' d='M2.25 12l8.954-8.955c.44-.439 1.152-.439 1.591 0L21.75 12M4.5 9.75v10.125c0 .621.504 1.125 1.125 1.125H9.75v-4.875c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125V21h4.125c.621 0 1.125-.504 1.125-1.125V9.75M8.25 21h8.25' /></svg>";
public const string Booking = "<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path fill='currentColor' d='M16 10H8c-.55 0-1 .45-1 1s.45 1 1 1h8c.55 0 1-.45 1-1s-.45-1-1-1zm3-7h-1V2c0-.55-.45-1-1-1s-1 .45-1 1v1H8V2c0-.55-.45-1-1-1s-1 .45-1 1v1H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zm-1 16H6c-.55 0-1-.45-1-1V8h14v10c0 .55-.45 1-1 1zm-5-5H8c-.55 0-1 .45-1 1s.45 1 1 1h5c.55 0 1-.45 1-1s-.45-1-1-1z'/></svg>";
public const string Coupon = "<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path fill='currentColor' d='M2 9.5V4a1 1 0 0 1 1-1h18a1 1 0 0 1 1 1v5.5a2.5 2.5 0 1 0 0 5V20a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1v-5.5a2.5 2.5 0 1 0 0-5zm2-1.532a4.5 4.5 0 0 1 0 8.064V19h16v-2.968a4.5 4.5 0 0 1 0-8.064V5H4v2.968zM9 9h6v2H9V9zm0 4h6v2H9v-2z' /></svg>";
public const string Post = "<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path fill='currentColor' d='M16 10H8c-.55 0-1 .45-1 1s.45 1 1 1h8c.55 0 1-.45 1-1s-.45-1-1-1zm3-7h-1V2c0-.55-.45-1-1-1s-1 .45-1 1v1H8V2c0-.55-.45-1-1-1s-1 .45-1 1v1H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zm-1 16H6c-.55 0-1-.45-1-1V8h14v10c0 .55-.45 1-1 1zm-5-5H8c-.55 0-1 .45-1 1s.45 1 1 1h5c.55 0 1-.45 1-1s-.45-1-1-1z'/></svg>";
public const string Stats = "<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'><path fill='currentColor' d='M8.143 15.857H5.57V9.43h2.572v6.428zm5.143 0h-2.572V3h2.572v12.857zm5.142 0h-2.571v-9h2.571v9z'/><path fill='currentColor' fill-rule='evenodd' d='M21 20.714H3v-2h18v2z' clip-rule='evenodd'/></svg>";
}
Loading

0 comments on commit e33f199

Please sign in to comment.