diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index 7d9ffd5d9..4e314fa48 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -23,6 +23,8 @@ public partial class HgService : IHgService, IHostedService private const string DELETED_REPO_FOLDER = "_____deleted_____"; private const string TEMP_REPO_FOLDER = "_____temp_____"; + private const string AllZeroHash = "0000000000000000000000000000000000000000"; + private readonly IOptions _options; private readonly Lazy _hgClient; private readonly ILogger _logger; @@ -67,6 +69,8 @@ await Task.Run(() => { InitRepoAt(new DirectoryInfo(PrefixRepoFilePath(code))); }); + await InvalidateDirCache(code); + await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } private void InitRepoAt(DirectoryInfo repoDirectory) @@ -104,6 +108,8 @@ public async Task ResetRepo(string code) await SoftDeleteRepo(code, $"{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}__reset"); //we must init the repo as uploading a zip is optional tmpRepo.MoveTo(PrefixRepoFilePath(code)); + await InvalidateDirCache(code); + await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } public async Task FinishReset(string code, Stream zipFile) @@ -137,6 +143,11 @@ await Task.Run(() => // Now we're ready to move the new repo into place, replacing the old one await DeleteRepo(code); tempRepo.MoveTo(PrefixRepoFilePath(code)); + await InvalidateDirCache(code); + // If someone uploaded an *empty* repo, we don't want to wait forever for a non-empty state + var changelogPath = Path.Join(PrefixRepoFilePath(code), ".hg", "store", "00changelog.i"); + var expectedState = File.Exists(changelogPath) ? RepoEmptyState.NonEmpty : RepoEmptyState.Empty; + await WaitForRepoEmptyState(code, expectedState); } /// @@ -262,6 +273,58 @@ public async Task ExecuteHgRecover(string code, CancellationToken t return response; } + public Task InvalidateDirCache(string code, CancellationToken token = default) + { + var repoPath = Path.Join(PrefixRepoFilePath(code)); + if (Directory.Exists(repoPath)) + { + // Invalidate NFS directory cache by forcing a write and re-read of the repo directory + var randomPath = Path.Join(repoPath, Path.GetRandomFileName()); + while (File.Exists(randomPath) || Directory.Exists(randomPath)) { randomPath = Path.Join(repoPath, Path.GetRandomFileName()); } + try + { + // Create and delete a directory since that's slightly safer than a file + var d = Directory.CreateDirectory(randomPath); + d.Delete(); + } + catch (Exception) { } + } + var result = ExecuteHgCommandServerCommand(code, "invalidatedircache", token); + return result; + } + + public async Task GetTipHash(string code, CancellationToken token = default) + { + var content = await ExecuteHgCommandServerCommand(code, "tip", token); + return await content.ReadAsStringAsync(); + } + + private async Task WaitForRepoEmptyState(string code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default) + { + // Set timeout so unforeseen errors can't cause an infinite loop + using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(token); + timeoutSource.CancelAfter(timeoutMs); + var done = false; + try + { + while (!done && !timeoutSource.IsCancellationRequested) + { + var hash = await GetTipHash(code, timeoutSource.Token); + var isEmpty = hash == AllZeroHash; + done = expectedState switch + { + RepoEmptyState.Empty => isEmpty, + RepoEmptyState.NonEmpty => !isEmpty + }; + if (!done) await Task.Delay(2500, timeoutSource.Token); + } + } + // We don't want to actually throw if we hit the timeout, because the operation *will* succeed eventually + // once the NFS caches synchronize, so we don't want to propagate an error message to the end user. So + // even if the timeout is hit, return as if we succeeded. + catch (OperationCanceledException) { } + } + public async Task GetLexEntryCount(string code, ProjectType projectType) { var command = projectType switch @@ -408,3 +471,9 @@ public class BrowseResponse { public BrowseFilesResponse[]? Files { get; set; } } + +public enum RepoEmptyState +{ + Empty, + NonEmpty +} diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index db8a3c0f8..5e351fc38 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -117,6 +117,10 @@ public async Task FinishReset(string code, Stream? zipFile = null) await hgService.FinishReset(code, zipFile); await UpdateProjectMetadata(project); } + else + { + await hgService.InvalidateDirCache(code); + } project.ResetStatus = ResetStatus.None; project.UpdateUpdatedDate(); await dbContext.SaveChangesAsync(); @@ -146,6 +150,10 @@ public async Task UpdateProjectMetadata(Project project) project.FlexProjectMetadata.LexEntryCount = count; } } + else + { + await hgService.InvalidateDirCache(project.Code); + } project.LastCommit = await hgService.GetLastCommitTimeFromHg(project.Code); // Caller is responsible for caling dbContext.SaveChangesAsync() diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index a5ddadd26..cc4703b7a 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -15,9 +15,11 @@ public interface IHgService Task ResetRepo(string code); Task FinishReset(string code, Stream zipFile); Task VerifyRepo(string code, CancellationToken token); + Task GetTipHash(string code, CancellationToken token = default); Task GetLexEntryCount(string code, ProjectType projectType); Task GetRepositoryIdentifier(Project project); Task ExecuteHgRecover(string code, CancellationToken token); + Task InvalidateDirCache(string code, CancellationToken token = default); bool HasAbandonedTransactions(string projectCode); Task HgCommandHealth(); } diff --git a/backend/Testing/LexCore/Services/HgServiceTests.cs b/backend/Testing/LexCore/Services/HgServiceTests.cs index c1bb0c6fa..502b0a3b1 100644 --- a/backend/Testing/LexCore/Services/HgServiceTests.cs +++ b/backend/Testing/LexCore/Services/HgServiceTests.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; +using Moq.Contrib.HttpClient; using Shouldly; using Testing.Fixtures; @@ -30,8 +31,16 @@ public HgServiceTests() HgResumableUrl = LexboxResumable, SendReceiveDomain = LexboxHgWeb }; + var handler = new Mock(MockBehavior.Strict); + + // This may need to become more sophisticated if our FinishReset tests are changed to include + // a Mercurial repo with actual commits in it, but this is good enough at the moment. + var AllZeroHash = "0000000000000000000000000000000000000000"; + handler.SetupAnyRequest().ReturnsResponse(AllZeroHash); + + var mockFactory = handler.CreateClientFactory(); _hgService = new HgService(new OptionsWrapper(_hgConfig), - Mock.Of(), + mockFactory, NullLogger.Instance); CleanUpTempDir(); } diff --git a/backend/Testing/Services/Utils.cs b/backend/Testing/Services/Utils.cs index 76cf059cd..dd8a6559d 100644 --- a/backend/Testing/Services/Utils.cs +++ b/backend/Testing/Services/Utils.cs @@ -84,11 +84,6 @@ public static async Task WaitForHgRefreshIntervalAsync() await Task.Delay(TestingEnvironmentVariables.HgRefreshInterval); } - public static async Task WaitForLexboxMetadataUpdateAsync() - { - await Task.Delay(3000); - } - private static string GetNewProjectDir(string projectCode, [CallerMemberName] string projectName = "") { diff --git a/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs b/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs index b2b927a85..baa0bd71e 100644 --- a/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs +++ b/backend/Testing/SyncReverseProxy/SendReceiveServiceTests.cs @@ -81,14 +81,10 @@ public async Task ModifyProjectData(HgProtocol protocol) var projectConfig = _srFixture.InitLocalFlexProjectWithRepo(); await using var project = await RegisterProjectInLexBox(projectConfig, _adminApiTester); - await WaitForHgRefreshIntervalAsync(); - // Push the project to the server var sendReceiveParams = new SendReceiveParams(protocol, projectConfig); _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); - await WaitForLexboxMetadataUpdateAsync(); - // Verify pushed and store last commit var lastCommitDate = await _adminApiTester.GetProjectLastCommit(projectConfig.Code); lastCommitDate.ShouldNotBeNullOrEmpty(); @@ -101,8 +97,6 @@ public async Task ModifyProjectData(HgProtocol protocol) // Push changes _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth, "Modify project data automated test"); - await WaitForLexboxMetadataUpdateAsync(); - // Verify the push updated the last commit date var lastCommitDateAfter = await _adminApiTester.GetProjectLastCommit(projectConfig.Code); lastCommitDateAfter.ShouldBeGreaterThan(lastCommitDate); @@ -117,8 +111,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) var projectConfig = _srFixture.InitLocalFlexProjectWithRepo(protocol, "SR_AfterReset"); await using var project = await RegisterProjectInLexBox(projectConfig, _adminApiTester); - await WaitForHgRefreshIntervalAsync(); - var sendReceiveParams = new SendReceiveParams(protocol, projectConfig); var srResult = _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); @@ -144,8 +136,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) await _adminApiTester.HttpClient.PostAsync($"{_adminApiTester.BaseUrl}/api/project/resetProject/{projectConfig.Code}", null); await _adminApiTester.HttpClient.PostAsync($"{_adminApiTester.BaseUrl}/api/project/finishResetProject/{projectConfig.Code}", null); - await WaitForHgRefreshIntervalAsync(); // TODO 765: Remove this - // Step 2: verify project is now empty, i.e. tip is "0000000..." response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri); jsonResult = await response.Content.ReadFromJsonAsync(); @@ -169,8 +159,6 @@ public async Task SendReceiveAfterProjectReset(HgProtocol protocol) var srResultStep3 = _sendReceiveService.SendReceiveProject(sendReceiveParams, AdminAuth); _output.WriteLine(srResultStep3); - await WaitForHgRefreshIntervalAsync(); // TODO 765: Remove this - // Step 4: verify project tip is same hash as original project tip response = await _adminApiTester.HttpClient.GetAsync(tipUri.Uri); jsonResult = await response.Content.ReadFromJsonAsync(); diff --git a/backend/Testing/Testing.csproj b/backend/Testing/Testing.csproj index 4ba343277..9f3bbc134 100644 --- a/backend/Testing/Testing.csproj +++ b/backend/Testing/Testing.csproj @@ -15,6 +15,7 @@ + diff --git a/frontend/tests/resetProject.test.ts b/frontend/tests/resetProject.test.ts index f809179cd..359909e7a 100644 --- a/frontend/tests/resetProject.test.ts +++ b/frontend/tests/resetProject.test.ts @@ -37,20 +37,14 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 2: Get tip hash and file list from hgweb, check some known values - // It can take a while for the server to pick up the new repo - let beforeResetJson: HgWebJson; - await expect(async () => { - const beforeResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - beforeResetJson = await beforeResetResponse.json() as HgWebJson; - expect(beforeResetJson).toHaveProperty('node'); - expect(beforeResetJson.node).not.toEqual(allZeroHash); - expect(beforeResetJson).toHaveProperty('files'); - expect(beforeResetJson.files).toHaveLength(1); - expect(beforeResetJson.files[0]).toHaveProperty('basename'); - expect(beforeResetJson.files[0].basename).toBe('hello.txt'); - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const beforeResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const beforeResetJson = await beforeResetResponse.json() as HgWebJson; + expect(beforeResetJson).toHaveProperty('node'); + expect(beforeResetJson.node).not.toEqual(allZeroHash); + expect(beforeResetJson).toHaveProperty('files'); + expect(beforeResetJson.files).toHaveLength(1); + expect(beforeResetJson.files[0]).toHaveProperty('basename'); + expect(beforeResetJson.files[0].basename).toBe('hello.txt'); // Step 3: reset project, do not upload zip file await projectPage.goto(); @@ -65,16 +59,11 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 4: confirm it's empty now - // It can take a while for the server to pick up the new repo - await expect(async () => { - const afterResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - const afterResetJson = await afterResetResponse.json() as HgWebJson; - expect(afterResetJson.node).toEqual(allZeroHash); - expect(afterResetJson).toHaveProperty('files'); - expect(afterResetJson.files).toHaveLength(0); - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const afterResetResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const afterResetJson = await afterResetResponse.json() as HgWebJson; + expect(afterResetJson.node).toEqual(allZeroHash); + expect(afterResetJson).toHaveProperty('files'); + expect(afterResetJson.files).toHaveLength(0); // Step 5: reset project again, uploading zip file downloaded from step 1 await projectPage.goto(); @@ -88,12 +77,7 @@ test('reset project and upload .zip file', async ({ page, tempProject, tempDir } await resetProjectModel.assertGone(); // Step 6: confirm tip hash and contents are same as before reset - // It can take a while for the server to pick up the new repo - await expect(async () => { - const afterUploadResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); - const afterResetJSon = await afterUploadResponse.json() as HgWebJson; - expect(afterResetJSon).toEqual(beforeResetJson); // NOT .toBe(), which would check that they're the same object. - }).toPass({ - intervals: [1_000, 3_000, 5_000], - }); + const afterUploadResponse = await page.request.get(`${testEnv.serverBaseUrl}/hg/${tempProject.code}/file/tip?style=json-lex`); + const afterResetJSon = await afterUploadResponse.json() as HgWebJson; + expect(afterResetJSon).toEqual(beforeResetJson); // NOT .toBe(), which would check that they're the same object. }); diff --git a/hgweb/command-runner.sh b/hgweb/command-runner.sh index bc511e19e..d03074d54 100644 --- a/hgweb/command-runner.sh +++ b/hgweb/command-runner.sh @@ -1,7 +1,7 @@ #!/bin/bash # Define the list of allowed commands -allowed_commands=("verify" "tip" "wesaylexentrycount" "lexentrycount" "recover" "healthz") +allowed_commands=("verify" "tip" "wesaylexentrycount" "lexentrycount" "recover" "healthz" "invalidatedircache") # Get the project code and command name from the URL IFS='/' read -ra PATH_SEGMENTS <<< "$PATH_INFO" @@ -44,6 +44,10 @@ echo "" # Run the hg command, simply output to stdout first_char=$(echo $project_code | cut -c1) +# Ensure NFS cache is refreshed in case project repo changed in another pod (e.g., project reset) +ls /var/hg/repos/$first_char/$project_code/.hg >/dev/null 2>/dev/null # Don't need output; this is enough to refresh NFS dir cache +# Sometimes invalidatedircache is called after deleting a project, so the cd would fail. So exit fast in that case. +[ "x$command_name" = "xinvalidatedircache" ] && exit 0 cd /var/hg/repos/$first_char/$project_code case $command_name in