Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore/tasks linting #2176

Merged
merged 17 commits into from
Dec 3, 2024
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CodeSnippetsPipeline.Test/CodeSnippetsPipeline.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.12.19">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="NUnit" Version="4.2.2" />
<PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
<PackageReference Include="NUnit.Analyzers" Version="4.4.0">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="9.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.12.19">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
Expand Down
47 changes: 20 additions & 27 deletions CodeSnippetsReflection.App/Program.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Text;
Expand All @@ -24,7 +25,7 @@ namespace CodeSnippetsReflection.App
/// </summary>
class Program
{
static void Main(string[] args)
static async Task Main(string[] args)
{
IConfiguration config = new ConfigurationBuilder()
.AddCommandLine(args)
Expand All @@ -36,7 +37,7 @@ static void Main(string[] args)
var generationArg = config.GetSection("Generation");
if (!snippetsPathArg.Exists() || !languagesArg.Exists())
{
Console.Error.WriteLine("Http snippets directory and languages should be specified");
await Console.Error.WriteLineAsync("Http snippets directory and languages should be specified");
Console.WriteLine(@"Example usage:
.\CodeSnippetReflection.App.exe --SnippetsPath C:\snippets --Languages c#,javascript --Generation odata|openapi");
return;
Expand All @@ -45,13 +46,13 @@ static void Main(string[] args)
var httpSnippetsDir = snippetsPathArg.Value;
if (!Directory.Exists(httpSnippetsDir))
{
Console.Error.WriteLine($@"Directory {httpSnippetsDir} does not exist!");
await Console.Error.WriteLineAsync($@"Directory {httpSnippetsDir} does not exist!");
return;
}

if (customMetadataPathArg.Exists() && !File.Exists(customMetadataPathArg.Value))
{
Console.Error.WriteLine($@"Metadata file {customMetadataPathArg.Value} does not exist!");
await Console.Error.WriteLineAsync($@"Metadata file {customMetadataPathArg.Value} does not exist!");
return;
}

Expand All @@ -67,12 +68,12 @@ static void Main(string[] args)
.GroupBy(l => ODataSnippetsGenerator.SupportedLanguages.Contains(l) || OpenApiSnippetsGenerator.SupportedLanguages.Contains(l))
.ToDictionary(g => g.Key, g => g.ToList());

var supportedLanguages = languageGroups.ContainsKey(true) ? languageGroups[true] : null;
var unsupportedLanguages = languageGroups.ContainsKey(false) ? languageGroups[false] : null;
var supportedLanguages = languageGroups.GetValueOrDefault(true, null);
var unsupportedLanguages = languageGroups.GetValueOrDefault(false, null);

if (supportedLanguages == null)
{
Console.Error.WriteLine($"None of the given languages are supported. Supported languages: {string.Join(" ", ODataSnippetsGenerator.SupportedLanguages)}");
await Console.Error.WriteLineAsync($"None of the given languages are supported. Supported languages: {string.Join(" ", ODataSnippetsGenerator.SupportedLanguages)}");
return;
}

Expand All @@ -94,18 +95,15 @@ static void Main(string[] args)

// cache the generators by generation rather than creating a new one on each generation to avoid multiple loads of the metadata.
var snippetGenerators = new ConcurrentDictionary<string, ISnippetsGenerator>();
Parallel.ForEach(supportedLanguages, language =>
await Task.WhenAll(supportedLanguages.Select(language =>
andrueastman marked this conversation as resolved.
Show resolved Hide resolved
{
//Generation will still be originalGeneration if language is java since it is not stable
//Remove the condition when java is stable
generation = (OpenApiSnippetsGenerator.SupportedLanguages.Contains(language)) ? "openapi" : originalGeneration;

var generator = snippetGenerators.GetOrAdd(generation, generationKey => GetSnippetsGenerator(generationKey, customMetadataPathArg));
Parallel.ForEach(files, file =>
{
ProcessFile(generator, language, file);
});
});
return files.Select(file => ProcessFileAsync(generator, language, file));
}).SelectMany(static t => t));
Console.WriteLine($"Processed {files.Count()} files.");
}

Expand All @@ -118,43 +116,38 @@ private static ISnippetsGenerator GetSnippetsGenerator(string generation, IConfi
};
}

private static void ProcessFile(ISnippetsGenerator generator, string language, string file)
private static async Task ProcessFileAsync(ISnippetsGenerator generator, string language, string file)
{
// convert http request into a type that works with SnippetGenerator.ProcessPayloadRequest()
// we are not altering the types as it should continue serving the HTTP endpoint as well
using var streamContent = new StreamContent(new MemoryStream(Encoding.UTF8.GetBytes(File.ReadAllText(file))));
using var streamContent = new StreamContent(new MemoryStream(Encoding.UTF8.GetBytes(await File.ReadAllTextAsync(file))));
streamContent.Headers.Add("Content-Type", "application/http;msgtype=request");

string snippet;
var filePath = file.Replace("-httpSnippet", $"---{language.ToLowerInvariant()}");
try
{
// This is a very fast operation, it is fine to make is synchronuous.
// With the parallel foreach in the main method, processing all snippets for C# in both Beta and V1 takes about 7 seconds.
// As of this writing, the code was processing 2650 snippets
// Using async-await is costlier as this operation is all in-memory and task creation and scheduling overhead is high for that.
// With async-await, the same operation takes 1 minute 7 seconds.
using var message = streamContent.ReadAsHttpRequestMessageAsync().Result;
snippet = generator.ProcessPayloadRequest(message, language);
using var message = await streamContent.ReadAsHttpRequestMessageAsync();
andrueastman marked this conversation as resolved.
Show resolved Hide resolved
snippet = await generator.ProcessPayloadRequestAsync(message, language);
}
catch (Exception e)
{
var message = $"Exception while processing {file}.{Environment.NewLine}{e.Message}{Environment.NewLine}{e.StackTrace}";
Console.Error.WriteLine(message);
File.WriteAllText(filePath + "-error", message);
await Console.Error.WriteLineAsync(message);
await File.WriteAllTextAsync(filePath + "-error", message);
return;
}

if (!string.IsNullOrWhiteSpace(snippet))
{
Console.WriteLine($"Writing snippet: {filePath}");
File.WriteAllText(filePath, snippet);
await File.WriteAllTextAsync(filePath, snippet);
}
else
{
var message = $"Failed to generate {language} snippets for {file}.";
File.WriteAllText(filePath + "-error", message);
Console.Error.WriteLine(message);
await File.WriteAllTextAsync(filePath + "-error", message);
await Console.Error.WriteLineAsync(message);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
<PackageReference Include="coverlet.collector" Version="6.0.2" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.12.19">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
Expand Down
Loading