From 5a8eac4a471863f8924bcb49b7455c93d7ece423 Mon Sep 17 00:00:00 2001 From: Ryan Jordan Date: Sun, 8 Dec 2024 17:11:38 -0500 Subject: [PATCH] sped up rating tool --- backend/src/common/__tests__/zip.test.ts | 4 +- backend/src/handlers/RegistryReset/index.ts | 1 + .../__tests__/end_to_end/getCost.test.ts | 4 +- .../end_to_end/packageByName.test.ts | 4 +- .../end_to_end/packageByRegex.test.ts | 4 +- .../end_to_end/packageCreate.test.ts | 4 +- .../end_to_end/packageDelete.test.ts | 4 +- .../__tests__/end_to_end/packageRate.test.ts | 4 +- .../end_to_end/packageRecommend.test.ts | 7 +- .../end_to_end/packageRetrieve.test.ts | 4 +- .../end_to_end/packageUpdate.test.ts | 4 +- .../__tests__/end_to_end/packagesList.test.ts | 4 +- .../end_to_end/registryReset.test.ts | 2 +- backend/src/services/rate/tools/api.ts | 243 +++++++++--------- backend/src/services/rate/utils/interfaces.ts | 11 + backend/template.yml | 2 +- 16 files changed, 165 insertions(+), 141 deletions(-) diff --git a/backend/src/common/__tests__/zip.test.ts b/backend/src/common/__tests__/zip.test.ts index bac2658..7061c6d 100644 --- a/backend/src/common/__tests__/zip.test.ts +++ b/backend/src/common/__tests__/zip.test.ts @@ -35,14 +35,14 @@ describe("File Operations and Compression Functions", () => { describe("cloneAndZipRepository", () => { it("should clone a repository and zip it", async () => { - const repoUrl = "https://github.com/isomorphic-git/isomorphic-git"; // Use a real repo URL + const repoUrl = "https://github.com/thejoshwolfe/yazl"; // Use a real repo URL const zipBuffer = await cloneAndZipRepository(repoUrl); const zip = await JSZip.loadAsync(zipBuffer); const files = Object.keys(zip.files); expect(files).toContain("README.md"); - }); + }, 60000); }); describe("zipDirectory", () => { diff --git a/backend/src/handlers/RegistryReset/index.ts b/backend/src/handlers/RegistryReset/index.ts index 29c619a..cc7d404 100644 --- a/backend/src/handlers/RegistryReset/index.ts +++ b/backend/src/handlers/RegistryReset/index.ts @@ -3,6 +3,7 @@ import { APIGatewayProxyHandler } from "aws-lambda"; import { S3Client } from "@aws-sdk/client-s3"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; + const commonPath = process.env.COMMON_PATH || '/opt/nodejs/common'; const { getEnvVariable } = require(`${commonPath}/utils`); const { clearDynamoDBTable } = require(`${commonPath}/dynamodb`); diff --git a/backend/src/handlers/__tests__/end_to_end/getCost.test.ts b/backend/src/handlers/__tests__/end_to_end/getCost.test.ts index 610b259..bb706f8 100644 --- a/backend/src/handlers/__tests__/end_to_end/getCost.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/getCost.test.ts @@ -12,7 +12,7 @@ describe("E2E Test for Get Cost Endpoint", () => { let circular2_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload yazl to the registry const requestBody: PackageData = { @@ -59,7 +59,7 @@ describe("E2E Test for Get Cost Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should only return total cost if dependency is false", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageByName.test.ts b/backend/src/handlers/__tests__/end_to_end/packageByName.test.ts index 08c4075..b5e6dd8 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageByName.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageByName.test.ts @@ -8,7 +8,7 @@ describe("E2E Test for Package By Name Endpoint", () => { let id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -43,7 +43,7 @@ describe("E2E Test for Package By Name Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, 90000); it("should return a 200 status for a package that exists", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageByRegex.test.ts b/backend/src/handlers/__tests__/end_to_end/packageByRegex.test.ts index 59ec640..f921c87 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageByRegex.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageByRegex.test.ts @@ -10,7 +10,7 @@ describe("E2E Test for PackageByRegEx Endpoint", () => { let url_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -41,7 +41,7 @@ describe("E2E Test for PackageByRegEx Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, 90000); it("should return a 200 status for a package that exists", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageCreate.test.ts b/backend/src/handlers/__tests__/end_to_end/packageCreate.test.ts index 9f98d1a..530f9ac 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageCreate.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageCreate.test.ts @@ -9,11 +9,11 @@ describe("E2E Test for Package Create Endpoint", () => { beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should return a 201 status for a package with Content set", async () => { const requestBody: PackageData = { diff --git a/backend/src/handlers/__tests__/end_to_end/packageDelete.test.ts b/backend/src/handlers/__tests__/end_to_end/packageDelete.test.ts index 1fdd0dd..c92ca02 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageDelete.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageDelete.test.ts @@ -10,7 +10,7 @@ describe("E2E Test for Package Delete Endpoint", () => { let url_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -39,7 +39,7 @@ describe("E2E Test for Package Delete Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should return a 200 status for a package uploaded via Content", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageRate.test.ts b/backend/src/handlers/__tests__/end_to_end/packageRate.test.ts index eb30029..ff1cd8d 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageRate.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageRate.test.ts @@ -10,7 +10,7 @@ describe("E2E Test for Package Rate Endpoint", () => { let url_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -39,7 +39,7 @@ describe("E2E Test for Package Rate Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should return a 200 status for a package with a rating", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageRecommend.test.ts b/backend/src/handlers/__tests__/end_to_end/packageRecommend.test.ts index 1c14a07..e26da02 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageRecommend.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageRecommend.test.ts @@ -10,18 +10,17 @@ async function upload(name: string, url: string) { URL: url, debloat: false }; - + console.log("Uploading package: ", name); await axios.post(`${baseUrl}/package`, requestBody); } describe("E2E Test for PackageRecommend Endpoint", () => { beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload packages to the registry await upload("yazl", "https://www.npmjs.com/package/yazl"); - await upload("express", "https://www.npmjs.com/package/express"); await upload("debug", "https://www.npmjs.com/package/debug"); await upload("inversify", "https://www.npmjs.com/package/inversify"); await upload("tslib", "https://www.npmjs.com/package/tslib"); @@ -33,7 +32,7 @@ describe("E2E Test for PackageRecommend Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, 90000); it("should return a list of 5 packages with express as the first", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageRetrieve.test.ts b/backend/src/handlers/__tests__/end_to_end/packageRetrieve.test.ts index 343a403..82ece4e 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageRetrieve.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageRetrieve.test.ts @@ -10,7 +10,7 @@ describe("E2E Test for Package Retrieve Endpoint", () => { let url_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -39,7 +39,7 @@ describe("E2E Test for Package Retrieve Endpoint", () => { }, 90000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should return a 200 status for a package uploaded via Content", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/packageUpdate.test.ts b/backend/src/handlers/__tests__/end_to_end/packageUpdate.test.ts index 2caeea4..9c0248c 100644 --- a/backend/src/handlers/__tests__/end_to_end/packageUpdate.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packageUpdate.test.ts @@ -9,7 +9,7 @@ describe("E2E Test for Package Update Endpoint", () => { let url_id: string; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -38,7 +38,7 @@ describe("E2E Test for Package Update Endpoint", () => { }, 60000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should return a 200 status for a package with Content set", async () => { const requestBody: Package = { diff --git a/backend/src/handlers/__tests__/end_to_end/packagesList.test.ts b/backend/src/handlers/__tests__/end_to_end/packagesList.test.ts index c5b1ffc..be8704c 100644 --- a/backend/src/handlers/__tests__/end_to_end/packagesList.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/packagesList.test.ts @@ -9,7 +9,7 @@ describe("E2E Test for Packages List Endpoint", () => { const ids: string[] = []; beforeAll(async () => { // Reset the registry before running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); // Upload a package with Content to the registry const requestBody: PackageData = { @@ -47,7 +47,7 @@ describe("E2E Test for Packages List Endpoint", () => { }, 240000); afterAll(async () => { // Reset the registry after running the tests - await axios.delete(`${baseUrl}/reset`); + await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); }, timeout); it("should fetch all packages and return 200 status", async () => { diff --git a/backend/src/handlers/__tests__/end_to_end/registryReset.test.ts b/backend/src/handlers/__tests__/end_to_end/registryReset.test.ts index fa3f8f9..1322e73 100644 --- a/backend/src/handlers/__tests__/end_to_end/registryReset.test.ts +++ b/backend/src/handlers/__tests__/end_to_end/registryReset.test.ts @@ -4,7 +4,7 @@ import { baseUrl } from "./config"; const timeout = 30000; describe("E2E Test for Registry Reset Endpoint", () => { it("should return a 200 status", async () => { - const response = await axios.delete(`${baseUrl}/reset`); + const response = await axios.delete(`${baseUrl}/reset`, { timeout: 60000 }); expect(response.status).toBe(200); }, timeout); }); \ No newline at end of file diff --git a/backend/src/services/rate/tools/api.ts b/backend/src/services/rate/tools/api.ts index fd1a03c..f96c55e 100644 --- a/backend/src/services/rate/tools/api.ts +++ b/backend/src/services/rate/tools/api.ts @@ -1,8 +1,8 @@ import * as fs from 'fs/promises'; import * as git from 'isomorphic-git'; import * as http from 'isomorphic-git/http/node'; -import axios, { AxiosInstance } from 'axios'; -import { gitData, npmData } from '../utils/interfaces'; +import axios, { AxiosInstance, AxiosResponse } from 'axios'; +import { gitData, GitHubFile, GitHubReview, npmData } from '../utils/interfaces'; import { logger } from './logging'; import { envVars } from '../utils/interfaces'; import * as semver from 'semver'; @@ -46,11 +46,11 @@ export class npmAnalysis { try { const oid = await git.resolveRef({ fs, dir, ref: 'HEAD' }); const { tree } = await git.readTree({ fs, dir, oid }); - - const readmeEntry = tree.find(entry => + + const readmeEntry = tree.find(entry => ['readme.md', 'readme', 'readme.txt', 'readme.rst'].includes(entry.path.toLowerCase()) ); - + let readmeContent: string | null = null; if (readmeEntry) { // Found a README file in the repository @@ -61,14 +61,14 @@ export class npmAnalysis { this.logger.logInfo(`No README file found in the repository tree. Trying to fetch via package URL...`); const readmeUrl = `${npmData.repoUrl}#readme`; // Construct URL to fetch README const response = await fetch(readmeUrl); - + if (response.ok) { readmeContent = await response.text(); } else { this.logger.logDebug(`Could not retrieve README from package URL ${readmeUrl} in ${dir}`); } } - + if (readmeContent) { npmData.documentation.hasReadme = true; npmData.documentation.numLines = readmeContent.split('\n').length; @@ -78,17 +78,17 @@ export class npmAnalysis { } catch { this.logger.logDebug(`Error retrieving the README content for ${npmData.repoUrl} in ${dir}`); } - } + } async lastCommitDate(dir: string, npmData: npmData): Promise { this.logger.logDebug(`Finding time since last commit...`); try { const commits = await git.log({ fs, dir, depth: 1 }); - const lastCommit = commits[0]; - + const lastCommit = commits[0]; + if (lastCommit) { - const lastCommitDate = new Date(lastCommit.commit.author.timestamp * 1000); - npmData.lastCommitDate = lastCommitDate.toDateString(); + const lastCommitDate = new Date(lastCommit.commit.author.timestamp * 1000); + npmData.lastCommitDate = lastCommitDate.toDateString(); } else { this.logger.logDebug(`No commits found in the repository ${npmData.repoUrl} in dir ${dir}`); } @@ -103,17 +103,17 @@ export class npmAnalysis { } - async analyzeDependencies(dir: string, npmData: npmData): Promise { + async analyzeDependencies(dir: string, npmData: npmData): Promise { try { const packageJsonPath = `${dir}/package.json`; const packageJson = await fs.readFile(packageJsonPath, 'utf-8'); const packageData = JSON.parse(packageJson); - const dependencies = {... packageData.dependencies, ... packageData.devDependencies}; + const dependencies = { ...packageData.dependencies, ...packageData.devDependencies }; //make sure these are strings for semver - - + + const totalDependencies = Object.keys(dependencies).length; if (totalDependencies === 0) { npmData.documentation.dependencies = { @@ -133,7 +133,7 @@ export class npmAnalysis { // console.log("unpinned"); } } - const fractionPinned = pinnedCount / totalDependencies; + const fractionPinned = pinnedCount / totalDependencies; npmData.documentation.dependencies = { total: totalDependencies, pinned: pinnedCount, @@ -168,11 +168,11 @@ export class npmAnalysis { const endTime = performance.now(); return endTime - startTime; } - - + + // Main function to run the tasks in order async runTasks(url: string, dest: number, version: string): Promise { - const repoDir = './dist/repoDir'+dest.toString(); + const repoDir = './dist/repoDir' + dest.toString(); console.log('Running npm tasks in', repoDir); this.logger.logDebug(`Running npm tasks in ${repoDir}...`); const npmData: npmData = { @@ -206,8 +206,8 @@ export class npmAnalysis { }; await this.cloneRepo(url, repoDir, version); - [ npmData.latency.lastCommitDate, - npmData.latency.documentation + [npmData.latency.lastCommitDate, + npmData.latency.documentation ] = await Promise.all([ this.executeTasks(this.lastCommitDate.bind(this), repoDir, npmData), this.executeTasks(this.getReadmeContent.bind(this), repoDir, npmData) @@ -215,7 +215,7 @@ export class npmAnalysis { npmData.latency.dependencies = await this.executeTasks(this.analyzeDependencies.bind(this), repoDir, npmData); await this.deleteRepo(repoDir); - + this.logger.logInfo(`All npm tasks completed in order within dir ${repoDir}`); return npmData; } @@ -225,7 +225,7 @@ export class gitAnalysis { private axiosInstance: AxiosInstance; private logger: logger; private token: string; - + //axios instance using tokens, loglevel, logfile from .env constructor(envVars: envVars) { this.logger = new logger(envVars); @@ -299,7 +299,7 @@ export class gitAnalysis { ): Promise { let retryCount = 0; let delay = initialDelay; - + while (retryCount <= maxRetries) { try { // Try the request function @@ -316,10 +316,10 @@ export class gitAnalysis { delay *= 2; // Exponential backoff: double the delay } } - + throw new Error(`Failed to complete the request after ${maxRetries} retries`); } - + //retrieve data for closed issues async fetchClosedIssues(gitData: gitData): Promise { this.logger.logDebug(`Fetching closed issues for ${gitData.repoName}...`); @@ -359,12 +359,12 @@ export class gitAnalysis { //retrieve data for number of contributors async fetchContributors(gitData: gitData): Promise { this.logger.logDebug(`Fetching contributors for ${gitData.repoName}...`); - + try { let page = 1; let contributorsCount = 0; let hasMorePages = true; - + // Fetch contributors with pagination and exponential backoff while (hasMorePages) { const endpoint = `/repos/${gitData.repoOwner}/${gitData.repoName}/contributors`; @@ -372,27 +372,27 @@ export class gitAnalysis { per_page: 100, // Fetch up to 100 contributors per page page: page }; - + // Use exponential backoff to handle retries on failure const response = await this.exponentialBackoff(() => this.axiosInstance.get(endpoint, { params }) ); - + // Update count and check for more pages contributorsCount += response.data.length; const linkHeader = response.headers['link']; hasMorePages = typeof linkHeader === 'string' && linkHeader.includes('rel="next"'); page++; } - + this.logger.logDebug(`Contributors Count fetched successfully for ${gitData.repoName}`); gitData.numberOfContributors = contributorsCount; - + } catch (error) { this.logger.logDebug(`Error fetching number of contributors for ${gitData.repoName}`, error); } } - + async fetchLicense(gitData: gitData): Promise { this.logger.logDebug(`Fetching license for ${gitData.repoName}...`); try { @@ -507,90 +507,103 @@ export class gitAnalysis { this.logger.logDebug(`Error fetching number of lines for ${gitData.repoName}`, error); } } - // Add to api.ts in gitAnalysis class + // Add to api.ts in gitAnalysis class - async fetchPullRequests(gitData: gitData): Promise { - this.logger.logDebug(`Fetching pull requests for ${gitData.repoName}...`); - - try { - if (!gitData.repoOwner || !gitData.repoName) { - throw new Error('Invalid repository owner or name'); + async fetchPullRequests(gitData: gitData): Promise { + this.logger.logDebug(`Fetching pull requests for ${gitData.repoName}...`); + + if (!gitData.repoOwner || !gitData.repoName) { + throw new Error("Invalid repository owner or name"); + } + + let totalReviewedAdditions = 0; + let totalAdditions = 0; + + try { + // Fetch up to 100 pull requests in a single request + const response: AxiosResponse = await this.axiosInstance.get( + `/repos/${gitData.repoOwner}/${gitData.repoName}/pulls`, + { + params: { + state: "closed", + per_page: 100, // Fetch up to 100 PRs + }, } - - let totalReviewedAdditions = 0; - let totalAdditions = 0; - - try { - // Fetch up to 100 pull requests in a single request - const response = await this.axiosInstance.get( - `/repos/${gitData.repoOwner}/${gitData.repoName}/pulls`, - { - params: { - state: "closed", - per_page: 100, // Fetch up to 100 PRs - }, - } - ); - - const prs = response.data; - - for (const pr of prs) { - try { - //fetch the files response - const filesResponse = await this.exponentialBackoff(() => - this.axiosInstance.get(`/repos/${gitData.repoOwner}/${gitData.repoName}/pulls/${pr.number}/files`) - ); - const files = filesResponse.data; - let additions = 0; - for (const file of files) { - additions += file.additions; - } - totalAdditions += additions; - - - // Fetch reviews for this PR - const reviewsResponse = await this.exponentialBackoff(() => - this.axiosInstance.get(`/repos/${gitData.repoOwner}/${gitData.repoName}/pulls/${pr.number}/reviews`) - ); - - if (reviewsResponse.data.length > 0) { - totalReviewedAdditions += additions; - this.logger.logDebug(`PR #${pr.number} has reviews.`); - this.logger.logDebug(`Total reviewed additions: ${totalReviewedAdditions}`); - } - } catch (prError) { - this.logger.logDebug(`Error processing PR ${pr.number}:`, prError); - continue; - } + ); + + const prs: GitHubPR[] = response.data; + + // Process PRs in parallel + const prMetrics = await Promise.all( + prs.map(async (pr): Promise<{ additions: number; reviewedAdditions: number }> => { + try { + // Fetch PR files and reviews concurrently + const [filesResponse, reviewsResponse]: [ + AxiosResponse, + AxiosResponse + ] = await Promise.all([ + this.exponentialBackoff(() => + this.axiosInstance.get( + `/repos/${gitData.repoOwner}/${gitData.repoName}/pulls/${pr.number}/files` + ) + ), + this.exponentialBackoff(() => + this.axiosInstance.get( + `/repos/${gitData.repoOwner}/${gitData.repoName}/pulls/${pr.number}/reviews` + ) + ), + ]); + + const files: GitHubFile[] = filesResponse.data; + const reviews: GitHubReview[] = reviewsResponse.data; + + // Calculate additions for this PR + const additions = files.reduce( + (sum: number, file: GitHubFile) => sum + file.additions, + 0 + ); + + const reviewedAdditions = reviews.length > 0 ? additions : 0; + + return { additions, reviewedAdditions }; + } catch (prError) { + this.logger.logDebug(`Error processing PR ${pr.number}:`, prError); + return { additions: 0, reviewedAdditions: 0 }; } - } catch (error) { - this.logger.logDebug(`Error fetching pull requests for ${gitData.repoName}:`, error); - } - - // Calculate metrics - this.logger.logInfo(`Pull request metrics for ${gitData.repoName} are: ${totalReviewedAdditions}, ${totalAdditions}, ${totalReviewedAdditions / totalAdditions}`); - - gitData.pullRequestMetrics = { - totalAdditions, - reviewedAdditions: totalReviewedAdditions, - reviewedFraction: totalAdditions > 0 + }) + ); + + // Aggregate metrics + for (const metric of prMetrics) { + totalAdditions += metric.additions; + totalReviewedAdditions += metric.reviewedAdditions; + } + + // Calculate and store metrics + gitData.pullRequestMetrics = { + totalAdditions, + reviewedAdditions: totalReviewedAdditions, + reviewedFraction: + totalAdditions > 0 ? parseFloat((totalReviewedAdditions / totalAdditions).toFixed(3)) : 0.0, - }; - - this.logger.logDebug(`Pull request metrics calculated successfully for ${gitData.repoName}`); - this.logger.logInfo(`Pull request metrics for ${gitData.repoName} are: ${gitData.pullRequestMetrics.reviewedAdditions}, ${gitData.pullRequestMetrics.totalAdditions}, ${gitData.pullRequestMetrics.reviewedFraction}`); - } catch (error) { - this.logger.logDebug(`Error fetching pull requests for ${gitData.repoName}:`, error); - gitData.pullRequestMetrics = { - totalAdditions: 0, - reviewedAdditions: 0, - reviewedFraction: 0.0, - }; - } + }; + + this.logger.logInfo( + `Pull request metrics for ${gitData.repoName}: ${gitData.pullRequestMetrics.reviewedAdditions}, ${gitData.pullRequestMetrics.totalAdditions}, ${gitData.pullRequestMetrics.reviewedFraction}` + ); + this.logger.logDebug(`Pull request metrics calculated successfully for ${gitData.repoName}`); + } catch (error) { + this.logger.logDebug(`Error fetching pull requests for ${gitData.repoName}:`, error); + gitData.pullRequestMetrics = { + totalAdditions: 0, + reviewedAdditions: 0, + reviewedFraction: 0.0, + }; } - - + } + + private async executeTasks(task: (gitData: gitData) => Promise, gitData: gitData): Promise { const startTime = performance.now(); await task(gitData); @@ -598,7 +611,7 @@ export class gitAnalysis { return endTime - startTime; } - async runTasks(url: string): Promise { + async runTasks(url: string): Promise { const gitData: gitData = { repoName: '', repoUrl: url, @@ -631,8 +644,8 @@ export class gitAnalysis { this.logger.logDebug(`Running git tasks for ${gitData.repoUrl}...`); if (await this.checkConnection(url)) { await this.getOwnerAndRepo(gitData); - [ gitData.latency.openIssues, - gitData.latency.licenses, + [gitData.latency.openIssues, + gitData.latency.licenses, ] = await Promise.all([ this.executeTasks(this.fetchOpenIssues.bind(this), gitData), this.executeTasks(this.fetchLicense.bind(this), gitData), diff --git a/backend/src/services/rate/utils/interfaces.ts b/backend/src/services/rate/utils/interfaces.ts index 6c0881b..a6dca28 100644 --- a/backend/src/services/rate/utils/interfaces.ts +++ b/backend/src/services/rate/utils/interfaces.ts @@ -31,6 +31,17 @@ export interface GitHubPR { [key: string]: any; // To accommodate any other fields returned by the API } +export interface GitHubFile { + additions: number; + [key: string]: any; // Additional optional fields +} + +export interface GitHubReview { + id: number; + state: string; + [key: string]: any; // Additional optional fields +} + export interface gitData { repoName: string; repoUrl: string; diff --git a/backend/template.yml b/backend/template.yml index 461742c..3c00281 100644 --- a/backend/template.yml +++ b/backend/template.yml @@ -222,7 +222,7 @@ Resources: Runtime: nodejs18.x CodeUri: dist/handlers/RegistryReset MemorySize: 128 - Timeout: 120 + Timeout: 30 Layers: - !Ref DependenciesLayer Events: