Merge pull request #8 from vladaviedov/dev

0.4.0

.eslintrc.json

@@ -13,7 +13,10 @@
     "rules": {
         "indent": [
             "error",
-            "tab"
+            "tab",
+            {
+                "SwitchCase": 1
+            }
         ],
         "linebreak-style": [
             "error",

README.md

@@ -1,17 +1,18 @@
 # gh-lang-stats
 
-Calculate Language Stats for your GitHub Account.
+Calculate language usage statistics for your GitHub Account.
 
 ## Running
-*TODO: Github Actions template*
+Github Actions
+- See [template repo](https://github.com/vladaviedov/ghls-template)
 
 Install Package
-* Install the package from GitHub repo
-* Copy a template and rename it `template.svg`
-* Generate a [personal access token](https://github.com/settings/tokens)
-* Run `npx gh-lang-stats` with the env `ACCESS_KEY` set to your PAT
+- Install the package from GitHub repo
+- Copy a template and rename it `template.svg`
+- Generate a [personal access token](https://github.com/settings/tokens)
+- Run `npx gh-lang-stats` with the env `ACCESS_KEY` set to your PAT
 
 ## TODO
-* GitHub Actions Template
-* Rate Limits
+* Documentation
+* More config
 * Caching

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vladaviedov/gh-lang-stats",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Github Lang Stats",
   "homepage": "https://github.com/vladaviedov/gh-lang-stats",
   "bugs": "https://github.com/vladaviedov/gh-lang-stats/issues",
@@ -28,6 +28,7 @@
     "eslint": "^8.22.0"
   },
   "dependencies": {
+    "@octokit/plugin-throttling": "^4.3.0",
     "octokit": "^2.0.7",
     "yaml": "^2.1.1"
   }

src/config.js

@@ -0,0 +1,6 @@
+export const config = {
+	token: process.env.TOKEN,
+	maxConcur: process.env.MAX_CONCUR ?? 5,
+	inputFile: process.env.INPUT_FILE ?? "template.svg",
+	outputFile: process.env.OUTPUT_FILE ?? "generated.svg"
+};

src/fill-template.js

@@ -1,15 +1,16 @@
 import { readFileSync, writeFileSync } from "fs";
+import { config } from "./config.js";
 
 export const fillTemplate = data => {
 	const total = data.Total;
 	delete data.Total;
 	const sorted = sortToArray(data);
 
-	let svg = readFileSync("template.svg", { encoding: "utf-8" });
+	let svg = readFileSync(config.inputFile, { encoding: "utf-8" });
 	svg = replaceBar(svg, sorted, total);
 	svg = replaceList(svg, sorted, total);
 
-	writeFileSync("generated.svg", svg, { encoding: "utf-8" });
+	writeFileSync(config.outputFile, svg, { encoding: "utf-8" });
 };
 
 const sortToArray = unsorted => {

src/github-api.js

@@ -72,7 +72,7 @@ export const qlFullList = async (client, id) => {
 			return r;
 		}));
 
-		return scan;
+		return scan.nodes;
 	} catch (ex) {
 		handleHttpErr(ex.response);
 	}
@@ -86,12 +86,18 @@ export const qlFullList = async (client, id) => {
  * @param {string} hash Commit Hash
  * @returns Commit information
  */
-export const restCommitInfo = (client, owner, repo, hash) => {
-	return client.request(reqCommitInfo, {
-		owner: owner,
-		repo: repo,
-		ref: hash 
-	});
+export const restCommitInfo = async (client, owner, repo, hash) => {
+	try {
+		const response = await client.request(reqCommitInfo, {
+			owner: owner,
+			repo: repo,
+			ref: hash 
+		});
+
+		return response;
+	} catch (ex) {
+		handleHttpErr(ex.response);
+	}
 };
 
 /**
@@ -115,6 +121,8 @@ export const rawLinguistYml = () => {
 const handleHttpErr = err => {
 	if (err.status == 401) {
 		console.error("Request is unauthorized.");
+	} else {
+		console.error("An unknown error has occured.");
 	}
 
 	process.exit(1);

src/load-commits.js

@@ -0,0 +1,60 @@
+import { restCommitInfo } from "./github-api.js";
+import { config } from "./config.js";
+
+const queue = [];
+
+export const loadCommits = async (client, repos) => {
+	const promises = repos.map(async r => {
+		const commits = await repoCommits(client, r);
+		return {
+			languages: r.languages.nodes,
+			commits: commits
+		};
+	});
+
+	processQueue();
+
+	const result = await Promise.all(promises);
+	return result;
+};
+
+const repoCommits = async (client, repo) => {
+	const owner = repo.owner.login;
+	const name = repo.name;
+	const commitHashes = repo.defaultBranchRef.target.history.nodes;
+
+	const promises = commitHashes.map(hash => enqueueHash(client, owner, name, hash.oid));
+
+	const commits = await Promise.all(promises);
+	return commits;
+};
+
+const enqueueHash = (client, owner, name, hash) => {
+	return new Promise(resolve => {
+		queue.push({
+			func: () => restCommitInfo(client, owner, name, hash),
+			callback: resolve
+		});
+	});
+};
+
+const processQueue = async () => {
+	const workers = [];
+
+	// Start initial workers
+	for (let i = 0; i < config.maxConcur; i++) {
+		workers.push(worker(queue.pop(), i));
+	}
+
+	while (queue.length > 0) {
+		// Wait for free slot
+		const res = await Promise.any(workers);
+		workers[res] = worker(queue.pop(), res);
+	}
+};
+
+const worker = async (request, index) => {
+	const response = await request.func();
+	request.callback(response);
+	return index;
+};

src/main.js

@@ -1,33 +1,29 @@
 #!/usr/bin/env node
 import { Octokit } from "octokit";
+import { throttling } from "@octokit/plugin-throttling";
 import { analyzeData } from "./analyze.js";
 import { fillTemplate } from "./fill-template.js";
-import { qlUserId, qlFullList, restCommitInfo } from "./github-api.js";
+import { qlUserId, qlFullList } from "./github-api.js";
+import { loadCommits } from "./load-commits.js";
+import { config } from "./config.js";
 
-const octokit = new Octokit({
-	auth: process.env.ACCESS_KEY
+const OctokitPlug = Octokit.plugin(throttling);
+const octokit = new OctokitPlug({
+	auth: config.token,
+	throttle: {
+		onRateLimit: retryAfter => {
+			console.error(`Ratelimit hit. Waiting ${retryAfter} seconds`);
+			return true;
+		},
+		onSecondaryRateLimit: retryAfter => {
+			console.error(`Secondary ratelimit hit. Waiting ${retryAfter} seconds`);
+			return true;
+		}
+	}
 });
 
-qlUserId(octokit).then(id => {
-	return qlFullList(octokit, id);
-}).then(response => {
-	const repos = response.nodes;
-	return Promise.all(repos.map(async r => {
-		return {
-			languages: r.languages.nodes,
-			commits: await repoCommits(r)
-		};
-	}));
-}).then(analyzeData).then(fillTemplate);
-
-const repoCommits = repo => {
-	const owner = repo.owner.login;
-	const name = repo.name;
-	const commitHashes = repo.defaultBranchRef.target.history.nodes;
-
-	const promises = [];
-	commitHashes.forEach(hash => {
-		promises.push(restCommitInfo(octokit, owner, name, hash.oid));
-	});
-	return Promise.all(promises);
-};
+qlUserId(octokit)
+	.then(id => qlFullList(octokit, id))
+	.then(list => loadCommits(octokit, list))
+	.then(analyzeData)
+	.then(fillTemplate);

yarn.lock

@@ -250,6 +250,14 @@
     "@octokit/types" "^7.0.0"
     bottleneck "^2.15.3"
 
+"@octokit/plugin-throttling@^4.3.0":
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/@octokit/plugin-throttling/-/plugin-throttling-4.3.0.tgz#8b45a4d3cca5f3d8ea6cbfbd185ee8d73daab869"
+  integrity sha512-4H0HlCQoJMkLputwruBvMNl4tBRiDHMkMZQyyR8Nh0KwLFrbMW0vuQjwSNPqTqMhiQ5JpP2XKPifhHr8gIs2jQ==
+  dependencies:
+    "@octokit/types" "^7.0.0"
+    bottleneck "^2.15.3"
+
 "@octokit/request-error@^2.1.0":
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-2.1.0.tgz#9e150357831bfc788d13a4fd4b1913d60c74d677"