feat(grzesiu): Use natural

package.json

@@ -32,6 +32,7 @@
     "isolated-vm": "4.2.0",
     "mongodb": "4.0.0-beta.5",
     "monitorss": "6.13.19",
+    "natural": "5.1.13",
     "node-cache": "5.1.2",
     "node-fetch": "2.6.1",
     "openai": "2.0.2",
@@ -47,6 +48,7 @@
     "@types/chai-as-promised": "7.1.4",
     "@types/dotenv": "8.2.0",
     "@types/mocha": "8.2.2",
+    "@types/natural": "5.1.0",
     "@types/nock": "11.1.0",
     "@types/node": "14.14.41",
     "@types/node-fetch": "2.5.10",

src/commands/grzesiu.ts

@@ -1,6 +1,13 @@
 import { Command } from '../types';
 import { Configuration, OpenAIApi } from 'openai';
 import grzesJson from '../../grzes.json';
+import Natural from 'natural';
+
+const tokenizer = new Natural.AggressiveTokenizerPl();
+const tfidf = grzesJson.reduce((tfidf, line) => {
+  tfidf.addDocument(tokenizer.tokenize(line));
+  return tfidf;
+}, new Natural.TfIdf());
 
 const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
@@ -24,30 +31,14 @@ const grzesiu: Command = {
   cooldown: COOLDOWN,
   async execute(msg, args) {
     const username = (msg.member?.displayName || msg.author.username).trim().split(/\s/)[0];
-    const prompt = await generateGrzesiuPrompt(username, args.join(' '));
-
-    const engine = 'text-davinci-001';
-    // const engine = 'text-babbage-001';
-    const response = await openai.createCompletion(engine, {
-      prompt,
-      temperature: 1,
-      max_tokens: RESPONSE_TOKENS,
-      top_p: 1,
-      frequency_penalty: 0,
-      presence_penalty: 2,
-      best_of: 4,
-    });
-
-    if (!response.data.choices?.[0]?.text) {
-      return msg.channel.send(`Niestety, Grzesiu nie miał nic do powiedzenia!`);
-    }
+    const question = args.join(' ');
 
-    const messages = response.data.choices[0].text
-      .split('\n')
-      .map((l) => l.trim())
-      .filter((l) => l.startsWith(`${GRZESIU_NAME}:`))
-      .flatMap((l) => l.split(`${GRZESIU_NAME}:`))
-      .filter((l) => l.trim().length > 0);
+    const generators = {
+      getGrzesiuAnswerFromOpenAI,
+      getRandomGrzesiuAnswers,
+    };
+
+    const messages = await generators.getRandomGrzesiuAnswers(username, question);
 
     if (!messages.length) {
       return msg.channel.send(`Niestety, Grzesiu nie miał nic do powiedzenia!`);
@@ -66,6 +57,77 @@ export default grzesiu;
 
 const getRandomInt = (len: number) => Math.floor(Math.random() * len);
 
+interface GrzesiuGenerator {
+  (username: string, question: string): Promise<string[]>;
+}
+
+// random
+const getRandomGrzesiuAnswers: GrzesiuGenerator = async (_username, question) => {
+  const g = grzesJson
+    .map((l) => l.trim())
+    .filter((line) => !BANNED_PATTERNS.test(line) && line.length > 0);
+
+  const potentialItems =
+    question.trim().length > 0
+      ? g.filter((l, idx) => {
+          const coeff = (tfidf.tfidf(question, idx) as unknown as number) || 1;
+          return coeff * Natural.DiceCoefficient(question, l) > 0.35;
+        })
+      : [];
+
+  const initialIndex =
+    potentialItems.length > 0 ? g.indexOf(potentialItems[getRandomInt(potentialItems.length)]) : -1;
+
+  const MAX_LINES = 5;
+  const numberOfLines = 5 - Math.floor(Math.log2(Math.floor(Math.random() * 2 ** MAX_LINES) + 1));
+
+  const lines = [];
+  let idx = initialIndex === -1 ? getRandomInt(g.length) : initialIndex;
+
+  for (let i = 0; i < numberOfLines; ++i) {
+    lines.push(g[idx]);
+
+    if (Math.random() < 0.9 && idx < g.length) {
+      ++idx;
+    } else {
+      idx = getRandomInt(g.length);
+    }
+  }
+
+  return lines;
+};
+
+// openAI
+const getGrzesiuAnswerFromOpenAI: GrzesiuGenerator = async (username, question) => {
+  const prompt = await generateGrzesiuPrompt(username, question);
+
+  const engine = 'text-davinci-001';
+  // const engine = 'text-babbage-001';
+  const response = await openai.createCompletion(engine, {
+    prompt,
+    temperature: 1,
+    max_tokens: RESPONSE_TOKENS,
+    top_p: 1,
+    frequency_penalty: 0,
+    presence_penalty: 2,
+    best_of: 4,
+  });
+
+  console.log(response);
+
+  if (!response.data.choices?.[0]?.text) {
+    return [];
+  }
+
+  const messages = response.data.choices[0].text
+    .split('\n')
+    .map((l) => l.trim())
+    .filter((l) => l.startsWith(`${GRZESIU_NAME}:`))
+    .flatMap((l) => l.split(`${GRZESIU_NAME}:`))
+    .filter((l) => l.trim().length > 0);
+  return messages;
+};
+
 const getRandomIndices = (num: number, max: number) => {
   const set = new Set<number>();
   while (set.size < num) set.add(getRandomInt(max));

yarn.lock

@@ -601,6 +601,13 @@
     "@types/bson" "*"
     "@types/node" "*"
 
+"@types/[email protected]":
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/@types/natural/-/natural-5.1.0.tgz#bb003b328db83db5bf138829ecf79ce26e9997c0"
+  integrity sha512-b6HzwfHcxK9himwAnhyth8Nz3CcbbgM/qu8bkSVEE/WzMUNR2wo2RHaVK5XVVakwP8EVcI5MAysaOkG8iWhEag==
+  dependencies:
+    "@types/node" "*"
+
 "@types/[email protected]":
   version "11.1.0"
   resolved "https://registry.yarnpkg.com/@types/nock/-/nock-11.1.0.tgz#0a8c1056a31ba32a959843abccf99626dd90a538"
@@ -848,6 +855,11 @@ addressparser@^1.0.1:
   resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
   integrity sha1-R6++GiqSYhkdtoOOT9HTm0CCF0Y=
 
+afinn-165@^1.0.2:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/afinn-165/-/afinn-165-1.0.4.tgz#3abf6b8922dd5db84d84e0abd155924381dd73a4"
+  integrity sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==
+
 aggregate-error@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a"
@@ -940,6 +952,13 @@ anymatch@~3.1.1:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
+apparatus@^0.0.10:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.10.tgz#81ea756772ada77863db54ceee8202c109bdca3e"
+  integrity sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==
+  dependencies:
+    sylvester ">= 0.0.8"
+
 append-transform@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-2.0.0.tgz#99d9d29c7b38391e6f428d28ce136551f0b77e12"
@@ -3608,6 +3627,18 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+[email protected]:
+  version "5.1.13"
+  resolved "https://registry.yarnpkg.com/natural/-/natural-5.1.13.tgz#61f3e5d15aacd0e6b5c270cb5e8ed1dba7e163a8"
+  integrity sha512-fbV7Jvr8X0S+Yb/iggoGxTBRMSbTxN1m2sgBspNI5dSnE6x3lN2P48O1ddyoiTtMC2q4OKDkHJAGuNRs9jM9WA==
+  dependencies:
+    afinn-165 "^1.0.2"
+    apparatus "^0.0.10"
+    safe-stable-stringify "^2.2.0"
+    sylvester "^0.0.12"
+    underscore "^1.9.1"
+    wordnet-db "^3.1.11"
+
 [email protected]:
   version "0.6.2"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -4523,6 +4554,11 @@ [email protected], safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, s
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
+safe-stable-stringify@^2.2.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73"
+  integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg==
+
 "safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -4941,6 +4977,16 @@ supports-color@^7.1.0, supports-color@^7.2.0:
   dependencies:
     has-flag "^4.0.0"
 
+"sylvester@>= 0.0.8":
+  version "0.0.21"
+  resolved "https://registry.yarnpkg.com/sylvester/-/sylvester-0.0.21.tgz#2987b1ce2bd2f38b0dce2a34388884bfa4400ea7"
+  integrity sha1-KYexzivS84sNzio0OIiEv6RADqc=
+
+sylvester@^0.0.12:
+  version "0.0.12"
+  resolved "https://registry.yarnpkg.com/sylvester/-/sylvester-0.0.12.tgz#5a884415cd2d002c57e7a3aac99462a75ce9fdb4"
+  integrity sha1-WohEFc0tACxX56OqyZRip1zp/bQ=
+
 table@^6.0.9:
   version "6.7.1"
   resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2"
@@ -5130,6 +5176,11 @@ unbox-primitive@^1.0.0, unbox-primitive@^1.0.1:
     has-symbols "^1.0.2"
     which-boxed-primitive "^1.0.2"
 
+underscore@^1.9.1:
+  version "1.13.2"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.2.tgz#276cea1e8b9722a8dbed0100a407dda572125881"
+  integrity sha512-ekY1NhRzq0B08g4bGuX4wd2jZx5GnKz6mKSqFL4nqBlfyMGiG10gDFhDTMEfYmDL6Jy0FUIZp7wiRB+0BP7J2g==
+
 [email protected], unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
@@ -5247,6 +5298,11 @@ word-wrap@^1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
+wordnet-db@^3.1.11:
+  version "3.1.14"
+  resolved "https://registry.yarnpkg.com/wordnet-db/-/wordnet-db-3.1.14.tgz#7ba1ec2cb5730393f0856efcc738a60085426199"
+  integrity sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==
+
 [email protected]:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.0.tgz#a8e038b4c94569596852de7a8ea4228eefdeb37b"