Skip to content

Commit

Permalink
[ Add ]
Browse files Browse the repository at this point in the history
  • Loading branch information
anasfik committed Nov 29, 2023
1 parent 345d0f6 commit b8b7db1
Show file tree
Hide file tree
Showing 26 changed files with 121 additions and 110 deletions.
3 changes: 2 additions & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
PORT=5559
OPENAI_API_KEY=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
# OPENAI_API_KEY=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
OPENAI_API_KEY=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
MANGODBURI="mongodb+srv://Gwhyyy:[email protected]/?retryWrites=true&w=majority"
3 changes: 2 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,6 @@
},
"[typescript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
}
},
"typescript.tsdk": "node_modules\\typescript\\lib"
}
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
"main": "index.js",
"scripts": {
"build": "tsc",
"start": "node ./dist/index.js",
"dev": "echo 'aaa' &&nodemon ./dist/index.js",
"start": "tsc && node ./dist/index.js",
"dev": "echo 'aaa' && nodemon ./dist/index.js",
"separator": "node ./src/controllers/utils/json_tiktoken_separator.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
Expand All @@ -33,4 +33,4 @@
"@types/express": "^4.17.18",
"typescript": "^5.2.2"
}
}
}
22 changes: 14 additions & 8 deletions src/adapters/json.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import fs from "fs";
import * as utils from "../controllers/utils/utils";
import uuid from "uuid";
import { v4 } from "uuid";
import { OpenAIClient } from "../ai_clients/openAI";
import { LangSyncLogger } from "../controllers/utils/logger";
import { loggingTypes } from "../enum";

export class JsonAdapter implements BaseAdapter {
constructor(private filePath: string) {}
Expand All @@ -14,12 +16,14 @@ export class JsonAdapter implements BaseAdapter {
allowMultipleUniqueIds: boolean = false;

deleteFile() {
fs.unlinkSync(this.filePath);
if (fs.existsSync(this.filePath)) {
fs.unlinkSync(this.filePath);
}
}

readFileAsString(): string {
let asString: string = fs.readFileSync(this.filePath, "utf8");
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: `File named ${this.filePath
.split("/")
.pop()} has been loaded as a string.`,
Expand All @@ -32,12 +36,12 @@ export class JsonAdapter implements BaseAdapter {
parseString(fileContent: string): any {
return JSON.parse(fileContent);
}

// @ts-ignore
async isHarming(options: HarmOptions): Promise<boolean> {
let isHarming: boolean = await this.aiClient.isHarming(options.fileContent);

if (isHarming) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message:
"The provided content violates our policy, and so it is unacceptable to be processed.",
});
Expand All @@ -49,7 +53,7 @@ export class JsonAdapter implements BaseAdapter {
} else {
}
} else {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "The provided content is acceptable to be processed.",
});

Expand All @@ -60,6 +64,7 @@ export class JsonAdapter implements BaseAdapter {
async asPartsForOpenAI(): Promise<string[]> {
let asString: string = this.readFileAsString();

// @ts-ignore
let isHarming: boolean = await this.isHarming({
fileContent: asString,
throwIfHarming: true,
Expand All @@ -70,7 +75,7 @@ export class JsonAdapter implements BaseAdapter {
let asParts: string[] =
await utils.parsedFileContentPartsSeparatorForOpenAI(parsed);

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: `File named ${this.filePath
.split("/")
.pop()} has been split into ${asParts.length} parts.`,
Expand All @@ -90,7 +95,8 @@ export class JsonAdapter implements BaseAdapter {
);
}

let generatedId = uuid.v4();
const generatedId = v4();

this.numberOfGeneratedUniqueIds++;

return generatedId;
Expand Down
14 changes: 8 additions & 6 deletions src/ai_clients/openAI.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { OpenAI } from "openai";
import configs from "../configs/openai";
import { LangSyncLogger } from "../controllers/utils/logger";
import { loggingTypes } from "../enum";
export class OpenAIClient implements ArtificialIntelligenceBase {
constructor() {
this.init(<string>configs.openAI);
Expand All @@ -19,7 +21,7 @@ export class OpenAIClient implements ArtificialIntelligenceBase {
input: content.toString(),
});

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "Moderation request has been processed by OpenAI.",
type: loggingTypes.info,
});
Expand All @@ -29,7 +31,7 @@ export class OpenAIClient implements ArtificialIntelligenceBase {
return typeof flagged === "boolean" ? flagged : false;
} catch (error: Error | any) {
if (error.status === 429) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message:
"OpenAI API rate limit reached, waiting 20 seconds for next moderation request",
type: loggingTypes.warning,
Expand All @@ -42,7 +44,7 @@ export class OpenAIClient implements ArtificialIntelligenceBase {
)
);

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "20 seconds passed, continuing request",
type: loggingTypes.info,
});
Expand All @@ -69,14 +71,14 @@ export class OpenAIClient implements ArtificialIntelligenceBase {
],
});

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: res.choices[0].message.content + "\n",
});

return res;
} catch (error: Error | any) {
if (error.status === 429) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message:
"OpenAI API rate limit reached, waiting 20 seconds for next request",
});
Expand All @@ -88,7 +90,7 @@ export class OpenAIClient implements ArtificialIntelligenceBase {
)
);

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "20 seconds passed, repeating request..",
});
return await this.process(messageToOpenAI);
Expand Down
3 changes: 2 additions & 1 deletion src/controllers/auth/validate_api_key_with_user_token.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { LangSyncDatabase } from "../database/database";
import { LangSyncLogger } from "../utils/logger";

export default async function verifyApiKeyWithUserAuthToken(
apiKey: ExtractedApiKey,
Expand All @@ -9,7 +10,7 @@ export default async function verifyApiKeyWithUserAuthToken(
if (!document) {
throw new Error("No user with this API key found.");
} else {
LangSyncLogger.instance.log({ message: "User found with this API key." });
new LangSyncLogger().log({ message: "User found with this API key." });

onVerified && onVerified();

Expand Down
6 changes: 4 additions & 2 deletions src/controllers/cli/process_exceptions.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import Joi from "joi";
import { Request, Response } from "express";
import { LangSyncDatabase } from "../database/database";
import { LangSyncLogger } from "../utils/logger";
import { loggingTypes } from "../../enum";

export default async function processCliException(req: Request, res: Response) {
let scheme = Joi.object({
Expand All @@ -16,7 +18,7 @@ export default async function processCliException(req: Request, res: Response) {
let { error, value } = scheme.validate(req.body);

if (error) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: error.toString(),
type: loggingTypes.error,
});
Expand All @@ -28,7 +30,7 @@ export default async function processCliException(req: Request, res: Response) {

return res.status(200).json({ message: "success" });
} catch (error: Error | any) {
LangSyncLogger.instance.log({ message: error, type: loggingTypes.error });
new LangSyncLogger().log({ message: error, type: loggingTypes.error });
return res.status(500).json({ error: error });
}
}
3 changes: 2 additions & 1 deletion src/controllers/database/client.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { MongoClient, ServerApiVersion } from "mongodb";
import dbConfig from "../../configs/db";
import { LangSyncLogger } from "../utils/logger";

export class LangSyncDatabaseClient {
constructor() {}
Expand All @@ -24,7 +25,7 @@ export class LangSyncDatabaseClient {

await this.client.db("admin").command({ ping: 1 });

LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "Database connection established successfully.",
});
} catch (err) {
Expand Down
8 changes: 1 addition & 7 deletions src/controllers/database/delete.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,9 @@
import { Filter } from "mongodb";
import { Document, Filter } from "mongodb";
import { LangSyncDatabaseClient } from "./client";

export class LangSyncDatabaseDelete {
constructor() {}

_instance: LangSyncDatabaseDelete = new LangSyncDatabaseDelete();

get instance(): LangSyncDatabaseDelete {
return this._instance;
}

delete(
databaseName: string,
collectionName: string,
Expand Down
8 changes: 1 addition & 7 deletions src/controllers/database/insert.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,16 @@
import { LangSyncDatabaseClient } from "./client";
import { LangSyncDatabase } from "./database";

export class LangSyncDatabaseInsert {
async cliException(value: any) {
return await this.insert("db", "cli_exceptions", value);
}
constructor() {}
_instance: LangSyncDatabaseInsert = new LangSyncDatabaseInsert();

get instance(): LangSyncDatabaseInsert {
return this._instance;
}

async fileOperation(doc: {
userId: any;
operationId: string;
createdAt: string;
jsonAsParts: Promise<string[]>;
jsonAsParts: string[];
}): Promise<any> {
return await this.insert("db", "jsonPartitions", doc);
}
Expand Down
8 changes: 1 addition & 7 deletions src/controllers/database/read.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Filter, WithId } from "mongodb";
import { Document, Filter } from "mongodb";
import { LangSyncDatabaseClient } from "./client";

export class LangSyncDatabaseRead {
Expand All @@ -24,12 +24,6 @@ export class LangSyncDatabaseRead {

constructor() {}

_instance: LangSyncDatabaseRead = new LangSyncDatabaseRead();

get instance(): LangSyncDatabaseRead {
return this._instance;
}

async userOperations(operationId: string): Promise<any[]> {
let docFilter = {
operationId: operationId,
Expand Down
10 changes: 2 additions & 8 deletions src/controllers/database/update.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@
import { Filter, UpdateFilter } from "mongodb";
import { Document, Filter, UpdateFilter } from "mongodb";
import { LangSyncDatabaseClient } from "./client";

export class LangSyncDatabaseUpdate {
constructor() {}

static _instance: LangSyncDatabaseUpdate = new LangSyncDatabaseUpdate();

static get instance(): LangSyncDatabaseUpdate {
return this._instance;
}

updateOperationDoc(
operationId: string,
resultTranslations: any[]
): Promise<any> {
const filterDoc = {
operationId: operationId,
};

return this.update("db", "jsonPartitions", filterDoc, {
$addToSet: {
//@ts-ignore
output: {
$each: resultTranslations,
},
Expand Down
6 changes: 4 additions & 2 deletions src/controllers/translate/process_translations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ import { Request, Response } from "express";
import { extractAndVerifyApiKeyExistence, sseEvent } from "../utils/utils";
import { LangSyncDatabase } from "../database/database";
import { TasksResolver } from "./tasks_resolver";
import { LangSyncLogger } from "../utils/logger";

// @ts-ignore
export default async function processTranslations(req: Request, res: Response) {
res.setHeader("Content-Type", "text/event-stream");
res.setHeader("Cache-Control", "no-cache");
Expand Down Expand Up @@ -78,7 +80,7 @@ export default async function processTranslations(req: Request, res: Response) {
})
);

const saveFileOperationDoc =
let saveFileOperationDoc =
await LangSyncDatabase.instance.read.savedFileByOperationId(operationId);

if (!saveFileOperationDoc) {
Expand Down Expand Up @@ -137,7 +139,7 @@ export default async function processTranslations(req: Request, res: Response) {
};

if (includeOutput) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: "including output in response..",
});
response.output = resultTranslations;
Expand Down
4 changes: 3 additions & 1 deletion src/controllers/users/file_operation_of_user.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { Request, Response } from "express";

import { LangSyncDatabase } from "../database/database";
import { extractAndVerifyApiKeyExistence } from "../utils/utils";
import { LangSyncLogger } from "../utils/logger";
import { loggingTypes } from "../../enum";

export default async function fileOperationOfUser(req: Request, res: Response) {
await extractAndVerifyApiKeyExistence(
Expand Down Expand Up @@ -41,7 +43,7 @@ export default async function fileOperationOfUser(req: Request, res: Response) {
return res.status(200).json(doc);
}
} catch (error: Error | any) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: error,
type: loggingTypes.error,
});
Expand Down
7 changes: 4 additions & 3 deletions src/controllers/users/get_user.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import Joi from "joi";
import { Request, Response } from "express";
import { LangSyncDatabase } from "../database/database";

import { LangSyncLogger } from "../utils/logger";
import { loggingTypes } from "../../enum";
// @ts-ignore
export default async function getUser(req: Request, res: Response) {
if (!req.headers["authorization"]) {
return res.status(400).json({
Expand Down Expand Up @@ -103,7 +104,7 @@ export default async function getUser(req: Request, res: Response) {
});
}
} catch (error: Error | any) {
LangSyncLogger.instance.log({
new LangSyncLogger().log({
message: error,
type: loggingTypes.error,
});
Expand Down
Loading

0 comments on commit b8b7db1

Please sign in to comment.