diff --git a/backend/package.json b/backend/package.json index a05d883a9..7cb96f5b6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,7 +11,8 @@ "coverage": "jest --coverage", "generate": "graphql-codegen --config codegen.ts", "lint": "tsc --noEmit && eslint --ext .ts,.tsx .", - "update:catalog": "node ./build/scripts/update-catalog.js" + "update:catalog": "node ./build/scripts/update-catalog.js", + "update:grades": "node ./build/scripts/update-grades.js" }, "repository": { "type": "git", diff --git a/backend/src/scripts/update-catalog.ts b/backend/src/scripts/update-catalog.ts index 7de48c726..620f0e4bd 100644 --- a/backend/src/scripts/update-catalog.ts +++ b/backend/src/scripts/update-catalog.ts @@ -11,8 +11,11 @@ import { SectionModel, SectionType } from "../models/section"; const SIS_COURSE_URL = "https://gateway.api.berkeley.edu/sis/v4/courses"; const SIS_CLASS_URL = "https://gateway.api.berkeley.edu/sis/v1/classes"; -const SIS_SECTION_URL = - "https://gateway.api.berkeley.edu/sis/v1/classes/sections"; +const SIS_SECTION_URL = "https://gateway.api.berkeley.edu/sis/v1/classes/sections"; + +type StrictOption = boolean | "throw"; + +const bulkWriteOptions: { strict?: StrictOption } = { strict: "throw" }; const semToTermId = (s: SemesterType) => { // term-id is computed by dropping the century digit of the year, then adding the term code @@ -26,182 +29,179 @@ const semToTermId = (s: SemesterType) => { }; const queryPages = async ( - url: string, - params: any, - headers: any, - field: string, - retries: number = 3 + url: string, + params: any, + headers: any, + field: string, + retries = 3, ) => { - let page = 1; - const values: T[] = []; - - console.log("Querying SIS API pages..."); - console.log(`URL: ${url}`); - console.log(`Params: ${JSON.stringify(params)}`); - console.log(`Headers: ${JSON.stringify(headers)}`); - while (true) { - let resp: AxiosResponse>; - - try { - resp = await axios.get(url, { - params: { "page-number": page, ...params }, - headers, - }); - } catch (err) { - if (axios.isAxiosError(err) && err.response?.status === 404) { - break; - } else { - console.log(`Unexpected err querying SIS API. Error: ${err}.`); - - if (retries > 0) { - retries--; - console.log(`Retrying...`); - continue; - } else { - console.log( - `Too many errors querying SIS API for courses. Terminating update...` - ); - throw err; + let page = 1; + const values: T[] = []; + + console.log("Querying SIS API pages..."); + console.log(`URL: ${url}`); + console.log(`Params: ${JSON.stringify(params)}`); + console.log(`Headers: ${JSON.stringify(headers)}`); + while (true) { + let resp: AxiosResponse>; + + try { + resp = await axios.get(url, { + params: { "page-number": page, ...params }, + headers, + }); + } catch (err) { + if (axios.isAxiosError(err) && err.response?.status === 404) { + break; + } else { + console.log(`Unexpected err querying SIS API. Error: ${err}.`); + + if (retries > 0) { + retries--; + console.log(`Retrying...`); + continue; + } else { + console.log( + `Too many errors querying SIS API for courses. Terminating update...`, + ); + throw err; + } + } } - } - } - values.push(...resp.data.apiResponse.response[field]); - page++; - } + values.push(...resp.data.apiResponse.response[field]); + page++; + } - console.log( - `Completed querying SIS API. Received ${values.length} objects in ${ - page - 1 - } pages.` - ); + console.log( + `Completed querying SIS API. Received ${values.length} objects in ${page - 1} pages.`, + ); - return values; + return values; }; const updateCourses = async () => { - const headers = { - app_id: config.sis.COURSE_APP_ID, - app_key: config.sis.COURSE_APP_KEY, - }; - - const params = { - "status-code": "ACTIVE", - "page-size": 100, - }; + const headers = { + app_id: config.sis.COURSE_APP_ID, + app_key: config.sis.COURSE_APP_KEY, + }; + const params = { + "status-code": "ACTIVE", + "page-size": 100, + }; - const courses = await queryPages( - SIS_COURSE_URL, - params, - headers, - "courses" - ); + const courses = await queryPages( + SIS_COURSE_URL, + params, + headers, + "courses", + ); - console.log("Updating database with new course data..."); + console.log("Updating database with new course data..."); - const bulkOps = courses.map((c) => ({ - replaceOne: { - filter: { classDisplayName: c.classDisplayName }, - replacement: c, - upsert: true, - }, - })); + const bulkOps = courses.map((c) => ({ + replaceOne: { + filter: { classDisplayName: c.classDisplayName }, + replacement: c, + upsert: true, + }, + })); - const options = {} as MongooseBulkWriteOptions; + const options = bulkWriteOptions as MongooseBulkWriteOptions; - const res = await CourseModel.bulkWrite(bulkOps, options); + const res = await CourseModel.bulkWrite(bulkOps, options); - console.log( - `Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.` - ); + console.log( + `Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`, + ); }; const updateClasses = async () => { - const headers = { - app_id: config.sis.CLASS_APP_ID, - app_key: config.sis.CLASS_APP_KEY, - }; + const headers = { + app_id: config.sis.CLASS_APP_ID, + app_key: config.sis.CLASS_APP_KEY, + }; - const activeSemesters = await SemesterModel.find({ active: true }).lean(); - const classes: ClassType[] = []; + const activeSemesters = await SemesterModel.find({ active: true }).lean(); + const classes: ClassType[] = []; - for (const s of activeSemesters) { - console.log(`Updating classses for ${s.term} ${s.year}...`); + for (const s of activeSemesters) { + console.log(`Updating classses for ${s.term} ${s.year}...`); - const params = { - "term-id": semToTermId(s), - "page-size": 100, - }; + const params = { + "term-id": semToTermId(s), + "page-size": 100, + }; - const semesterClasses = await queryPages( - SIS_CLASS_URL, - params, - headers, - "classes" - ); - classes.push(...semesterClasses); - } + const semesterClasses = await queryPages( + SIS_CLASS_URL, + params, + headers, + "classes", + ); + classes.push(...semesterClasses); + } - console.log("Updating database with new class data..."); - const bulkOps = classes.map((c) => ({ - replaceOne: { - filter: { displayName: c.displayName }, - replacement: c, - upsert: true, - }, - })); + console.log("Updating database with new class data..."); + const bulkOps = classes.map((c) => ({ + replaceOne: { + filter: { displayName: c.displayName }, + replacement: c, + upsert: true, + }, + })); - const options = {} as MongooseBulkWriteOptions; + const options = bulkWriteOptions as MongooseBulkWriteOptions; - const res = await ClassModel.bulkWrite(bulkOps, options); + const res = await ClassModel.bulkWrite(bulkOps, options); - console.log( - `Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.` - ); + console.log( + `Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`, + ); }; const updateSections = async () => { - const headers = { - app_id: config.sis.CLASS_APP_ID, - app_key: config.sis.CLASS_APP_KEY, - }; + const headers = { + app_id: config.sis.CLASS_APP_ID, + app_key: config.sis.CLASS_APP_KEY, + }; - const activeSemesters = await SemesterModel.find({ active: true }).lean(); - const sections: SectionType[] = []; + const activeSemesters = await SemesterModel.find({ active: true }).lean(); + const sections: SectionType[] = []; - for (const s of activeSemesters) { - console.log(`Updating sections for ${s.term} ${s.year}...`); + for (const s of activeSemesters) { + console.log(`Updating sections for ${s.term} ${s.year}...`); - const params = { - "term-id": semToTermId(s), - "page-size": 100, - }; + const params = { + "term-id": semToTermId(s), + "page-size": 100, + }; - const semesterClasses = await queryPages( - SIS_SECTION_URL, - params, - headers, - "classSections" - ); - sections.push(...semesterClasses); - } + const semesterClasses = await queryPages( + SIS_SECTION_URL, + params, + headers, + "classSections", + ); + sections.push(...semesterClasses); + } - console.log("Updating database with new section data..."); - const bulkOps = sections.map((s) => ({ - replaceOne: { - filter: { displayName: s.displayName }, - replacement: s, - upsert: true, - }, - })); + console.log("Updating database with new section data..."); + const bulkOps = sections.map((s) => ({ + replaceOne: { + filter: { displayName: s.displayName }, + replacement: s, + upsert: true, + }, + })); - const options = {} as MongooseBulkWriteOptions; + const options = bulkWriteOptions as MongooseBulkWriteOptions; - const res = await SectionModel.bulkWrite(bulkOps, options); + const res = await SectionModel.bulkWrite(bulkOps, options); - console.log( - `Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.` - ); + console.log( + `Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`, + ); }; (async () => { diff --git a/backend/src/scripts/update-grades.ts b/backend/src/scripts/update-grades.ts new file mode 100644 index 000000000..3de5fddaa --- /dev/null +++ b/backend/src/scripts/update-grades.ts @@ -0,0 +1,102 @@ +import mongooseLoader from '../bootstrap/loaders/mongoose'; +import * as Fs from 'fs'; +import { GradeModel } from '../models/grade'; + +const resourcesPath = "/backend/src/resources/grades/raw" + +const getAllGradeFiles = async () => { + return Fs.readdirSync(resourcesPath) +} + +const capitalizeFirstLetter = (s : string) : string => { + return s.charAt(0).toUpperCase() + s.slice(1); +} + +let accessed = false; + +const getOrDefault = (values : string[], m : Map, key : string) : string | null => { + if (!m.has(key)) return null; + const ind = m.get(key) + if (ind === undefined) return null; + return values[m.get(key)!]; +} + +const fileToMongo = async (file : string) => { + const matches = /(fall|spring|summer)_([0-9]{4})/.exec(file) + + if (matches == null) throw new Error("Invalid files found") + + const semester = matches[1] + const year = parseInt(matches[2]) + + const headerMap : Map = new Map() + const objects:any[] = [] + + const data = Fs.readFileSync(`${resourcesPath}/${file}`) + + console.log("Starting: " + file, "Lines: " + data.length) + let lines = data.toString().split("\n") + const header = lines[0].split(",") + lines = lines.slice(1) + + header.forEach((name, index) => { + headerMap.set(name.replace(/^[0-9\s]*|[+*\r\n]/g, ""), index) + }) + lines.forEach((line) => { + try { + if (!line) return + const values = line.split(",") + const obj:any = { + CourseControlNbr: parseInt(getOrDefault(values, headerMap, "Course Control Nbr") ?? ""), + CourseNumber: getOrDefault(values, headerMap, "Course Number"), + CourseSubjectShortNm: getOrDefault(values, headerMap, "Course Subject Short Nm"), + CourseTitleName: getOrDefault(values, headerMap, "Course Title Nm"), + EnrollmentCnt: parseInt(getOrDefault(values, headerMap, "Enrollment Cnt") ?? ""), + GradeNm: getOrDefault(values, headerMap, "Grade Nm"), + GradeSortNbr: getOrDefault(values, headerMap, "Grade Sort Nbr"), + GradeSubtypeDesc: getOrDefault(values, headerMap, "Grade Subtype Desc"), + GradeTypeDesc: getOrDefault(values, headerMap, "Grade Type Desc"), + InstructorName: (getOrDefault(values, headerMap, "Instructor Name") ?? "").split("; "), + SectionNbr: getOrDefault(values, headerMap, "Section Nbr"), + term: { + semester: capitalizeFirstLetter(semester), + year: year + } + } + Object.keys(obj).forEach((k) => obj[k] == null || obj[k] == "" || Number.isNaN(obj[k]) && delete obj[k]); + objects.push(obj) + } catch(err) { + console.log(file, err, line) + } + }) + + console.log(`Processing for ${file} complete, ${objects.length} documents to be added`) + await GradeModel.insertMany(objects) + + console.log(`Successfully saved ${file}`) +} + +(async () => { + try { + + await mongooseLoader(); + + await GradeModel.deleteMany({}) + + const files = await getAllGradeFiles() + for(const file of files) { + await fileToMongo(file) + } + + console.log(`Loaded ${(await GradeModel.find()).length} entries`) + + console.log("SUCCESS") + + } catch (err) { + await GradeModel.deleteMany({}) + console.error(err); + process.exit(1); + } + + process.exit(0); +})(); \ No newline at end of file