diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml new file mode 100644 index 0000000..8f51ca8 --- /dev/null +++ b/.github/workflows/node.js.yml @@ -0,0 +1,27 @@ +# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions + +name: Node.js CI + +on: + pull_request: + branches: ['master', 'main'] +jobs: + build: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + + steps: + - uses: actions/checkout@v4 + - name: uses NodeJS ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: './scripts/package-lock.json' + - run: npm ci && npm run build:no_rcbc # rcbc needs to be fixed + working-directory: ./scripts/typescript diff --git a/scripts/typescript/package.json b/scripts/typescript/package.json index e8d047b..1dc2b9a 100644 --- a/scripts/typescript/package.json +++ b/scripts/typescript/package.json @@ -5,6 +5,7 @@ "main": "index.js", "scripts": { "build": "npx tsc", + "build:no_rcbc": "npm run build -- --project tsconfig.no_rcbc.json", "validate": "npx ts-node ./src/validate", "afrl": "npx ts-node ./src/afrl", "bcdb": "npx ts-node ./src/bcdb", diff --git a/scripts/typescript/src/afrl/afrl-csv-to-json.ts b/scripts/typescript/src/afrl/afrl-csv-to-json.ts index 9c48572..982d68f 100644 --- a/scripts/typescript/src/afrl/afrl-csv-to-json.ts +++ b/scripts/typescript/src/afrl/afrl-csv-to-json.ts @@ -1,7 +1,10 @@ -import { AFRLData } from "./types" -import { ICitation, ICollection, ICondition, IInventory, IMaterial, IProject, IProperty, IReference } from "../types/cript" +import { AFRLData } from "./types"; +import { ICitation, ICollection, ICondition, IInventory, IMaterial, IProject, IProperty, IReference } from "@cript"; import csvtojson from "csvtojson"; -import * as fs from 'stream'; +import * as fs from "stream"; +import { CriptGraph, CriptValidator } from "@utilities"; +import { EdgeUID } from "../types/cript/Edges"; + /** * This script is a typescript port of @see https://github.com/C-Accel-CRIPT/criptscripts/tree/master/scripts/python_sdk_scripts/AFRL * The main difference is the source data is now read from the original CSV and not a preprocessed *.js. @@ -27,20 +30,22 @@ export class AFRLtoJSON { private readonly mixtures = new Map(); // The project data will be stored in - private project: IProject; + private readonly project: IProject; // The collection data will be stored in - private collection: ICollection; + private readonly collection: ICollection; // The inventories data will be stored in - private inventory_solvents: IInventory; - private inventory_polymers: IInventory; - private inventory_mixtures: IInventory; + private readonly inventory_solvents: IInventory; + private readonly inventory_polymers: IInventory; + private readonly inventory_mixtures: IInventory; private error_stream: fs.Writable; - constructor(config: Config) { + private validator: CriptValidator; + constructor(config: Config) { + this.validator = new CriptValidator(); this.error_stream = config.error_stream ?? process.stderr; this.error_stream.on('error', e => console.error(e)); @@ -81,6 +86,7 @@ export class AFRLtoJSON { // Create a project with the collection in it this.project = { + uuid: CriptGraph.make_uuid(), name: config.project_name, node: ['Project'], collection: [this.collection], @@ -89,6 +95,24 @@ export class AFRLtoJSON { } + /** + * Check if the property is valid before to push it into the properties. + * In case property is invalid, an exception is thrown including information to debug + */ + private add_property( properties: IProperty[], property: IProperty) { + this.validator.validate_or_throw(property) + properties.push(property) + } + + /** + * Check if the condition is valid before to push it into the conditions. + * In case condition is invalid, an exception is thrown including information to debug + */ + private add_condition( conditions: ICondition[], condition: ICondition) { + this.validator.validate_or_throw(condition) + conditions.push(condition) + } + private get_citation(row: AFRLData): ICitation | undefined { if(row.reference === undefined) { @@ -98,12 +122,12 @@ export class AFRLtoJSON { // Check if citation was already created const existing_citation = this.citations.get(row.reference); if (existing_citation) { - console.log(`-- Found existing reference: ${existing_citation.reference.title}`) + console.log(`-- Found existing reference: ${JSON.stringify(existing_citation.reference)}`) return existing_citation; } // Create citation - const citation: ICitation = { + const citation = { node: ['Citation'], reference: { title: row.reference, @@ -111,7 +135,7 @@ export class AFRLtoJSON { node: ['Reference'], } as IReference, type: 'reference', - } as ICitation; + } satisfies ICitation; // get DOI and authors const DOI_DOT_ORG = "doi.org/"; @@ -127,7 +151,8 @@ export class AFRLtoJSON { // .map( a => a.trim() ); // remove empty pre/post spaces } - // Store in hashmap + // Check and store node in an index + this.validator.validate_or_throw(citation) this.citations.set(row.reference, citation); return citation; @@ -146,7 +171,8 @@ export class AFRLtoJSON { } // Pull solvent from server - // Blocker: the script assumes the solvent already exists on the backend, but we probably don't have it. + // BLOCKER: The API does not allow to search by chemical_id (a.k.a. CAS). + // Still true on 12/14/2023. /* try { solvent = api.get( @@ -171,13 +197,15 @@ export class AFRLtoJSON { // Temporary solution: we create a new Solvent... const solvent: IMaterial = { + uid: CriptGraph.make_uid(), node: ['Material'], name: row.solvent, - cas, + chemical_id: cas, // CRIPT does not have a "cas" field, we store it as "chemical_id" instead. } as IMaterial; this.record_error(`Search material from "cas" is not implemented, creating a local solvent for ${JSON.stringify(solvent)}`) - // Store in hashmap + // Check and store node in an index + this.validator.validate_or_throw(solvent) this.solvents.set(cas, solvent); return solvent; @@ -205,52 +233,51 @@ export class AFRLtoJSON { // Create properties const properties: IProperty[] = []; if (mw_w && !isNaN(mw_w)) - properties.push({ + this.add_property(properties, { key: "mw_w", - value: String(mw_w), // FIXME: backend does not accept numbers + value: mw_w, unit: "g/mol", citation, node: ['Property'], - type: 'value' // FIXME: is this correct from a chemist point of view? - } as IProperty) + type: 'value' + }) + if (mw_d && !isNaN(mw_d)) properties.push({ key: "mw_d", - value: String(mw_d), // FIXME: backend does not accept numbers, + value: mw_d, unit: "", citation, node: ['Property'], - type: 'value' // FIXME: is this correct from a chemist point of view? + type: 'value' } as IProperty) // Create new material object const polymer: IMaterial = { + uid: CriptGraph.make_uid(), name: unique_name, property: properties, node: ['Material'], } as IMaterial; // Create identifiers - // - // note: the new API does not have a concept for the legacy's API Identifiers. - // We have to set those directly on the Material node. - // + if (name) { - polymer.names = [name]; // Not sure about that, waiting for Brilant's answer. - //identifiers.push(cript.Identifier(key="prefered_name", value=name)) + polymer.names = [name]; } if (cas) { - polymer.cas = cas; - //identifiers.push(cript.Identifier(key="cas", value=cas)) + polymer.chemical_id = cas; } if (bigsmiles) { polymer.bigsmiles = bigsmiles; - //identifiers.push(cript.Identifier(key="bigsmiles", value=bigsmiles)) } + // Check and store node in an index + this.validator.validate_or_throw(polymer) this.polymers.set(unique_name, polymer); + return polymer } @@ -267,16 +294,15 @@ export class AFRLtoJSON { // Create new material object const mixture = { + uid: CriptGraph.make_uid(), node: ['Material'], name: unique_name, // "identifiers": identifiers, deprecated, see explanation below - component: [ - polymer, - solvent - ], + component: [polymer, solvent].map( CriptGraph.make_edge_uid ), property: new Array, names: [unique_name], } satisfies IMaterial; + this.validator.validate_or_throw(mixture) // Create identifiers // @@ -292,29 +318,27 @@ export class AFRLtoJSON { // Create properties if (conc_vol_fraction && !isNaN(conc_vol_fraction)) { - const property: IProperty = { + this.add_property(mixture.property, { key: "conc_vol_fraction", value: conc_vol_fraction, - component: [polymer], + component: [polymer].map(CriptGraph.make_edge_uid), citation, node: ['Property'], - type: 'value', // FIXME: is this correct from a chemist point of view? + type: 'value', unit: null, - }; - mixture.property.push(property); + }); } if (conc_mass_fraction && !isNaN(conc_mass_fraction)) { - const property: IProperty = { + this.add_property(mixture.property,{ key: "conc_mass_fraction", value: conc_mass_fraction, - component: [polymer], + component: [polymer].map(CriptGraph.make_edge_uid), citation, node: ['Property'], - type: 'value', // FIXME: is this correct from a chemist point of view? + type: 'value', unit: null, - } - mixture.property.push(property); + }) } if (temp_cloud && !isNaN(temp_cloud)) { @@ -322,10 +346,10 @@ export class AFRLtoJSON { const temp_cloud_property = { key: "temp_cloud", value: temp_cloud, - component: [polymer], + component: [polymer].map(CriptGraph.make_edge_uid), citation, node: ['Property'], - type: 'value', // FIXME: is this correct from a chemist point of view? + type: 'value', unit: "degC", condition: new Array() // will be filled below... } satisfies IProperty; @@ -333,27 +357,29 @@ export class AFRLtoJSON { // If present, add conditions if (pressure) - temp_cloud_property.condition.push({ + this.add_condition(temp_cloud_property.condition, { node: ['Condition'], key: "pressure", - value: String(pressure), // FIXME: typings are wrong, we should be able to use a number + value: pressure, unit: "MPa", }); if (one_phase_direction) { - temp_cloud_property.condition.push({ - node: ['Condition'], - key: "+one_phase_direction", - type: 'value', - value: one_phase_direction, - } as ICondition); - - this.record_error(`one_phase_direction cannot be stored in CRIPT, +one_phase_direction vocab is not allowed.`) + const one_phase_direction_condition = { + node: ['Condition'], + key: "+one_phase_direction", + type: 'value', + value: one_phase_direction, + } satisfies ICondition + + // Still true on 12/14/2023: + this.record_error(`${one_phase_direction_condition.key} cannot be stored in CRIPT, we don't have a key for that yet. Skipping it.`) + // temp_cloud_property.condition.push(one_phase_direction_condition); } // Add property to the mixture - mixture.property.push(temp_cloud_property); + this.add_property(mixture.property, temp_cloud_property); } // Store in hashmap @@ -427,11 +453,15 @@ export class AFRLtoJSON { add_material(material: IMaterial, inventory: IInventory) { if(!this.project.material) this.project.material = []; if(!inventory.material) inventory.material = []; - // note: here I do not check if inventry is a part of a collection in this.project + + // note: here I do not check if inventory is a part of a collection in this.project // but by design (cf. constructor) the inventory should be a part of it. + // Check if material is valid (sooner is better) + this.validator.validate_or_throw(material) + this.project.material.push(material); - inventory.material.push(material); + inventory.material.push(CriptGraph.make_edge_uid(material)); } private static load_config(): Config { @@ -470,7 +500,8 @@ export class AFRLtoJSON { * @returns */ async load_csv(csv_file_path: string): Promise { - console.log(`Loading file and converting to javascript data types: ${csv_file_path} ...`) + + console.log(`Loading file and converting to AFRLData objects: ${csv_file_path} ...`) const raw_json: { [key: string]: string }[] = await csvtojson().fromFile(csv_file_path) console.log(`Found ${raw_json.length} rows found in CSV. File is now converted as a { [key: string]: string }[] `) @@ -484,14 +515,8 @@ export class AFRLtoJSON { } console.log(`Logging samples randomly DONE`); - - // Checking data against type - console.warn(`Assigning default value for string properties ...`) - console.warn(`Data validation is not 100% safe, some fields might be missing. TODO: install AJV and create a schema for AFRLData type.`) - // Use typecheck to list all (unique) fields existing on AFRLdata const empty_data: AFRLData = { - cloud_point_temp: 0, mixture_id: 0, one_phase_direction: "", @@ -581,7 +606,13 @@ export class AFRLtoJSON { * @param data * @returns */ - load_data(data: AFRLData[]): IProject { + async make_project(data: AFRLData[]): Promise { + + console.log(`Initializing CriptValidator ...`) + if( !await this.validator.init() ) { + throw new Error(`Initializing CriptValidator FAILED.`) + } + console.log(`Initializing CriptValidator ...`) // load data console.log('Loading data ...') @@ -604,6 +635,10 @@ export class AFRLtoJSON { } console.log('Loading data OK') + + // Ensure Project is valid before to return it + this.validator.validate_or_throw(this.project) + return this.project; } } \ No newline at end of file diff --git a/scripts/typescript/src/afrl/index.ts b/scripts/typescript/src/afrl/index.ts index 9d5f759..196684b 100644 --- a/scripts/typescript/src/afrl/index.ts +++ b/scripts/typescript/src/afrl/index.ts @@ -5,10 +5,7 @@ import { output_dir_path, write_json_helper } from "@utilities"; const file_name = 'AFRL_linear_polymer_3pdb_data_csv_4_5_2023.csv'; // TODO: read this from command line argument -// Call main as a promise because low level await are not allowed. -main().then( () => process.exit() ) - -async function main() { +(async () => { // Immediately called async (no await at top level on this nodejs version) console.log('=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=') console.log('=-=-=-=-=-=-=-=-=-=-=-=-= AFRL CSV to JSON -=-=-=-=-=-==-=-=-=-=-=-=-=') @@ -32,20 +29,23 @@ async function main() { const afrl_data = await serializer.load_csv(input_csv_file_path); // Load the AFRLData[] to a Project - const project = serializer.load_data(afrl_data); + const project = await serializer.make_project(afrl_data); // Ensure output directory exists console.log('Ensure output folder path exists ...') console.log(`Path: ${output_dir_path}`) if (!fs.existsSync(output_dir_path)) { console.log(`Output folder path does not exist, creating it ...`) - fs.mkdirSync(output_dir_path, { recursive: true }); + if( !fs.mkdirSync(output_dir_path, { recursive: true }) ) { + throw new Error(`Unable to make output directory! (path: ${output_dir_path})`) + } console.log(`Output folder path created.`) } - console.log('Output folder is ready') // Write JSON files - await write_json_helper(project, 'afrl', 'minified'); await write_json_helper(project, 'afrl', 'human-readable'); -} + await write_json_helper(project, 'afrl.min', 'minified'); + + console.log(`Files saved. Browse ${output_dir_path} folder.`) +})() diff --git a/scripts/typescript/src/bcdb/bcdb.ts b/scripts/typescript/src/bcdb/bcdb.ts index bdb9fdb..0df76df 100644 --- a/scripts/typescript/src/bcdb/bcdb.ts +++ b/scripts/typescript/src/bcdb/bcdb.ts @@ -8,9 +8,15 @@ * See link to google drive files above (column_meanings.xlsx) */ import * as XLSX from "xlsx"; -import { Column } from "./types/column"; +import { Column } from "./types"; import { ICitation, ICollection, ICondition, IMaterial, IProject, IProperty, IReference } from "@cript"; -import { CriptGraphOptimizer, CriptValidator, LogLevel, Logger, LoggerOptions, OptimizedProject } from "@utilities"; +import { + CriptValidator, + LogLevel, + Logger, + LoggerOptions, + CriptGraph +} from "@utilities"; export class BCDBLoader { readonly logger: Logger; @@ -34,13 +40,13 @@ export class BCDBLoader { input_file_path: string; sheets: ['blocks', 'diblock']; // added for the user to understand, but only expect a const array limitSheetRows: number; - }): Promise { + }): Promise { //-- Download DB schema try { await this.validator.init(); } catch(error: any) { - throw new Error('Unable to initialize the validator', error); + throw new Error('Unable to initialize the validator'); } //-- Load desired sheets @@ -393,7 +399,7 @@ export class BCDBLoader { project.material.push(block2); this.logger.info(`Add blocks to process as ingredients ...`); - polymer.component = [block1, block2]; + polymer.component = [block1, block2].map( CriptGraph.make_edge_uid ); }; } this.logger.prefix = null; @@ -418,18 +424,17 @@ export class BCDBLoader { }; // Optimise the project object (uses uids, create Edges, etc..) - const optimizer = new CriptGraphOptimizer(); - const optimized_project: OptimizedProject = optimizer.get_optimized(project); + const optimized_project = CriptGraph.optimize_project(project, 'make-edge-uid'); // Validate against DB schema - const is_valid = await this.validator.validate('ProjectPost', optimized_project.project); + const is_valid = this.validator.validate('ProjectPost', optimized_project); if(!is_valid) { this.logger.error(this.validator.errorsAsString(100)); - this.logger.error(`Project '${optimized_project.project.name}' is NOT valid, see errors in logs above!`) + this.logger.error(`Project '${optimized_project.name}' is NOT valid, see errors in logs above!`) throw new Error(`Project is NOT valid`); } else { - this.logger.info(`Project '${optimized_project.project.name}' is valid.`) + this.logger.info(`Project '${optimized_project.name}' is valid.`) } return optimized_project; diff --git a/scripts/typescript/src/types/cript/IInventory.ts b/scripts/typescript/src/types/cript/IInventory.ts index 62eb9b3..4728596 100644 --- a/scripts/typescript/src/types/cript/IInventory.ts +++ b/scripts/typescript/src/types/cript/IInventory.ts @@ -1,10 +1,11 @@ -import { ICollection, IMaterial } from './index'; +import { EdgeUUID, ICollection, IMaterial } from "./index"; +import { EdgeUID } from "./Edges"; export interface IInventory { name: string; notes?: string; material_count?: number; - material?: IMaterial[]; + material?: (IMaterial|EdgeUID|EdgeUUID)[]; public?: boolean; uid?: string; uuid?: string; diff --git a/scripts/typescript/src/types/cript/IMaterial.ts b/scripts/typescript/src/types/cript/IMaterial.ts index 820f3ce..3e6f86c 100644 --- a/scripts/typescript/src/types/cript/IMaterial.ts +++ b/scripts/typescript/src/types/cript/IMaterial.ts @@ -24,12 +24,9 @@ export interface IMaterial { component?: EdgeUUID[] | EdgeUID[]; component_count?: number; computational_forcefield?: IComputationalForcefield; - //identifiers (3/20/2023: Fatjon and I decided to rely on typings not on CV) identifier_count?: number; bigsmiles?: string; smiles?: string; - /** @deprecated, use chem_formula instead*/ - cas?: string; inchi?: string; inchi_key?: string; mol_form?: string; diff --git a/scripts/typescript/src/types/cript/IProperty.ts b/scripts/typescript/src/types/cript/IProperty.ts index a7d2509..3e5282e 100644 --- a/scripts/typescript/src/types/cript/IProperty.ts +++ b/scripts/typescript/src/types/cript/IProperty.ts @@ -1,10 +1,7 @@ -import { Edge, EdgeUUID } from "@cript"; import { ICitation } from "./ICitation"; import { IComputation } from "./IComputation"; import { ICondition } from "./ICondition"; -import { IData } from "./IData"; -import { IMaterial } from "./IMaterial"; -import { EdgeUID } from "./Edges"; +import { EdgeUID, EdgeUUID} from "./Edges"; export interface IProperty { readonly node: ['Property']; diff --git a/scripts/typescript/src/types/cript/index.ts b/scripts/typescript/src/types/cript/index.ts index e41f864..38fac3f 100644 --- a/scripts/typescript/src/types/cript/index.ts +++ b/scripts/typescript/src/types/cript/index.ts @@ -22,7 +22,7 @@ import { ISoftwareConfiguration } from './ISoftwareConfiguration'; import { IVocabBase, IVocabIdentifier, IVocabProperty, IVocabQuantity, IVocab, IControlledVocabulary } from './IVocab'; import { IQuantity } from './IQuantity'; import { IEquipment } from './IEquipment'; -import { Edge, EdgeUUID } from './Edges'; +import { EdgeUID, EdgeUUID } from './Edges'; export type IPrimary = | ICollection @@ -66,7 +66,7 @@ IVocabIdentifier | IVocabProperty | IVocabQuantity | IControlledVocabulary | -Edge | +EdgeUID | EdgeUUID; export type Item = IPrimary | ICitation | IReference | IUser; @@ -101,7 +101,7 @@ export type { IVocabProperty, IVocabQuantity, IControlledVocabulary, - Edge, + EdgeUID, EdgeUUID, }; diff --git a/scripts/typescript/src/utilities/cript-graph.ts b/scripts/typescript/src/utilities/cript-graph.ts index 5cb853f..d3067c3 100644 --- a/scripts/typescript/src/utilities/cript-graph.ts +++ b/scripts/typescript/src/utilities/cript-graph.ts @@ -52,7 +52,7 @@ export namespace CriptGraph { * @param node must have an uid and be present in uids Set */ export function make_edge_uid(node: any): EdgeUID { - if ( !has_uid(node) ) throw new Error("Node should have an uuid to be converted to an EdgeUUID"); + if ( !has_uid(node) ) throw new Error("Node should have an uid to be converted to an EdgeUID"); return { uid: node.uid } } diff --git a/scripts/typescript/tsconfig.no_rcbc.json b/scripts/typescript/tsconfig.no_rcbc.json new file mode 100644 index 0000000..0b11710 --- /dev/null +++ b/scripts/typescript/tsconfig.no_rcbc.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "src/rcbc", /* Some work is required in order to compile. Many nodes are uncompleted */ + ] +}