Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Setup linting and CI #21

Merged
merged 4 commits into from
Nov 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .eslintrc.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
'use strict';

module.exports = {
root: true,
parserOptions: {
ecmaVersion: '2022',
sourceType: 'module',
requireConfigFile: false,
},
extends: ['eslint:recommended', 'plugin:n/recommended', 'plugin:prettier/recommended'],
env: {
node: true,
browser: false,
},
rules: {},
};
24 changes: 24 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: CI

on:
push:
branches:
- main
- master
pull_request: {}

concurrency:
group: ci-${{ github.head_ref || github.ref }}
cancel-in-progress: true

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 18
- run: npm ci
- run: npm run lint
# - run: npm run test # TODO add some tests 🙈
2 changes: 1 addition & 1 deletion .prettierrc
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"singleQuote": true,
"semi": false,
"printWidth": 100,
"trailingComma": "es5"
}
19 changes: 8 additions & 11 deletions index.js
Original file line number Diff line number Diff line change
@@ -1,23 +1,20 @@
import program from 'commander';

import { runApi } from './lib/api.js'
import { runApi } from './lib/api.js';

program
.option(
'-c, --clear-index',
'Whether indexes of the project should be cleared while processing'
)
.option('-j, --json-driver', 'Use the json driver instead of algolia')
.option('-c, --clear-index', 'Whether indexes of the project should be cleared while processing')
.option('-j, --json-driver', 'Use the json driver instead of algolia');

program.on('--help', function() {
program.on('--help', function () {
console.log(`
Examples:
$ npm start
$ npm start -- -j # to write to fs
$ npm start -- -c # Clear indexes before populating content
`)
})
`);
});

program.parse(process.argv)
program.parse(process.argv);

runApi(program.clearIndex, program.jsonDriver)
runApi(program.clearIndex, program.jsonDriver);
191 changes: 88 additions & 103 deletions lib/api.js
Original file line number Diff line number Diff line change
@@ -1,82 +1,77 @@
import 'dotenv/config'
import 'dotenv/config';

import fsExtra from 'fs-extra'
import { difference } from 'lodash-es'
import { gt, compare as compareSemVers } from 'semver'
import fsExtra from 'fs-extra';
import { difference } from 'lodash-es';
import { gt, compare as compareSemVers } from 'semver';

import algoliaDriver from './drivers/algolia.js'
import jsonDriver from './drivers/json.js'
import schemas from './schemas/index.js'
import algoliaDriver from './drivers/algolia.js';
import jsonDriver from './drivers/json.js';
import schemas from './schemas/index.js';

const { readJsonSync } = fsExtra;

const apiIndexes = ['modules', 'classes', 'methods', 'versions']
const apiIndexes = ['modules', 'classes', 'methods', 'versions'];

export async function runApi(clearIndex = false, useJsonDriver = false) {
let driver = useJsonDriver ? jsonDriver : algoliaDriver
let driver = useJsonDriver ? jsonDriver : algoliaDriver;

apiIndexes.map(driver.init)
apiIndexes.map(driver.init);

if (clearIndex) {
apiIndexes.map(driver.clear)
apiIndexes.map(driver.clear);
}

await Promise.all([
processDocs(driver, 'ember'),
processDocs(driver, 'ember-data'),
])
await Promise.all([processDocs(driver, 'ember'), processDocs(driver, 'ember-data')]);
}

async function processDocs(driver, project) {
let prevIndexedVersions = await driver.getPreviouslyIndexedVersions(project)
let prevIndexedVersions = await driver.getPreviouslyIndexedVersions(project);

const {
meta: { availableVersions },
} = readJsonSync(`../ember-api-docs-data/rev-index/${project}.json`)
} = readJsonSync(`../ember-api-docs-data/rev-index/${project}.json`);

let versionsToProcess = difference(availableVersions, prevIndexedVersions)
let versionsToProcess = difference(availableVersions, prevIndexedVersions);

if (versionsToProcess.length === 0) {
console.log(`No new versions to process for ${project}`)
return
console.log(`No new versions to process for ${project}`);
return;
}

try {
// iterate versions and drop latest minor of each major in buckets
// make an array of the latest minors you get
let latestPatches = Object.values(versionsToProcess.reduce(addIfLatestPatch, {}));
console.log(`Processing ${project} for versions: ${latestPatches}`)
console.log(`Processing ${project} for versions: ${latestPatches}`);
await latestPatches
.filter(version => filterMissingRevs(version, project))
.map(version => readIndexFileForVersion(version, project))
.filter((version) => filterMissingRevs(version, project))
.map((version) => readIndexFileForVersion(version, project))
// Fetch all public modules and public classes
.map(versionIndexObject =>
fetchPublicModuleClassesForVersion(versionIndexObject, project)
)
.map((versionIndexObject) => fetchPublicModuleClassesForVersion(versionIndexObject, project))
// Run the schema against all data stored
.map(mapDataForVersion)
.map(content => writeToDriver(driver, content))
let versions = [...prevIndexedVersions, ...versionsToProcess].sort(
compareSemVers
)
.map((content) => writeToDriver(driver, content));
let versions = [...prevIndexedVersions, ...versionsToProcess].sort(compareSemVers);

await driver.write(
'versions',
[{
id: project,
name: project,
index_date_timestamp: Date.now(),
versions
}],
[
{
id: project,
name: project,
index_date_timestamp: Date.now(),
versions,
},
],
project
)
);
} catch (err) {
console.log('Error:: ', err)
console.log('Error:: ', err);
}
}

function addIfLatestPatch(latestPatches, version) {
let semvers = version.split('.')
let semvers = version.split('.');
let major = semvers[0];
let minor = semvers[1];
let minorVersion = `${major}.${minor}`;
Expand All @@ -89,56 +84,52 @@ function addIfLatestPatch(latestPatches, version) {
}

function filterMissingRevs(version, libName) {
const emberVersionJSONPath = `../ember-api-docs-data/rev-index/${libName}-${version}.json`
let isIncluded = true
const emberVersionJSONPath = `../ember-api-docs-data/rev-index/${libName}-${version}.json`;
let isIncluded = true;
try {
readJsonSync(emberVersionJSONPath)
} catch(e) {
isIncluded = false
readJsonSync(emberVersionJSONPath);
} catch (e) {
isIncluded = false;
}
return isIncluded
return isIncluded;
}

function readIndexFileForVersion(version, libName) {
const emberVersionJSONPath = `../ember-api-docs-data/rev-index/${libName}-${version}.json`
console.debug(`OPENING:: ${emberVersionJSONPath}`)
return readJsonSync(emberVersionJSONPath)
const emberVersionJSONPath = `../ember-api-docs-data/rev-index/${libName}-${version}.json`;
console.debug(`OPENING:: ${emberVersionJSONPath}`);
return readJsonSync(emberVersionJSONPath);
}

function fetchPublicModuleClassesForVersion(versionIndexObject, libName) {
const publicModules = versionIndexObject.data.relationships[
'public-modules'
].data.map(module => {
const id = module.id;
if(!versionIndexObject.meta.module[id]){
console.warn(`Skipping processing module ${id} because it's missing a meta entry`);
return null;
const publicModules = versionIndexObject.data.relationships['public-modules'].data
.map((module) => {
const id = module.id;
if (!versionIndexObject.meta.module[id]) {
console.warn(`Skipping processing module ${id} because it's missing a meta entry`);
return null;
}
const modulePath = `../ember-api-docs-data/json-docs/${libName}/${versionIndexObject.data.attributes.version}/modules/${versionIndexObject.meta.module[id]}.json`;

console.debug(`OPENING:: ${modulePath}`);
return readJsonSync(modulePath);
})
.filter(Boolean);

const publicClasses = versionIndexObject.data.relationships['public-classes'].data.map(
(classObj) => {
const id = classObj.id;
const classPath = `../ember-api-docs-data/json-docs/${libName}/${versionIndexObject.data.attributes.version}/classes/${versionIndexObject.meta.class[id]}.json`;

console.debug(`OPENING:: ${classPath}`);
return readJsonSync(classPath);
}
const modulePath = `../ember-api-docs-data/json-docs/${libName}/${
versionIndexObject.data.attributes.version
}/modules/${versionIndexObject.meta.module[id]}.json`

console.debug(`OPENING:: ${modulePath}`)
return readJsonSync(modulePath)
}).filter(Boolean)

const publicClasses = versionIndexObject.data.relationships[
'public-classes'
].data.map(classObj => {
const id = classObj.id;
const classPath = `../ember-api-docs-data/json-docs/${libName}/${
versionIndexObject.data.attributes.version
}/classes/${versionIndexObject.meta.class[id]}.json`

console.debug(`OPENING:: ${classPath}`)
return readJsonSync(classPath)
})
);

return {
version: versionIndexObject,
publicModules,
publicClasses,
}
};
}

/**
Expand All @@ -148,39 +139,33 @@ function fetchPublicModuleClassesForVersion(versionIndexObject, libName) {
* @returns {object} - Extended version object with methods & mapped schemas
*/
function mapDataForVersion(versionObject) {
const staticFunctions = extractStaticFunctionsFromModules(
versionObject.publicModules
)
const methods = extractMethodsFromClasses(versionObject.publicClasses)
const staticFunctions = extractStaticFunctionsFromModules(versionObject.publicModules);
const methods = extractMethodsFromClasses(versionObject.publicClasses);

return {
...versionObject,
methods: [...methods, ...staticFunctions],
publicModules: versionObject.publicModules.map(schemas.moduleSchema),
publicClasses: versionObject.publicClasses.map(schemas.classSchema),
}
};
}

function writeToDriver(driver, versionObject) {
const { id } = versionObject.version.data
const { id } = versionObject.version.data;

let tokens = id.split('-')
let version = tokens.pop()
let projectName = tokens.join('-')
let tokens = id.split('-');
let version = tokens.pop();
let projectName = tokens.join('-');

console.info(
`version: ${id}, public classes: ${
versionObject.publicClasses.length
}, public modules: ${versionObject.publicModules.length}, methods: ${
versionObject.methods.length
}`
)
`version: ${id}, public classes: ${versionObject.publicClasses.length}, public modules: ${versionObject.publicModules.length}, methods: ${versionObject.methods.length}`
);

return Promise.all([
driver.write('modules', versionObject.publicModules, projectName, version),
driver.write('classes', versionObject.publicClasses, projectName, version),
driver.write('methods', versionObject.methods, projectName, version),
])
]);
}

/**
Expand All @@ -195,34 +180,34 @@ function extractMethodsFromClasses(classes) {
currentClass.data.attributes.methods
.reduce((classMethods, currentMethod) => {
// Transform the current method and push on to methods.
classMethods.push(schemas.methodSchema(currentMethod, currentClass))
return classMethods
classMethods.push(schemas.methodSchema(currentMethod, currentClass));
return classMethods;
}, [])
// Merge all methods of all classes into a single array
.concat(methods)
)
}, [])
);
}, []);
}

function extractStaticFunctionsFromModules(modules) {
return modules.reduce((methods, currentModule) => {
const staticfunctionsObj = currentModule.data.attributes.staticfunctions
const staticfunctionsObj = currentModule.data.attributes.staticfunctions;

// Guard against staticfunctions not existing.
if (!staticfunctionsObj) return methods
if (!staticfunctionsObj) return methods;
// Extract all the static functions from inside their sub-modules
const moduleStaticFunctions = Object.keys(staticfunctionsObj).reduce(
(prevStaticFunctions, currModuleName) => {
return prevStaticFunctions.concat(staticfunctionsObj[currModuleName])
return prevStaticFunctions.concat(staticfunctionsObj[currModuleName]);
},
[]
)
);

return moduleStaticFunctions
.reduce((moduleStaticFunctions, currentStaticFunction) => {
moduleStaticFunctions.push(schemas.methodSchema(currentStaticFunction, currentModule))
return moduleStaticFunctions
moduleStaticFunctions.push(schemas.methodSchema(currentStaticFunction, currentModule));
return moduleStaticFunctions;
}, [])
.concat(methods)
}, [])
.concat(methods);
}, []);
}
Loading