-
-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Remove couch usage & generate archives instead (#21)
* Drop couch usage & generate archives instead * Some fixes & upgrades * Rearranged folder heirarchy * Update docs * fix travis script * Output msgs to stop timeout in travis * Keep versioned doc ids for ember data * Incremental processing - Ability to pass project name and/or version to index - Reorganized the logic so that we process one version of a doc at a time, ensuring that we don't need lot of memory * Add engines section * Add more info in package.json * Update dependency * Sync to local before start * Skip previously indexed docs * Sync to s3 at the end if requested * Fix formatting & update travis token's with siva's * Use the AWS env variable conventions * Update existing docs * Fix indentation
- Loading branch information
1 parent
2a3e451
commit d8c217d
Showing
23 changed files
with
862 additions
and
1,555 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,28 +1,17 @@ | ||
| language: node_js | ||
| node_js: '6' | ||
|
|
||
| env: | ||
| global: | ||
| - COUCH_URL="https://emberjs.cloudant.com/ember-api-docs" | ||
| # export COUCH_USERNAME=xxxxx | ||
| - secure: e80GjZi18denINfipJnduwS0C/7fYf3lf9XMmlYRsSObhXbja/wYmqZfZsXUC5mDZFfutTY0sJDmnF3ibc8ZkvY1LSJ/22K48KSOGYQZ95RJBAK+nHa5m29ZOocx9UJwuo/n4UFTqS0+u5gvg5ArPAUftPA6BUaMszxGURDroTtQPjCzUUkclbul4Nb8OD2/S9Kb959VUsZnC9VezQzNMrym7Fw8b7c5UiriwBAc58wrrgdIBfQII38SuMSgz0OERScFBUVRs+6ETmuVgG/YMLYzISzg5rzCmrDqfHDEElCRfasKJ0ukc+g/tM/onBSMJSNX7r4NjNUvIyZjG8djbm4mrTfiSSxikXXa6/EDsgHrWbl+50wRb12c9iEkQDb+qNOxemvzsXJyKFPSeC0r9Ta1C3YSe2bOWNUAxH3a3GKoSEuIbNfgzX2K6GiiKHzgLYCK+acMQJrtz4Qs/D2u/EI5ST1FWKyIIu2qWJ84LPWJKLGByTjOjEDQ+arpQWYn6ACTDl4Tv1luzagQ7txbJxqPLcFG6yOBPXH1R1sXJEJwB2wkSCvAC6bwN37Yx/lUewTkzqI17W9HtboabQ9Hk4pqPSlFGI6bZZM4ASeopWMLQqGMLd3kUbneRr8BWCuPhgiX/rj9GqDOQoAZMoejuBirWU+tn2SXX0JJbt9i1lw= | ||
| # export COUCH_PASSWORD=xxxxx | ||
| - secure: bpaoN/CqbrtWQXKqUznMPmqsHr4H7+6nT2SnRCc56hBWsh9dnM6KcpaJnVvnpXKF2Rj5Teiv5t/V0ycoLb7yrj327A5yLsHEI30sO5CrVu2b/7DS/wvF2FPc7bd1eoLlvrP5WbgQikP2hempRACs0dOwTLE1lhBVQjJr0xRb7LPl7+ZE2EaNnU8je+WWeGNWqAT1VHD6pBKDwYhtyfke7tRV2QvvDAijELLGjvAzEv4FhZnCL7CMK8wAgL6QDeMCndyXxJMudel33AL/Lb8hWxBPEnZRWyGAjfYzLSXnsELnUEEtM83uFeKexVkxmQWGQ8BOK8GOK0e/GRctCKVTqKgDnX1LATergOv3mjCsROHiLzqkqIUUYlJNGMYtl1IEJZ+SgmExFirPio8ORVbw/I/bwsaNpq+d7HzEa/WbsblikWdNKecjnWGeVKH2boZIrReb3CF5cGhJo7ZjH15C9M3jXLxc4LkY9vHPY90/8cKqJ4sWpu58qIQIn210+WiiZ6L6gOSuMAEQYtg1PxtnpoyLG3s68PhyX2LxSEU+Gt97IiPszGwo4saqIC7y+jUj+kG+C9I5DoarUXENGveH/kZja2Pllm2YUndIoWxKoIGy3K4NqsiPfxGyqnIVq0bW4aGul/XTjCBitkD+hrpcg8EcV7QUgr50BnmyY1cFNV8= | ||
| # export AWS_ACCESS_KEY_ID=xxxxxx | ||
| - secure: QXKfIbHOdsLhhSq2KY4trFXc+eWUoYQAbMfAjA9nQuVN0RjVksQUyK4s5TI3dMcXjh7N95Jb7F2P3AC17SgVaz/ydUkrYwPUaglSAOuOECGVxqtNV/y3TnZjquz8XnuPLW2ZLxIqoN2lbNisomVxegLSMGgeo+IX/2mgSWZCKlUNdlCCJ8DUv7Vex+L43DPfOsbmrUkXFiimYtg4NjsxKWjerHM3iWxQlZ1m8pzhKhOfMJs9JlAEKZmRXt4Mw2IbTyc+g5tVfUb5l7yQdtMkoFZ3tqYCm59P8erocMn2m0PYPy/IHd+YM9xaCtZFcMwyEywMvZtVtP1ByscIDKoNTNSgy1IUgR/lJHeHN4/v/CEkSAUIvMKoGGDIFbfHPKJZ2HepupFYVJwFfSZbbiaipIMVMerKPsyJKyTkqx/Xke7ZfgW2RQvVjOjEyvD9zbfLte1iNbdWqVrI3cPAInNRNgdHuYZgjcvX3aKbqk4Xpn81+4B6O1keS8iU30vfnd1Xw7Wye5BxraVq2ajdq/jHkkK9ULlZZKmyebW/f3I5JsLRCbSJy+sb/zUAawEXLr0qaYKOfQxg1iuOwswMmgSVFYVZUKfRIlLo3A/e1e8/YR9oTMYaXdiDVW813cK5ZnNbr5qoK9PSAf38hDRZtpmldC3m4c0R9OyQhtrFW8KaLF0= | ||
| # export AWS_SECRET_ACCESS_KEY=xxxxx | ||
| - secure: jXcVmvHqS9BD2tX9/5ZYE3YY4VrVSHimOWLDI2xY+j071Ngcsr/70bjoEqD+ja2YI/fDsZrblelUV8a+ettlqC2PlM7s869zxt1cDcqRwQiopyz3b7ihp4JsbxHF4C1AYg+3POSuC5Knvh9sIimebrwMRgrMEED5t37kfSSD3JBprhodBARIOw7jo2JgXmqxWv/4YpqA5IBGTTwmGB2DsuZnqLutZeT9Fhcstkm4aaBwN5sjC18lugcEh6vrvmCZS6gbiG2v2MIsvaGzWXLZgl66pk4JLzhzA68YtNNRgye9hn5BVJXhkZK14Q1BWhol4v8B+mSBaB5R4yikBIvODzeTpXbacFc84EHZQlw/9ZNTm3MvdpBLd4guPPJk6oKFUTXwdPaWccVFli+NdPz16e4Y4/xUYS1fKfjHLF2gejZ27gHxosD6M+ss3HIrmMLzatHC9iHk7juICP67M5bRcPuwCzm7K6pBmC5QWULd01Ppn3h4V+xUilCA8OmfkrGBLJcl8d2t7H0Q7ebei4SSdrbtkuRRdo8OV1CU7VVJdU5MvYCtDZs16qUozILvXn9Oj05GZ4Uu7wy4TmwF5+6Ye+Fherc2UXQ8CWdjKqYoKY6usVCK0E251vlwk3cjkaZhc+xpLLIPEuVLnGC8CzxMJqaxa6zOrSjLXpZGXnRH3Lw= | ||
|
|
||
| cache: | ||
| directories: | ||
| - "$HOME/.yarn-cache" | ||
| - node_modules | ||
| - tmp | ||
|
|
||
| script: | ||
| - yarn run test | ||
| - 'if [ "$TRAVIS_PULL_REQUEST" = false ]; then yarn start; else echo "We dont publish docs via PRs"; fi' | ||
|
|
||
| - if [ "$TRAVIS_PULL_REQUEST" = false ]; then AWS_SHOULD_PUBLISH=yes yarn start; else echo "We dont publish docs via PRs"; fi | ||
| branches: | ||
| only: | ||
| - master | ||
| env: | ||
| global: | ||
| - secure: NqKxv7m1J21O2z1QfNIKce85KW70IeID8SshF/Ekzn4F0eg3PGQHHVLwgYmiOTznRu0kb+m2lbD5C3KuXJcdXHQXaG8ByAOYytMd6+n/1IJdM8Ao1nhWPlzN0xcU9TqPtlbKlJeUw9+G7zFsBK6nGAc866/CO4zJblqdOh8pztJlP4iI8QqzenK/YWJg2AacLjxQHNqjPQtK48ItgV1pdrXgVFGj7si8VmOz9uGytcqB+aQCWx+sP/08hkdJoGaxixjMhe1oAJpEccCGTuJ8EXdCZpssz0aZeyVUQBG+19eDUraYMBIXm6nZhX9OK1aHGTT5A6/pcDbLpmz8gs8DG3j9semKp0DwRWg2iIvunKEk5IKogCkbaebDw1U6l4V6lP9BZPZ+ab8NnJHEUL9vrkKPaZ1qPUvmXVoDErciUzmNWP26aTMRfFIe4a2Ax6sSyzbl06STEyVaBTonfpNgcbpDZiR6C7ngsKYFVyUdf0kSHdAlchF+CEX99pyGJPgmrjromcbzSA9x3Uye5puT/2KQXFfKXRmMIflTePqi7PlLQ9t/Z1roWocfORaIsgobv1RfKwsHv5dw+7j8OVsk6//dZu5GFbNcqqCMqoRz0mQ1yG3t9oa2zfpDLyIGvxAvi8Owdcc3TgQQJ5xcZ63ziqgB4KtE/5COiccoEjNWT4k= | ||
| - secure: QCpb0SMkTCAGUIEiv2Bhv4QRz6GOtUmRBq2Kjvorr0PP1pFUND3j2qI5QvBlzgZo6G23KGUQ08E0l0BwInoqYGSHBIAX1PhiTwnqnNOy0n2RJUaKqsED6vfqUbcFtyE9HtvMT3xwz38L2aHIHAdYEkfI2fNX5/SAhTz0+Ovtbd7kKbZvX4V21FOmqdUnZyoA66m5YTs8dPv+/52iHA7oTF06HvejQKbsiu2wbZ8pP/cKVLOxcPkb2mI9Y03cNlHWsEwKgYTIFYuz3UXa/v24TSFmDOj2YcYKYml2Pk3e0VZnw36+N/1yx6trkpceeodLGGe9EM8kIqstcYdGnMfA7S6ADu8Fh8F88lBBItVqqE984b9LVlXE+ujr88DQVOMu8YkzFN/RGF6fonvgPNZ5M9Q/n59+ycCJOqOW+LFzEs+liH2Z9Hbgft+LlYaT3QiJXNXqqgd/j4fNMGdCalFk0s7pVmMtN6gJzrbGbeLd3R7gJ8HFey9oycWsZO5eXKKQ/thfXfk2bdfuusyluPyqqU2O6W4fbpB80hIC5T4VClrT08ikuLHtuj7vFJVmxn4cVIecxHX+DmR9NXvZgS8M/r5uxp57purLt/RGQGBk/uJFLflOgoFBy/K9QQD280MvF5VrgzifyUvNR4wcCzE1XEVQmKZFeuGGDbB74eW1V4c= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,84 +1,67 @@ | ||
| 'use strict' | ||
| const RSVP = require('rsvp') | ||
| const fs = require('graceful-fs') | ||
| const argv = require('minimist')(process.argv.slice(2)) | ||
|
|
||
| let RSVP = require('rsvp') | ||
| let _ = require('lodash') | ||
| let rm = require('rimraf') | ||
| let PouchDB = require('pouchdb') | ||
| let fs = require('fs') | ||
| let glob = require('glob') | ||
| const markup = require('./lib/markup') | ||
| const readDocs = require('./lib/read-docs') | ||
| const fetchYuiDocs = require('./lib/fetch-yui-docs') | ||
| const createClassesOnDisk = require('./lib/create-classes') | ||
| const transformYuiObject = require('./lib/transform-yui-object') | ||
| const normalizeEmberDependencies = require('./lib/normalize-ember-dependencies') | ||
| const getVersionIndex = require('./lib/get-version-index') | ||
| const saveDoc = require('./lib/save-document') | ||
| const { syncToLocal, syncToS3 } = require('./lib/s3-sync') | ||
|
|
||
| let fetch = require('./lib/fetch') | ||
| let readDocs = require('./lib/read-docs') | ||
| let addSinceTags = require('./lib/add-since-tags') | ||
| let addInheritedItems = require('./lib/add-inherited-items') | ||
| let transformModules = require('./lib/modules-transform') | ||
| let putClassesInCouch = require('./lib/classes-in-couch') | ||
| let createVersionIndex = require('./lib/create-version-index') | ||
| let normalizeEmberDependencies = require('./lib/normalize-ember-dependencies') | ||
| let normalizeIDs = require('./lib/normalize-ids') | ||
| let markup = require('./lib/markup') | ||
| let batchUpdate = require('./lib/batch-update') | ||
|
|
||
| require('marked') | ||
|
|
||
| let db = new PouchDB(process.env.COUCH_URL, { | ||
| auth: { | ||
| username: process.env.COUCH_USERNAME, | ||
| password: process.env.COUCH_PASSWORD | ||
| } | ||
| RSVP.on('error', function (reason) { | ||
| console.log(reason) | ||
| process.exit(1) | ||
| }) | ||
|
|
||
| if (fs.existsSync('tmp/docs')) { | ||
| rm.sync('tmp/docs') | ||
| } | ||
|
|
||
| function transformProjectFiles (projectName) { | ||
| console.log('reading docs for ' + projectName) | ||
| let promise = RSVP.resolve(readDocs(projectName)) | ||
| .then((stuff) => { | ||
| console.log('transforming modules for ' + projectName) | ||
| return transformModules(stuff) | ||
| }).then((stuff) => { | ||
| console.log('adding since tags for ' + projectName) | ||
| return addSinceTags(stuff) | ||
| }).then((stuff) => { | ||
| console.log('adding inherited items for ' + projectName) | ||
| return addInheritedItems(stuff) | ||
| }).then(yuidocs => { | ||
| console.log('normalizing yuidocs for ' + projectName) | ||
| return normalizeIDs(yuidocs, projectName) | ||
| }).then(doc => { | ||
| console.log('creating version index for ' + projectName) | ||
| return createVersionIndex(db, projectName, doc).then(() => doc) | ||
| }).then(doc => { | ||
| console.log('converting markdown to html for ' + projectName) | ||
| return markup(doc) | ||
| }) | ||
| let possibleProjects = ['ember', 'ember-data'] | ||
| let projects = argv.project && possibleProjects.includes(argv.project) ? [argv.project] : possibleProjects | ||
| let specificDocsVersion = argv.version ? argv.version : '' | ||
|
|
||
| return promise | ||
| } | ||
| let docsVersionMsg = specificDocsVersion !== '' ? '. For version ' + specificDocsVersion : '' | ||
| console.log(`Downloading docs for ${projects.join(' & ')}${docsVersionMsg}`) | ||
|
|
||
| let projects = ['ember', 'ember-data'] | ||
| let releaseToGenDocFor = process.argv[2] ? process.argv[2] : '' | ||
| syncToLocal() | ||
| .then(() => fetchYuiDocs(projects, specificDocsVersion)) | ||
| .then(() => readDocs(projects, specificDocsVersion)) | ||
| .then(docs => { | ||
| return RSVP.map(projects, projectName => { | ||
| return RSVP.map(docs[projectName], doc => { | ||
| let docVersion = doc.version | ||
| console.log(`Starting to process ${projectName}-${docVersion}`) | ||
| return transformYuiObject([doc], projectName).then(markup).then(doc => { | ||
| let giantDocument = { | ||
| data: doc.data | ||
| } | ||
| console.log('normalizing dependencies') | ||
| return normalizeEmberDependencies(giantDocument) | ||
| }).then(doc => { | ||
| return createClassesOnDisk(doc, projectName, docVersion) | ||
| }).then(doc => { | ||
| console.log(`Finished processing ${projectName}-${docVersion}`) | ||
| return getVersionIndex(doc, projectName) | ||
| }) | ||
| }).then((docs) => { | ||
| let [docToSave, ...remainingDocs] = docs.filter(doc => doc.data.id === projectName) | ||
|
|
||
| console.log('downloading docs for ' + projects.join(' & ')) | ||
| if (!docToSave) { | ||
| return Promise.resolve() | ||
| } | ||
|
|
||
| fetch(db, releaseToGenDocFor).then(downloadedFiles => { | ||
| RSVP.map(projects, transformProjectFiles).then(docs => { | ||
| let giantDocument = { | ||
| data: _.flatten(docs.map(doc => doc.data)) | ||
| } | ||
| console.log('normalizing dependencies') | ||
| normalizeEmberDependencies(giantDocument) | ||
| let existingDoc = `tmp/json-docs/${projectName}/projects/${projectName}.json` | ||
| if (fs.existsSync(existingDoc)) { | ||
| existingDoc = JSON.parse(fs.readFileSync(existingDoc)) | ||
| docToSave.data.relationships['project-versions'].data = docToSave.data.relationships['project-versions'].data.concat(existingDoc.data.relationships['project-versions'].data) | ||
| } | ||
|
|
||
| return putClassesInCouch(giantDocument, db) | ||
| }).then(function () { | ||
| let docs = glob.sync('tmp/docs/**/*.json') | ||
|
|
||
| console.log('putting document in CouchDB') | ||
| return batchUpdate(db, docs) | ||
| }).catch(function (err) { | ||
| console.warn('err!', err, err.stack) | ||
| process.exit(1) | ||
| remainingDocs.forEach(d => { | ||
| docToSave.data.relationships['project-versions'].data = docToSave.data.relationships['project-versions'].data.concat(d.data.relationships['project-versions'].data) | ||
| }) | ||
| return saveDoc(docToSave, projectName).then(() => projectName) | ||
| }) | ||
| }) | ||
| }) | ||
| }) | ||
| .then(syncToS3) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.