Remove couch usage & generate archives instead

.travis.yml

@@ -1,28 +1,17 @@
 language: node_js
 node_js: '6'
-
-env:
-  global:
-  - COUCH_URL="https://emberjs.cloudant.com/ember-api-docs"
-  # export COUCH_USERNAME=xxxxx
-  - secure: e80GjZi18denINfipJnduwS0C/7fYf3lf9XMmlYRsSObhXbja/wYmqZfZsXUC5mDZFfutTY0sJDmnF3ibc8ZkvY1LSJ/22K48KSOGYQZ95RJBAK+nHa5m29ZOocx9UJwuo/n4UFTqS0+u5gvg5ArPAUftPA6BUaMszxGURDroTtQPjCzUUkclbul4Nb8OD2/S9Kb959VUsZnC9VezQzNMrym7Fw8b7c5UiriwBAc58wrrgdIBfQII38SuMSgz0OERScFBUVRs+6ETmuVgG/YMLYzISzg5rzCmrDqfHDEElCRfasKJ0ukc+g/tM/onBSMJSNX7r4NjNUvIyZjG8djbm4mrTfiSSxikXXa6/EDsgHrWbl+50wRb12c9iEkQDb+qNOxemvzsXJyKFPSeC0r9Ta1C3YSe2bOWNUAxH3a3GKoSEuIbNfgzX2K6GiiKHzgLYCK+acMQJrtz4Qs/D2u/EI5ST1FWKyIIu2qWJ84LPWJKLGByTjOjEDQ+arpQWYn6ACTDl4Tv1luzagQ7txbJxqPLcFG6yOBPXH1R1sXJEJwB2wkSCvAC6bwN37Yx/lUewTkzqI17W9HtboabQ9Hk4pqPSlFGI6bZZM4ASeopWMLQqGMLd3kUbneRr8BWCuPhgiX/rj9GqDOQoAZMoejuBirWU+tn2SXX0JJbt9i1lw=
-  # export COUCH_PASSWORD=xxxxx
-  - secure: bpaoN/CqbrtWQXKqUznMPmqsHr4H7+6nT2SnRCc56hBWsh9dnM6KcpaJnVvnpXKF2Rj5Teiv5t/V0ycoLb7yrj327A5yLsHEI30sO5CrVu2b/7DS/wvF2FPc7bd1eoLlvrP5WbgQikP2hempRACs0dOwTLE1lhBVQjJr0xRb7LPl7+ZE2EaNnU8je+WWeGNWqAT1VHD6pBKDwYhtyfke7tRV2QvvDAijELLGjvAzEv4FhZnCL7CMK8wAgL6QDeMCndyXxJMudel33AL/Lb8hWxBPEnZRWyGAjfYzLSXnsELnUEEtM83uFeKexVkxmQWGQ8BOK8GOK0e/GRctCKVTqKgDnX1LATergOv3mjCsROHiLzqkqIUUYlJNGMYtl1IEJZ+SgmExFirPio8ORVbw/I/bwsaNpq+d7HzEa/WbsblikWdNKecjnWGeVKH2boZIrReb3CF5cGhJo7ZjH15C9M3jXLxc4LkY9vHPY90/8cKqJ4sWpu58qIQIn210+WiiZ6L6gOSuMAEQYtg1PxtnpoyLG3s68PhyX2LxSEU+Gt97IiPszGwo4saqIC7y+jUj+kG+C9I5DoarUXENGveH/kZja2Pllm2YUndIoWxKoIGy3K4NqsiPfxGyqnIVq0bW4aGul/XTjCBitkD+hrpcg8EcV7QUgr50BnmyY1cFNV8=
-  # export AWS_ACCESS_KEY_ID=xxxxxx
-  - secure: QXKfIbHOdsLhhSq2KY4trFXc+eWUoYQAbMfAjA9nQuVN0RjVksQUyK4s5TI3dMcXjh7N95Jb7F2P3AC17SgVaz/ydUkrYwPUaglSAOuOECGVxqtNV/y3TnZjquz8XnuPLW2ZLxIqoN2lbNisomVxegLSMGgeo+IX/2mgSWZCKlUNdlCCJ8DUv7Vex+L43DPfOsbmrUkXFiimYtg4NjsxKWjerHM3iWxQlZ1m8pzhKhOfMJs9JlAEKZmRXt4Mw2IbTyc+g5tVfUb5l7yQdtMkoFZ3tqYCm59P8erocMn2m0PYPy/IHd+YM9xaCtZFcMwyEywMvZtVtP1ByscIDKoNTNSgy1IUgR/lJHeHN4/v/CEkSAUIvMKoGGDIFbfHPKJZ2HepupFYVJwFfSZbbiaipIMVMerKPsyJKyTkqx/Xke7ZfgW2RQvVjOjEyvD9zbfLte1iNbdWqVrI3cPAInNRNgdHuYZgjcvX3aKbqk4Xpn81+4B6O1keS8iU30vfnd1Xw7Wye5BxraVq2ajdq/jHkkK9ULlZZKmyebW/f3I5JsLRCbSJy+sb/zUAawEXLr0qaYKOfQxg1iuOwswMmgSVFYVZUKfRIlLo3A/e1e8/YR9oTMYaXdiDVW813cK5ZnNbr5qoK9PSAf38hDRZtpmldC3m4c0R9OyQhtrFW8KaLF0=
-  # export AWS_SECRET_ACCESS_KEY=xxxxx
-  - secure: jXcVmvHqS9BD2tX9/5ZYE3YY4VrVSHimOWLDI2xY+j071Ngcsr/70bjoEqD+ja2YI/fDsZrblelUV8a+ettlqC2PlM7s869zxt1cDcqRwQiopyz3b7ihp4JsbxHF4C1AYg+3POSuC5Knvh9sIimebrwMRgrMEED5t37kfSSD3JBprhodBARIOw7jo2JgXmqxWv/4YpqA5IBGTTwmGB2DsuZnqLutZeT9Fhcstkm4aaBwN5sjC18lugcEh6vrvmCZS6gbiG2v2MIsvaGzWXLZgl66pk4JLzhzA68YtNNRgye9hn5BVJXhkZK14Q1BWhol4v8B+mSBaB5R4yikBIvODzeTpXbacFc84EHZQlw/9ZNTm3MvdpBLd4guPPJk6oKFUTXwdPaWccVFli+NdPz16e4Y4/xUYS1fKfjHLF2gejZ27gHxosD6M+ss3HIrmMLzatHC9iHk7juICP67M5bRcPuwCzm7K6pBmC5QWULd01Ppn3h4V+xUilCA8OmfkrGBLJcl8d2t7H0Q7ebei4SSdrbtkuRRdo8OV1CU7VVJdU5MvYCtDZs16qUozILvXn9Oj05GZ4Uu7wy4TmwF5+6Ye+Fherc2UXQ8CWdjKqYoKY6usVCK0E251vlwk3cjkaZhc+xpLLIPEuVLnGC8CzxMJqaxa6zOrSjLXpZGXnRH3Lw=
-
 cache:
   directories:
   - "$HOME/.yarn-cache"
   - node_modules
   - tmp
-
 script:
 - yarn run test
-- 'if [ "$TRAVIS_PULL_REQUEST" = false ]; then yarn start; else echo "We dont publish docs via PRs"; fi'
-
+- if [ "$TRAVIS_PULL_REQUEST" = false ]; then AWS_SHOULD_PUBLISH=yes yarn start; else echo "We dont publish docs via PRs"; fi
 branches:
   only:
   - master
+env:
+  global:
+  - secure: NqKxv7m1J21O2z1QfNIKce85KW70IeID8SshF/Ekzn4F0eg3PGQHHVLwgYmiOTznRu0kb+m2lbD5C3KuXJcdXHQXaG8ByAOYytMd6+n/1IJdM8Ao1nhWPlzN0xcU9TqPtlbKlJeUw9+G7zFsBK6nGAc866/CO4zJblqdOh8pztJlP4iI8QqzenK/YWJg2AacLjxQHNqjPQtK48ItgV1pdrXgVFGj7si8VmOz9uGytcqB+aQCWx+sP/08hkdJoGaxixjMhe1oAJpEccCGTuJ8EXdCZpssz0aZeyVUQBG+19eDUraYMBIXm6nZhX9OK1aHGTT5A6/pcDbLpmz8gs8DG3j9semKp0DwRWg2iIvunKEk5IKogCkbaebDw1U6l4V6lP9BZPZ+ab8NnJHEUL9vrkKPaZ1qPUvmXVoDErciUzmNWP26aTMRfFIe4a2Ax6sSyzbl06STEyVaBTonfpNgcbpDZiR6C7ngsKYFVyUdf0kSHdAlchF+CEX99pyGJPgmrjromcbzSA9x3Uye5puT/2KQXFfKXRmMIflTePqi7PlLQ9t/Z1roWocfORaIsgobv1RfKwsHv5dw+7j8OVsk6//dZu5GFbNcqqCMqoRz0mQ1yG3t9oa2zfpDLyIGvxAvi8Owdcc3TgQQJ5xcZ63ziqgB4KtE/5COiccoEjNWT4k=
+  - secure: QCpb0SMkTCAGUIEiv2Bhv4QRz6GOtUmRBq2Kjvorr0PP1pFUND3j2qI5QvBlzgZo6G23KGUQ08E0l0BwInoqYGSHBIAX1PhiTwnqnNOy0n2RJUaKqsED6vfqUbcFtyE9HtvMT3xwz38L2aHIHAdYEkfI2fNX5/SAhTz0+Ovtbd7kKbZvX4V21FOmqdUnZyoA66m5YTs8dPv+/52iHA7oTF06HvejQKbsiu2wbZ8pP/cKVLOxcPkb2mI9Y03cNlHWsEwKgYTIFYuz3UXa/v24TSFmDOj2YcYKYml2Pk3e0VZnw36+N/1yx6trkpceeodLGGe9EM8kIqstcYdGnMfA7S6ADu8Fh8F88lBBItVqqE984b9LVlXE+ujr88DQVOMu8YkzFN/RGF6fonvgPNZ5M9Q/n59+ycCJOqOW+LFzEs+liH2Z9Hbgft+LlYaT3QiJXNXqqgd/j4fNMGdCalFk0s7pVmMtN6gJzrbGbeLd3R7gJ8HFey9oycWsZO5eXKKQ/thfXfk2bdfuusyluPyqqU2O6W4fbpB80hIC5T4VClrT08ikuLHtuj7vFJVmxn4cVIecxHX+DmR9NXvZgS8M/r5uxp57purLt/RGQGBk/uJFLflOgoFBy/K9QQD280MvF5VrgzifyUvNR4wcCzE1XEVQmKZFeuGGDbB74eW1V4c=

README.md

@@ -2,63 +2,26 @@
 
 This app is for turning ember API doc build output into [json api](http://jsonapi.org/) compliant data for use in various applications seeking to use the Ember API.
 
-The script pulls yuidoc build output from all Ember versions from Amazon S3, converts it to json api format and pushes it to CouchDB.
-You can setup a CouchDB instance either [locally](#setting-up-a-local-couchdb-instance) or [on cloundant](#setting-up-a-couchdb-on-cloudant) for testing out changes to this app.
-
-
-## Setting up a local CouchDB instance
-
-1. Install the latest release of [CouchDB](http://couchdb.apache.org/).
-1. Once installed, visit the [CouchDB UI](http://localhost:5984/_utils). From the top navigation section, Click "_Create Database_" and type in "*ember-api-docs*".
-1. Create a CouchDB admin user.  Got to the section in the left nav called "_Admin Party!_", and supply a username and password.
-1. Set up CORS for CouchDB
-   An easy way to do this is to use the script at https://github.com/pouchdb/add-cors-to-couchdb
-
-    ```shell
-    npm install -g add-cors-to-couchdb
-    add-cors-to-couchdb -u <your admin couchdb username> -p <your admin couchdb password>
-    ```
-
-
-## Setting up a CouchDB on cloudant
-
-1. A [cloudant](https://cloudant.com/) account is neccessary before proceeding with the following steps.
-1. Create a new Database and give it the name "*ember-api-docs*".
-1. Goto the "_Permissions_" tab in the nav.
-1. Click on "_Generate API Key_", save the credentials showed on the UI as we'll be using this in the next section as the CouchDB credentials.
-1. Click on the _"_writer"_ checkbox for the API user so that it can push docs to cloudant.
-1. Navigate to your account settings & in the "_CORS_" tab ensure that its enabled for all domains. 
-
+The script pulls yuidoc build output from all Ember versions from Amazon S3, converts it to json api format and creates an archive.
 
 ## Running the app
 
 1. Fork/Clone [ember-jsonapi-docs](https://github.com/ember-learn/ember-jsonapi-docs)
-1. Run `npm install` or `yarn`
-1. Set the following environment variables:
-
-   ```shell
-   export COUCH_URL=http://localhost:5984/ember-api-docs # or COUCH_URL=https://user-name.cloudant.com/ember-api-docs
-   export COUCH_USERNAME=<your couchdb username>
-   export COUCH_PASSWORD=<your couchdb password>
-   ```
+1. Run `yarn` or `npm install` (Needs node 6+)
 1. Set up AWS access
     ```shell
-    export AWS_ACCESS_KEY_ID=xxxxxx
-    export AWS_SECRET_ACCESS_KEY=xxxxx
+    export AWS_ACCESS_KEY=xxxxxx
+    export AWS_SECRET_KEY=xxxxx
     ```
     The app accesses builds.emberjs.com (an Amazon S3 bucket) in read-only mode, which is public. This requires any valid AWS credentials.
 
     You can get your credentials by logging into your [AWS console](https://console.aws.amazon.com) and navigating to "_My Security Credentials_" under your profile name. You can generate a new pair under the "_Access Keys (Access Key ID and Secret Access Key)_" section.
 1. To test your changes in the app run,
-   ```node index.js 2.11```
-   This will run the app only for the specified version of the docs. Once complete, if no errors you should see your doc database 
-   populated with data from your CouchDB admin ui. When no version is passed, the app will try to process all 
-   ember versions since 1.0 which takes high memory & time to complete. If you intend it, then run `node --max_old_space_size=8192 index.js`.
+   ```node index.js```
+   Once complete, if no errors you should see a docs.tar file inside the `tmp` folder. The app tries to process all 
+   ember & ember-data versions since 1.0 which takes high memory & time to complete. If you intend it, then run `node --max_old_space_size=8192 index.js`.
    You are setting your node max heap space to 8GB, so make sure you have that much space available on your machine.
 
 
-## Setting up a new production instance
-
-1. Follow the steps from [setting up a CouchDB on cloudant section](#setting-up-a-couchdb-on-cloudant)
-1. Run `npm start` or `yarn start` from your local. This is required since travis kills a build if [a task crosses 50mins](https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts), & the first run will take 50+ mins to complete.
-1. On Travis UI [enable cron jobs](https://docs.travis-ci.com/user/cron-jobs/) for daily/hourly basis so that newer docs get indexed incrementally.
+## To Generate docs for a specific project and/or version for development 
+You can do this by passing `--project ember/ember-data --version 2.11.1` as an argument to the index script. e.g., `yarn start -- --project ember --version 2.11.0`

index.js

@@ -1,84 +1,67 @@
-'use strict'
+const RSVP = require('rsvp')
+const fs = require('graceful-fs')
+const argv = require('minimist')(process.argv.slice(2))
 
-let RSVP = require('rsvp')
-let _ = require('lodash')
-let rm = require('rimraf')
-let PouchDB = require('pouchdb')
-let fs = require('fs')
-let glob = require('glob')
+const markup = require('./lib/markup')
+const readDocs = require('./lib/read-docs')
+const fetchYuiDocs = require('./lib/fetch-yui-docs')
+const createClassesOnDisk = require('./lib/create-classes')
+const transformYuiObject = require('./lib/transform-yui-object')
+const normalizeEmberDependencies = require('./lib/normalize-ember-dependencies')
+const getVersionIndex = require('./lib/get-version-index')
+const saveDoc = require('./lib/save-document')
+const { syncToLocal, syncToS3 } = require('./lib/s3-sync')
 
-let fetch = require('./lib/fetch')
-let readDocs = require('./lib/read-docs')
-let addSinceTags = require('./lib/add-since-tags')
-let addInheritedItems = require('./lib/add-inherited-items')
-let transformModules = require('./lib/modules-transform')
-let putClassesInCouch = require('./lib/classes-in-couch')
-let createVersionIndex = require('./lib/create-version-index')
-let normalizeEmberDependencies = require('./lib/normalize-ember-dependencies')
-let normalizeIDs = require('./lib/normalize-ids')
-let markup = require('./lib/markup')
-let batchUpdate = require('./lib/batch-update')
-
-require('marked')
-
-let db = new PouchDB(process.env.COUCH_URL, {
-  auth: {
-    username: process.env.COUCH_USERNAME,
-    password: process.env.COUCH_PASSWORD
-  }
+RSVP.on('error', function (reason) {
+  console.log(reason)
+  process.exit(1)
 })
 
-if (fs.existsSync('tmp/docs')) {
-  rm.sync('tmp/docs')
-}
-
-function transformProjectFiles (projectName) {
-  console.log('reading docs for ' + projectName)
-  let promise = RSVP.resolve(readDocs(projectName))
-    .then((stuff) => {
-      console.log('transforming modules for ' + projectName)
-      return transformModules(stuff)
-    }).then((stuff) => {
-      console.log('adding since tags for ' + projectName)
-      return addSinceTags(stuff)
-    }).then((stuff) => {
-      console.log('adding inherited items for ' + projectName)
-      return addInheritedItems(stuff)
-    }).then(yuidocs => {
-      console.log('normalizing yuidocs for ' + projectName)
-      return normalizeIDs(yuidocs, projectName)
-    }).then(doc => {
-      console.log('creating version index for ' + projectName)
-      return createVersionIndex(db, projectName, doc).then(() => doc)
-    }).then(doc => {
-      console.log('converting markdown to html for ' + projectName)
-      return markup(doc)
-    })
+let possibleProjects = ['ember', 'ember-data']
+let projects = argv.project && possibleProjects.includes(argv.project) ? [argv.project] : possibleProjects
+let specificDocsVersion = argv.version ? argv.version : ''
 
-  return promise
-}
+let docsVersionMsg = specificDocsVersion !== '' ? '. For version ' + specificDocsVersion : ''
+console.log(`Downloading docs for ${projects.join(' & ')}${docsVersionMsg}`)
 
-let projects = ['ember', 'ember-data']
-let releaseToGenDocFor = process.argv[2] ? process.argv[2] : ''
+syncToLocal()
+  .then(() => fetchYuiDocs(projects, specificDocsVersion))
+  .then(() => readDocs(projects, specificDocsVersion))
+  .then(docs => {
+    return RSVP.map(projects, projectName => {
+      return RSVP.map(docs[projectName], doc => {
+        let docVersion = doc.version
+        console.log(`Starting to process ${projectName}-${docVersion}`)
+        return transformYuiObject([doc], projectName).then(markup).then(doc => {
+          let giantDocument = {
+            data: doc.data
+          }
+          console.log('normalizing dependencies')
+          return normalizeEmberDependencies(giantDocument)
+        }).then(doc => {
+          return createClassesOnDisk(doc, projectName, docVersion)
+        }).then(doc => {
+          console.log(`Finished processing ${projectName}-${docVersion}`)
+          return getVersionIndex(doc, projectName)
+        })
+      }).then((docs) => {
+        let [docToSave, ...remainingDocs] = docs.filter(doc => doc.data.id === projectName)
 
-console.log('downloading docs for ' + projects.join(' & '))
+        if (!docToSave) {
+          return Promise.resolve()
+        }
 
-fetch(db, releaseToGenDocFor).then(downloadedFiles => {
-  RSVP.map(projects, transformProjectFiles).then(docs => {
-    let giantDocument = {
-      data: _.flatten(docs.map(doc => doc.data))
-    }
-    console.log('normalizing dependencies')
-    normalizeEmberDependencies(giantDocument)
+        let existingDoc = `tmp/json-docs/${projectName}/projects/${projectName}.json`
+        if (fs.existsSync(existingDoc)) {
+          existingDoc = JSON.parse(fs.readFileSync(existingDoc))
+          docToSave.data.relationships['project-versions'].data = docToSave.data.relationships['project-versions'].data.concat(existingDoc.data.relationships['project-versions'].data)
+        }
 
-    return putClassesInCouch(giantDocument, db)
-  }).then(function () {
-    let docs = glob.sync('tmp/docs/**/*.json')
-
-    console.log('putting document in CouchDB')
-    return batchUpdate(db, docs)
-  }).catch(function (err) {
-    console.warn('err!', err, err.stack)
-    process.exit(1)
+        remainingDocs.forEach(d => {
+          docToSave.data.relationships['project-versions'].data = docToSave.data.relationships['project-versions'].data.concat(d.data.relationships['project-versions'].data)
+        })
+        return saveDoc(docToSave, projectName).then(() => projectName)
+      })
+    })
   })
-})
+  .then(syncToS3)

lib/add-inherited-items.js

@@ -12,7 +12,6 @@ module.exports = function addInheritedItems (docSets) {
 
     classes.forEach(function (klass) {
       let parents = getParents(klass, classes)
-      // let thisClassItems = classItems.filter(item => item.class === klass.name)
 
       for (let i = 0; i < parents.length; ++i) {
         parents = parents.concat(getParents(parents[i], classes))

lib/classes-in-couch.js → lib/create-classes.js

@@ -1,12 +1,10 @@
 'use strict'
 
 let RSVP = require('rsvp')
-let Queue = require('promise-queue')
 let saveDoc = require('./save-document')
 
-module.exports = function (document, db) {
+module.exports = function (document, projectName, projectVersion) {
   let things = document.data
-  let queue = new Queue(10)
 
   return RSVP.map(things, klass => {
     if (!klass.id) {
@@ -15,12 +13,10 @@ module.exports = function (document, db) {
       process.exit(1)
     }
     let document = {
-      _id: klass.id,
       data: klass
     }
 
-    return queue.add(() => {
-      return saveDoc(document, db)
-    })
+    console.log(`Creating ${klass.id} in ${projectName}-${projectVersion}`)
+    return saveDoc(document, projectName, projectVersion)
   }).then(() => document)
 }

lib/create-project-versions.js

@@ -1,14 +1,11 @@
 'use strict'
 
 let tojsonapi = require('yuidoc-to-jsonapi/lib/converter')
-let Queue = require('promise-queue')
 let RSVP = require('rsvp')
 let saveDoc = require('./save-document')
 let updateIDs = require('./update-with-versions-and-project')
 
 module.exports = function createProjectVersions (versions, projectName, db) {
-  let queue = new Queue(10)
-
   return RSVP.map(versions, version => {
     let jsonapidoc = updateIDs(tojsonapi(version.data), version.version)
 
@@ -21,9 +18,7 @@ module.exports = function createProjectVersions (versions, projectName, db) {
       }
     }
 
-    let id = `project-version-${projectName}-${version.version}`
     let versionDocument = {
-      _id: id,
       data: {
         id: `${projectName}-${version.version}`,
         type: 'project-version',
@@ -51,6 +46,6 @@ module.exports = function createProjectVersions (versions, projectName, db) {
       included: [projectData]
     }
 
-    return queue.add(() => saveDoc(versionDocument, projectName, version.version, db))
+    return saveDoc(versionDocument, projectName, version.version)
   })
 }