Initial commit

.build_scripts/deploy.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# Create iron.json and then deploy
+echo '{"token": "'$IRON_TOKEN'", "project_id": "'$IRON_PROJECT_ID'"}' > updater/iron.json
+docker-compose run deploy

.gitignore

@@ -0,0 +1,5 @@
+node_modules
+payload.json
+iron.json
+updater/*.geojson
+npm-debug.log

.travis.yml

@@ -0,0 +1,16 @@
+sudo: false
+language: node_js
+node_js:
+- '4.1'
+services:
+- docker
+env:
+  global:
+  - secure: KQL/r2DFrDbrWf7TMeowxZDAd1FKb2HfP9gSGnCnREl3QEEXEPGdaBsBPzPZg4nGOmfDWi5oH3MVG73nFhZo3qO+1Vb3m4FyD2Qjr9IdvldIVIcp1YGz31zG5slB5mUZ5nP32cYVUo9ADTJmqcKOxRfthLd4Smwfc+e7NPsMM/+a9d/aZcKFKA9Rt7KAlmHdRHkP9iEBg1QqPbPhKoOvUO9EVgKkCw/LwsDtzAwCX7j5+kzKSwqeO2zaZFZQ6gQn9VdCI0FkbAOJ7V9ayzW3bLe89dAvPL2oGmAGxlgSYc3JgG/lNdI+diKbmTLVlDq72pGkS4kb++dbEv6ZJTJ3uYs2wUoSstksR/v7npt71zCWDy7qc769bA/xp+SfQvcR+U7XIuzDMPETBR4Q7UXNy33GmDISwNr2bNSYtGKnBEcDiL09CINLuv4q00teuQa8B1dnkLAv/hwTfORrh48pckH8qFKaWMO/iC700skTS1Cyi8Qri9Z5EP0eSxAVv/51Y54hIbcH/8knUXEsmPF8XIoSQ6y5UMRIGTCC50gVoEl7aRPBGnN2yBz2VVpLSdQw7Ifx4cYmP1d7Z7WSya/ubNT00jN2xcDng+hRi1VS4eRZjZykgowU1rsxnR+EcA2uhqN1I2sztcSFpVj8yN7nTfKfTCUBjUmqGLeIleCMv84= // IRON_TOKEN
+  - secure: zvglZkgmaN8DQrALVcBYBlJ7bQB5qE1bDzk/CdQRrr8dVFhnER2TD4u1GFhL6vlS/2v53tRrO19zA4zEURTHcTQtghnez58gP8XNw/DfpIvjI03xFjHiL+wKjOjVRdnAD8k2AZV9cOQsf1oSVJRbfj8AvlVWt2hnXnFDt8Q3xII72AEKUNr790ezQ2kDzkbCNnwDdVAgt8zbHVWSQiJG3Nk6ZEZqxIhhQ4Z052NrRKpq3MgkOATuI6Tpf8isHJ4UtsVDW1yLG6Ypip9KJL7NA4VscthjMp8fvvBm6jG++2xTVE+FSpbR5/WuMokQuElxk41rZ8OD/rMTev/bms6crQWQytdxzTLpVDZsZbWD+dcOgwgqOdabihmMM/lUGCar2OA4XcWGsB/kysTYaLUA6fglfGe7AZbXRssuCPKHIgant/kk0k8eKDJvV8u+qgA1mi9sq5AOUqjLjT+Dd3v44S6CTbxedjNSUqggtW+4sDJO+qzBTlpfOcZiso4avHCsyMHa+xA883B6s4eWSxI6+aiL2F9LhHkxnQjCwQymU5EwGOXCSSbE9PZFoHHXmOSmGrp/aqCvj8cI8IKlEb269jgbdMcSOAANMLvjkS2g62fRC00NiHNAlGOvvxF+zGxAUTPOoHRRGV7d8cOvv5jOpAxHQ7qm1fAwirj94bGJZHs= // IRON_PROJECT_ID
+install: echo nothing to install
+script: docker-compose run --rm test
+deploy:
+  provider: script
+  script: ".build_scripts/deploy.sh"
+  on: master

Dockerfile

@@ -0,0 +1,7 @@
+FROM iron/images:node-4.1
+WORKDIR /worker
+ADD ./updater /worker
+RUN apt-get update
+RUN apt-get install zip
+RUN /bin/bash -c 'cd /worker; npm install;'
+RUN curl -sSL http://get.iron.io/cli | sh

README.md

@@ -0,0 +1,33 @@
+### sentinel-api vector tiles updater
+
+This application is designed to run on IronWorker from Iron.io and update a Mapbox vector tile set on a regular basis to power the frontend for ad-next.
+
+### Installation
+
+- Make sure latest version of `docker` with `docker-compose` is installed and launched
+- Install latest version of Iron.io CLI
+
+        $ curl -sSL http://get.iron.io/cli | sh
+
+- Create `iron.json` for upload to iron.io. Use `iron.json.sample` as template
+- Create `payload.json` if you plan to test the tool locally. Use `payload.json.sample` as a template.
+
+
+### Local test
+
+    $ docker-compose run test
+
+### Local shell
+
+    $ docker-compose run shell
+
+### Automatic Deployment
+
+Changes made on `master` will be deployed automatically by Travis.
+
+### Manual Deployment
+
+You should copy `iron.json` to the updater folder before running this command.
+
+    $ docker-compose run deploy
+

docker-compose.yml

@@ -0,0 +1,17 @@
+test:
+  build: .
+  command: npm run test
+  container_name: ad-sentinel-vector-tile-updater
+  working_dir: /worker
+
+shell:
+  build: .
+  command: /bin/bash
+  container_name: ad-sentinel-vector-tile-updater
+  working_dir: /worker
+
+deploy:
+  build: .
+  command: bash -c 'ls; zip -qr updater.zip .; iron worker upload --name ad-sentinel-vector-tile-updater --zip updater.zip iron/images:node-4.1 node index.js;'
+  container_name: ad-sentinel-vector-tile-updater
+  working_dir: /worker

updater/.babelrc

@@ -0,0 +1,3 @@
+{
+ "presets": [ "es2015" ]
+}

updater/.eslintrc

@@ -0,0 +1,12 @@
+{
+  "extends": ["standard"],
+  "env": {
+    "es6": true,
+    "node": true
+  },
+  "rules": {
+    "semi": [2, "always"],
+    "no-extra-semi": 2,
+    "semi-spacing": [2, { "before": false, "after": true }]
+  }
+}

updater/index.js

@@ -0,0 +1,5 @@
+// only ES5 is allowed in this file
+require('babel-register')();
+
+// load the update and run it
+require('./updater').doTheThing();

updater/iron.json.sample

@@ -0,0 +1 @@
+{"token":"yourtoken","project_id":"yourprojectid"}

updater/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "ad-sentinel-vector-tile-updater",
+  "version": "1.0.0",
+  "description": "An updater for vector tiles used for ad-next, pulled from sat-api",
+  "main": "index.js",
+  "scripts": {
+    "tape": "tape -r babel-register test/**/*.js",
+    "test": "nyc npm run tape",
+    "lint": "eslint . --ext .js"
+  },
+  "author": "Development Seed",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/AstroDigital/ad-sentinel-vector-tile-updater.git"
+  },
+  "license": "BSD",
+  "dependencies": {
+    "async": "^1.5.2",
+    "babel": "^6.1.18",
+    "babel-preset-es2015": "^6.3.13",
+    "babel-register": "^6.3.13",
+    "elasticsearch": "^10.1.3",
+    "elasticsearch-streams": "0.0.9",
+    "iron_worker": "^0.1.6",
+    "mapbox-upload": "^4.2.1",
+    "moment": "^2.11.2",
+    "winston": "^2.1.1"
+  },
+  "devDependencies": {
+    "nyc": "^5.5.0",
+    "tape": "^4.4.0",
+    "eslint": "^2.0.0",
+    "eslint-config-standard": "^5.1.0",
+    "eslint-plugin-promise": "^1.0.8",
+    "eslint-plugin-standard": "^1.3.2"
+  }
+}

updater/payload.json.sample

@@ -0,0 +1,9 @@
+{
+    "esURL": "http://esurl.com",
+    "logLevel": "info",
+    "groupings": [
+        {"pattern": "2016", "mapboxAccount": "astrodigital", "mapboxID": "sentinel_meta_2016"},
+        {"pattern": "2015", "mapboxAccount": "astrodigital", "mapboxID": "sentinel_meta_2015"}
+    ],
+    "mapboxToken": "write_token"
+}

updater/test/basic.js

@@ -0,0 +1,12 @@
+'use strict';
+
+let test = require('tape');
+import { zp } from '../updater';
+
+test('Test zero padding', (t) => {
+  t.equal(zp('a', 3), '00a');
+  t.equal(zp('aa', 3), '0aa');
+  t.equal(zp('aaa', 3), 'aaa');
+  t.equal(zp('a', 1), 'a');
+  t.end();
+});

updater/updater.js

@@ -0,0 +1,170 @@
+/**
+ * A script to create vector tiles from generated GeoJSON, using a sat-api
+ * database as the source.
+ */
+'use strict';
+
+import elasticsearch from 'elasticsearch';
+import iron_worker from 'iron_worker'; // eslint-disable-line camelcase
+import { crossTest, warpArray } from './warp.js';
+import { series } from 'async';
+import { writeFileSync } from 'fs';
+import upload from 'mapbox-upload';
+import moment from 'moment';
+import { join } from 'path';
+import { ReadableSearch } from 'elasticsearch-streams';
+
+let Winston = require('winston');
+let winston = new (Winston.Logger)({
+  transports: [
+    new (Winston.transports.Console)({'timestamp': true, colorize: true})
+  ]
+});
+
+const params = iron_worker.params() || {};
+const groupings = params.groupings || [];
+const mapboxToken = params.mapboxToken;
+const esURL = params.esURL || '';
+const queryLimit = params.queryLimit || 1000;
+
+// Set logging level
+winston.level = params.logLevel || 'info';
+
+// Create ES client
+const es = new elasticsearch.Client({
+  host: esURL
+});
+
+/**
+* Zero padding function
+*
+* @param {string} n The string to be padded
+* @param {number} c The length of final string
+* @return {string} A zero-padded string
+*/
+export function zp (n, c) {
+  var s = String(n);
+  if (s.length < c) {
+    return zp(`0${n}`, c);
+  } else {
+    return s;
+  }
+}
+
+/**
+ * The main run loop for the script, this will start up Mongo, grab the data,
+ * build the GeoJSON and upload to Mabox for rendering as a vector tile.
+ */
+export function doTheThing () {
+  // Grab all daytime items given our regex pattern and return the geojson
+  let buildGeoJSON = function (pattern, skipToday = false, cb) {
+    const searchExec = function searchExec (from, callback) {
+      es.search({
+        index: 'sat-api',
+        type: 'sentinel2',
+        q: `date:[${pattern}-01-01 TO ${pattern}-12-31]`,
+        from: from,
+        size: queryLimit
+      }, callback);
+    };
+
+    // Create ES results stream
+    const rs = new ReadableSearch(searchExec);
+    let count = 0;
+    let geojson = {
+      type: 'FeatureCollection',
+      features: []
+    };
+
+    rs.on('data', (d) => {
+      d = d._source;
+      // Make sure we have all needed fields
+      if (d.cloud_coverage === undefined || !d.date || !d.tile_geometry ||
+          d.utm_zone === undefined || !d.latitude_band || !d.grid_square || !d.path) {
+        return;
+      }
+
+      const feature = {
+        type: 'Feature',
+        geometry: d.tile_geometry,
+        properties: {
+          c: d.cloud_coverage,
+          d: Number(moment(d.date, 'YYYY-MM-DD').format('DDD')),
+          s: `${zp(d.utm_zone, 2)}${d.latitude_band}${d.grid_square}${d.path.slice(-1)}`
+        }
+      };
+
+      geojson.features.push(feature);
+
+      count++;
+      if (count % 1000 === 0) {
+        winston.info(`Processed ${count} records.`);
+        // cb(geojson);
+      }
+    });
+    rs.on('error', (e) => {
+      winston.error(e);
+    });
+    rs.on('end', () => {
+      winston.verbose('ES stream ended');
+      winston.verbose('World wrapping started');
+      for (const f of geojson.features) {
+        const coordArray = f.geometry.coordinates[0];
+        const crosses = crossTest(coordArray);
+        if (crosses) {
+          f.geometry.coordinates[0] = warpArray(coordArray);
+        }
+      }
+      winston.verbose('World wrapping complete');
+      cb(geojson);
+    });
+  };
+
+  // Build up task groups
+  let groups = groupings.map((g) => {
+    return function (done) {
+      winston.info(`Running for grouping ${g.pattern} and uploading to ${g.mapboxID}`);
+      buildGeoJSON(g.pattern, g.skipToday, (geojson) => {
+        // Save it to disk so we can upload to Mapbox, we're already blocked
+        // here so just do it sync
+        let filename = `${g.mapboxID}.geojson`;
+        winston.info(`Saving geojson to disk at ${filename}`);
+        writeFileSync(filename, JSON.stringify(geojson));
+
+        // We have the geojson, upload to Mapbox
+        winston.info(`Started uploading to ${g.mapboxAccount}.${g.mapboxID}`);
+        let progress = upload({
+          file: join(__dirname, `/${filename}`),
+          account: g.mapboxAccount,
+          accesstoken: mapboxToken,
+          mapid: `${g.mapboxAccount}.${g.mapboxID}`
+        });
+
+        progress.once('error', (err) => {
+          done(err);
+        });
+
+        progress.on('progress', (p) => {
+          winston.verbose(`Upload progress for ${g.mapboxID}: ${p.percentage}`);
+        });
+
+        progress.once('finished', () => {
+          winston.info(`Finished uploading to ${g.mapboxID}`);
+          done(null);
+        });
+      });
+    };
+  });
+
+  // Run in a series
+  series(groups, (err, results) => {
+    if (err) {
+      winston.error('Exiting with an error');
+      winston.error(err);
+      process.exit(1);
+    }
+
+    winston.info('All vectorization processes have finished.');
+    process.exit(0);
+  });
+}

updater/warp.js

@@ -0,0 +1,24 @@
+export function crossTest (array) {
+  let lonUnderMinus160 = false;
+  let lonOverPlus160 = false;
+  for (let i = 0; i < array.length; i++) {
+    if (array[i][0] > 160) {
+      lonOverPlus160 = true;
+    } else if (array[i][0] < -160) {
+      lonUnderMinus160 = true;
+    }
+  }
+
+  return lonOverPlus160 && lonUnderMinus160;
+}
+
+export function warpArray (array) {
+  let newArray = array.slice(0);
+  for (var i = 0; i < newArray.length; i++) {
+    if (newArray[i][0] < -170) {
+      newArray[i][0] = newArray[i][0] + 360;
+    }
+  }
+
+  return newArray;
+}