import

#!/usr/bin/env node

var url = require('url'),
    path = require('path'),
    http = require('http'),
    flat = require('flat'),
    csv = require('fast-csv');

var importer,
    reqOpts,
    options = {
      wait: 500, // wait between requests
    },
    stats = {rows: 0, requests:0, responses:{}},
    self = process.argv[1].split(path.sep).pop(),
    indexes = {};

var supportedTypes = {
  users: {
    required: ['name', 'username', 'password'],
    constraints: {
      unique: ['username']
    }
  },
  people: {
    required: ['uuid','name'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'person'
  },
  'places-level-0': {
    required: ['uuid','name'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'national_office'
  },
  'places-level-1': {
    required: ['uuid','name'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'district_hospital'
  },
  'places-level-2': {
    required: ['uuid','name'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'health_center'
  },
  'places-level-3': {
    required: ['uuid','name'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'clinic'
  },
  'places-update': {
    required: ['uuid']
  },
  'records': {
    required: ['uuid', 'reported_date'],
    constraints: {
      unique: ['uuid']
    },
    doc_type: 'data_record'
  }
};

var supportedTypeHelp = function() {
  return Object.keys(supportedTypes).join('\n      ');
};

var outputError = function() {
  var RED = '\033[0;31m';
  var NC = '\033[0m';

  var args = Array.prototype.slice.call(arguments);

  args[0] = RED + arguments[0] + NC;

  console.error.apply(null, args);
};

// requires node >= 4 because of the multi-line string.
if (process.version.match(/^v(\d+\.\d+)/)[1] < 4) {
  outputError('Please upgrade your NodeJS to >= 4.');
  process.exit(1);
}

var usageInfo = `
Usage:

  cat file.csv | ./${self} [options] type 

Description:

  Create or update data from a CSV formatted file using the /api/v1 API,
  requires webapp version 2.6.2 or later.  Returns errors and is silent on
  success.

  You must specify the document type as a parameter.  Each type has its own
  contraints and required columns. 

    Currently supported types: 

      ${supportedTypeHelp()}

  The column names in your source data are mapped directly to property names on
  the JSON POST body or can be mapped to different strings with the \`-c\`
  option.  To understand more about the special meanings of property names see
  the project-templates or the medic-api documentation.

  Column names that begin with \`place.\` or \`contact.\` are handled
  specially.  This allows you to specify custom properties on the contact or
  place objects that are associated with the data.  For example if you define a
  column labeled \`place.supervisor\`, then the values in those cells are set
  on the \`supervisor\` property of the place object during user creation.

Options:

  -h  This help information

  -w  Wait between requests (ms).  Default 500

  -d  Dry run, only prints request body.

  -p  Set the \`place\` UUID value on the command line rather than in the input
      data. This can be convenient if all the rows in your data are associated
      with the same place.

  -c  Comma separated list of columns to import.  Also supports a colon
      separator to map the spreadsheet column name to the obj property
      name.  If not specified attempts to import all columns.

      Examples:
        
        Import only the 'uuid' and 'name' columns:

          -c uuid,name 

        Map the 'ID' column name to the 'uuid' property name on the object.

          -c ID:uuid,Village:contact_name

Environment Variables:

  COUCH_URL
  
    Specifies the URL of the project.

    Examples:

      export COUCH_URL=https://admin:secret@project.app.medicmobile.org

Usage Examples:

  Users import dry run:

    cat 'Group 2 Tula.csv' | ./${self} users -d

  Users import dry run with a place (district/branch) identifier:

    cat 'Group 2 Tula.csv' | \\
      ./${self} users -d 8fe84af17cb7ac863a92e884e50440c3

  Users import:

    cat 'Group 2 Tula.csv' | \\
      ./${self} users 8fe84af17cb7ac863a92e884e50440c3

  Use head command to only create first record:

    head -2 'Group 2 Tula.csv' | \\
      ./${self} users 8fe84af17cb7ac863a92e884e50440c3

  Import people:

    cat 'Supervisors.csv' | \\
      ./${self} people 8fe84af17cb7ac863a92e884e50440c3

  Import people and ignore the concession and gender columns:

    cat 'Supervisors.csv' | \\
      ./${self} people -i concession,gender 8fe84af17cb7ac863a92e884e50440c3

`;

var usage = function() {
  console.info(usageInfo);
  process.exit(1);
};

if (process.argv.indexOf('-h') !== -1) {
  usage();
}

if (process.argv.indexOf('-w') !== -1) {
  options.wait = process.argv.splice(process.argv.indexOf('-w'), 2)[1];
}

if (process.argv.indexOf('-d') !== -1) {
  options.dryrun = true;
  process.argv.splice(process.argv.indexOf('-d'), 1);
}

if (process.argv.indexOf('-p') !== -1) {
  options.placeUUID = process.argv.splice(process.argv.indexOf('-p'), 2)[1];
}

if (process.argv.indexOf('-c') !== -1) {
  options.columns = [];
  process.argv.splice(process.argv.indexOf('-c'), 2)[1].split(',')
    .forEach(function(col) {
      var parts = col.split(':');
      options.columns.push({
        source: parts[0],
        target: parts[1]
      });
    });
}

if (process.env.COUCH_URL) {
  reqOpts = url.parse(process.env.COUCH_URL);
} else {
  console.error('Missing COUCH_URL');
  usage();
}

options.recordType = process.argv[2];
if (options.recordType) {
  if (!supportedTypes[options.recordType]) {
    throw new Error(`Record type not supported: ${options.recordType}`);
  }
} else {
  usage();
}

if (reqOpts.protocol === 'https:') {
  http = require('https');
} else if (reqOpts.protocol === 'http:') {
  console.warn('using insecure protocol: http');
} else {
  throw new Error(`Unsupported protocol: ${reqOpts.protocol}`);
}

var recordStat = function(obj) {
  var resCode = obj.statusCode;
  if (resCode) {
    if (typeof stats.responses[resCode] === 'undefined') {
      stats.responses[resCode] = 0;
    }
    stats.responses[resCode]++;
  }
};

var validate = function(obj) {
  var config = supportedTypes[options.recordType];
  config.required.forEach(function(field) {
    if (typeof obj[field] === 'undefined' || obj[field] === null) {
      throw new Error(`Missing required field: ${field}`);
    }
  });
  if (config.constraints && config.constraints.unique) {
    config.constraints.unique.forEach(function(col) {
      if (!indexes[col]) {
        indexes[col] = {};
      }
      if (indexes[col][obj[col]]) {
        outputError(JSON.stringify(obj));
        throw new Error(`Failed unique constraint: ${col} ${obj[col]}`);
      } else {
        indexes[col][obj[col]] = 1;
      }
    });
  }
};

var importUser = function(obj) {
  var user = {
    username: obj.username,
    password: obj.password,
    type: obj.type || 'district-manager',
    language: obj.lang || obj.language || 'en',
    known: typeof obj.known === 'undefined' ? true: Boolean(obj.known),
    external_id: obj.external_id,
    place: obj.place || options.placeUUID,
    contact:  obj.contact || { name: obj.name, phone: obj.phone }
  };
  // support `contact` or `place` properties
  Object.keys(obj).forEach(function(key) {
    var prop;
    if (key.match(/^place\.\w+/)) {
      prop = key.split('.')[1];
      if (prop === 'uuid') {
        user.place._id = obj[key];
      } else {
        user.place[prop] = obj[key];
      }
    } else if (key.match(/^contact\.\w+/)) {
      prop = key.split('.')[1];
      if (prop === 'uuid') {
        user.contact._id = obj[key];
      } else {
        user.contact[prop] = obj[key];
      }
    }
  });
  setTimeout(function() {
    reqOpts.method = 'POST';
    reqOpts.path = '/api/v1/users';
    reqOpts.headers = {
      'content-type': 'application/json'
    };
    if (options.dryrun) {
      console.log(reqOpts.method, reqOpts.path);
      return console.log(JSON.stringify(user, null, 2));
    }
    stats.requests++;
    var req = http.request(reqOpts, function(res) {
      recordStat(res);
      if (res.statusCode != 200 && res.statusCode != 201) {
        res.setEncoding('utf8');
        res.on('data', function (chunk) {
          outputError('  ' + chunk);
        });
        return outputError(
          'failed to import user %s status %s %s:',
          obj.username,
          res.statusCode,
          res.statusMessage
        );
      }
    });
    req.on('error', function(e) {
      if (e) throw e;
    });
    req.write(JSON.stringify(user));
    req.end();
  }, stats.rows * options.wait);
};

var importPerson = function(obj) {
  var config = supportedTypes[options.recordType],
      person = {
        imported_date: new Date().toISOString()
      };
  Object.keys(obj).forEach(function(k) {
    if (k === 'uuid') {
      person._id = obj[k];
    } else {
      person[k] = obj[k];
    }
  });
  // support values on `place.foo` columns
  Object.keys(obj).forEach(function(key) {
    var prop;
    if (key.match(/^place\.\w+/)) {
      prop = key.split('.')[1];
      if (prop === 'uuid') {
        person.place._id = obj[key];
      } else {
        person.place[prop] = obj[key];
      }
    }
  });
  if (!person.place || Object.keys(person.place).length === 0) {
    person.place = obj.place || options.placeUUID;
  }
  person.type = config.doc_type;
  setTimeout(function() {
    reqOpts.method = 'POST';
    reqOpts.path = '/api/v1/people';
    reqOpts.headers = {
      'content-type': 'application/json'
    };
    if (options.dryrun) {
      console.log(reqOpts.method, reqOpts.path);
      return console.log(JSON.stringify(person, null, 2));
    }
    stats.requests++;
    var req = http.request(reqOpts, function(res) {
      recordStat(res);
      if (res.statusCode != 200 && res.statusCode != 201) {
        res.setEncoding('utf8');
        res.on('data', function (chunk) {
          outputError('  ' + chunk);
        });
        return outputError(
          'failed to import person %s status %s %s:',
          obj.name,
          res.statusCode,
          res.statusMessage
        );
      }
    });
    req.on('error', function(e) {
      if (e) throw e;
    });
    req.write(JSON.stringify(person));
    req.end();
  }, stats.rows * options.wait);
};

var importPlace = function(obj) {
  var config = supportedTypes[options.recordType],
      place = {
        imported_date: new Date().toISOString(),
        parent: obj.parent || options.placeUUID
      };
  Object.keys(obj).forEach(function(key) {
    if (key === 'uuid') {
      place._id = obj[key];
    } else if (key.match(/^contact\.\w+/) || key.match(/^parent\.\w+/)) {
      var props = key.split('.');
      if (!place[props[0]]) {
        place[props[0]] = {};
      }
      if (props[1] === 'uuid') {
        place[props[0]]._id = obj[key];
      } else {
        place[props[0]][props[1]] = obj[key];
      }
    } else {
      place[key] = obj[key];
    }
  });
  place.type = config.doc_type;
  setTimeout(function() {
    reqOpts.method = 'POST';
    reqOpts.path = '/api/v1/places';
    reqOpts.headers = {
      'content-type': 'application/json'
    };
    if (options.dryrun) {
      console.log(reqOpts.method, reqOpts.path);
      return console.log(JSON.stringify(place, null, 2));
    }
    stats.requests++;
    var req = http.request(reqOpts, function(res) {
      recordStat(res);
      if (res.statusCode != 200 && res.statusCode != 201) {
        res.setEncoding('utf8');
        res.on('data', function (chunk) {
          outputError('  ' + chunk);
        });
        return outputError(
          'failed to import place %s status %s %s:',
          obj.name,
          res.statusCode,
          res.statusMessage
        );
      }
    });
    req.on('error', function(e) {
      if (e) throw e;
    });
    req.write(JSON.stringify(place));
    req.end();
  }, stats.rows * options.wait);
};

var updatePlace = function(obj) {
  var config = supportedTypes[options.recordType],
      reqBody = {};
  Object.keys(obj).forEach(function(key) {
    if (key === 'uuid') {
      return;
    } else {
      reqBody[key] = obj[key];
    }
  });
  setTimeout(function() {
    reqOpts.method = 'POST';
    reqOpts.path = '/api/v1/places/' + obj.uuid;
    reqOpts.headers = {
      'content-type': 'application/json'
    };
    if (options.dryrun) {
      console.log(reqOpts.method, reqOpts.path);
      return console.log(JSON.stringify(reqBody, null, 2));
    }
    stats.requests++;
    var req = http.request(reqOpts, function(res) {
      recordStat(res);
      if (res.statusCode != 200 && res.statusCode != 201) {
        res.setEncoding('utf8');
        res.on('data', function (chunk) {
          outputError('  ' + chunk);
        });
        return outputError(
          'failed to update place %s status %s %s:',
          obj.uuid,
          res.statusCode,
          res.statusMessage
        );
      }
    });
    req.on('error', function(e) {
      if (e) throw e;
    });
    req.write(JSON.stringify(reqBody));
    req.end();
  }, stats.rows * options.wait);
};

var importRecord = function(obj, type) {
  var config = supportedTypes[options.recordType],
      record = {
        imported_date: new Date().toISOString(),
      };
  Object.keys(obj).forEach(function(key) {
    if (key === 'uuid') {
      record._id = obj[key];
    } else if (key === 'reported_date') {
      record.reported_date = Number(obj[key]);
    } else {
      record[key] = obj[key];
    }
  });
  record = flat.unflatten(record); // support dot notation in keys
  record.type = config.doc_type;
  record.content_type = type || 'xml';
  setTimeout(function() {
    reqOpts.method = 'POST';
    reqOpts.path = '/medic/';
    reqOpts.headers = {
      'content-type': 'application/json'
    };
    if (options.dryrun) {
      console.log(reqOpts.method, reqOpts.path);
      return console.log(JSON.stringify(record, null, 2));
    }
    stats.requests++;
    var req = http.request(reqOpts, function(res) {
      recordStat(res);
      if (res.statusCode != 200 && res.statusCode != 201) {
        res.setEncoding('utf8');
        res.on('data', function (chunk) {
          outputError('  ' + chunk);
        });
        return outputError(
          'failed to import record %s status %s %s:',
          obj.name,
          res.statusCode,
          res.statusMessage
        );
      }
    });
    req.on('error', function(e) {
      if (e) throw e;
    });
    req.write(JSON.stringify(record));
    req.end();
  }, stats.rows * options.wait);
};

if (options.recordType === 'users') {
  importer = importUser;
} else if (options.recordType === 'people') {
  importer = importPerson;
} else if (options.recordType.match(/places-level-\d/)) {
  importer = importPlace;
} else if (options.recordType === 'places-update') {
  importer = updatePlace;
} else if (options.recordType === 'records') {
  importer = importRecord;
}

var onDataHandler = function(obj) {
  stats.rows++;
  if (options.columns) {
    var newObj = {};
    options.columns.forEach(function(col) {
      // if target field was passed in then use it
      newObj[col.target || col.source] = obj[col.source];
    });
    obj = newObj;
  }
  validate(obj);
  importer(obj);
};

csv
  .fromStream(process.stdin, {headers:true})
  .on('data', onDataHandler)
  .on('end', function() {
    console.info(
      'processing %s rows, will take about %s minutes.',
      stats.rows,
      (((stats.rows * options.wait)/1000) / 60).toFixed(1)
    );
  });

var responsesContainErrors = function(responses) {
  return Object.keys(responses).some(function(key) {
    return !key.startsWith('2');
  });
};

process.on('exit', function() {
  console.info('');
  console.info(stats);
  if (stats.responses && responsesContainErrors(stats.responses)) {
    outputError('IMPORTANT: Some errors occured during this import, please read the output above!');
  }
});

process.on('SIGINT', function() {
  process.exit();
});