Instead of async mkdirp in stream, mkdirp happens at the beginning,

replace all the forbidden character in potential file names with underscore
SEL-Columbia · Aug 15, 2014 · 0da836b · 0da836b
1 parent 80d329c
commit 0da836b
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 30 deletions.
diff --git a/lib/cli.js b/lib/cli.js
@@ -1,4 +1,6 @@
 var fs = require('fs'),
+    mkdirp = require('mkdirp'),
+    path = require('path'),
     program = require('commander'),
     formhub_dl = require('./pull_fh'),
     splitter = require('./csv_split');
@@ -25,11 +27,28 @@ var action = function() {
         if (fh_dataset) {
             var input_stream = formhub_dl(fh_dataset);
         }
+        return ensure_dir(program.output, function() {
+            var split = new splitter(input_stream,
+                                     program.output,
+                                     program.groupby);
+            split.run();
+        });
+};
 
-        var split = new splitter(input_stream,
-                                 program.output,
-                                 program.groupby);
-        return split.run();
-    };
+var ensure_dir = function(file_path, cb) {
+    var dirname = path.resolve(file_path);
+    fs.exists(dirname, function(exist) {
+        if (exist) { cb(); }
+        else {
+            mkdirp(dirname, function(err) {
+                if(err) { 
+                    console.log(err); 
+                    process.exit(1);
+                }
+                cb();
+            });
+        }
+    });
+};
 
 action();
diff --git a/lib/csv_split.js b/lib/csv_split.js
@@ -1,7 +1,6 @@
 var fs = require("fs"),
     through = require("through"),
     csv = require("csv-streamify"),
-    mkdirp = require("mkdirp"),
     path = require("path");
 
 var Splitter = function(input_stream, output_dir, group_by) {
@@ -25,7 +24,7 @@ Splitter.prototype.write_csv_line = function(arr) {
 Splitter.prototype.split = function() {
     var self = this;
     return through(function(data) {
-        var name = data[self.group_sequence];
+        var name = clean_name(data[self.group_sequence]);
         var write_obj = {"name" : name,
                          "data" : data
                         };
@@ -37,16 +36,14 @@ Splitter.prototype.drain = function(data) {
     var self = this;
     var name = data.name;
     var outfile = path.resolve(this.output_dir, name + ".csv");
-    ensure_dir(outfile, function() {
-        if (self.write_streams[name]) {
-            self.write_streams[name].write(self.write_csv_line(data.data));
-        } else {
-            var ws = fs.createWriteStream(outfile);
-            self.write_streams[name] = ws;
-            ws.write(self.write_csv_line(self.header));
-            ws.write(self.write_csv_line(data.data));
-        }
-    });
+    if (this.write_streams[name]) {
+        this.write_streams[name].write(this.write_csv_line(data.data));
+    } else {
+        var ws = fs.createWriteStream(outfile);
+        this.write_streams[name] = ws;
+        ws.write(this.write_csv_line(this.header));
+        ws.write(this.write_csv_line(data.data));
+    }
 };
 
 Splitter.prototype.parse = function(group_name) {
@@ -71,18 +68,15 @@ Splitter.prototype.run = function() {
         });
 };
 
-var ensure_dir = function(file_path, cb) {
-    var dirname = path.dirname(file_path)
-    fs.exists(dirname, function(exist) {
-        if (exist) {
-            cb();
-        }
-        else {
-            mkdirp(dirname, function(err) {
-                if(err) { console.log(err); }
-                cb();
-            });
-        }
-    });
+var clean_name = function(file_name) {
+    // file_name cannot have '/', '\' in it
+    // convert them into underscore
+    var forbidden_char = /(\/|\\)/g;
+    var replacement = '_';
+    if (file_name.match(forbidden_char)){
+        return file_name.replace(forbidden_char, replacement);
+    }
+    return file_name;
 };
+
 module.exports = Splitter;