[errors] relax error handling in ingestion

logv · Jun 16, 2018 · 800b607 · 800b607
1 parent 12aca8a
commit 800b607
Showing 1 changed file with 8 additions and 5 deletions.
diff --git a/src/cmd/cmd_ingest.go b/src/cmd/cmd_ingest.go
@@ -70,8 +70,8 @@ func ingestDictionary(r *sybil.Record, recordmap *Dictionary, prefix string) err
 			sybil.Debug(fmt.Sprintf("TYPE %T IS UNKNOWN FOR FIELD", iv), keyName)
 		}
 		if err != nil {
-			// TODO: consider if there should be a relaxed mode
-			return errors.Wrap(err, fmt.Sprintf("issue loading %v", keyName))
+			// TODO: collect error counters?
+			sybil.Debug("INGEST RECORD ISSUE:", errors.Wrap(err, fmt.Sprintf("issue with record %v", i)))
 		}
 	}
 	return nil
@@ -128,7 +128,8 @@ func importCsvRecords() error {
 		}
 
 		if err := t.ChunkAndSave(); err != nil {
-			return err
+			// TODO: collect error counters?
+			sybil.Debug("INGEST RECORD ISSUE:", err)
 		}
 	}
 	return nil
@@ -209,10 +210,12 @@ func importJSONRecords(jsonPath string) error {
 				err = ingestDictionary(r, &dict, "")
 			}
 			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("issue with record %v", i))
+				// TODO: collect error counters?
+				sybil.Debug("INGEST RECORD ISSUE:", errors.Wrap(err, fmt.Sprintf("issue with record %v", i)))
 			}
 			if err := t.ChunkAndSave(); err != nil {
-				return err
+				// TODO: collect error counters?
+				sybil.Debug("INGEST RECORD ISSUE:", err)
 			}
 		}