diff --git a/README.md b/README.md
index 7be154d..b8137b8 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,57 @@
-# sq
+
sq
+Convert and query JSON, JSONL, CSV, and SQLite with ease!
-Like `jq` , but for SQLite.
+`sq` is a simple yet powerful command line tool for query and converting from and to SQLite, CSV, JSON, and JSONL files, heavily inspired by [ `jq` ](https://jqlang.github.io/jq/).
-## TODO
+## Features
-* [ ] Fix JSON input
-* [ ] Finish README
-* [ ] Write some docs
-* [ ] Make a build for all major platforms
+* Convert between SQLite, CSV, JSON, and JSONL files.
+* Query CSV, JSON, and JSONL using real SQLite queries.
+* Allows for rapid scripting and conversion of data.
+* Pipe in data or read from files.
+
+## Installation
+
+```bash
+go install github.com/chand1012/sq@latest
+```
+
+## Examples
+
+Query some orders from a CSV file. Column names for CSV files are converted to lower case and spaces are replaced with underscores.
+
+```bash
+$ sq -r orders.csv 'select country from sq where seller_amount > 20 not null;'
+United States of America
+United States of America
+United States of America
+United States of America
+United States of America
+United States of America
+United Kingdom
+Canada
+United States of America
+United States of America
+United States of America
+United States of America
+United States of America
+Canada
+United States of America
+Canada
+Canada
+Australia
+United States of America
+...
+```
+
+Download and query some JSONL datasets.
+
+```bash
+$ curl https://raw.githubusercontent.com/TimeSurgeLabs/llm-finetuning/4e934ce602f34f62f4d803c40cd1e7825d216192/data/fingpt-sentiment-1k.jsonl | sq 'select * from sq where output = "positive";' -f jsonl > positive.jsonl
+```
+
+You can even use it with `jq` !
+
+```bash
+$ curl https://api.gogopool.com/stakers | jq '.stakers' | sq -t stakers 'SELECT stakerAddr,avaxValidating FROM stakers WHERE avaxValidating > 0;' -f json > stakers.json
+```
diff --git a/cmd/root.go b/cmd/root.go
index 667e3ff..7f67686 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -80,6 +80,7 @@ func run(cmd *cobra.Command, args []string) {
log.Debugf("Input file path: %s", inputFilePath)
d, _, err = db.LoadFile(inputFilePath)
if err != nil {
+ log.Debug("Input file is not a SQLite database, reading as a regular file")
file, err := os.ReadFile(inputFilePath)
if err != nil {
logger.HandlePanic(log, err, verbose)
diff --git a/pkg/db/from_json.go b/pkg/db/from_json.go
index 4de6277..5e4fdaa 100644
--- a/pkg/db/from_json.go
+++ b/pkg/db/from_json.go
@@ -44,8 +44,8 @@ func FromJSON(b []byte, tableName string) (*sql.DB, string, error) {
columns, types := utils.BreakOutMap(typeMap)
- // preprocess the column names
- columns = processColumnNames(columns)
+ // // preprocess the column names
+ // columns = processColumnNames(columns)
createQuery := genCreateTableQuery(tableName, columns, types)
diff --git a/pkg/db/from_jsonl.go b/pkg/db/from_jsonl.go
index 8d87a51..377ed0c 100644
--- a/pkg/db/from_jsonl.go
+++ b/pkg/db/from_jsonl.go
@@ -48,8 +48,8 @@ func FromJSONL(b []byte, tableName string) (*sql.DB, string, error) {
columns, types := utils.BreakOutMap(typeMap)
- // preprocess the column names
- columns = processColumnNames(columns)
+ // // preprocess the column names
+ // columns = processColumnNames(columns)
createQuery := genCreateTableQuery(tableName, columns, types)
diff --git a/pkg/db/load_sql.go b/pkg/db/load_sql.go
index 10bbf71..923b964 100644
--- a/pkg/db/load_sql.go
+++ b/pkg/db/load_sql.go
@@ -2,6 +2,7 @@ package db
import (
"database/sql"
+ "errors"
"os"
_ "github.com/glebarez/go-sqlite"
@@ -22,6 +23,19 @@ func createTempDB() (*sql.DB, string, error) {
return db, tmpFile.Name(), nil
}
+// check if the file is a valid sqlite db
+func IsValidDB(fileName string) bool {
+ db, err := sql.Open("sqlite", fileName)
+ if err != nil {
+ return false
+ }
+ defer db.Close()
+
+ // try to query the db
+ _, err = db.Query("SELECT * FROM sqlite_master")
+ return err == nil
+}
+
// load a sql file from bytes
func LoadStdin(bytes []byte) (*sql.DB, string, error) {
tmpFile, err := os.CreateTemp(os.TempDir(), "sq-*.sql")
@@ -44,6 +58,9 @@ func LoadStdin(bytes []byte) (*sql.DB, string, error) {
}
func LoadFile(fileName string) (*sql.DB, string, error) {
+ if !IsValidDB(fileName) {
+ return nil, "", errors.New("file is not a valid SQLite database")
+ }
db, err := sql.Open("sqlite", fileName)
if err != nil {
return nil, fileName, err