Skip to content

Commit

Permalink
Clean.
Browse files Browse the repository at this point in the history
  • Loading branch information
Tang8330 committed Nov 21, 2024
1 parent 6c66925 commit 89337c8
Showing 1 changed file with 0 additions and 56 deletions.
56 changes: 0 additions & 56 deletions lib/parquetutil/generate_schema.go
Original file line number Diff line number Diff line change
@@ -1,65 +1,9 @@
package parquetutil

import (
"encoding/json"
"strings"

"github.com/artie-labs/transfer/lib/typing"
"github.com/artie-labs/transfer/lib/typing/columns"
)

type ParquetColumn struct {
originalName string
cleanedName string
column columns.Column
}

func (pc ParquetColumn) OriginalName() string {
return pc.originalName
}

func (pc ParquetColumn) CleanedName() string {
return pc.cleanedName
}

func (pc ParquetColumn) Column() columns.Column {
return pc.column
}

func NewParquetColumn(colName string, column columns.Column) ParquetColumn {
return ParquetColumn{
originalName: colName,
cleanedName: strings.TrimPrefix(colName, "__"),
column: column,
}
}

func GenerateJSONSchema(columns []ParquetColumn) (string, error) {
var fields []typing.Field
for _, column := range columns {
// We don't need to escape the column name here.
field, err := column.column.KindDetails.ParquetAnnotation(column.cleanedName)
if err != nil {
return "", err
}

fields = append(fields, *field)
}

schemaBytes, err := json.Marshal(
typing.Field{
Tag: typing.FieldTag{Name: "parquet-go-root", RepetitionType: typing.ToPtr("REQUIRED")}.String(),
Fields: fields,
},
)

if err != nil {
return "", err
}

return string(schemaBytes), nil
}

func GenerateCSVSchema(columns []columns.Column) ([]string, error) {
var fields []string
for _, column := range columns {
Expand Down

0 comments on commit 89337c8

Please sign in to comment.