From c23f6d4f68b3051575cde593806b24f0bb134cf5 Mon Sep 17 00:00:00 2001 From: leko Date: Mon, 7 Aug 2023 17:03:16 +0200 Subject: [PATCH] feat: add custom xsd --- app/api/client.ts | 19 +++ app/api/types.ts | 11 ++ app/components/CustomConfiguration.tsx | 172 ++++++++++++++++++++----- app/components/FileUpload.tsx | 12 +- app/components/ValidationResult.tsx | 4 +- app/pages/jobs/[id]/custom.tsx | 4 +- app/styles/theme.ts | 17 ++- builtin/xsd.js | 7 +- cmd/file.go | 2 +- cmd/server.go | 32 +++++ cmd/session.go | 72 ++++++++--- cmd/xsd.go | 120 +++++++++++++++++ 12 files changed, 410 insertions(+), 62 deletions(-) create mode 100644 cmd/xsd.go diff --git a/app/api/client.ts b/app/api/client.ts index b833969..fd74c3e 100644 --- a/app/api/client.ts +++ b/app/api/client.ts @@ -104,6 +104,25 @@ class ApiClient { }) } + async xsdUpload (id: string, file: File, onProgress?: (p: any) => void): Promise { + const { checksum } = await calculateChecksum(file) + const data = new FormData() + + data.append('file', file) + + return await axios({ + method: 'post', + url: this.withUrl(`sessions/${id}/upload/xsd`), + data, + params: { checksum }, + onUploadProgress: (p) => { + if (onProgress !== undefined) { + onProgress(p) + } + } + }) + } + async validate (id: string): Promise { return await axios({ method: 'get', diff --git a/app/api/types.ts b/app/api/types.ts index 175bad5..59ce669 100644 --- a/app/api/types.ts +++ b/app/api/types.ts @@ -23,6 +23,16 @@ export interface ScriptConfigOption { options?: any[] } +export interface XSDUploadFile { + id: string + name: string +} + +export interface XSDUpload { + name: string + files?: XSDUploadFile[] +} + export interface Session { id: string name: string @@ -30,6 +40,7 @@ export interface Session { created: number stopped: number files: string[] + xsdFiles?: XSDUpload[] status: string results: any[] profile?: Profile diff --git a/app/components/CustomConfiguration.tsx b/app/components/CustomConfiguration.tsx index 40d1f26..addd007 100644 --- a/app/components/CustomConfiguration.tsx +++ b/app/components/CustomConfiguration.tsx @@ -17,18 +17,22 @@ import { Stack, Typography, OutlinedInput, - DialogActions + DialogActions, + Card } from '@mui/material' import { grey } from '@mui/material/colors' import TuneOutlineIcon from '@mui/icons-material/TuneOutlined' import ChevronRightIcon from '@mui/icons-material/ChevronRight' import React from 'react' -import type { Profile, Script } from '../api/types' +import FileUpload, { type FileList } from './FileUpload' +import type { Profile, Script, Session, XSDUploadFile } from '../api/types' import scriptData from '../public/scripts.json' +import useApiClient from '../hooks/useApiClient' const scriptOptions = scriptData.filter(v => v.name !== 'xsd') export interface CustomConfigurationProps { + session: Session | null onNext: (profile: Profile) => void disabled?: boolean } @@ -129,10 +133,18 @@ const ScriptRow = ({ ) } -const CustomConfiguration = (props: CustomConfigurationProps): JSX.Element => { +const CustomConfiguration = ({ + session, + disabled, + onNext +}: CustomConfigurationProps): JSX.Element => { const [schema, setSchema] = React.useState('netex@1.2-nc') + const [schemaEntry, setSchemaEntry] = React.useState('') const [scripts, setScripts] = React.useState(scriptOptions.map(v => v.name)) const [scriptOpts, setScriptOpts] = React.useState>>({}) + const [fileList, setFileList] = React.useState>({}) + const [schemaFiles, setSchemaFiles] = React.useState([]) + const apiClient = useApiClient() const handleSelectSchema = (event: SelectChangeEvent): void => { if (event.target?.name === '') { @@ -142,6 +154,14 @@ const CustomConfiguration = (props: CustomConfigurationProps): JSX.Element => { setSchema(event.target.value) } + const handleSelectSchemaEntry = (event: SelectChangeEvent): void => { + if (event.target?.name === '') { + return + } + + setSchemaEntry(event.target.value) + } + const handleScriptToggle = (v: Script): void => { const i = scripts.indexOf(v.name) const s = [...scripts] @@ -163,16 +183,18 @@ const CustomConfiguration = (props: CustomConfigurationProps): JSX.Element => { } const handleNextClick = (): void => { - if (props.disabled === true) { + if (disabled === true) { return } const xsdScript = { ...scriptData.find(v => v.name === 'xsd'), - config: { schema } + config: schema === 'custom' + ? { schema: 'custom', entry: schemaEntry } + : { schema } } - props.onNext({ + onNext({ name: 'custom', description: 'Custom configuration', longDescription: '', @@ -189,36 +211,118 @@ const CustomConfiguration = (props: CustomConfigurationProps): JSX.Element => { }) } + React.useEffect(() => { + if (session == null) { + return + } + if (session.xsdFiles != null) { + const { xsdFiles } = session + + setFileList(xsdFiles.reduce((o: Record, { name }) => { + o[name] = { + name, + status: 'uploaded', + progress: 100 + } + return o + }, {}) ?? {}) + setSchemaFiles(xsdFiles.reduce((o: XSDUploadFile[], v) => { + if (v.files != null) { + o.push(...v.files) + } + return o + }, []) ?? []) + } + if (session.profile != null) { + const { profile } = session + const xsdScript = profile.scripts?.find(v => v.name === 'xsd') + + setSchema(xsdScript?.config?.schema) + setSchemaEntry(xsdScript?.config?.entry) + } + }, [session, setFileList]) + return ( - - - Profile - 1. Begin by selecting which profile to use for validation - -
    -
  • NeTEx - The full NeTEx schema (more info)
    • -
  • NeTEx Fast - NeTEx schema without constraint (more info)
    • -
  • EPIP - NeTEx European Passenger Information Profile (more info)
    • -
  • EPIP Light - NeTEx European Passenger Information Profile
    • -
- - - Schema - - + + + + Profile + 1. Begin by selecting which profile to use for validation + +
    +
  • NeTEx - The full NeTEx schema (more info)
    • +
  • NeTEx Fast - NeTEx schema without constraint (more info)
    • +
  • EPIP - NeTEx European Passenger Information Profile (more info)
    • +
  • EPIP Light - NeTEx European Passenger Information Profile
    • +
+ + + Profile + + + + {schema === 'custom' && ( + + + + Upload custom profile + Select which file to use as profile 'Select file(s)' + + + { + await apiClient.xsdUpload(session?.id ?? '', file, cb) + .then(res => { + setSchemaFiles(res.data?.xsdFiles?.reduce((o: XSDUploadFile[], v: any) => { + if (v.files != null) { + o.push(...v.files) + } + return o + }, []) ?? []) + }) + }} + onChange={(fileList: FileList) => { + setFileList({ ...fileList }) + }} + onError={({ errorMessage }: { errorMessage: string }) => { + return `Error caught uploading file, message: ${errorMessage}` + }} + /> + + + Main entry point + + + + + )} - Rules + Rules 2. In addition to the schema validation, we have also included a few optional rules that validate the consistency of the documents { diff --git a/app/pages/jobs/[id]/custom.tsx b/app/pages/jobs/[id]/custom.tsx index 8180c68..5ff90ab 100644 --- a/app/pages/jobs/[id]/custom.tsx +++ b/app/pages/jobs/[id]/custom.tsx @@ -84,8 +84,8 @@ const Custom: NextPage = () => { - Custom configuration - + Custom configuration + diff --git a/app/styles/theme.ts b/app/styles/theme.ts index c052c4d..419e13d 100644 --- a/app/styles/theme.ts +++ b/app/styles/theme.ts @@ -211,7 +211,19 @@ theme.components.MuiPaper = { styleOverrides: { root: { borderRadius: 0, - boxShadow: 'none' + boxShadow: 'none', + background: 'white' + } + } +} + +theme.components.MuiCard = { + styleOverrides: { + root: { + borderStyle: 'solid', + borderRadius: 8, + borderWidth: 1, + borderColor: 'rgba(0, 0, 0, 0.12)' } } } @@ -315,6 +327,7 @@ theme.components.MuiListItemText = { theme.components.MuiOutlinedInput = { styleOverrides: { root: { + background: 'white' } } } @@ -334,7 +347,7 @@ theme.components.MuiLink = { styleOverrides: { root: { color: 'inherit', - fontWeight: 500, + fontWeight: 500 } } } diff --git a/builtin/xsd.js b/builtin/xsd.js index fb87ef9..879c890 100644 --- a/builtin/xsd.js +++ b/builtin/xsd.js @@ -13,7 +13,10 @@ const types = require("types"); * @return {errors.ScriptError[]?} */ function main(ctx) { - ctx.log.debug(`validation using schema "${ctx.config.schema}"`); + const {config} = ctx; + const schema = config.schema === "custom" ? config.entry : config.schema; - return ctx.xsd.validate(ctx.config.schema).get() + ctx.log.debug(`validation using schema "${config.schema}"`); + + return ctx.xsd.validate(config.schema).get(); } diff --git a/cmd/file.go b/cmd/file.go index b338c9b..d247ce2 100644 --- a/cmd/file.go +++ b/cmd/file.go @@ -77,7 +77,7 @@ func (c *FileContext) Open(name string, r io.Reader) error { } func (c *FileContext) Add(name string, r io.Reader) (*FileInfo, error) { - tempFile, err := os.CreateTemp("", "") + tempFile, err := os.CreateTemp("", "greenlight") if err != nil { return nil, err } diff --git a/cmd/server.go b/cmd/server.go index 16e7ccf..7bbaa21 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -184,6 +184,38 @@ func startServer(cmd *cobra.Command, args []string) { return c.JSON(code, session) }) + e.POST("/api/sessions/:sid/upload/xsd", func(c echo.Context) error { + session := sessions.Get(c.Param("sid")) + if session == nil { + return fmt.Errorf("session not found") + } else if session.Status != "created" { + return fmt.Errorf("session already processed") + } + + form, err := c.MultipartForm() + if err != nil { + return err + } + + for _, files := range form.File { + for _, file := range files { + f, err := file.Open() + if err != nil { + return err + } + + xsd, err := NewXSDUpload(file.Filename, f) + if err != nil { + return err + } + + session.XSDFiles = append(session.XSDFiles, xsd) + } + } + + return c.JSON(http.StatusOK, session) + }) + e.GET("/api/sessions/:sid/validate", func(c echo.Context) error { session := sessions.Get(c.Param("sid")) if session == nil { diff --git a/cmd/session.go b/cmd/session.go index 025fe29..67c29ed 100644 --- a/cmd/session.go +++ b/cmd/session.go @@ -3,12 +3,13 @@ package main import ( "context" "encoding/json" + "path/filepath" "sync" "time" "github.com/concreteit/greenlight" - "github.com/dustinkirkland/golang-petname" - "github.com/matoous/go-nanoid" + petname "github.com/dustinkirkland/golang-petname" + gonanoid "github.com/matoous/go-nanoid" ) func init() { @@ -49,13 +50,14 @@ func (s *SessionMap) New() (*Session, error) { } type Session struct { - ID string `json:"id"` - Name string `json:"name"` - Created time.Time `json:"created"` - Stopped time.Time `json:"stopped"` - Status string `json:"status"` - Profile *Profile `json:"profile"` - Results []*greenlight.ValidationResult + ID string `json:"id"` + Name string `json:"name"` + Created time.Time `json:"created"` + Stopped time.Time `json:"stopped"` + Status string `json:"status"` + Profile *Profile `json:"profile"` + XSDFiles []*XSDUpload `json:"xsdFiles"` + Results []*greenlight.ValidationResult fileContext *FileContext `json:"-"` } @@ -67,6 +69,7 @@ func (s *Session) NewValidation() (*greenlight.Validation, error) { } s.Results = []*greenlight.ValidationResult{} + xsdConfig := s.xsdConfig() for _, file := range s.fileContext.Find("xml") { if err := validation.AddFile(file.Name, file.FilePath); err != nil { @@ -76,11 +79,14 @@ func (s *Session) NewValidation() (*greenlight.Validation, error) { file.File.Seek(0, 0) rvs := []*greenlight.RuleValidation{} - for _, script := range s.Profile.Scripts { for name, s := range scripts { if name == script.Name { - validation.AddScript(s, script.Config) + if name == "xsd" { + validation.AddScript(s, xsdConfig) + } else { + validation.AddScript(s, script.Config) + } rvs = append(rvs, &greenlight.RuleValidation{ Name: script.Name, @@ -102,15 +108,45 @@ func (s *Session) NewValidation() (*greenlight.Validation, error) { return validation, nil } +func (s *Session) xsdConfig() map[string]any { + config := map[string]any{} + for _, script := range s.Profile.Scripts { + if script.Name == "xsd" { + config = script.Config + if schema, ok := script.Config["schema"].(string); ok && schema == "custom" && s.XSDFiles != nil { + if entry, ok := script.Config["entry"].(string); ok { + for _, xsdFile := range s.XSDFiles { + if xsdFile.Files == nil { + continue + } + for _, file := range xsdFile.Files { + if file.ID != entry { + continue + } + + config = map[string]any{ + "schema": filepath.Join(xsdFile.dirPath, file.Name), + } + } + } + } + } + return config + } + } + return config +} + func (s Session) MarshalJSON() ([]byte, error) { obj := map[string]interface{}{ - "id": s.ID, - "name": s.Name, - "created": s.Created.Unix(), - "files": s.fileContext.Find("xml"), - "status": s.Status, - "results": s.Results, - "profile": s.Profile, + "id": s.ID, + "name": s.Name, + "created": s.Created.Unix(), + "files": s.fileContext.Find("xml"), + "status": s.Status, + "results": s.Results, + "profile": s.Profile, + "xsdFiles": s.XSDFiles, } if s.Status != "created" && s.Status != "running" { diff --git a/cmd/xsd.go b/cmd/xsd.go new file mode 100644 index 0000000..10718b9 --- /dev/null +++ b/cmd/xsd.go @@ -0,0 +1,120 @@ +package main + +import ( + "archive/zip" + "io" + "os" + "path/filepath" + "strings" + + "github.com/h2non/filetype" + gonanoid "github.com/matoous/go-nanoid/v2" +) + +type XSDUploadFile struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type XSDUpload struct { + dirPath string + + Name string `json:"name"` + Files []XSDUploadFile `json:"files,omitempty"` +} + +func NewXSDUpload(name string, r io.Reader) (*XSDUpload, error) { + dirPath, err := os.MkdirTemp(os.TempDir(), "greenlight") + if err != nil { + return nil, err + } + + filePath := filepath.Join(dirPath, name) + f, err := os.Create(filePath) + if err != nil { + return nil, err + } + defer f.Close() + + if _, err := io.Copy(f, r); err != nil { + return nil, err + } + + if _, err := f.Seek(0, 0); err != nil { + return nil, err + } + + fileType, err := filetype.MatchReader(f) + if err != nil { + return nil, err + } + + files := []XSDUploadFile{} + switch fileType.Extension { + case "xml": + id, err := gonanoid.New() + if err != nil { + return nil, err + } + + files = append(files, XSDUploadFile{ + ID: id, + Name: name, + }) + case "zip": + if files, err = unzip(filePath, dirPath); err != nil { + return nil, err + } + } + + return &XSDUpload{ + dirPath: dirPath, + Name: name, + Files: files, + }, nil +} + +func unzip(src, dst string) ([]XSDUploadFile, error) { + r, err := zip.OpenReader(src) + if err != nil { + return nil, err + } + defer r.Close() + + files := []XSDUploadFile{} + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return nil, err + } + + filePath := filepath.Join(dst, f.Name) + if f.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, f.Mode()); err != nil { + return nil, err + } + continue + } + + df, err := os.Create(filePath) + if err != nil { + continue + } + defer df.Close() + + if _, err := io.Copy(df, rc); err != nil { + return nil, err + } + + id, err := gonanoid.New() + if err != nil { + return nil, err + } + files = append(files, XSDUploadFile{ + ID: id, + Name: strings.TrimPrefix(filePath, dst+"/"), + }) + } + + return files, nil +}