diff --git a/config.example b/config.example index aa18b83..a12e0a6 100644 --- a/config.example +++ b/config.example @@ -101,6 +101,9 @@ map-types zero.String string zero.Time string +map-packages + github.com/errors errors + # Map some times to an OpenAPI format. # # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#dataTypeFormat diff --git a/docparse/docparse.go b/docparse/docparse.go index ee8957e..eb337e0 100644 --- a/docparse/docparse.go +++ b/docparse/docparse.go @@ -30,9 +30,10 @@ type Program struct { // Config for the program. type Config struct { // Kommentaar control. - Packages []string - Output func(io.Writer, *Program) error - Debug bool + Packages []string + Output func(io.Writer, *Program) error + Paths, Methods []string + Debug bool // General information. Title string @@ -50,6 +51,7 @@ type Config struct { Basepath string StructTag string MapTypes map[string]string + MapPackages map[string]string MapFormats map[string]string } @@ -75,6 +77,7 @@ func NewProgram(dbg bool) *Program { DefaultResponseCt: "application/json", MapTypes: make(map[string]string), MapFormats: make(map[string]string), + MapPackages: make(map[string]string), // Override from commandline. Debug: dbg, @@ -82,6 +85,29 @@ func NewProgram(dbg bool) *Program { } } +func (p Program) ShouldExport(e Endpoint) bool { + okPath := len(p.Config.Paths) == 0 + okMethod := len(p.Config.Methods) == 0 + if okPath && okMethod { + return true + } + + for _, path := range p.Config.Paths { + if path == e.Path { + okPath = true + break + } + } + for _, method := range p.Config.Methods { + if method == e.Method { + okMethod = true + break + } + } + + return okPath && okMethod +} + var printDebug bool func dbg(s string, a ...interface{}) { @@ -124,7 +150,7 @@ type Ref struct { // Main reason to store as a string (and Refs as a map) for now is so that // it looks pretties in the pretty.Print() output. May not want to keep // this. - Reference string //*Reference + Reference *Reference //*Reference } // Param is a path, query, or form parameter. @@ -156,6 +182,10 @@ type Reference struct { Fields []Param // Struct fields. } +func (r Reference) String() string { + return fmt.Sprintf("%s.%s", r.Package, r.Name) +} + const ( ctxForm = "form" ctxPath = "path" @@ -186,6 +216,10 @@ func parseComment(prog *Program, comment, pkgPath, filePath string) ([]*Endpoint if e.Method == "" { return nil, 0, nil } + if !prog.ShouldExport(*e) { + // fmt.Fprintf(os.Stderr, "skipping %s %s\n", e.Method, e.Path) + return nil, 0, nil + } // Find more start lines. i := 1 @@ -245,7 +279,7 @@ func parseComment(prog *Program, comment, pkgPath, filePath string) ([]*Endpoint e.Request.Path, err = parseRefValue(prog, "path", h[2], filePath) if err == nil { - pathRef, err := GetReference(prog, "query", false, e.Request.Path.Reference, filePath) + pathRef, err := GetReference(prog, "query", false, e.Request.Path.Reference.Lookup, filePath) if err != nil { return nil, i, err } @@ -507,7 +541,7 @@ func parseRefValue(prog *Program, context, value, filePath string) (*Ref, error) // intermediate format (otherwise pretty.Print() show the full object, which // is kind of noisy). We should probably store it as a pointer once I'm done // with the docparse part. - params.Reference = ref.Lookup + params.Reference = ref return params, nil } @@ -581,6 +615,38 @@ func MapType(prog *Program, in string) (kind, format string) { return kind, format } +func MapPackage(prog *Program, pkg string) string { + if v, ok := prog.Config.MapPackages[pkg]; ok { + return v + } + + // fallback to previous behavior(trim lib name) + i := strings.LastIndex(pkg, "/") + if i == -1 { + return pkg + } + + return pkg[i+1:] +} + +func FixSchemaPackage(s *Schema) { + if s == nil { + return + } + + // fmt.Fprintf(os.Stderr, " ==== fixing schema %s\n", s.Reference) + FixSchemaProportiesPackage(s.Properties) +} + +func FixSchemaProportiesPackage(props map[string]*Schema) { + for _, prop := range props { + // fmt.Fprintf(os.Stderr, " ==== ==== %s -- %s\n", i, prop.Reference) + FixSchemaPackage(prop.Items) + FixSchemaPackage(prop.AdditionalProperties) + FixSchemaProportiesPackage(prop.Properties) + } +} + func exprToString(node ast.Expr) string { switch n := node.(type) { case *ast.BasicLit: diff --git a/docparse/find.go b/docparse/find.go index c81a93a..e0611bc 100644 --- a/docparse/find.go +++ b/docparse/find.go @@ -7,9 +7,11 @@ import ( "go/parser" "go/token" "io" + "os" "path" "path/filepath" "reflect" + "runtime/debug" "sort" "strings" @@ -37,6 +39,7 @@ func FindComments(w io.Writer, prog *Program) error { for _, fullPath := range pkg.GoFiles { // Print as just / in errors instead of full path. + //printDebug = strings.Contains(fullPath, "/thread_controller.go") relPath := pkg.PkgPath + "/" + filepath.Base(fullPath) fset := token.NewFileSet() f, err := parser.ParseFile(fset, fullPath, nil, parser.ParseComments) @@ -46,6 +49,7 @@ func FindComments(w io.Writer, prog *Program) error { } for _, c := range f.Comments { + //printDebug = printDebug || strings.Contains(c.Text(), "PATCH /v2/tickets/{ticketId}/messages/{id}.json Messages") e, relLine, err := parseComment(prog, c.Text(), pkg.PkgPath, fullPath) if err != nil { p := fset.Position(c.Pos()) @@ -69,10 +73,23 @@ func FindComments(w io.Writer, prog *Program) error { } prog.Endpoints = append(prog.Endpoints, e...) + //printDebug = false } } } + for i, ref := range prog.References { + old := ref.Package + ref.Package = MapPackage(prog, ref.Package) + delete(prog.References, i) + i = strings.Replace(i, old, ref.Package, 1) + if prev, ok := prog.References[i]; ok { + return fmt.Errorf("name collition: %s.%s <-> %s.%s(%s)", old, ref.Name, prev.Package, prev.Name, prev.File) + } + prog.References[i] = ref + FixSchemaPackage(ref.Schema) + } + if len(allErr) > 0 { msg := "" for _, err := range allErr { @@ -118,11 +135,11 @@ func findType(currentFile, pkgPath, name string) ( importPath string, err error, ) { - dbg("findType: file: %#v, pkgPath: %#v, name: %#v", currentFile, pkgPath, name) resolvedPath, pkg, err := resolvePackage(currentFile, pkgPath) if err != nil { return nil, "", "", fmt.Errorf("could not resolve package: %v", err) } + dbg("findType: file: %#v, pkgPath: %#v, name: %#v resolvedPath:%s pkg:%s", currentFile, pkgPath, name, resolvedPath, pkg.Name) decls, err := getDecls(pkg, resolvedPath) if err != nil { @@ -133,17 +150,22 @@ func findType(currentFile, pkgPath, name string) ( if ts.ts == nil { continue } + if ts.ts.Name.Name == name { - impPath := pkg.ImportPath - if impPath == "." { - impPath = pkg.Name - } - return ts.ts, ts.file, impPath, nil + dbg("findType: declaration file:%s importpath: %s ts.name:%s name:%s", ts.file, pkg.ImportPath, ts.ts.Name.Name, name) + // impPath := pkg.ImportPath + // if impPath == "." { + // impPath = pkg.Name + // } + return ts.ts, ts.file, pkg.ImportPath, nil } } - return nil, "", "", fmt.Errorf("could not find type %#v in package %#v", - name, resolvedPath) + if name == "pathParams" { + debug.PrintStack() + } + return nil, "", "", fmt.Errorf("could not find type %#v in package %#v:%s", + name, resolvedPath, pkg.ImportPath) } func findValue(currentFile, pkgPath, name string) ( @@ -205,7 +227,7 @@ func resolvePackage(currentFile, pkgPath string) ( func getDecls(pkg *build.Package, pkgPath string) ([]declCache, error) { // Try to load from cache. - decls, ok := declsCache[pkgPath] + decls, ok := declsCache[pkg.ImportPath] if ok { return decls, nil } @@ -260,7 +282,7 @@ func getDecls(pkg *build.Package, pkgPath string) ([]declCache, error) { } } - declsCache[pkgPath] = decls + declsCache[pkg.ImportPath] = decls return decls, nil } @@ -289,7 +311,7 @@ func (err ErrNotStruct) Error() string { // // A GetReference("Foo", "") call will add two entries to prog.References: Foo // and Bar (but only Foo is returned). -func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath string) (*Reference, error) { +func GetReference(prog *Program, context string, isEmbed bool, lookup, sourceFilePath string) (*Reference, error) { wrapper := "" isSlice := false if strings.HasPrefix(lookup, "[") && strings.HasSuffix(lookup, "]") && strings.Contains(lookup, ":") { @@ -302,26 +324,26 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath lookup = lookup[2:] } - dbg("getReference: lookup: %#v -> filepath: %#v", lookup, filePath) - name, pkg := ParseLookup(lookup, filePath) - dbg("getReference: pkg: %#v -> name: %#v", pkg, name) + dbg("getReference: lookup: %#v -> filepath: %#v", lookup, sourceFilePath) + refName, pkgAlias := ParseLookup(lookup, sourceFilePath) + dbg("getReference: pkg: %#v -> name: %#v", pkgAlias, refName) + + // Find type. + ts, refFilePath, refPkg, err := findType(sourceFilePath, pkgAlias, refName) + if err != nil { + return nil, err + } // Already parsed this one, don't need to do it again. - if ref, ok := prog.References[lookup]; ok { + if ref, ok := prog.References[Reference{Name: refName, Package: refPkg}.String()]; ok { // Update context: some structs are embedded but also referenced // directly. if ref.IsEmbed { - prog.References[lookup] = ref + prog.References[ref.String()] = ref } return &ref, nil } - // Find type. - ts, foundPath, pkg, err := findType(filePath, pkg, name) - if err != nil { - return nil, err - } - var st *ast.StructType switch typ := ts.Type.(type) { case *ast.StructType: @@ -334,21 +356,28 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath if wrapper != "" { arLookup = fmt.Sprintf("[%v:%v]", wrapper, arLookup) } - return GetReference(prog, context, isEmbed, arLookup, filePath) + return GetReference(prog, context, isEmbed, arLookup, sourceFilePath) default: return nil, ErrNotStruct{ts, fmt.Sprintf( - "%v is not a struct or interface but a %T", name, ts.Type)} + "%v is not a struct or interface but a %T", refName, ts.Type)} } ref := Reference{ - Name: name, - Package: pkg, - Lookup: filepath.Base(pkg) + "." + name, - File: foundPath, + Name: refName, + Package: refPkg, + // NOTE: Is it needed? there's a lookup param which i think it's the same + // Lookup looks like the way a variable is defined in the doc + // i.e. Request body: >>>> threadupdate.Message <<<< this + //Lookup: filepath.Base(refPkg) + "." + refName, + Lookup: lookup, + File: refFilePath, Context: context, IsEmbed: isEmbed, IsSlice: isSlice, } + + prog.References[ref.String()] = ref + if ts.Doc != nil { ref.Info = strings.TrimSpace(ts.Doc.Text()) } @@ -382,18 +411,23 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath continue } + var ( + ref *Reference + err error + ) switch t := f.Type.(type) { case *ast.Ident: - err = resolveType(prog, context, false, t, "", pkg) + ref, err = resolveType(prog, context, false, t, refFilePath, refPkg) case *ast.StarExpr: ex, _ := t.X.(*ast.Ident) - err = resolveType(prog, context, false, ex, "", pkg) + ref, err = resolveType(prog, context, false, ex, refFilePath, refPkg) } if err != nil { return nil, fmt.Errorf("could not lookup %s in %s: %s", err, f.Type, lookup) } + lookup = ref.String() } // Names is an array in cases like "Foo, Bar string". @@ -403,7 +437,7 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath tag := reflect.StructTag(strings.Trim(f.Tag.Value, "`")).Get(tagName) if tag != "" { return nil, fmt.Errorf("not exported but has %q tag: %s.%s field %v", - tagName, pkg, name, f.Names) + tagName, refPkg, refName, f.Names) } } @@ -417,7 +451,7 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath } } - prog.References[ref.Lookup] = ref + prog.References[ref.String()] = ref var ( nested []string nestedTagged []*ast.Field @@ -453,7 +487,7 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath } } - nestLookup, err := findNested(prog, context, isEmbed, f, foundPath, pkg) + nestLookup, err := findNested(prog, context, isEmbed, f, refFilePath, refPkg) if err != nil { return nil, fmt.Errorf("\n findNested: %v", err) } @@ -479,9 +513,9 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath } // Convert to JSON Schema. - schema, err := structToSchema(prog, name, tagName, ref) + schema, err := structToSchema(prog, refName, tagName, ref) if err != nil { - return nil, fmt.Errorf("%v can not be converted to JSON schema: %v", name, err) + return nil, fmt.Errorf("%v can not be converted to JSON schema: %v", refName, err) } ref.Schema = schema @@ -508,7 +542,7 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath } // Find the referenced struct - reference, err := GetReference(prog, context, false, lookupStruct+f.Name, filePath) + reference, err := GetReference(prog, context, false, lookupStruct+f.Name, sourceFilePath) if err != nil { return nil, fmt.Errorf("could not get referenced struct %s", lookupStruct+f.Name) } @@ -544,9 +578,9 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath // If the fields have been changed, regenerate the schema with the new fields if changed { - schema, err = structToSchema(prog, name, tagName, ref) + schema, err = structToSchema(prog, refName, tagName, ref) if err != nil { - return nil, fmt.Errorf("%v can not be converted to JSON schema: %v", name, err) + return nil, fmt.Errorf("%v can not be converted to JSON schema: %v", refName, err) } ref.Schema = schema } @@ -583,7 +617,7 @@ func GetReference(prog *Program, context string, isEmbed bool, lookup, filePath ref.Schema = wrappedSchema } - prog.References[ref.Lookup] = ref + prog.References[ref.String()] = ref return &ref, nil } @@ -679,37 +713,42 @@ start: } lookup := pkg + "." + name.Name - if i := strings.LastIndex(pkg, "/"); i > -1 { - lookup = pkg[i+1:] + "." + name.Name - } + + // if i := strings.LastIndex(pkg, "/"); i > -1 { + // lookup = pkg[i+1:] + "." + name.Name + // } // Don't need to add stuff we map to Go primitives. if x, _ := MapType(prog, lookup); x != "" { return lookup, nil } if _, ok := prog.References[lookup]; !ok { - err := resolveType(prog, context, isEmbed, name, filePath, pkg) + ref, err := resolveType(prog, context, isEmbed, name, filePath, pkg) if err != nil { return "", fmt.Errorf("%v.%v: %v", pkg, name, err) } + + if ref != nil { + return ref.String(), nil + } } return lookup, nil } // Add the type declaration to references. -func resolveType(prog *Program, context string, isEmbed bool, typ *ast.Ident, filePath, pkg string) error { +func resolveType(prog *Program, context string, isEmbed bool, typ *ast.Ident, filePath, pkg string) (*Reference, error) { var ts *ast.TypeSpec if typ.Obj == nil { var err error ts, _, _, err = findType(filePath, pkg, typ.Name) if err != nil { - return err + return nil, err } } else { var ok bool ts, ok = typ.Obj.Decl.(*ast.TypeSpec) if !ok { - return fmt.Errorf("resolveType: not a type declaration but %T", + return nil, fmt.Errorf("resolveType: not a type declaration but %T", typ.Obj.Decl) } } @@ -718,13 +757,15 @@ func resolveType(prog *Program, context string, isEmbed bool, typ *ast.Ident, fi // simply a string. _, ok := ts.Type.(*ast.StructType) if !ok { - return nil + return nil, nil } // This sets prog.References lookup := pkg + "." + typ.Name - _, err := GetReference(prog, context, isEmbed, lookup, filePath) - return err + ref, err := GetReference(prog, context, isEmbed, lookup, filePath) + fmt.Fprintf(os.Stderr, "=====resolved %s -> %s\n", lookup, ref.String()) + + return ref, err } // ParseLookup for the package and name, if lookup is an imported path e.g diff --git a/docparse/jsonschema.go b/docparse/jsonschema.go index 5819c3d..892f302 100644 --- a/docparse/jsonschema.go +++ b/docparse/jsonschema.go @@ -432,9 +432,10 @@ start: } } - if i := strings.LastIndex(lookup, "/"); i > -1 { - lookup = pkg[i+1:] + "." + name.Name - } + fmt.Fprintf(os.Stderr, "=======fieldToSchema %s\n", lookup) + //if i := strings.LastIndex(lookup, "/"); i > -1 { + //lookup = pkg[i+1:] + "." + name.Name + //} p.Description = "" // SwaggerHub will complain if both Description and $ref are set. p.Reference = lookup @@ -580,7 +581,7 @@ arrayStart: // Add to prog.References if not there already rName, rPkg := ParseLookup(lookup, ref.File) - if _, ok := prog.References[filepath.Base(rPkg)+"."+rName]; !ok { + if _, ok := prog.References[rPkg+"."+rName]; !ok { _, err = GetReference(prog, ref.Context, false, lookup, ref.File) } return err diff --git a/main.go b/main.go index 9cf0806..ab0c530 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "os" "runtime" "runtime/pprof" + "strings" "github.com/teamwork/kommentaar/docparse" "github.com/teamwork/kommentaar/kconfig" @@ -33,6 +34,8 @@ func main() { var stdout io.Writer = os.Stdout func start() (bool, error) { + paths := flag.String("paths", "", "only these paths") + methods := flag.String("methods", "", "only these methods") config := flag.String("config", "", "configuration file") debug := flag.Bool("debug", false, "print debug output to stderr") addr := flag.String("serve", "", "serve HTML output on this address, instead of writing to\n"+ @@ -61,6 +64,12 @@ func start() (bool, error) { } prog := docparse.NewProgram(*debug) + if *methods != "" { + prog.Config.Methods = strings.Split(*methods, ",") + } + if *paths != "" { + prog.Config.Paths = strings.Split(*paths, ",") + } if *config != "" { err := kconfig.Load(prog, *config) diff --git a/openapi2/openapi2.go b/openapi2/openapi2.go index dc6e041..9ab22dc 100644 --- a/openapi2/openapi2.go +++ b/openapi2/openapi2.go @@ -202,14 +202,20 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { // track which defs are referenced so we can remove unreferenced ones, e.g embedded // but also handle where it is both embedded and named referencedDefs := map[string]struct{}{} - ref := func(s string) string { - s = strings.TrimPrefix(s, "#/definitions/") + refMapper := func(ref *docparse.Reference) string { + ref.Package = docparse.MapPackage(prog, ref.Package) + s := strings.TrimPrefix(ref.String(), "#/definitions/") referencedDefs[s] = struct{}{} return "#/definitions/" + s } // Add endpoints. for _, e := range prog.Endpoints { + // dbg := e.Path == "/v2/tickets/{ticketId}/messages/{id}.json" //&& e.Method == "PATCH" + // if !dbg { + // continue + // } + e.Path = prog.Config.Prefix + e.Path op := Operation{ @@ -231,21 +237,22 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { if e.Request.Path != nil { // TODO: Don't access prog.References directly. This probably // shouldn't be there anyway. - ref := prog.References[e.Request.Path.Reference] + ref := prog.References[e.Request.Path.Reference.String()] + if ref.Schema != nil { + for name, p := range ref.Schema.Properties { + if p.OmitDoc { + // path is required, so just blank description. + p.Description = "" + } - for name, p := range ref.Schema.Properties { - if p.OmitDoc { - // path is required, so just blank description. - p.Description = "" + op.Parameters = append(op.Parameters, Parameter{ + Name: name, + In: "path", + Description: p.Description, + Type: p.Type, + Required: true, + }) } - - op.Parameters = append(op.Parameters, Parameter{ - Name: name, - In: "path", - Description: p.Description, - Type: p.Type, - Required: true, - }) } } @@ -253,7 +260,7 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { if e.Request.Query != nil { // TODO: Don't access prog.References directly. This probably // shouldn't be there anyway. - ref := prog.References[e.Request.Query.Reference] + ref := prog.References[e.Request.Query.Reference.String()] for _, f := range ref.Fields { // TODO: this should be done in docparse. @@ -310,7 +317,7 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { if e.Request.Form != nil { // TODO: Don't access prog.References directly. This probably // shouldn't be there anyway. - ref := prog.References[e.Request.Form.Reference] + ref := prog.References[e.Request.Form.Reference.String()] for _, f := range ref.Fields { // TODO: this should be done in docparse @@ -371,12 +378,12 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { if e.Request.Body != nil { op.Parameters = append(op.Parameters, Parameter{ // TODO: name required, is there a better value to use? - Name: e.Request.Body.Reference, + Name: e.Request.Body.Reference.Lookup, In: "body", Description: e.Request.Body.Description, Required: true, Schema: &docparse.Schema{ - Reference: ref(e.Request.Body.Reference), + Reference: refMapper(e.Request.Body.Reference), }, }) op.Consumes = append(op.Consumes, e.Request.ContentType) @@ -400,13 +407,13 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { } // Link reference. - if resp.Body != nil && resp.Body.Reference != "" { + if resp.Body != nil && resp.Body.Reference != nil && resp.Body.Reference.Lookup != "" { r.Schema = &docparse.Schema{ - Reference: ref(resp.Body.Reference), + Reference: refMapper(resp.Body.Reference), } } else if dr, ok := prog.Config.DefaultResponse[code]; ok { r.Schema = &docparse.Schema{ - Reference: ref(dr.Body.Reference), + Reference: refMapper(dr.Body.Reference), } if dr.ContentType != "" { resp.ContentType = dr.ContentType @@ -456,7 +463,7 @@ func write(outFormat string, w io.Writer, prog *docparse.Program) error { if v.Schema == nil { return fmt.Errorf("schema is nil for %q", k) } - prefixPropertyReferences(v.Schema.Properties, ref) + prefixPropertyReferences(v.Schema.Properties, refMapper) out.Definitions[k] = *v.Schema } // Remove unreferenced definitions. @@ -510,17 +517,17 @@ func appendIfNotExists(xs []string, y string) []string { return append(xs, y) } -func prefixPropertyReferences(properties map[string]*docparse.Schema, getRef func(string) string) { +func prefixPropertyReferences(properties map[string]*docparse.Schema, getRef func(*docparse.Reference) string) { var rm []string for k, s := range properties { if s.Reference != "" { - s.Reference = getRef(s.Reference) + s.Reference = getRef(&docparse.Reference{Lookup: s.Reference}) } if s.Items != nil && s.Items.Reference != "" { - s.Items.Reference = getRef(s.Items.Reference) + s.Items.Reference = getRef(&docparse.Reference{Lookup: s.Items.Reference}) } if s.AdditionalProperties != nil && s.AdditionalProperties.Reference != "" { - s.AdditionalProperties.Reference = getRef(s.AdditionalProperties.Reference) + s.AdditionalProperties.Reference = getRef(&docparse.Reference{Lookup: s.AdditionalProperties.Reference}) } if s.OmitDoc {