diff --git a/go.mod b/go.mod index 29c62833..60af8ff6 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/anchore/go-logger v0.0.0-20230531193951-db5ae83e7dbe github.com/anchore/stereoscope v0.0.0-20230727211946-d1f3d766295e - github.com/anchore/syft v0.87.1 + github.com/anchore/syft v0.88.0 github.com/in-toto/in-toto-golang v0.9.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 @@ -59,13 +59,14 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-containerregistry v0.16.1 // indirect github.com/google/licensecheck v0.3.1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/jinzhu/copier v0.3.5 // indirect + github.com/jinzhu/copier v0.4.0 // indirect + github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/pgzip v1.2.5 // indirect @@ -90,6 +91,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/saferwall/pe v1.4.4 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sassoftware/go-rpmutils v0.2.0 // indirect github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect diff --git a/go.sum b/go.sum index e04abab4..ff5d461e 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 h1:AV7qjwM github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= github.com/anchore/stereoscope v0.0.0-20230727211946-d1f3d766295e h1:S6IhYpsBCpvphlHA1tN0glSG/kjVvFzC6OJuU2qW5Pc= github.com/anchore/stereoscope v0.0.0-20230727211946-d1f3d766295e/go.mod h1:0LsgHgXO4QFnk2hsYwtqd3fR18PIZXlFLIl2qb9tu3g= -github.com/anchore/syft v0.87.1 h1:S9xAQzvSRjGTqnkHXxGNX7TQAU0vzr2HAbuD2Sklc1s= -github.com/anchore/syft v0.87.1/go.mod h1:FweiDRmNb6zq/+vD8rernOfCdkV6KBfppIKFX207eCY= +github.com/anchore/syft v0.88.0 h1:QRPcXwbQnxcOIfSZ5Sd6psfVQ756VICvx/HUMsIJEBw= +github.com/anchore/syft v0.88.0/go.mod h1:6GgbZflKWC7ph2Zjb5wgq0ORKhwDaHG3xcjG84FSMPo= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= @@ -316,8 +316,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -373,8 +373,8 @@ github.com/in-toto/in-toto-golang v0.9.0/go.mod h1:xsBVrVsHNsB61++S6Dy2vWosKhuA3 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -382,6 +382,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= +github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -519,6 +521,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/saferwall/pe v1.4.4 h1:Ml++7/2/Z1iKwV4zCsd1nIqTEAdUQKAetwbbcCarhOg= github.com/saferwall/pe v1.4.4/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ= diff --git a/vendor/github.com/anchore/syft/internal/constants.go b/vendor/github.com/anchore/syft/internal/constants.go index 354a5427..259b2d77 100644 --- a/vendor/github.com/anchore/syft/internal/constants.go +++ b/vendor/github.com/anchore/syft/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "10.0.0" + JSONSchemaVersion = "10.0.1" ) diff --git a/vendor/github.com/anchore/syft/syft/formats/common/cyclonedxhelpers/format.go b/vendor/github.com/anchore/syft/syft/formats/common/cyclonedxhelpers/format.go index 9582ba85..e36149e1 100644 --- a/vendor/github.com/anchore/syft/syft/formats/common/cyclonedxhelpers/format.go +++ b/vendor/github.com/anchore/syft/syft/formats/common/cyclonedxhelpers/format.go @@ -5,6 +5,7 @@ import ( "github.com/CycloneDX/cyclonedx-go" "github.com/google/uuid" + "golang.org/x/exp/slices" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" @@ -139,7 +140,7 @@ func isExpressiblePackageRelationship(ty artifact.RelationshipType) bool { } func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependency { - result := make([]cyclonedx.Dependency, 0) + dependencies := map[string]*cyclonedx.Dependency{} for _, r := range relationships { exists := isExpressiblePackageRelationship(r.Type) if !exists { @@ -160,15 +161,32 @@ func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependenc continue } - // ind dep + toRef := deriveBomRef(toPkg) + dep := dependencies[toRef] + if dep == nil { + dep = &cyclonedx.Dependency{ + Ref: toRef, + Dependencies: &[]string{}, + } + dependencies[toRef] = dep + } - innerDeps := []string{} - innerDeps = append(innerDeps, deriveBomRef(fromPkg)) - result = append(result, cyclonedx.Dependency{ - Ref: deriveBomRef(toPkg), - Dependencies: &innerDeps, - }) + fromRef := deriveBomRef(fromPkg) + if !slices.Contains(*dep.Dependencies, fromRef) { + *dep.Dependencies = append(*dep.Dependencies, fromRef) + } } + + result := make([]cyclonedx.Dependency, 0, len(dependencies)) + for _, dep := range dependencies { + slices.Sort(*dep.Dependencies) + result = append(result, *dep) + } + + slices.SortFunc(result, func(a, b cyclonedx.Dependency) bool { + return a.Ref < b.Ref + }) + return result } diff --git a/vendor/github.com/anchore/syft/syft/internal/fileresolver/directory_indexer.go b/vendor/github.com/anchore/syft/syft/internal/fileresolver/directory_indexer.go index c9b5567a..b6432f00 100644 --- a/vendor/github.com/anchore/syft/syft/internal/fileresolver/directory_indexer.go +++ b/vendor/github.com/anchore/syft/syft/internal/fileresolver/directory_indexer.go @@ -349,9 +349,13 @@ func (r directoryIndexer) addSymlinkToIndex(p string, info os.FileInfo) (string, } if filepath.IsAbs(linkTarget) { + linkTarget = filepath.Clean(linkTarget) // if the link is absolute (e.g, /bin/ls -> /bin/busybox) we need to - // resolve relative to the root of the base directory - linkTarget = filepath.Join(r.base, filepath.Clean(linkTarget)) + // resolve relative to the root of the base directory, if it is not already + // prefixed with a volume name + if filepath.VolumeName(linkTarget) == "" { + linkTarget = filepath.Join(r.base, filepath.Clean(linkTarget)) + } } else { // if the link is not absolute (e.g, /dev/stderr -> fd/2 ) we need to // resolve it relative to the directory in question (e.g. resolve to diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/binary/default_classifiers.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/binary/default_classifiers.go index 93ed25d3..24cb4321 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/binary/default_classifiers.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/binary/default_classifiers.go @@ -269,6 +269,20 @@ var defaultClassifiers = []classifier{ cpe.Must("cpe:2.3:a:nginx:nginx:*:*:*:*:*:*:*:*"), }, }, + { + Class: "bash-binary", + FileGlob: "**/bash", + EvidenceMatcher: fileContentsVersionMatcher( + // @(#)Bash version 5.2.15(1) release GNU + // @(#)Bash version 5.2.0(1) alpha GNU + // @(#)Bash version 5.2.0(1) beta GNU + // @(#)Bash version 5.2.0(1) rc4 GNU + `(?m)@\(#\)Bash version (?P[0-9]+\.[0-9]+\.[0-9]+)\([0-9]\) [a-z0-9]+ GNU`, + ), + Package: "bash", + PURL: mustPURL("pkg:generic/bash@version"), + CPEs: singleCPE("cpe:2.3:a:gnu:bash:*:*:*:*:*:*:*:*"), + }, } // in both binaries and shared libraries, the version pattern is [NUL]3.11.2[NUL] diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json index 5e357e2b..7ffc387c 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/dictionary/data/cpe-index.json @@ -283,6 +283,7 @@ "360class.jansenhm": "cpe:2.3:a:360class.jansenhm_project:360class.jansenhm:*:*:*:*:*:node.js:*:*", "626": "cpe:2.3:a:626_project:626:*:*:*:*:*:node.js:*:*", "@actions/core": "cpe:2.3:a:toolkit_project:toolkit:*:*:*:*:*:node.js:*:*", + "@antfu/utils": "cpe:2.3:a:antfu:utils:*:*:*:*:*:node.js:*:*", "@awsui/components-react": "cpe:2.3:a:amazon:awsui\\/components-react:*:*:*:*:*:node.js:*:*", "@azure/ms-rest-nodeauth": "cpe:2.3:a:microsoft:ms-rest-nodeauth:*:*:*:*:*:node.js:*:*", "@backstage/plugin-auth-backend": "cpe:2.3:a:linuxfoundation:auth_backend:*:*:*:*:*:node.js:*:*", @@ -1081,6 +1082,7 @@ "spacy": "cpe:2.3:a:explosion:spacy:*:*:*:*:*:python:*:*", "sqlparse": "cpe:2.3:a:sqlparse_project:sqlparse:*:*:*:*:*:python:*:*", "tkvideoplayer": "cpe:2.3:a:python:tkvideoplayer:*:*:*:*:*:*:*:*", + "togglee": "cpe:2.3:a:togglee:togglee:*:*:*:*:*:pypi:*:*", "urllib3": "cpe:2.3:a:python:urllib3:*:*:*:*:*:*:*:*", "validators": "cpe:2.3:a:validators_project:validators:*:*:*:*:*:python:*:*", "vault-cli": "cpe:2.3:a:vault-cli_project:vault-cli:*:*:*:*:*:python:*:*", diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java.go index 2e838de3..7a1a8f7b 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java.go @@ -21,16 +21,16 @@ var ( "be", } - primaryJavaManifestGroupIDFields = []string{ + PrimaryJavaManifestGroupIDFields = []string{ + "Bundle-SymbolicName", "Extension-Name", "Specification-Vendor", "Implementation-Vendor", - "Bundle-SymbolicName", "Implementation-Vendor-Id", "Implementation-Title", "Bundle-Activator", } - secondaryJavaManifestGroupIDFields = []string{ + SecondaryJavaManifestGroupIDFields = []string{ "Automatic-Module-Name", "Main-Class", "Package", @@ -168,7 +168,7 @@ func artifactIDFromJavaPackage(p pkg.Package) string { } artifactID := strings.TrimSpace(metadata.PomProperties.ArtifactID) - if startsWithTopLevelDomain(artifactID) && len(strings.Split(artifactID, ".")) > 1 { + if looksLikeGroupID(artifactID) && len(strings.Split(artifactID, ".")) > 1 { // there is a strong indication that the artifact ID is really a group ID, don't use it return "" } @@ -184,6 +184,9 @@ func GroupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) { return GroupIDsFromJavaMetadata(p.Name, metadata) } +// GroupIDsFromJavaMetadata returns the possible group IDs for a Java package +// This function is similar to GroupIDFromJavaPackage, but returns all possible group IDs and is less strict +// It is used as a way to generate possible candidates for CPE matching. func GroupIDsFromJavaMetadata(pkgName string, metadata pkg.JavaMetadata) (groupIDs []string) { groupIDs = append(groupIDs, groupIDsFromPomProperties(metadata.PomProperties)...) groupIDs = append(groupIDs, groupIDsFromPomProject(metadata.PomProject)...) @@ -242,7 +245,7 @@ func addGroupIDsFromGroupIDsAndArtifactID(groupID, artifactID string) (groupIDs } func groupIDsFromJavaManifest(pkgName string, manifest *pkg.JavaManifest) []string { - if groupID, ok := defaultArtifactIDToGroupID[pkgName]; ok { + if groupID, ok := DefaultArtifactIDToGroupID[pkgName]; ok { return []string{groupID} } @@ -251,7 +254,7 @@ func groupIDsFromJavaManifest(pkgName string, manifest *pkg.JavaManifest) []stri } // try the common manifest fields first for a set of candidates - groupIDs := getManifestFieldGroupIDs(manifest, primaryJavaManifestGroupIDFields) + groupIDs := GetManifestFieldGroupIDs(manifest, PrimaryJavaManifestGroupIDFields) if len(groupIDs) != 0 { return groupIDs @@ -262,10 +265,10 @@ func groupIDsFromJavaManifest(pkgName string, manifest *pkg.JavaManifest) []stri // for more info see pkg:maven/commons-io/commons-io@2.8.0 within cloudbees/cloudbees-core-mm:2.263.4.2 // at /usr/share/jenkins/jenkins.war:WEB-INF/plugins/analysis-model-api.hpi:WEB-INF/lib/commons-io-2.8.0.jar // as well as the ant package from cloudbees/cloudbees-core-mm:2.277.2.4-ra. - return getManifestFieldGroupIDs(manifest, secondaryJavaManifestGroupIDFields) + return GetManifestFieldGroupIDs(manifest, SecondaryJavaManifestGroupIDFields) } -func getManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (groupIDs []string) { +func GetManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (groupIDs []string) { if manifest == nil { return nil } @@ -302,3 +305,7 @@ func removeOSCIDirectives(groupID string) string { func startsWithTopLevelDomain(value string) bool { return internal.HasAnyOfPrefixes(value, domains...) } + +func looksLikeGroupID(value string) bool { + return strings.Contains(value, ".") +} diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java_groupid_map.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java_groupid_map.go index 9e36db18..5205c81e 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java_groupid_map.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/common/cpe/java_groupid_map.go @@ -1,6 +1,6 @@ package cpe -var defaultArtifactIDToGroupID = map[string]string{ +var DefaultArtifactIDToGroupID = map[string]string{ "ant": "org.apache.ant", "ant-antlr": "org.apache.ant", "ant-antunit": "org.apache.ant", diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/package.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/package.go index dbbdd0b9..b093c928 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/package.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/package.go @@ -8,23 +8,67 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func newConanfilePackage(m pkg.ConanMetadata, locations ...file.Location) *pkg.Package { - fields := strings.Split(strings.TrimSpace(m.Ref), "/") - if len(fields) < 2 { - return nil +type conanRef struct { + Name string + Version string + User string + Channel string + Revision string + Timestamp string +} + +func splitConanRef(ref string) *conanRef { + // Conan ref format is: + // pkg/0.1@user/channel#rrev%timestamp + // This method is based on conan's ref.loads method: + // https://github.com/conan-io/conan/blob/release/2.0/conans/model/recipe_ref.py#L93C21-L93C21 + + var cref conanRef + + // timestamp + tokens := strings.Split(ref, "%") + text := tokens[0] + if len(tokens) == 2 { + cref.Timestamp = tokens[1] } - pkgName, pkgVersion := fields[0], fields[1] + // revision + tokens = strings.Split(text, "#") + ref = tokens[0] + if len(tokens) == 2 { + cref.Revision = tokens[1] + } - if pkgName == "" || pkgVersion == "" { + // name and version are always given + tokens = strings.Split(ref, "@") + nameAndVersion := strings.Split(tokens[0], "/") + if len(nameAndVersion) < 2 || nameAndVersion[0] == "" || nameAndVersion[1] == "" { + return nil + } + cref.Name = nameAndVersion[0] + cref.Version = nameAndVersion[1] + // user and channel + if len(tokens) == 2 && tokens[1] != "" { + tokens = strings.Split(tokens[1], "/") + if len(tokens) == 2 { + cref.User = tokens[0] + cref.Channel = tokens[1] + } + } + return &cref +} + +func newConanfilePackage(m pkg.ConanMetadata, locations ...file.Location) *pkg.Package { + ref := splitConanRef(m.Ref) + if ref == nil { return nil } p := pkg.Package{ - Name: pkgName, - Version: pkgVersion, + Name: ref.Name, + Version: ref.Version, Locations: file.NewLocationSet(locations...), - PURL: packageURL(pkgName, pkgVersion), + PURL: packageURL(ref), Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -37,22 +81,16 @@ func newConanfilePackage(m pkg.ConanMetadata, locations ...file.Location) *pkg.P } func newConanlockPackage(m pkg.ConanLockMetadata, locations ...file.Location) *pkg.Package { - fields := strings.Split(strings.Split(m.Ref, "@")[0], "/") - if len(fields) < 2 { - return nil - } - - pkgName, pkgVersion := fields[0], fields[1] - - if pkgName == "" || pkgVersion == "" { + ref := splitConanRef(m.Ref) + if ref == nil { return nil } p := pkg.Package{ - Name: pkgName, - Version: pkgVersion, + Name: ref.Name, + Version: ref.Version, Locations: file.NewLocationSet(locations...), - PURL: packageURL(pkgName, pkgVersion), + PURL: packageURL(ref), Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanLockMetadataType, @@ -64,13 +102,20 @@ func newConanlockPackage(m pkg.ConanLockMetadata, locations ...file.Location) *p return &p } -func packageURL(name, version string) string { +func packageURL(ref *conanRef) string { + qualifiers := packageurl.Qualifiers{} + if ref.Channel != "" { + qualifiers = append(qualifiers, packageurl.Qualifier{ + Key: "channel", + Value: ref.Channel, + }) + } return packageurl.NewPackageURL( packageurl.TypeConan, - "", - name, - version, - nil, // TODO: no qualifiers (...yet) + ref.User, + ref.Name, + ref.Version, + qualifiers, "", ).ToString() } diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/parse_conanfile.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/parse_conanfile.go index f9ae172f..bf60706d 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/parse_conanfile.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/cpp/parse_conanfile.go @@ -36,7 +36,7 @@ func parseConanfile(_ file.Resolver, _ *generic.Environment, reader file.Locatio switch { case strings.Contains(line, "[requires]"): inRequirements = true - case strings.ContainsAny(line, "[]#"): + case strings.ContainsAny(line, "[]") || strings.HasPrefix(strings.TrimSpace(line), "#"): inRequirements = false } diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/package.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/package.go index 30ba083b..37942563 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/package.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/package.go @@ -11,7 +11,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debug.Module, mainModule, goVersion, architecture string, buildSettings map[string]string, locations ...file.Location) pkg.Package { +func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debug.Module, mainModule, goVersion, architecture string, buildSettings map[string]string, cryptoSettings []string, locations ...file.Location) pkg.Package { if dep.Replace != nil { dep = dep.Replace } @@ -36,6 +36,7 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu Architecture: architecture, BuildSettings: buildSettings, MainModule: mainModule, + GoCryptoSettings: cryptoSettings, }, } diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/parse_go_binary.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/parse_go_binary.go index 89ed4ba0..6da18583 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/parse_go_binary.go @@ -66,7 +66,7 @@ func (c *goBinaryCataloger) parseGoBinary(resolver file.Resolver, _ *generic.Env return pkgs, nil, nil } -func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *debug.BuildInfo, arch string, location file.Location) pkg.Package { +func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location) pkg.Package { gbs := getBuildSettings(mod.Settings) main := c.newGoBinaryPackage( resolver, @@ -75,6 +75,7 @@ func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *debug mod.GoVersion, arch, gbs, + mod.cryptoSettings, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) @@ -153,7 +154,7 @@ func extractVersionFromLDFlags(ldflags string) (majorVersion string, fullVersion // getArchs finds a binary architecture by two ways: // 1) reading build info from binaries compiled by go1.18+ // 2) reading file headers from binaries compiled by < go1.18 -func getArchs(readers []io.ReaderAt, builds []*debug.BuildInfo) []string { +func getArchs(readers []io.ReaderAt, builds []*extendedBuildInfo) []string { if len(readers) != len(builds) { log.Trace("golang cataloger: bin parsing: number of builds and readers doesn't match") return nil @@ -255,7 +256,7 @@ func createMainModuleFromPath(path string) (mod debug.Module) { return } -func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *debug.BuildInfo, arch string) []pkg.Package { +func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string) []pkg.Package { var pkgs []pkg.Package if mod == nil { return pkgs @@ -277,6 +278,7 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file mod.GoVersion, arch, nil, + mod.cryptoSettings, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) if pkg.IsValid(&p) { diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/scan_binary.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/scan_binary.go index c010b2a8..b19e2427 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/scan_binary.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/golang/scan_binary.go @@ -6,12 +6,19 @@ import ( "io" "runtime/debug" + "github.com/kastenhq/goversion/version" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader" ) +type extendedBuildInfo struct { + *debug.BuildInfo + cryptoSettings []string +} + // scanFile scans file to try to report the Go and module versions. -func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildInfo, []string) { +func scanFile(reader unionreader.UnionReader, filename string) ([]*extendedBuildInfo, []string) { // NOTE: multiple readers are returned to cover universal binaries, which are files // with more than one binary readers, err := unionreader.GetReaders(reader) @@ -20,7 +27,7 @@ func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildIn return nil, nil } - var builds []*debug.BuildInfo + var builds []*extendedBuildInfo for _, r := range readers { bi, err := getBuildInfo(r) if err != nil { @@ -30,7 +37,14 @@ func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildIn if bi == nil { continue } - builds = append(builds, bi) + + v, err := getCryptoInformation(r) + if err != nil { + log.WithFields("file", filename, "error", err).Trace("unable to read golang version info") + continue + } + + builds = append(builds, &extendedBuildInfo{bi, v}) } archs := getArchs(readers, builds) @@ -38,6 +52,29 @@ func scanFile(reader unionreader.UnionReader, filename string) ([]*debug.BuildIn return builds, archs } +func getCryptoInformation(reader io.ReaderAt) ([]string, error) { + v, err := version.ReadExeFromReader(reader) + if err != nil { + return nil, err + } + + return getCryptoSettingsFromVersion(v), nil +} + +func getCryptoSettingsFromVersion(v version.Version) []string { + cryptoSettings := []string{} + if v.StandardCrypto { + cryptoSettings = append(cryptoSettings, "standard-crypto") + } + if v.BoringCrypto { + cryptoSettings = append(cryptoSettings, "boring-crypto") + } + if v.FIPSOnly { + cryptoSettings = append(cryptoSettings, "crypto/tls/fipsonly") + } + return cryptoSettings +} + func getBuildInfo(r io.ReaderAt) (bi *debug.BuildInfo, err error) { defer func() { if r := recover(); r != nil { diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/archive_parser.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/archive_parser.go index ea216e90..a9662be6 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/archive_parser.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/archive_parser.go @@ -62,11 +62,11 @@ func parseJavaArchive(_ file.Resolver, _ *generic.Environment, reader file.Locat } // uniquePkgKey creates a unique string to identify the given package. -func uniquePkgKey(p *pkg.Package) string { +func uniquePkgKey(groupID string, p *pkg.Package) string { if p == nil { return "" } - return fmt.Sprintf("%s|%s", p.Name, p.Version) + return fmt.Sprintf("%s|%s|%s", groupID, p.Name, p.Version) } // newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover @@ -371,13 +371,27 @@ func pomProjectByParentPath(archivePath string, location file.Location, extractP return projectByParentPath, nil } -// packagesFromPomProperties processes a single Maven POM properties for a given parent package, returning all listed Java packages found and +// newPackageFromMavenData processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. Note the pom.xml is optional, the pom.properties is not. func newPackageFromMavenData(pomProperties pkg.PomProperties, pomProject *pkg.PomProject, parentPkg *pkg.Package, location file.Location) *pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" - if !strings.HasPrefix(pomProperties.ArtifactID, parentPkg.Name) { - vPathSuffix += ":" + pomProperties.ArtifactID + groupID := "" + if parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata); ok { + groupID = groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + } + + parentKey := fmt.Sprintf("%s:%s:%s", groupID, parentPkg.Name, parentPkg.Version) + // Since we don't have a package yet, it's important to use the same `field: value` association that we used when creating the parent package + // See below where Name => pomProperties.ArtifactID and Version => pomProperties.Version. We want to check for potentially nested identical + // packages and create equal virtual paths so they are de duped in the future + pomProjectKey := fmt.Sprintf("%s:%s:%s", pomProperties.GroupID, pomProperties.ArtifactID, pomProperties.Version) + if parentKey != pomProjectKey { + // build a new virtual path suffix for the package that is different from the parent package + // we want to use the GroupID and ArtifactID here to preserve uniqueness + // Some packages have the same name but different group IDs (e.g. "org.glassfish.jaxb/jaxb-core", "com.sun.xml.bind/jaxb-core") + // https://github.com/anchore/syft/issues/1944 + vPathSuffix += ":" + pomProperties.GroupID + ":" + pomProperties.ArtifactID } virtualPath := location.AccessPath() + vPathSuffix @@ -408,21 +422,26 @@ func newPackageFromMavenData(pomProperties pkg.PomProperties, pomProject *pkg.Po } func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool { - // the name/version pair matches... - if uniquePkgKey(&p) == uniquePkgKey(parentPkg) { - return true - } - metadata, ok := p.Metadata.(pkg.JavaMetadata) - if !ok { - log.WithFields("package", p.String()).Warn("unable to extract java metadata to check for matching package identity") - return false + parentMetadata, parentOk := parentPkg.Metadata.(pkg.JavaMetadata) + if !ok || !parentOk { + switch { + case !ok: + log.WithFields("package", p.String()).Trace("unable to extract java metadata to check for matching package identity for package: %s", p.Name) + case !parentOk: + log.WithFields("package", parentPkg.String()).Trace("unable to extract java metadata to check for matching package identity for package: %s", parentPkg.Name) + } + // if we can't extract metadata, we can check for matching identities via the package name + // this is not ideal, but it's better than nothing - this should not be used if we have Metadata + + return uniquePkgKey("", &p) == uniquePkgKey("", parentPkg) } - parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata) - if !ok { - log.WithFields("package", p.String()).Warn("unable to extract java metadata from parent for verifying virtual path") - return false + // try to determine identity with the metadata + groupID := groupIDFromJavaMetadata(p.Name, metadata) + parentGroupID := groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + if uniquePkgKey(groupID, &p) == uniquePkgKey(parentGroupID, parentPkg) { + return true } // the virtual path matches... @@ -434,10 +453,14 @@ func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool { // note: you CANNOT use name-is-subset-of-artifact-id or vice versa --this is too generic. Shaded jars are a good // example of this: where the package name is "cloudbees-analytics-segment-driver" and a child is "analytics", but // they do not indicate the same package. - if metadata.PomProperties.ArtifactID != "" && parentPkg.Name == metadata.PomProperties.ArtifactID { - return true + // NOTE: artifactId might not be a good indicator of uniqueness since archives can contain forks with the same name + // from different groups (e.g. "org.glassfish.jaxb.jaxb-core" and "com.sun.xml.bind.jaxb-core") + // we will use this check as a last resort + if metadata.PomProperties != nil { + if metadata.PomProperties.ArtifactID != "" && parentPkg.Name == metadata.PomProperties.ArtifactID { + return true + } } - return false } diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/package_url.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/package_url.go index adf05ff7..df1baf79 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/package_url.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/package_url.go @@ -1,6 +1,8 @@ package java import ( + "strings" + "github.com/anchore/packageurl-go" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" @@ -9,9 +11,9 @@ import ( // PackageURL returns the PURL for the specific java package (see https://github.com/package-url/purl-spec) func packageURL(name, version string, metadata pkg.JavaMetadata) string { var groupID = name - groupIDs := cpe.GroupIDsFromJavaMetadata(name, metadata) - if len(groupIDs) > 0 { - groupID = groupIDs[0] + + if gID := groupIDFromJavaMetadata(name, metadata); gID != "" { + groupID = gID } pURL := packageurl.NewPackageURL( @@ -23,3 +25,121 @@ func packageURL(name, version string, metadata pkg.JavaMetadata) string { "") return pURL.ToString() } + +// GroupIDFromJavaPackage returns the authoritative group ID for a Java package. +// The order of precedence is: +// 1. The group ID from the POM properties +// 2. The group ID from the POM project +// 3. The group ID from a select map of known group IDs +// 4. The group ID from the Java manifest +func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaMetadata) (groupID string) { + if groupID = groupIDFromPomProperties(metadata.PomProperties); groupID != "" { + return groupID + } + + if groupID = groupIDFromPomProject(metadata.PomProject); groupID != "" { + return groupID + } + + if groupID = groupIDFromKnownPackageList(pkgName); groupID != "" { + return groupID + } + + if groupID = groupIDFromJavaManifest(metadata.Manifest); groupID != "" { + return groupID + } + + return groupID +} + +func groupIDFromKnownPackageList(pkgName string) (groupID string) { + if groupID, ok := cpe.DefaultArtifactIDToGroupID[pkgName]; ok { + return groupID + } + return groupID +} + +func groupIDFromJavaManifest(manifest *pkg.JavaManifest) (groupID string) { + if manifest == nil { + return groupID + } + + groupIDS := cpe.GetManifestFieldGroupIDs(manifest, cpe.PrimaryJavaManifestGroupIDFields) + // assumes that primaryJavaManifestNameFields are ordered by priority + if len(groupIDS) != 0 { + return groupIDS[0] + } + + groupIDS = cpe.GetManifestFieldGroupIDs(manifest, cpe.SecondaryJavaManifestGroupIDFields) + + if len(groupIDS) != 0 { + return groupIDS[0] + } + + return groupID +} + +func groupIDFromPomProperties(properties *pkg.PomProperties) (groupID string) { + if properties == nil { + return groupID + } + + if properties.GroupID != "" { + return cleanGroupID(properties.GroupID) + } + + // sometimes the publisher puts the group ID in the artifact ID field unintentionally + if looksLikeGroupID(properties.ArtifactID) { + // there is a strong indication that the artifact ID is really a group ID + return cleanGroupID(properties.ArtifactID) + } + + return groupID +} + +func groupIDFromPomProject(project *pkg.PomProject) (groupID string) { + if project == nil { + return groupID + } + + // check the project details + if project.GroupID != "" { + return cleanGroupID(project.GroupID) + } + + // sometimes the publisher puts the group ID in the artifact ID field unintentionally + if looksLikeGroupID(project.ArtifactID) { + // there is a strong indication that the artifact ID is really a group ID + return cleanGroupID(project.ArtifactID) + } + + // let's check the parent details + // if the current project does not have a group ID, but the parent does, we'll use the parent's group ID + if project.Parent != nil { + if project.Parent.GroupID != "" { + return cleanGroupID(project.Parent.GroupID) + } + + // sometimes the publisher puts the group ID in the artifact ID field unintentionally + if looksLikeGroupID(project.Parent.ArtifactID) { + // there is a strong indication that the artifact ID is really a group ID + return cleanGroupID(project.Parent.ArtifactID) + } + } + + return groupID +} +func looksLikeGroupID(value string) bool { + return strings.Contains(value, ".") +} + +func cleanGroupID(groupID string) string { + return strings.TrimSpace(removeOSCIDirectives(groupID)) +} + +func removeOSCIDirectives(groupID string) string { + // for example: + // org.bar;uses:=“org.foo” -> org.bar + // more about OSGI directives see https://spring.io/blog/2008/10/20/understanding-the-osgi-uses-directive/ + return strings.Split(groupID, ";")[0] +} diff --git a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/parse_pom_xml.go b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/parse_pom_xml.go index 704cb559..a6f5340c 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/vendor/github.com/anchore/syft/syft/pkg/cataloger/java/parse_pom_xml.go @@ -1,6 +1,7 @@ package java import ( + "bytes" "encoding/xml" "fmt" "io" @@ -8,9 +9,11 @@ import ( "regexp" "strings" + "github.com/saintfish/chardet" "github.com/vifraa/gopom" "golang.org/x/net/html/charset" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -58,12 +61,13 @@ func newPomProject(path string, p gopom.Project) *pkg.PomProject { artifactID := safeString(p.ArtifactID) name := safeString(p.Name) projectURL := safeString(p.URL) + log.WithFields("path", path, "artifactID", artifactID, "name", name, "projectURL", projectURL).Trace("parsing pom.xml") return &pkg.PomProject{ Path: path, Parent: pomParent(p, p.Parent), - GroupID: resolveProperty(p, p.GroupID), + GroupID: resolveProperty(p, p.GroupID, "groupId"), ArtifactID: artifactID, - Version: resolveProperty(p, p.Version), + Version: resolveProperty(p, p.Version, "version"), Name: name, Description: cleanDescription(p.Description), URL: projectURL, @@ -73,14 +77,14 @@ func newPomProject(path string, p gopom.Project) *pkg.PomProject { func newPackageFromPom(pom gopom.Project, dep gopom.Dependency, locations ...file.Location) pkg.Package { m := pkg.JavaMetadata{ PomProperties: &pkg.PomProperties{ - GroupID: resolveProperty(pom, dep.GroupID), - ArtifactID: resolveProperty(pom, dep.ArtifactID), - Scope: resolveProperty(pom, dep.Scope), + GroupID: resolveProperty(pom, dep.GroupID, "groupId"), + ArtifactID: resolveProperty(pom, dep.ArtifactID, "artifactId"), + Scope: resolveProperty(pom, dep.Scope, "scope"), }, } name := safeString(dep.ArtifactID) - version := resolveProperty(pom, dep.Version) + version := resolveProperty(pom, dep.Version, "version") p := pkg.Package{ Name: name, @@ -99,9 +103,15 @@ func newPackageFromPom(pom gopom.Project, dep gopom.Dependency, locations ...fil } func decodePomXML(content io.Reader) (project gopom.Project, err error) { - decoder := xml.NewDecoder(content) - // prevent against warnings for "xml: encoding "iso-8859-1" declared but Decoder.CharsetReader is nil" + inputReader, err := getUtf8Reader(content) + if err != nil { + return project, fmt.Errorf("unable to read pom.xml: %w", err) + } + + decoder := xml.NewDecoder(inputReader) + // when an xml file has a character set declaration (e.g. '') read that and use the correct decoder decoder.CharsetReader = charset.NewReaderLabel + if err := decoder.Decode(&project); err != nil { return project, fmt.Errorf("unable to unmarshal pom.xml: %w", err) } @@ -109,6 +119,33 @@ func decodePomXML(content io.Reader) (project gopom.Project, err error) { return project, nil } +func getUtf8Reader(content io.Reader) (io.Reader, error) { + pomContents, err := io.ReadAll(content) + if err != nil { + return nil, err + } + + detector := chardet.NewTextDetector() + detection, err := detector.DetectBest(pomContents) + + var inputReader io.Reader + if err == nil && detection != nil { + if detection.Charset == "UTF-8" { + inputReader = bytes.NewReader(pomContents) + } else { + inputReader, err = charset.NewReaderLabel(detection.Charset, bytes.NewReader(pomContents)) + if err != nil { + return nil, fmt.Errorf("unable to get encoding: %w", err) + } + } + } else { + // we could not detect the encoding, but we want a valid file to read. Replace unreadable + // characters with the UTF-8 replacement character. + inputReader = strings.NewReader(strings.ToValidUTF8(string(pomContents), "�")) + } + return inputReader, nil +} + func pomParent(pom gopom.Project, parent *gopom.Parent) (result *pkg.PomParent) { if parent == nil { return nil @@ -116,9 +153,9 @@ func pomParent(pom gopom.Project, parent *gopom.Parent) (result *pkg.PomParent) artifactID := safeString(parent.ArtifactID) result = &pkg.PomParent{ - GroupID: resolveProperty(pom, parent.GroupID), + GroupID: resolveProperty(pom, parent.GroupID, "groupId"), ArtifactID: artifactID, - Version: resolveProperty(pom, parent.Version), + Version: resolveProperty(pom, parent.Version, "version"), } if result.GroupID == "" && result.ArtifactID == "" && result.Version == "" { @@ -147,10 +184,11 @@ func cleanDescription(original *string) (cleaned string) { // If no match is found, the entire expression including ${} is returned // //nolint:gocognit -func resolveProperty(pom gopom.Project, property *string) string { +func resolveProperty(pom gopom.Project, property *string, propertyName string) string { propertyCase := safeString(property) + log.WithFields("existingPropertyValue", propertyCase, "propertyName", propertyName).Trace("resolving property") return propertyMatcher.ReplaceAllStringFunc(propertyCase, func(match string) string { - propertyName := strings.TrimSpace(match[2 : len(match)-1]) + propertyName := strings.TrimSpace(match[2 : len(match)-1]) // remove leading ${ and trailing } entries := pomProperties(pom) if value, ok := entries[propertyName]; ok { return value @@ -172,14 +210,26 @@ func resolveProperty(pom gopom.Project, property *string) string { for fieldNum := 0; fieldNum < pomValueType.NumField(); fieldNum++ { f := pomValueType.Field(fieldNum) tag := f.Tag.Get("xml") - tag = strings.TrimSuffix(tag, ",omitempty") + tag = strings.Split(tag, ",")[0] + // a segment of the property name matches the xml tag for the field, + // so we need to recurse down the nested structs or return a match + // if we're done. if part == tag { pomValue = pomValue.Field(fieldNum) pomValueType = pomValue.Type() if pomValueType.Kind() == reflect.Ptr { + // we were recursing down the nested structs, but one of the steps + // we need to take is a nil pointer, so give up and return the original match + if pomValue.IsNil() { + return match + } pomValue = pomValue.Elem() - pomValueType = pomValue.Type() + if !pomValue.IsZero() { + // we found a non-zero value whose tag matches this part of the property name + pomValueType = pomValue.Type() + } } + // If this was the last part of the property name, return the value if partNum == numParts-1 { return fmt.Sprintf("%v", pomValue.Interface()) } diff --git a/vendor/github.com/anchore/syft/syft/pkg/golang_metadata.go b/vendor/github.com/anchore/syft/syft/pkg/golang_metadata.go index fd730958..791b7fcc 100644 --- a/vendor/github.com/anchore/syft/syft/pkg/golang_metadata.go +++ b/vendor/github.com/anchore/syft/syft/pkg/golang_metadata.go @@ -7,6 +7,7 @@ type GolangBinMetadata struct { Architecture string `json:"architecture" cyclonedx:"architecture"` H1Digest string `json:"h1Digest,omitempty" cyclonedx:"h1Digest"` MainModule string `json:"mainModule,omitempty" cyclonedx:"mainModule"` + GoCryptoSettings []string `json:"goCryptoSettings,omitempty" cyclonedx:"goCryptoSettings"` } // GolangModMetadata represents all captured data for a Golang source scan with go.mod/go.sum diff --git a/vendor/github.com/google/uuid/.travis.yml b/vendor/github.com/google/uuid/.travis.yml deleted file mode 100644 index d8156a60..00000000 --- a/vendor/github.com/google/uuid/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: go - -go: - - 1.4.3 - - 1.5.3 - - tip - -script: - - go test -v ./... diff --git a/vendor/github.com/google/uuid/CHANGELOG.md b/vendor/github.com/google/uuid/CHANGELOG.md new file mode 100644 index 00000000..2bd78667 --- /dev/null +++ b/vendor/github.com/google/uuid/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## [1.3.1](https://github.com/google/uuid/compare/v1.3.0...v1.3.1) (2023-08-18) + + +### Bug Fixes + +* Use .EqualFold() to parse urn prefixed UUIDs ([#118](https://github.com/google/uuid/issues/118)) ([574e687](https://github.com/google/uuid/commit/574e6874943741fb99d41764c705173ada5293f0)) + +## Changelog diff --git a/vendor/github.com/google/uuid/CONTRIBUTING.md b/vendor/github.com/google/uuid/CONTRIBUTING.md index 04fdf09f..55668887 100644 --- a/vendor/github.com/google/uuid/CONTRIBUTING.md +++ b/vendor/github.com/google/uuid/CONTRIBUTING.md @@ -2,6 +2,22 @@ We definitely welcome patches and contribution to this project! +### Tips + +Commits must be formatted according to the [Conventional Commits Specification](https://www.conventionalcommits.org). + +Always try to include a test case! If it is not possible or not necessary, +please explain why in the pull request description. + +### Releasing + +Commits that would precipitate a SemVer change, as desrcibed in the Conventional +Commits Specification, will trigger [`release-please`](https://github.com/google-github-actions/release-please-action) +to create a release candidate pull request. Once submitted, `release-please` +will create a release. + +For tips on how to work with `release-please`, see its documentation. + ### Legal requirements In order to protect both you and ourselves, you will need to sign the diff --git a/vendor/github.com/google/uuid/README.md b/vendor/github.com/google/uuid/README.md index f765a46f..3e9a6188 100644 --- a/vendor/github.com/google/uuid/README.md +++ b/vendor/github.com/google/uuid/README.md @@ -1,6 +1,6 @@ -# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master) +# uuid The uuid package generates and inspects UUIDs based on -[RFC 4122](http://tools.ietf.org/html/rfc4122) +[RFC 4122](https://datatracker.ietf.org/doc/html/rfc4122) and DCE 1.1: Authentication and Security Services. This package is based on the github.com/pborman/uuid package (previously named @@ -9,10 +9,12 @@ a UUID is a 16 byte array rather than a byte slice. One loss due to this change is the ability to represent an invalid UUID (vs a NIL UUID). ###### Install -`go get github.com/google/uuid` +```sh +go get github.com/google/uuid +``` ###### Documentation -[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid) +[![Go Reference](https://pkg.go.dev/badge/github.com/google/uuid.svg)](https://pkg.go.dev/github.com/google/uuid) Full `go doc` style documentation for the package can be viewed online without installing this package by using the GoDoc site here: diff --git a/vendor/github.com/google/uuid/node_js.go b/vendor/github.com/google/uuid/node_js.go index 24b78edc..b2a0bc87 100644 --- a/vendor/github.com/google/uuid/node_js.go +++ b/vendor/github.com/google/uuid/node_js.go @@ -7,6 +7,6 @@ package uuid // getHardwareInterface returns nil values for the JS version of the code. -// This remvoves the "net" dependency, because it is not used in the browser. +// This removes the "net" dependency, because it is not used in the browser. // Using the "net" library inflates the size of the transpiled JS code by 673k bytes. func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go index a57207ae..a56138cc 100644 --- a/vendor/github.com/google/uuid/uuid.go +++ b/vendor/github.com/google/uuid/uuid.go @@ -69,7 +69,7 @@ func Parse(s string) (UUID, error) { // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx case 36 + 9: - if strings.ToLower(s[:9]) != "urn:uuid:" { + if !strings.EqualFold(s[:9], "urn:uuid:") { return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) } s = s[9:] @@ -101,7 +101,8 @@ func Parse(s string) (UUID, error) { 9, 11, 14, 16, 19, 21, - 24, 26, 28, 30, 32, 34} { + 24, 26, 28, 30, 32, 34, + } { v, ok := xtob(s[x], s[x+1]) if !ok { return uuid, errors.New("invalid UUID format") @@ -117,7 +118,7 @@ func ParseBytes(b []byte) (UUID, error) { switch len(b) { case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { + if !bytes.EqualFold(b[:9], []byte("urn:uuid:")) { return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) } b = b[9:] @@ -145,7 +146,8 @@ func ParseBytes(b []byte) (UUID, error) { 9, 11, 14, 16, 19, 21, - 24, 26, 28, 30, 32, 34} { + 24, 26, 28, 30, 32, 34, + } { v, ok := xtob(b[x], b[x+1]) if !ok { return uuid, errors.New("invalid UUID format") diff --git a/vendor/github.com/jinzhu/copier/.gitignore b/vendor/github.com/jinzhu/copier/.gitignore new file mode 100644 index 00000000..6d742b37 --- /dev/null +++ b/vendor/github.com/jinzhu/copier/.gitignore @@ -0,0 +1,2 @@ +.idea/ +ttt/ diff --git a/vendor/github.com/jinzhu/copier/README.md b/vendor/github.com/jinzhu/copier/README.md index ec04b4be..079dc578 100644 --- a/vendor/github.com/jinzhu/copier/README.md +++ b/vendor/github.com/jinzhu/copier/README.md @@ -30,7 +30,7 @@ type User struct { Name string Role string Age int32 - EmployeCode int64 `copier:"EmployeNum"` // specify field name + EmployeeCode int64 `copier:"EmployeeNum"` // specify field name // Explicitly ignored in the destination struct. Salary int @@ -53,7 +53,7 @@ type Employee struct { Salary int `copier:"-"` DoubleAge int32 - EmployeId int64 `copier:"EmployeNum"` // specify field name + EmployeeId int64 `copier:"EmployeeNum"` // specify field name SuperRole string } @@ -86,7 +86,7 @@ func main() { fmt.Printf("%#v \n", employees) // []Employee{ - // {Name: "Jinzhu", Age: 18, Salary:0, DoubleAge: 36, EmployeId: 0, SuperRole: "Super Admin"} + // {Name: "Jinzhu", Age: 18, Salary:0, DoubleAge: 36, EmployeeId: 0, SuperRole: "Super Admin"} // } // Copy slice to slice @@ -95,8 +95,8 @@ func main() { fmt.Printf("%#v \n", employees) // []Employee{ - // {Name: "Jinzhu", Age: 18, Salary:0, DoubleAge: 36, EmployeId: 0, SuperRole: "Super Admin"}, - // {Name: "jinzhu 2", Age: 30, Salary:0, DoubleAge: 60, EmployeId: 0, SuperRole: "Super Dev"}, + // {Name: "Jinzhu", Age: 18, Salary:0, DoubleAge: 36, EmployeeId: 0, SuperRole: "Super Admin"}, + // {Name: "jinzhu 2", Age: 30, Salary:0, DoubleAge: 60, EmployeeId: 0, SuperRole: "Super Dev"}, // } // Copy map to map diff --git a/vendor/github.com/jinzhu/copier/copier.go b/vendor/github.com/jinzhu/copier/copier.go index 6dc9600c..43a14f1a 100644 --- a/vendor/github.com/jinzhu/copier/copier.go +++ b/vendor/github.com/jinzhu/copier/copier.go @@ -3,10 +3,10 @@ package copier import ( "database/sql" "database/sql/driver" - "errors" "fmt" "reflect" "strings" + "sync" "unicode" ) @@ -37,15 +37,35 @@ const ( type Option struct { // setting this value to true will ignore copying zero values of all the fields, including bools, as well as a // struct having all it's fields set to their zero values respectively (see IsZero() in reflect/value.go) - IgnoreEmpty bool - DeepCopy bool - Converters []TypeConverter + IgnoreEmpty bool + CaseSensitive bool + DeepCopy bool + Converters []TypeConverter + // Custom field name mappings to copy values with different names in `fromValue` and `toValue` types. + // Examples can be found in `copier_field_name_mapping_test.go`. + FieldNameMapping []FieldNameMapping +} + +func (opt Option) converters() map[converterPair]TypeConverter { + var converters = map[converterPair]TypeConverter{} + + // save converters into map for faster lookup + for i := range opt.Converters { + pair := converterPair{ + SrcType: reflect.TypeOf(opt.Converters[i].SrcType), + DstType: reflect.TypeOf(opt.Converters[i].DstType), + } + + converters[pair] = opt.Converters[i] + } + + return converters } type TypeConverter struct { SrcType interface{} DstType interface{} - Fn func(src interface{}) (interface{}, error) + Fn func(src interface{}) (dst interface{}, err error) } type converterPair struct { @@ -53,6 +73,27 @@ type converterPair struct { DstType reflect.Type } +func (opt Option) fieldNameMapping() map[converterPair]FieldNameMapping { + var mapping = map[converterPair]FieldNameMapping{} + + for i := range opt.FieldNameMapping { + pair := converterPair{ + SrcType: reflect.TypeOf(opt.FieldNameMapping[i].SrcType), + DstType: reflect.TypeOf(opt.FieldNameMapping[i].DstType), + } + + mapping[pair] = opt.FieldNameMapping[i] + } + + return mapping +} + +type FieldNameMapping struct { + SrcType interface{} + DstType interface{} + Mapping map[string]string +} + // Tag Flags type flags struct { BitFlags map[string]uint8 @@ -82,23 +123,10 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) amount = 1 from = indirect(reflect.ValueOf(fromValue)) to = indirect(reflect.ValueOf(toValue)) - converters map[converterPair]TypeConverter + converters = opt.converters() + mappings = opt.fieldNameMapping() ) - // save convertes into map for faster lookup - for i := range opt.Converters { - if converters == nil { - converters = make(map[converterPair]TypeConverter) - } - - pair := converterPair{ - SrcType: reflect.TypeOf(opt.Converters[i].SrcType), - DstType: reflect.TypeOf(opt.Converters[i].DstType), - } - - converters[pair] = opt.Converters[i] - } - if !to.CanAddr() { return ErrInvalidCopyDestination } @@ -147,7 +175,11 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) for _, k := range from.MapKeys() { toKey := indirect(reflect.New(toType.Key())) - if !set(toKey, k, opt.DeepCopy, converters) { + isSet, err := set(toKey, k, opt.DeepCopy, converters) + if err != nil { + return err + } + if !isSet { return fmt.Errorf("%w map, old key: %v, new key: %v", ErrNotSupported, k.Type(), toType.Key()) } @@ -156,7 +188,11 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) elemType, _ = indirectType(elemType) } toValue := indirect(reflect.New(elemType)) - if !set(toValue, from.MapIndex(k), opt.DeepCopy, converters) { + isSet, err = set(toValue, from.MapIndex(k), opt.DeepCopy, converters) + if err != nil { + return err + } + if !isSet { if err = copier(toValue.Addr().Interface(), from.MapIndex(k).Interface(), opt); err != nil { return err } @@ -174,26 +210,30 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) return } - if from.Kind() == reflect.Slice && to.Kind() == reflect.Slice && fromType.ConvertibleTo(toType) { + if from.Kind() == reflect.Slice && to.Kind() == reflect.Slice { if to.IsNil() { slice := reflect.MakeSlice(reflect.SliceOf(to.Type().Elem()), from.Len(), from.Cap()) to.Set(slice) } - - for i := 0; i < from.Len(); i++ { - if to.Len() < i+1 { - to.Set(reflect.Append(to, reflect.New(to.Type().Elem()).Elem())) - } - - if !set(to.Index(i), from.Index(i), opt.DeepCopy, converters) { - // ignore error while copy slice element - err = copier(to.Index(i).Addr().Interface(), from.Index(i).Interface(), opt) + if fromType.ConvertibleTo(toType) { + for i := 0; i < from.Len(); i++ { + if to.Len() < i+1 { + to.Set(reflect.Append(to, reflect.New(to.Type().Elem()).Elem())) + } + isSet, err := set(to.Index(i), from.Index(i), opt.DeepCopy, converters) if err != nil { - continue + return err + } + if !isSet { + // ignore error while copy slice element + err = copier(to.Index(i).Addr().Interface(), from.Index(i).Interface(), opt) + if err != nil { + continue + } } } + return } - return } if fromType.Kind() != reflect.Struct || toType.Kind() != reflect.Struct { @@ -201,6 +241,13 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) return } + if len(converters) > 0 { + if ok, e := set(to, from, opt.DeepCopy, converters); e == nil && ok { + // converter supported + return + } + } + if from.Kind() == reflect.Slice || to.Kind() == reflect.Slice { isSlice = true if from.Kind() == reflect.Slice { @@ -225,6 +272,27 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) dest = indirect(to) } + if len(converters) > 0 { + if ok, e := set(dest, source, opt.DeepCopy, converters); e == nil && ok { + if isSlice { + // FIXME: maybe should check the other types? + if to.Type().Elem().Kind() == reflect.Ptr { + to.Index(i).Set(dest.Addr()) + } else { + if to.Len() < i+1 { + reflect.Append(to, dest) + } else { + to.Index(i).Set(dest) + } + } + } else { + to.Set(dest) + } + + continue + } + } + destKind := dest.Kind() initDest := false if destKind == reflect.Interface { @@ -248,19 +316,22 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) name := field.Name // Get bit flags for field - fieldFlags, _ := flgs.BitFlags[name] + fieldFlags := flgs.BitFlags[name] // Check if we should ignore copying if (fieldFlags & tagIgnore) != 0 { continue } - srcFieldName, destFieldName := getFieldName(name, flgs) - if fromField := source.FieldByName(srcFieldName); fromField.IsValid() && !shouldIgnore(fromField, opt.IgnoreEmpty) { + fieldNamesMapping := getFieldNamesMapping(mappings, fromType, toType) + + srcFieldName, destFieldName := getFieldName(name, flgs, fieldNamesMapping) + if fromField := fieldByNameOrZeroValue(source, srcFieldName); fromField.IsValid() && !shouldIgnore(fromField, opt.IgnoreEmpty) { // process for nested anonymous field destFieldNotSet := false if f, ok := dest.Type().FieldByName(destFieldName); ok { - for idx := range f.Index { + // only initialize parent embedded struct pointer in the path + for idx := range f.Index[:len(f.Index)-1] { destField := dest.FieldByIndex(f.Index[:idx+1]) if destField.Kind() != reflect.Ptr { @@ -285,10 +356,14 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) break } - toField := dest.FieldByName(destFieldName) + toField := fieldByName(dest, destFieldName, opt.CaseSensitive) if toField.IsValid() { if toField.CanSet() { - if !set(toField, fromField, opt.DeepCopy, converters) { + isSet, err := set(toField, fromField, opt.DeepCopy, converters) + if err != nil { + return err + } + if !isSet { if err := copier(toField.Addr().Interface(), fromField.Interface(), opt); err != nil { return err } @@ -317,7 +392,7 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) // Copy from from method to dest field for _, field := range deepFields(toType) { name := field.Name - srcFieldName, destFieldName := getFieldName(name, flgs) + srcFieldName, destFieldName := getFieldName(name, flgs, getFieldNamesMapping(mappings, fromType, toType)) var fromMethod reflect.Value if source.CanAddr() { @@ -327,7 +402,7 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) } if fromMethod.IsValid() && fromMethod.Type().NumIn() == 0 && fromMethod.Type().NumOut() == 1 && !shouldIgnore(fromMethod, opt.IgnoreEmpty) { - if toField := dest.FieldByName(destFieldName); toField.IsValid() && toField.CanSet() { + if toField := fieldByName(dest, destFieldName, opt.CaseSensitive); toField.IsValid() && toField.CanSet() { values := fromMethod.Call([]reflect.Value{}) if len(values) >= 1 { set(toField, values[0], opt.DeepCopy, converters) @@ -342,7 +417,11 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) if to.Len() < i+1 { to.Set(reflect.Append(to, dest.Addr())) } else { - if !set(to.Index(i), dest.Addr(), opt.DeepCopy, converters) { + isSet, err := set(to.Index(i), dest.Addr(), opt.DeepCopy, converters) + if err != nil { + return err + } + if !isSet { // ignore error while copy slice element err = copier(to.Index(i).Addr().Interface(), dest.Addr().Interface(), opt) if err != nil { @@ -354,7 +433,11 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) if to.Len() < i+1 { to.Set(reflect.Append(to, dest)) } else { - if !set(to.Index(i), dest, opt.DeepCopy, converters) { + isSet, err := set(to.Index(i), dest, opt.DeepCopy, converters) + if err != nil { + return err + } + if !isSet { // ignore error while copy slice element err = copier(to.Index(i).Addr().Interface(), dest.Interface(), opt) if err != nil { @@ -373,6 +456,31 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) return } +func getFieldNamesMapping(mappings map[converterPair]FieldNameMapping, fromType reflect.Type, toType reflect.Type) map[string]string { + var fieldNamesMapping map[string]string + + if len(mappings) > 0 { + pair := converterPair{ + SrcType: fromType, + DstType: toType, + } + if v, ok := mappings[pair]; ok { + fieldNamesMapping = v.Mapping + } + } + return fieldNamesMapping +} + +func fieldByNameOrZeroValue(source reflect.Value, fieldName string) (value reflect.Value) { + defer func() { + if err := recover(); err != nil { + value = reflect.Value{} + } + }() + + return source.FieldByName(fieldName) +} + func copyUnexportedStructFields(to, from reflect.Value) { if from.Kind() != reflect.Struct || to.Kind() != reflect.Struct || !from.Type().AssignableTo(to.Type()) { return @@ -392,14 +500,20 @@ func copyUnexportedStructFields(to, from reflect.Value) { } func shouldIgnore(v reflect.Value, ignoreEmpty bool) bool { - if !ignoreEmpty { - return false - } - - return v.IsZero() + return ignoreEmpty && v.IsZero() } +var deepFieldsLock sync.RWMutex +var deepFieldsMap = make(map[reflect.Type][]reflect.StructField) + func deepFields(reflectType reflect.Type) []reflect.StructField { + deepFieldsLock.RLock() + cache, ok := deepFieldsMap[reflectType] + deepFieldsLock.RUnlock() + if ok { + return cache + } + var res []reflect.StructField if reflectType, _ = indirectType(reflectType); reflectType.Kind() == reflect.Struct { fields := make([]reflect.StructField, 0, reflectType.NumField()) @@ -416,11 +530,13 @@ func deepFields(reflectType reflect.Type) []reflect.StructField { } } } - - return fields + res = fields } - return nil + deepFieldsLock.Lock() + deepFieldsMap[reflectType] = res + deepFieldsLock.Unlock() + return res } func indirect(reflectValue reflect.Value) reflect.Value { @@ -438,98 +554,101 @@ func indirectType(reflectType reflect.Type) (_ reflect.Type, isPtr bool) { return reflectType, isPtr } -func set(to, from reflect.Value, deepCopy bool, converters map[converterPair]TypeConverter) bool { - if from.IsValid() { - if ok, err := lookupAndCopyWithConverter(to, from, converters); err != nil { - return false - } else if ok { - return true - } - - if to.Kind() == reflect.Ptr { - // set `to` to nil if from is nil - if from.Kind() == reflect.Ptr && from.IsNil() { - to.Set(reflect.Zero(to.Type())) - return true - } else if to.IsNil() { - // `from` -> `to` - // sql.NullString -> *string - if fromValuer, ok := driverValuer(from); ok { - v, err := fromValuer.Value() - if err != nil { - return false - } - // if `from` is not valid do nothing with `to` - if v == nil { - return true - } +func set(to, from reflect.Value, deepCopy bool, converters map[converterPair]TypeConverter) (bool, error) { + if !from.IsValid() { + return true, nil + } + if ok, err := lookupAndCopyWithConverter(to, from, converters); err != nil { + return false, err + } else if ok { + return true, nil + } + + if to.Kind() == reflect.Ptr { + // set `to` to nil if from is nil + if from.Kind() == reflect.Ptr && from.IsNil() { + to.Set(reflect.Zero(to.Type())) + return true, nil + } else if to.IsNil() { + // `from` -> `to` + // sql.NullString -> *string + if fromValuer, ok := driverValuer(from); ok { + v, err := fromValuer.Value() + if err != nil { + return true, nil + } + // if `from` is not valid do nothing with `to` + if v == nil { + return true, nil } - // allocate new `to` variable with default value (eg. *string -> new(string)) - to.Set(reflect.New(to.Type().Elem())) } - // depointer `to` - to = to.Elem() + // allocate new `to` variable with default value (eg. *string -> new(string)) + to.Set(reflect.New(to.Type().Elem())) } + // depointer `to` + to = to.Elem() + } - if deepCopy { - toKind := to.Kind() - if toKind == reflect.Interface && to.IsNil() { - if reflect.TypeOf(from.Interface()) != nil { - to.Set(reflect.New(reflect.TypeOf(from.Interface())).Elem()) - toKind = reflect.TypeOf(to.Interface()).Kind() - } - } - if from.Kind() == reflect.Ptr && from.IsNil() { - return true - } - if toKind == reflect.Struct || toKind == reflect.Map || toKind == reflect.Slice { - return false + if deepCopy { + toKind := to.Kind() + if toKind == reflect.Interface && to.IsNil() { + if reflect.TypeOf(from.Interface()) != nil { + to.Set(reflect.New(reflect.TypeOf(from.Interface())).Elem()) + toKind = reflect.TypeOf(to.Interface()).Kind() } } + if from.Kind() == reflect.Ptr && from.IsNil() { + return true, nil + } + if _, ok := to.Addr().Interface().(sql.Scanner); !ok && (toKind == reflect.Struct || toKind == reflect.Map || toKind == reflect.Slice) { + return false, nil + } + } - if from.Type().ConvertibleTo(to.Type()) { - to.Set(from.Convert(to.Type())) - } else if toScanner, ok := to.Addr().Interface().(sql.Scanner); ok { - // `from` -> `to` - // *string -> sql.NullString - if from.Kind() == reflect.Ptr { - // if `from` is nil do nothing with `to` - if from.IsNil() { - return true - } - // depointer `from` - from = indirect(from) + if from.Type().ConvertibleTo(to.Type()) { + to.Set(from.Convert(to.Type())) + } else if toScanner, ok := to.Addr().Interface().(sql.Scanner); ok { + // `from` -> `to` + // *string -> sql.NullString + if from.Kind() == reflect.Ptr { + // if `from` is nil do nothing with `to` + if from.IsNil() { + return true, nil } - // `from` -> `to` - // string -> sql.NullString - // set `to` by invoking method Scan(`from`) - err := toScanner.Scan(from.Interface()) - if err != nil { - return false - } - } else if fromValuer, ok := driverValuer(from); ok { - // `from` -> `to` - // sql.NullString -> string - v, err := fromValuer.Value() - if err != nil { - return false - } - // if `from` is not valid do nothing with `to` - if v == nil { - return true - } - rv := reflect.ValueOf(v) - if rv.Type().AssignableTo(to.Type()) { - to.Set(rv) - } - } else if from.Kind() == reflect.Ptr { - return set(to, from.Elem(), deepCopy, converters) - } else { - return false + // depointer `from` + from = indirect(from) } + // `from` -> `to` + // string -> sql.NullString + // set `to` by invoking method Scan(`from`) + err := toScanner.Scan(from.Interface()) + if err != nil { + return false, nil + } + } else if fromValuer, ok := driverValuer(from); ok { + // `from` -> `to` + // sql.NullString -> string + v, err := fromValuer.Value() + if err != nil { + return false, nil + } + // if `from` is not valid do nothing with `to` + if v == nil { + return true, nil + } + rv := reflect.ValueOf(v) + if rv.Type().AssignableTo(to.Type()) { + to.Set(rv) + } else if to.CanSet() && rv.Type().ConvertibleTo(to.Type()) { + to.Set(rv.Convert(to.Type())) + } + } else if from.Kind() == reflect.Ptr { + return set(to, from.Elem(), deepCopy, converters) + } else { + return false, nil } - return true + return true, nil } // lookupAndCopyWithConverter looks up the type pair, on success the TypeConverter Fn func is called to copy src to dst field. @@ -541,7 +660,6 @@ func lookupAndCopyWithConverter(to, from reflect.Value, converters map[converter if cnv, ok := converters[pair]; ok { result, err := cnv.Fn(from.Interface()) - if err != nil { return false, err } @@ -574,7 +692,7 @@ func parseTags(tag string) (flg uint8, name string, err error) { if unicode.IsUpper([]rune(t)[0]) { name = strings.TrimSpace(t) } else { - err = errors.New("copier field name tag must be start upper case") + err = ErrFieldNameTagStartNotUpperCase } } } @@ -651,8 +769,14 @@ func checkBitFlags(flagsList map[string]uint8) (err error) { return } -func getFieldName(fieldName string, flgs flags) (srcFieldName string, destFieldName string) { +func getFieldName(fieldName string, flgs flags, fieldNameMapping map[string]string) (srcFieldName string, destFieldName string) { // get dest field name + if name, ok := fieldNameMapping[fieldName]; ok { + srcFieldName = fieldName + destFieldName = name + return + } + if srcTagName, ok := flgs.SrcNames.FieldNameToTag[fieldName]; ok { destFieldName = srcTagName if destTagName, ok := flgs.DestNames.TagToFieldName[srcTagName]; ok { @@ -686,7 +810,6 @@ func getFieldName(fieldName string, flgs flags) (srcFieldName string, destFieldN } func driverValuer(v reflect.Value) (i driver.Valuer, ok bool) { - if !v.CanAddr() { i, ok = v.Interface().(driver.Valuer) return @@ -695,3 +818,11 @@ func driverValuer(v reflect.Value) (i driver.Valuer, ok bool) { i, ok = v.Addr().Interface().(driver.Valuer) return } + +func fieldByName(v reflect.Value, name string, caseSensitive bool) reflect.Value { + if caseSensitive { + return v.FieldByName(name) + } + + return v.FieldByNameFunc(func(n string) bool { return strings.EqualFold(n, name) }) +} diff --git a/vendor/github.com/jinzhu/copier/errors.go b/vendor/github.com/jinzhu/copier/errors.go index cf7c5e74..f50ea32b 100644 --- a/vendor/github.com/jinzhu/copier/errors.go +++ b/vendor/github.com/jinzhu/copier/errors.go @@ -3,8 +3,9 @@ package copier import "errors" var ( - ErrInvalidCopyDestination = errors.New("copy destination is invalid") - ErrInvalidCopyFrom = errors.New("copy from is invalid") - ErrMapKeyNotMatch = errors.New("map's key type doesn't match") - ErrNotSupported = errors.New("not supported") + ErrInvalidCopyDestination = errors.New("copy destination must be non-nil and addressable") + ErrInvalidCopyFrom = errors.New("copy from must be non-nil and addressable") + ErrMapKeyNotMatch = errors.New("map's key type doesn't match") + ErrNotSupported = errors.New("not supported") + ErrFieldNameTagStartNotUpperCase = errors.New("copier field name tag must be start upper case") ) diff --git a/vendor/github.com/kastenhq/goversion/LICENSE b/vendor/github.com/kastenhq/goversion/LICENSE new file mode 100644 index 00000000..6a66aea5 --- /dev/null +++ b/vendor/github.com/kastenhq/goversion/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/kastenhq/goversion/version/asm.go b/vendor/github.com/kastenhq/goversion/version/asm.go new file mode 100644 index 00000000..3d5122b0 --- /dev/null +++ b/vendor/github.com/kastenhq/goversion/version/asm.go @@ -0,0 +1,349 @@ +// Copyright 2017 The Go Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package version + +import ( + "encoding/binary" + "fmt" + "os" +) + +type matcher [][]uint32 + +const ( + pWild uint32 = 0xff00 + pAddr uint32 = 0x10000 + pEnd uint32 = 0x20000 + pRelAddr uint32 = 0x30000 + + opMaybe = 1 + iota + opMust + opDone + opAnchor = 0x100 + opSub8 = 0x200 + opFlags = opAnchor | opSub8 +) + +var amd64Matcher = matcher{ + {opMaybe | opAnchor, + // __rt0_amd64_darwin: + // JMP __rt0_amd64 + 0xe9, pWild | pAddr, pWild, pWild, pWild | pEnd, 0xcc, 0xcc, 0xcc, + }, + {opMaybe, + // _rt0_amd64_linux: + // lea 0x8(%rsp), %rsi + // mov (%rsp), %rdi + // lea ADDR(%rip), %rax # main + // jmpq *%rax + 0x48, 0x8d, 0x74, 0x24, 0x08, + 0x48, 0x8b, 0x3c, 0x24, 0x48, + 0x8d, 0x05, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0xff, 0xe0, + }, + {opMaybe, + // _rt0_amd64_linux: + // lea 0x8(%rsp), %rsi + // mov (%rsp), %rdi + // mov $ADDR, %eax # main + // jmpq *%rax + 0x48, 0x8d, 0x74, 0x24, 0x08, + 0x48, 0x8b, 0x3c, 0x24, + 0xb8, pWild | pAddr, pWild, pWild, pWild, + 0xff, 0xe0, + }, + {opMaybe, + // __rt0_amd64: + // mov (%rsp), %rdi + // lea 8(%rsp), %rsi + // jmp runtime.rt0_g0 + 0x48, 0x8b, 0x3c, 0x24, + 0x48, 0x8d, 0x74, 0x24, 0x08, + 0xe9, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0xcc, 0xcc, + }, + {opMaybe, + // _start (toward end) + // lea __libc_csu_fini(%rip), %r8 + // lea __libc_csu_init(%rip), %rcx + // lea ADDR(%rip), %rdi # main + // callq *xxx(%rip) + 0x4c, 0x8d, 0x05, pWild, pWild, pWild, pWild, + 0x48, 0x8d, 0x0d, pWild, pWild, pWild, pWild, + 0x48, 0x8d, 0x3d, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0xff, 0x15, + }, + {opMaybe, + // _start (toward end) + // push %rsp (1) + // mov $__libc_csu_fini, %r8 (7) + // mov $__libc_csu_init, %rcx (7) + // mov $ADDR, %rdi # main (7) + // callq *xxx(%rip) + 0x54, + 0x49, 0xc7, 0xc0, pWild, pWild, pWild, pWild, + 0x48, 0xc7, 0xc1, pWild, pWild, pWild, pWild, + 0x48, 0xc7, 0xc7, pAddr | pWild, pWild, pWild, pWild, + }, + {opMaybe | opAnchor, + // main: + // lea ADDR(%rip), %rax # rt0_go + // jmpq *%rax + 0x48, 0x8d, 0x05, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0xff, 0xe0, + }, + {opMaybe | opAnchor, + // main: + // mov $ADDR, %eax + // jmpq *%rax + 0xb8, pWild | pAddr, pWild, pWild, pWild, + 0xff, 0xe0, + }, + {opMaybe | opAnchor, + // main: + // JMP runtime.rt0_go(SB) + 0xe9, pWild | pAddr, pWild, pWild, pWild | pEnd, 0xcc, 0xcc, 0xcc, + }, + {opMust | opAnchor, + // rt0_go: + // mov %rdi, %rax + // mov %rsi, %rbx + // sub %0x27, %rsp + // and $0xfffffffffffffff0,%rsp + // mov %rax,0x10(%rsp) + // mov %rbx,0x18(%rsp) + 0x48, 0x89, 0xf8, + 0x48, 0x89, 0xf3, + 0x48, 0x83, 0xec, 0x27, + 0x48, 0x83, 0xe4, 0xf0, + 0x48, 0x89, 0x44, 0x24, 0x10, + 0x48, 0x89, 0x5c, 0x24, 0x18, + }, + {opMust, + // later in rt0_go: + // mov %eax, (%rsp) + // mov 0x18(%rsp), %rax + // mov %rax, 0x8(%rsp) + // callq runtime.args + // callq runtime.osinit + // callq runtime.schedinit (ADDR) + 0x89, 0x04, 0x24, + 0x48, 0x8b, 0x44, 0x24, 0x18, + 0x48, 0x89, 0x44, 0x24, 0x08, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild, pWild, pWild, pWild, + }, + {opMaybe, + // later in rt0_go: + // mov %eax, (%rsp) + // mov 0x18(%rsp), %rax + // mov %rax, 0x8(%rsp) + // callq runtime.args + // callq runtime.osinit + // callq runtime.schedinit (ADDR) + // lea other(%rip), %rdi + 0x89, 0x04, 0x24, + 0x48, 0x8b, 0x44, 0x24, 0x18, + 0x48, 0x89, 0x44, 0x24, 0x08, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0x48, 0x8d, 0x05, + }, + {opMaybe, + // later in rt0_go: + // mov %eax, (%rsp) + // mov 0x18(%rsp), %rax + // mov %rax, 0x8(%rsp) + // callq runtime.args + // callq runtime.osinit + // callq runtime.hashinit + // callq runtime.schedinit (ADDR) + // pushq $main.main + 0x89, 0x04, 0x24, + 0x48, 0x8b, 0x44, 0x24, 0x18, + 0x48, 0x89, 0x44, 0x24, 0x08, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild, pWild, pWild, pWild, + 0xe8, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0x68, + }, + {opDone | opSub8, + // schedinit (toward end) + // mov ADDR(%rip), %rax + // test %rax, %rax + // jne + // movq $0x7, ADDR(%rip) + // + 0x48, 0x8b, 0x05, pWild, pWild, pWild, pWild, + 0x48, 0x85, 0xc0, + 0x75, pWild, + 0x48, 0xc7, 0x05, pWild | pAddr, pWild, pWild, pWild, 0x07, 0x00, 0x00, 0x00 | pEnd, + }, + {opDone | opSub8, + // schedinit (toward end) + // mov ADDR(%rip), %rbx + // cmp $0x0, %rbx + // jne + // lea "unknown"(%rip), %rbx + // mov %rbx, ADDR(%rip) + // movq $7, (ADDR+8)(%rip) + 0x48, 0x8b, 0x1d, pWild, pWild, pWild, pWild, + 0x48, 0x83, 0xfb, 0x00, + 0x75, pWild, + 0x48, 0x8d, 0x1d, pWild, pWild, pWild, pWild, + 0x48, 0x89, 0x1d, pWild, pWild, pWild, pWild, + 0x48, 0xc7, 0x05, pWild | pAddr, pWild, pWild, pWild, 0x07, 0x00, 0x00, 0x00 | pEnd, + }, + {opDone, + // schedinit (toward end) + // cmpq $0x0, ADDR(%rip) + // jne + // lea "unknown"(%rip), %rax + // mov %rax, ADDR(%rip) + // lea ADDR(%rip), %rax + // movq $7, 8(%rax) + 0x48, 0x83, 0x3d, pWild | pAddr, pWild, pWild, pWild, 0x00, + 0x75, pWild, + 0x48, 0x8d, 0x05, pWild, pWild, pWild, pWild, + 0x48, 0x89, 0x05, pWild, pWild, pWild, pWild, + 0x48, 0x8d, 0x05, pWild | pAddr, pWild, pWild, pWild | pEnd, + 0x48, 0xc7, 0x40, 0x08, 0x07, 0x00, 0x00, 0x00, + }, + {opDone, + // schedinit (toward end) + // cmpq $0x0, ADDR(%rip) + // jne + // movq $0x7, ADDR(%rip) + 0x48, 0x83, 0x3d, pWild | pAddr, pWild, pWild, pWild, 0x00, + 0x75, pWild, + 0x48, 0xc7, 0x05 | pEnd, pWild | pAddr, pWild, pWild, pWild, 0x07, 0x00, 0x00, 0x00, + }, + {opDone, + // test %eax, %eax + // jne + // lea "unknown"(RIP), %rax + // mov %rax, ADDR(%rip) + 0x48, 0x85, 0xc0, 0x75, pWild, 0x48, 0x8d, 0x05, pWild, pWild, pWild, pWild, 0x48, 0x89, 0x05, pWild | pAddr, pWild, pWild, pWild | pEnd, + }, + {opDone, + // schedinit (toward end) + // mov ADDR(%rip), %rcx + // test %rcx, %rcx + // jne + // movq $0x7, ADDR(%rip) + // + 0x48, 0x8b, 0x0d, pWild, pWild, pWild, pWild, + 0x48, 0x85, 0xc9, + 0x75, pWild, + 0x48, 0xc7, 0x05 | pEnd, pWild | pAddr, pWild, pWild, pWild, 0x07, 0x00, 0x00, 0x00, + }, +} + +var DebugMatch bool + +func (m matcher) match(f exe, addr uint64) (uint64, bool) { + data, err := f.ReadData(addr, 512) + if DebugMatch { + fmt.Fprintf(os.Stderr, "data @%#x: %x\n", addr, data[:16]) + } + if err != nil { + if DebugMatch { + fmt.Fprintf(os.Stderr, "match: %v\n", err) + } + return 0, false + } + if DebugMatch { + fmt.Fprintf(os.Stderr, "data: %x\n", data[:32]) + } +Matchers: + for pc, p := range m { + op := p[0] + p = p[1:] + Search: + for i := 0; i <= len(data)-len(p); i++ { + a := -1 + e := -1 + if i > 0 && op&opAnchor != 0 { + break + } + for j := 0; j < len(p); j++ { + b := byte(p[j]) + m := byte(p[j] >> 8) + if data[i+j]&^m != b { + continue Search + } + if p[j]&pAddr != 0 { + a = j + } + if p[j]&pEnd != 0 { + e = j + 1 + } + } + // matched + if DebugMatch { + fmt.Fprintf(os.Stderr, "match (%d) %#x+%d %x %x\n", pc, addr, i, p, data[i:i+len(p)]) + } + if a != -1 { + val := uint64(int32(binary.LittleEndian.Uint32(data[i+a:]))) + if e == -1 { + addr = val + } else { + addr += uint64(i+e) + val + } + if op&opSub8 != 0 { + addr -= 8 + } + } + if op&^opFlags == opDone { + if DebugMatch { + fmt.Fprintf(os.Stderr, "done %x\n", addr) + } + return addr, true + } + if a != -1 { + // changed addr, so reload + data, err = f.ReadData(addr, 512) + if err != nil { + return 0, false + } + if DebugMatch { + fmt.Fprintf(os.Stderr, "reload @%#x: %x\n", addr, data[:32]) + } + } + continue Matchers + } + // not matched + if DebugMatch { + fmt.Fprintf(os.Stderr, "no match (%d) %#x %x %x\n", pc, addr, p, data[:32]) + } + if op&^opFlags == opMust { + return 0, false + } + } + // ran off end of matcher + return 0, false +} + +func readBuildVersionX86Asm(f exe) (isGo bool, buildVersion string) { + entry := f.Entry() + if entry == 0 { + if DebugMatch { + fmt.Fprintf(os.Stderr, "missing entry!\n") + } + return + } + addr, ok := amd64Matcher.match(f, entry) + if !ok { + return + } + v, err := readBuildVersion(f, addr, 16) + if err != nil { + return + } + return true, v +} diff --git a/vendor/github.com/kastenhq/goversion/version/exe.go b/vendor/github.com/kastenhq/goversion/version/exe.go new file mode 100644 index 00000000..61c530bd --- /dev/null +++ b/vendor/github.com/kastenhq/goversion/version/exe.go @@ -0,0 +1,297 @@ +// Copyright 2017 The Go Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package version + +import ( + "bytes" + "debug/elf" + "debug/macho" + "debug/pe" + "encoding/binary" + "fmt" + "io" +) + +type sym struct { + Name string + Addr uint64 + Size uint64 +} + +type exe interface { + AddrSize() int // bytes + ReadData(addr, size uint64) ([]byte, error) + Symbols() ([]sym, error) + SectionNames() []string + ByteOrder() binary.ByteOrder + Entry() uint64 + TextRange() (uint64, uint64) + RODataRange() (uint64, uint64) +} + +func parseExe(f io.ReaderAt) (exe, error) { + data := make([]byte, 16) + if _, err := f.ReadAt(data, 0); err != nil { + return nil, err + } + if bytes.HasPrefix(data, []byte("\x7FELF")) { + e, err := elf.NewFile(f) + if err != nil { + return nil, err + } + return &elfExe{e}, nil + } + if bytes.HasPrefix(data, []byte("MZ")) { + e, err := pe.NewFile(f) + if err != nil { + return nil, err + } + return &peExe{e}, nil + } + if bytes.HasPrefix(data, []byte("\xFE\xED\xFA")) || bytes.HasPrefix(data[1:], []byte("\xFA\xED\xFE")) { + e, err := macho.NewFile(f) + if err != nil { + return nil, err + } + return &machoExe{e}, nil + } + return nil, fmt.Errorf("unrecognized executable format") +} + +type elfExe struct { + f *elf.File +} + +func (x *elfExe) AddrSize() int { return 0 } + +func (x *elfExe) ByteOrder() binary.ByteOrder { return x.f.ByteOrder } + +func (x *elfExe) Entry() uint64 { return x.f.Entry } + +func (x *elfExe) ReadData(addr, size uint64) ([]byte, error) { + for _, prog := range x.f.Progs { + if prog.Vaddr <= addr && addr <= prog.Vaddr+prog.Filesz-1 { + n := prog.Vaddr + prog.Filesz - addr + if n > size { + n = size + } + data := make([]byte, n) + _, err := prog.ReadAt(data, int64(addr-prog.Vaddr)) + if err != nil { + return nil, err + } + return data, nil + } + } + return nil, fmt.Errorf("address not mapped") +} + +func (x *elfExe) Symbols() ([]sym, error) { + syms, err := x.f.Symbols() + if err != nil { + return nil, err + } + var out []sym + for _, s := range syms { + out = append(out, sym{s.Name, s.Value, s.Size}) + } + return out, nil +} + +func (x *elfExe) SectionNames() []string { + var names []string + for _, sect := range x.f.Sections { + names = append(names, sect.Name) + } + return names +} + +func (x *elfExe) TextRange() (uint64, uint64) { + for _, p := range x.f.Progs { + if p.Type == elf.PT_LOAD && p.Flags&elf.PF_X != 0 { + return p.Vaddr, p.Vaddr + p.Filesz + } + } + return 0, 0 +} + +func (x *elfExe) RODataRange() (uint64, uint64) { + for _, p := range x.f.Progs { + if p.Type == elf.PT_LOAD && p.Flags&(elf.PF_R|elf.PF_W|elf.PF_X) == elf.PF_R { + return p.Vaddr, p.Vaddr + p.Filesz + } + } + for _, p := range x.f.Progs { + if p.Type == elf.PT_LOAD && p.Flags&(elf.PF_R|elf.PF_W|elf.PF_X) == (elf.PF_R|elf.PF_X) { + return p.Vaddr, p.Vaddr + p.Filesz + } + } + return 0, 0 +} + +type peExe struct { + f *pe.File +} + +func (x *peExe) imageBase() uint64 { + switch oh := x.f.OptionalHeader.(type) { + case *pe.OptionalHeader32: + return uint64(oh.ImageBase) + case *pe.OptionalHeader64: + return oh.ImageBase + } + return 0 +} + +func (x *peExe) AddrSize() int { + if x.f.Machine == pe.IMAGE_FILE_MACHINE_AMD64 { + return 8 + } + return 4 +} + +func (x *peExe) ByteOrder() binary.ByteOrder { return binary.LittleEndian } + +func (x *peExe) Entry() uint64 { + switch oh := x.f.OptionalHeader.(type) { + case *pe.OptionalHeader32: + return uint64(oh.ImageBase + oh.AddressOfEntryPoint) + case *pe.OptionalHeader64: + return oh.ImageBase + uint64(oh.AddressOfEntryPoint) + } + return 0 +} + +func (x *peExe) ReadData(addr, size uint64) ([]byte, error) { + addr -= x.imageBase() + data := make([]byte, size) + for _, sect := range x.f.Sections { + if uint64(sect.VirtualAddress) <= addr && addr+size-1 <= uint64(sect.VirtualAddress+sect.Size-1) { + _, err := sect.ReadAt(data, int64(addr-uint64(sect.VirtualAddress))) + if err != nil { + return nil, err + } + return data, nil + } + } + return nil, fmt.Errorf("address not mapped") +} + +func (x *peExe) Symbols() ([]sym, error) { + base := x.imageBase() + var out []sym + for _, s := range x.f.Symbols { + if s.SectionNumber <= 0 || int(s.SectionNumber) > len(x.f.Sections) { + continue + } + sect := x.f.Sections[s.SectionNumber-1] + out = append(out, sym{s.Name, uint64(s.Value) + base + uint64(sect.VirtualAddress), 0}) + } + return out, nil +} + +func (x *peExe) SectionNames() []string { + var names []string + for _, sect := range x.f.Sections { + names = append(names, sect.Name) + } + return names +} + +func (x *peExe) TextRange() (uint64, uint64) { + // Assume text is first non-empty section. + for _, sect := range x.f.Sections { + if sect.VirtualAddress != 0 && sect.Size != 0 { + return uint64(sect.VirtualAddress) + x.imageBase(), uint64(sect.VirtualAddress+sect.Size) + x.imageBase() + } + } + return 0, 0 +} + +func (x *peExe) RODataRange() (uint64, uint64) { + return x.TextRange() +} + +type machoExe struct { + f *macho.File +} + +func (x *machoExe) AddrSize() int { + if x.f.Cpu&0x01000000 != 0 { + return 8 + } + return 4 +} + +func (x *machoExe) ByteOrder() binary.ByteOrder { return x.f.ByteOrder } + +func (x *machoExe) Entry() uint64 { + for _, load := range x.f.Loads { + b, ok := load.(macho.LoadBytes) + if !ok { + continue + } + // TODO: Other thread states. + bo := x.f.ByteOrder + const x86_THREAD_STATE64 = 4 + cmd, siz := macho.LoadCmd(bo.Uint32(b[0:4])), bo.Uint32(b[4:8]) + if cmd == macho.LoadCmdUnixThread && siz == 184 && bo.Uint32(b[8:12]) == x86_THREAD_STATE64 { + return bo.Uint64(b[144:]) + } + } + return 0 +} + +func (x *machoExe) ReadData(addr, size uint64) ([]byte, error) { + data := make([]byte, size) + for _, load := range x.f.Loads { + seg, ok := load.(*macho.Segment) + if !ok { + continue + } + if seg.Addr <= addr && addr+size-1 <= seg.Addr+seg.Filesz-1 { + if seg.Name == "__PAGEZERO" { + continue + } + _, err := seg.ReadAt(data, int64(addr-seg.Addr)) + if err != nil { + return nil, err + } + return data, nil + } + } + return nil, fmt.Errorf("address not mapped") +} + +func (x *machoExe) Symbols() ([]sym, error) { + var out []sym + for _, s := range x.f.Symtab.Syms { + out = append(out, sym{s.Name, s.Value, 0}) + } + return out, nil +} + +func (x *machoExe) SectionNames() []string { + var names []string + for _, sect := range x.f.Sections { + names = append(names, sect.Name) + } + return names +} + +func (x *machoExe) TextRange() (uint64, uint64) { + // Assume text is first non-empty segment. + for _, load := range x.f.Loads { + seg, ok := load.(*macho.Segment) + if ok && seg.Name != "__PAGEZERO" && seg.Addr != 0 && seg.Filesz != 0 { + return seg.Addr, seg.Addr + seg.Filesz + } + } + return 0, 0 +} + +func (x *machoExe) RODataRange() (uint64, uint64) { + return x.TextRange() +} diff --git a/vendor/github.com/kastenhq/goversion/version/read.go b/vendor/github.com/kastenhq/goversion/version/read.go new file mode 100644 index 00000000..ea151646 --- /dev/null +++ b/vendor/github.com/kastenhq/goversion/version/read.go @@ -0,0 +1,254 @@ +// Copyright 2017 The Go Authors. All Rights Reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package version reports the Go version used to build program executables. +package version + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +// Version is the information reported by ReadExe. +type Version struct { + Release string // Go version (runtime.Version in the program) + ModuleInfo string // program's module information + BoringCrypto bool // program uses BoringCrypto + StandardCrypto bool // program uses standard crypto (replaced by BoringCrypto) + FIPSOnly bool // program imports "crypto/tls/fipsonly" +} + +// ReadExe reports information about the Go version used to build +// the program executable named by file. +func ReadExe(file string) (Version, error) { + f, err := os.Open(file) + if err != nil { + return Version{}, err + } + defer f.Close() + + return ReadExeFromReader(f) +} + +func ReadExeFromReader(reader io.ReaderAt) (Version, error) { + var v Version + f, err := parseExe(reader) + if err != nil { + return v, err + } + isGo := false + for _, name := range f.SectionNames() { + if name == ".note.go.buildid" { + isGo = true + } + } + syms, symsErr := f.Symbols() + isGccgo := false + for _, sym := range syms { + name := sym.Name + if name == "runtime.main" || name == "main.main" { + isGo = true + } + if strings.HasPrefix(name, "runtime.") && strings.HasSuffix(name, "$descriptor") { + isGccgo = true + } + if name == "runtime.buildVersion" { + isGo = true + release, err := readBuildVersion(f, sym.Addr, sym.Size) + if err != nil { + return v, err + } + v.Release = release + } + // Note: Using strings.HasPrefix because Go 1.17+ adds ".abi0" to many of these symbols. + if strings.Contains(name, "_Cfunc__goboringcrypto_") || strings.HasPrefix(name, "crypto/internal/boring/sig.BoringCrypto") { + v.BoringCrypto = true + } + if strings.HasPrefix(name, "crypto/internal/boring/sig.FIPSOnly") { + v.FIPSOnly = true + } + for _, re := range standardCryptoNames { + if re.MatchString(name) { + v.StandardCrypto = true + } + } + if strings.HasPrefix(name, "crypto/internal/boring/sig.StandardCrypto") { + v.StandardCrypto = true + } + } + + if DebugMatch { + v.Release = "" + } + if err := findModuleInfo(&v, f); err != nil { + return v, err + } + if v.Release == "" { + g, release := readBuildVersionX86Asm(f) + if g { + isGo = true + v.Release = release + if err := findCryptoSigs(&v, f); err != nil { + return v, err + } + } + } + if isGccgo && v.Release == "" { + isGo = true + v.Release = "gccgo (version unknown)" + } + if !isGo && symsErr != nil { + return v, symsErr + } + + if !isGo { + return v, errors.New("not a Go executable") + } + if v.Release == "" { + v.Release = "unknown Go version" + } + return v, nil +} + +var re = regexp.MustCompile + +var standardCryptoNames = []*regexp.Regexp{ + re(`^crypto/sha1\.\(\*digest\)`), + re(`^crypto/sha256\.\(\*digest\)`), + re(`^crypto/rand\.\(\*devReader\)`), + re(`^crypto/rsa\.encrypt(\.abi.)?$`), + re(`^crypto/rsa\.decrypt(\.abi.)?$`), +} + +func readBuildVersion(f exe, addr, size uint64) (string, error) { + if size == 0 { + size = uint64(f.AddrSize() * 2) + } + if size != 8 && size != 16 { + return "", fmt.Errorf("invalid size for runtime.buildVersion") + } + data, err := f.ReadData(addr, size) + if err != nil { + return "", fmt.Errorf("reading runtime.buildVersion: %v", err) + } + + if size == 8 { + addr = uint64(f.ByteOrder().Uint32(data)) + size = uint64(f.ByteOrder().Uint32(data[4:])) + } else { + addr = f.ByteOrder().Uint64(data) + size = f.ByteOrder().Uint64(data[8:]) + } + if size > 1000 { + return "", fmt.Errorf("implausible string size %d for runtime.buildVersion", size) + } + + data, err = f.ReadData(addr, size) + if err != nil { + return "", fmt.Errorf("reading runtime.buildVersion string data: %v", err) + } + return string(data), nil +} + +// Code signatures that indicate BoringCrypto or crypto/internal/fipsonly. +// These are not byte literals in order to avoid the actual +// byte signatures appearing in the goversion binary, +// because on some systems you can't tell rodata from text. +var ( + sigBoringCrypto, _ = hex.DecodeString("EB1DF448F44BF4B332F52813A3B450D441CC2485F001454E92101B1D2F1950C3") + sigStandardCrypto, _ = hex.DecodeString("EB1DF448F44BF4BAEE4DFA9851CA56A91145E83E99C59CF911CB8E80DAF12FC3") + sigFIPSOnly, _ = hex.DecodeString("EB1DF448F44BF4363CB9CE9D68047D31F28D325D5CA5873F5D80CAF6D6151BC3") +) + +func findCryptoSigs(v *Version, f exe) error { + const maxSigLen = 1 << 10 + start, end := f.TextRange() + for addr := start; addr < end; { + size := uint64(1 << 20) + if end-addr < size { + size = end - addr + } + data, err := f.ReadData(addr, size) + if err != nil { + return fmt.Errorf("reading text: %v", err) + } + if haveSig(data, sigBoringCrypto) { + v.BoringCrypto = true + } + if haveSig(data, sigFIPSOnly) { + v.FIPSOnly = true + } + if haveSig(data, sigStandardCrypto) { + v.StandardCrypto = true + } + if addr+size < end { + size -= maxSigLen + } + addr += size + } + return nil +} + +func haveSig(data, sig []byte) bool { + const align = 16 + for { + i := bytes.Index(data, sig) + if i < 0 { + return false + } + if i&(align-1) == 0 { + return true + } + // Found unaligned match; unexpected but + // skip to next aligned boundary and keep searching. + data = data[(i+align-1)&^(align-1):] + } +} + +func findModuleInfo(v *Version, f exe) error { + const maxModInfo = 128 << 10 + start, end := f.RODataRange() + for addr := start; addr < end; { + size := uint64(4 << 20) + if end-addr < size { + size = end - addr + } + data, err := f.ReadData(addr, size) + if err != nil { + return fmt.Errorf("reading text: %v", err) + } + if haveModuleInfo(data, v) { + return nil + } + if addr+size < end { + size -= maxModInfo + } + addr += size + } + return nil +} + +var ( + infoStart, _ = hex.DecodeString("3077af0c9274080241e1c107e6d618e6") + infoEnd, _ = hex.DecodeString("f932433186182072008242104116d8f2") +) + +func haveModuleInfo(data []byte, v *Version) bool { + i := bytes.Index(data, infoStart) + if i < 0 { + return false + } + j := bytes.Index(data[i:], infoEnd) + if j < 0 { + return false + } + v.ModuleInfo = string(data[i+len(infoStart) : i+j]) + return true +} diff --git a/vendor/github.com/saintfish/chardet/2022.go b/vendor/github.com/saintfish/chardet/2022.go new file mode 100644 index 00000000..e667225e --- /dev/null +++ b/vendor/github.com/saintfish/chardet/2022.go @@ -0,0 +1,102 @@ +package chardet + +import ( + "bytes" +) + +type recognizer2022 struct { + charset string + escapes [][]byte +} + +func (r *recognizer2022) Match(input *recognizerInput) (output recognizerOutput) { + return recognizerOutput{ + Charset: r.charset, + Confidence: r.matchConfidence(input.input), + } +} + +func (r *recognizer2022) matchConfidence(input []byte) int { + var hits, misses, shifts int +input: + for i := 0; i < len(input); i++ { + c := input[i] + if c == 0x1B { + for _, esc := range r.escapes { + if bytes.HasPrefix(input[i+1:], esc) { + hits++ + i += len(esc) + continue input + } + } + misses++ + } else if c == 0x0E || c == 0x0F { + shifts++ + } + } + if hits == 0 { + return 0 + } + quality := (100*hits - 100*misses) / (hits + misses) + if hits+shifts < 5 { + quality -= (5 - (hits + shifts)) * 10 + } + if quality < 0 { + quality = 0 + } + return quality +} + +var escapeSequences_2022JP = [][]byte{ + {0x24, 0x28, 0x43}, // KS X 1001:1992 + {0x24, 0x28, 0x44}, // JIS X 212-1990 + {0x24, 0x40}, // JIS C 6226-1978 + {0x24, 0x41}, // GB 2312-80 + {0x24, 0x42}, // JIS X 208-1983 + {0x26, 0x40}, // JIS X 208 1990, 1997 + {0x28, 0x42}, // ASCII + {0x28, 0x48}, // JIS-Roman + {0x28, 0x49}, // Half-width katakana + {0x28, 0x4a}, // JIS-Roman + {0x2e, 0x41}, // ISO 8859-1 + {0x2e, 0x46}, // ISO 8859-7 +} + +var escapeSequences_2022KR = [][]byte{ + {0x24, 0x29, 0x43}, +} + +var escapeSequences_2022CN = [][]byte{ + {0x24, 0x29, 0x41}, // GB 2312-80 + {0x24, 0x29, 0x47}, // CNS 11643-1992 Plane 1 + {0x24, 0x2A, 0x48}, // CNS 11643-1992 Plane 2 + {0x24, 0x29, 0x45}, // ISO-IR-165 + {0x24, 0x2B, 0x49}, // CNS 11643-1992 Plane 3 + {0x24, 0x2B, 0x4A}, // CNS 11643-1992 Plane 4 + {0x24, 0x2B, 0x4B}, // CNS 11643-1992 Plane 5 + {0x24, 0x2B, 0x4C}, // CNS 11643-1992 Plane 6 + {0x24, 0x2B, 0x4D}, // CNS 11643-1992 Plane 7 + {0x4e}, // SS2 + {0x4f}, // SS3 +} + +func newRecognizer_2022JP() *recognizer2022 { + return &recognizer2022{ + "ISO-2022-JP", + escapeSequences_2022JP, + } +} + +func newRecognizer_2022KR() *recognizer2022 { + return &recognizer2022{ + "ISO-2022-KR", + escapeSequences_2022KR, + } +} + +func newRecognizer_2022CN() *recognizer2022 { + return &recognizer2022{ + "ISO-2022-CN", + escapeSequences_2022CN, + } +} diff --git a/vendor/github.com/saintfish/chardet/AUTHORS b/vendor/github.com/saintfish/chardet/AUTHORS new file mode 100644 index 00000000..842d0216 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/AUTHORS @@ -0,0 +1 @@ +Sheng Yu (yusheng dot sjtu at gmail dot com) diff --git a/vendor/github.com/saintfish/chardet/LICENSE b/vendor/github.com/saintfish/chardet/LICENSE new file mode 100644 index 00000000..35ee796b --- /dev/null +++ b/vendor/github.com/saintfish/chardet/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2012 chardet Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Partial of the Software is derived from ICU project. See icu-license.html for +license of the derivative portions. diff --git a/vendor/github.com/saintfish/chardet/README.md b/vendor/github.com/saintfish/chardet/README.md new file mode 100644 index 00000000..41f58ee0 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/README.md @@ -0,0 +1,10 @@ +# chardet + +chardet is library to automatically detect +[charset](http://en.wikipedia.org/wiki/Character_encoding) of texts for [Go +programming language](http://golang.org/). It's based on the algorithm and data +in [ICU](http://icu-project.org/)'s implementation. + +## Documentation and Usage + +See [pkgdoc](https://pkg.go.dev/github.com/saintfish/chardet) diff --git a/vendor/github.com/saintfish/chardet/detector.go b/vendor/github.com/saintfish/chardet/detector.go new file mode 100644 index 00000000..e11c222e --- /dev/null +++ b/vendor/github.com/saintfish/chardet/detector.go @@ -0,0 +1,136 @@ +// Package chardet ports character set detection from ICU. +package chardet + +import ( + "errors" + "sort" +) + +// Result contains all the information that charset detector gives. +type Result struct { + // IANA name of the detected charset. + Charset string + // IANA name of the detected language. It may be empty for some charsets. + Language string + // Confidence of the Result. Scale from 1 to 100. The bigger, the more confident. + Confidence int +} + +// Detector implements charset detection. +type Detector struct { + recognizers []recognizer + stripTag bool +} + +// List of charset recognizers +var recognizers = []recognizer{ + newRecognizer_utf8(), + newRecognizer_utf16be(), + newRecognizer_utf16le(), + newRecognizer_utf32be(), + newRecognizer_utf32le(), + newRecognizer_8859_1_en(), + newRecognizer_8859_1_da(), + newRecognizer_8859_1_de(), + newRecognizer_8859_1_es(), + newRecognizer_8859_1_fr(), + newRecognizer_8859_1_it(), + newRecognizer_8859_1_nl(), + newRecognizer_8859_1_no(), + newRecognizer_8859_1_pt(), + newRecognizer_8859_1_sv(), + newRecognizer_8859_2_cs(), + newRecognizer_8859_2_hu(), + newRecognizer_8859_2_pl(), + newRecognizer_8859_2_ro(), + newRecognizer_8859_5_ru(), + newRecognizer_8859_6_ar(), + newRecognizer_8859_7_el(), + newRecognizer_8859_8_I_he(), + newRecognizer_8859_8_he(), + newRecognizer_windows_1251(), + newRecognizer_windows_1256(), + newRecognizer_KOI8_R(), + newRecognizer_8859_9_tr(), + + newRecognizer_sjis(), + newRecognizer_gb_18030(), + newRecognizer_euc_jp(), + newRecognizer_euc_kr(), + newRecognizer_big5(), + + newRecognizer_2022JP(), + newRecognizer_2022KR(), + newRecognizer_2022CN(), + + newRecognizer_IBM424_he_rtl(), + newRecognizer_IBM424_he_ltr(), + newRecognizer_IBM420_ar_rtl(), + newRecognizer_IBM420_ar_ltr(), +} + +// NewTextDetector creates a Detector for plain text. +func NewTextDetector() *Detector { + return &Detector{recognizers, false} +} + +// NewHtmlDetector creates a Detector for Html. +func NewHtmlDetector() *Detector { + return &Detector{recognizers, true} +} + +var ( + NotDetectedError = errors.New("Charset not detected.") +) + +// DetectBest returns the Result with highest Confidence. +func (d *Detector) DetectBest(b []byte) (r *Result, err error) { + var all []Result + if all, err = d.DetectAll(b); err == nil { + r = &all[0] + } + return +} + +// DetectAll returns all Results which have non-zero Confidence. The Results are sorted by Confidence in descending order. +func (d *Detector) DetectAll(b []byte) ([]Result, error) { + input := newRecognizerInput(b, d.stripTag) + outputChan := make(chan recognizerOutput) + for _, r := range d.recognizers { + go matchHelper(r, input, outputChan) + } + outputs := make([]recognizerOutput, 0, len(d.recognizers)) + for i := 0; i < len(d.recognizers); i++ { + o := <-outputChan + if o.Confidence > 0 { + outputs = append(outputs, o) + } + } + if len(outputs) == 0 { + return nil, NotDetectedError + } + + sort.Sort(recognizerOutputs(outputs)) + dedupOutputs := make([]Result, 0, len(outputs)) + foundCharsets := make(map[string]struct{}, len(outputs)) + for _, o := range outputs { + if _, found := foundCharsets[o.Charset]; !found { + dedupOutputs = append(dedupOutputs, Result(o)) + foundCharsets[o.Charset] = struct{}{} + } + } + if len(dedupOutputs) == 0 { + return nil, NotDetectedError + } + return dedupOutputs, nil +} + +func matchHelper(r recognizer, input *recognizerInput, outputChan chan<- recognizerOutput) { + outputChan <- r.Match(input) +} + +type recognizerOutputs []recognizerOutput + +func (r recognizerOutputs) Len() int { return len(r) } +func (r recognizerOutputs) Less(i, j int) bool { return r[i].Confidence > r[j].Confidence } +func (r recognizerOutputs) Swap(i, j int) { r[i], r[j] = r[j], r[i] } diff --git a/vendor/github.com/saintfish/chardet/icu-license.html b/vendor/github.com/saintfish/chardet/icu-license.html new file mode 100644 index 00000000..d078d057 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/icu-license.html @@ -0,0 +1,51 @@ + + + + +ICU License - ICU 1.8.1 and later + + + +

ICU License - ICU 1.8.1 and later

+ +

COPYRIGHT AND PERMISSION NOTICE

+ +

+Copyright (c) 1995-2012 International Business Machines Corporation and others +

+

+All rights reserved. +

+

+Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies +of the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. +

+

+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, +OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE +USE OR PERFORMANCE OF THIS SOFTWARE. +

+

+Except as contained in this notice, the name of a copyright holder shall not be +used in advertising or otherwise to promote the sale, use or other dealings in +this Software without prior written authorization of the copyright holder. +

+ +
+

+All trademarks and registered trademarks mentioned herein are the property of their respective owners. +

+ + diff --git a/vendor/github.com/saintfish/chardet/multi_byte.go b/vendor/github.com/saintfish/chardet/multi_byte.go new file mode 100644 index 00000000..1fab34ce --- /dev/null +++ b/vendor/github.com/saintfish/chardet/multi_byte.go @@ -0,0 +1,345 @@ +package chardet + +import ( + "errors" + "math" +) + +type recognizerMultiByte struct { + charset string + language string + decoder charDecoder + commonChars []uint16 +} + +type charDecoder interface { + DecodeOneChar([]byte) (c uint16, remain []byte, err error) +} + +func (r *recognizerMultiByte) Match(input *recognizerInput) (output recognizerOutput) { + return recognizerOutput{ + Charset: r.charset, + Language: r.language, + Confidence: r.matchConfidence(input), + } +} + +func (r *recognizerMultiByte) matchConfidence(input *recognizerInput) int { + raw := input.raw + var c uint16 + var err error + var totalCharCount, badCharCount, singleByteCharCount, doubleByteCharCount, commonCharCount int + for c, raw, err = r.decoder.DecodeOneChar(raw); len(raw) > 0; c, raw, err = r.decoder.DecodeOneChar(raw) { + totalCharCount++ + if err != nil { + badCharCount++ + } else if c <= 0xFF { + singleByteCharCount++ + } else { + doubleByteCharCount++ + if r.commonChars != nil && binarySearch(r.commonChars, c) { + commonCharCount++ + } + } + if badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount { + return 0 + } + } + + if doubleByteCharCount <= 10 && badCharCount == 0 { + if doubleByteCharCount == 0 && totalCharCount < 10 { + return 0 + } else { + return 10 + } + } + + if doubleByteCharCount < 20*badCharCount { + return 0 + } + if r.commonChars == nil { + confidence := 30 + doubleByteCharCount - 20*badCharCount + if confidence > 100 { + confidence = 100 + } + return confidence + } + maxVal := math.Log(float64(doubleByteCharCount) / 4) + scaleFactor := 90 / maxVal + confidence := int(math.Log(float64(commonCharCount)+1)*scaleFactor + 10) + if confidence > 100 { + confidence = 100 + } + if confidence < 0 { + confidence = 0 + } + return confidence +} + +func binarySearch(l []uint16, c uint16) bool { + start := 0 + end := len(l) - 1 + for start <= end { + mid := (start + end) / 2 + if c == l[mid] { + return true + } else if c < l[mid] { + end = mid - 1 + } else { + start = mid + 1 + } + } + return false +} + +var eobError = errors.New("End of input buffer") +var badCharError = errors.New("Decode a bad char") + +type charDecoder_sjis struct { +} + +func (charDecoder_sjis) DecodeOneChar(input []byte) (c uint16, remain []byte, err error) { + if len(input) == 0 { + return 0, nil, eobError + } + first := input[0] + c = uint16(first) + remain = input[1:] + if first <= 0x7F || (first > 0xA0 && first <= 0xDF) { + return + } + if len(remain) == 0 { + return c, remain, badCharError + } + second := remain[0] + remain = remain[1:] + c = c<<8 | uint16(second) + if (second >= 0x40 && second <= 0x7F) || (second >= 0x80 && second <= 0xFE) { + } else { + err = badCharError + } + return +} + +var commonChars_sjis = []uint16{ + 0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0, + 0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5, + 0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc, + 0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341, + 0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389, + 0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa, +} + +func newRecognizer_sjis() *recognizerMultiByte { + return &recognizerMultiByte{ + "Shift_JIS", + "ja", + charDecoder_sjis{}, + commonChars_sjis, + } +} + +type charDecoder_euc struct { +} + +func (charDecoder_euc) DecodeOneChar(input []byte) (c uint16, remain []byte, err error) { + if len(input) == 0 { + return 0, nil, eobError + } + first := input[0] + remain = input[1:] + c = uint16(first) + if first <= 0x8D { + return uint16(first), remain, nil + } + if len(remain) == 0 { + return 0, nil, eobError + } + second := remain[0] + remain = remain[1:] + c = c<<8 | uint16(second) + if first >= 0xA1 && first <= 0xFE { + if second < 0xA1 { + err = badCharError + } + return + } + if first == 0x8E { + if second < 0xA1 { + err = badCharError + } + return + } + if first == 0x8F { + if len(remain) == 0 { + return 0, nil, eobError + } + third := remain[0] + remain = remain[1:] + c = c<<0 | uint16(third) + if third < 0xa1 { + err = badCharError + } + } + return +} + +var commonChars_euc_jp = []uint16{ + 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2, + 0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3, + 0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4, + 0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de, + 0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef, + 0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af, + 0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7, + 0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1, + 0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee, + 0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1, +} + +var commonChars_euc_kr = []uint16{ + 0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc, + 0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9, + 0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce, + 0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce, + 0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba, + 0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee, + 0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7, + 0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6, + 0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6, + 0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad, +} + +func newRecognizer_euc_jp() *recognizerMultiByte { + return &recognizerMultiByte{ + "EUC-JP", + "ja", + charDecoder_euc{}, + commonChars_euc_jp, + } +} + +func newRecognizer_euc_kr() *recognizerMultiByte { + return &recognizerMultiByte{ + "EUC-KR", + "ko", + charDecoder_euc{}, + commonChars_euc_kr, + } +} + +type charDecoder_big5 struct { +} + +func (charDecoder_big5) DecodeOneChar(input []byte) (c uint16, remain []byte, err error) { + if len(input) == 0 { + return 0, nil, eobError + } + first := input[0] + remain = input[1:] + c = uint16(first) + if first <= 0x7F || first == 0xFF { + return + } + if len(remain) == 0 { + return c, nil, eobError + } + second := remain[0] + remain = remain[1:] + c = c<<8 | uint16(second) + if second < 0x40 || second == 0x7F || second == 0xFF { + err = badCharError + } + return +} + +var commonChars_big5 = []uint16{ + 0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446, + 0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3, + 0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548, + 0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8, + 0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da, + 0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3, + 0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59, + 0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c, + 0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44, + 0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f, +} + +func newRecognizer_big5() *recognizerMultiByte { + return &recognizerMultiByte{ + "Big5", + "zh", + charDecoder_big5{}, + commonChars_big5, + } +} + +type charDecoder_gb_18030 struct { +} + +func (charDecoder_gb_18030) DecodeOneChar(input []byte) (c uint16, remain []byte, err error) { + if len(input) == 0 { + return 0, nil, eobError + } + first := input[0] + remain = input[1:] + c = uint16(first) + if first <= 0x80 { + return + } + if len(remain) == 0 { + return 0, nil, eobError + } + second := remain[0] + remain = remain[1:] + c = c<<8 | uint16(second) + if first >= 0x81 && first <= 0xFE { + if (second >= 0x40 && second <= 0x7E) || (second >= 0x80 && second <= 0xFE) { + return + } + + if second >= 0x30 && second <= 0x39 { + if len(remain) == 0 { + return 0, nil, eobError + } + third := remain[0] + remain = remain[1:] + if third >= 0x81 && third <= 0xFE { + if len(remain) == 0 { + return 0, nil, eobError + } + fourth := remain[0] + remain = remain[1:] + if fourth >= 0x30 && fourth <= 0x39 { + c = c<<16 | uint16(third)<<8 | uint16(fourth) + return + } + } + } + err = badCharError + } + return +} + +var commonChars_gb_18030 = []uint16{ + 0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac, + 0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4, + 0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4, + 0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6, + 0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6, + 0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7, + 0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7, + 0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5, + 0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2, + 0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0, +} + +func newRecognizer_gb_18030() *recognizerMultiByte { + return &recognizerMultiByte{ + "GB-18030", + "zh", + charDecoder_gb_18030{}, + commonChars_gb_18030, + } +} diff --git a/vendor/github.com/saintfish/chardet/recognizer.go b/vendor/github.com/saintfish/chardet/recognizer.go new file mode 100644 index 00000000..1bf8461c --- /dev/null +++ b/vendor/github.com/saintfish/chardet/recognizer.go @@ -0,0 +1,83 @@ +package chardet + +type recognizer interface { + Match(*recognizerInput) recognizerOutput +} + +type recognizerOutput Result + +type recognizerInput struct { + raw []byte + input []byte + tagStripped bool + byteStats []int + hasC1Bytes bool +} + +func newRecognizerInput(raw []byte, stripTag bool) *recognizerInput { + input, stripped := mayStripInput(raw, stripTag) + byteStats := computeByteStats(input) + return &recognizerInput{ + raw: raw, + input: input, + tagStripped: stripped, + byteStats: byteStats, + hasC1Bytes: computeHasC1Bytes(byteStats), + } +} + +func mayStripInput(raw []byte, stripTag bool) (out []byte, stripped bool) { + const inputBufferSize = 8192 + out = make([]byte, 0, inputBufferSize) + var badTags, openTags int32 + var inMarkup bool = false + stripped = false + if stripTag { + stripped = true + for _, c := range raw { + if c == '<' { + if inMarkup { + badTags += 1 + } + inMarkup = true + openTags += 1 + } + if !inMarkup { + out = append(out, c) + if len(out) >= inputBufferSize { + break + } + } + if c == '>' { + inMarkup = false + } + } + } + if openTags < 5 || openTags/5 < badTags || (len(out) < 100 && len(raw) > 600) { + limit := len(raw) + if limit > inputBufferSize { + limit = inputBufferSize + } + out = make([]byte, limit) + copy(out, raw[:limit]) + stripped = false + } + return +} + +func computeByteStats(input []byte) []int { + r := make([]int, 256) + for _, c := range input { + r[c] += 1 + } + return r +} + +func computeHasC1Bytes(byteStats []int) bool { + for _, count := range byteStats[0x80 : 0x9F+1] { + if count > 0 { + return true + } + } + return false +} diff --git a/vendor/github.com/saintfish/chardet/single_byte.go b/vendor/github.com/saintfish/chardet/single_byte.go new file mode 100644 index 00000000..efe41c90 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/single_byte.go @@ -0,0 +1,882 @@ +package chardet + +// Recognizer for single byte charset family +type recognizerSingleByte struct { + charset string + hasC1ByteCharset string + language string + charMap *[256]byte + ngram *[64]uint32 +} + +func (r *recognizerSingleByte) Match(input *recognizerInput) recognizerOutput { + var charset string = r.charset + if input.hasC1Bytes && len(r.hasC1ByteCharset) > 0 { + charset = r.hasC1ByteCharset + } + return recognizerOutput{ + Charset: charset, + Language: r.language, + Confidence: r.parseNgram(input.input), + } +} + +type ngramState struct { + ngram uint32 + ignoreSpace bool + ngramCount, ngramHit uint32 + table *[64]uint32 +} + +func newNgramState(table *[64]uint32) *ngramState { + return &ngramState{ + ngram: 0, + ignoreSpace: false, + ngramCount: 0, + ngramHit: 0, + table: table, + } +} + +func (s *ngramState) AddByte(b byte) { + const ngramMask = 0xFFFFFF + if !(b == 0x20 && s.ignoreSpace) { + s.ngram = ((s.ngram << 8) | uint32(b)) & ngramMask + s.ignoreSpace = (s.ngram == 0x20) + s.ngramCount++ + if s.lookup() { + s.ngramHit++ + } + } + s.ignoreSpace = (b == 0x20) +} + +func (s *ngramState) HitRate() float32 { + if s.ngramCount == 0 { + return 0 + } + return float32(s.ngramHit) / float32(s.ngramCount) +} + +func (s *ngramState) lookup() bool { + var index int + if s.table[index+32] <= s.ngram { + index += 32 + } + if s.table[index+16] <= s.ngram { + index += 16 + } + if s.table[index+8] <= s.ngram { + index += 8 + } + if s.table[index+4] <= s.ngram { + index += 4 + } + if s.table[index+2] <= s.ngram { + index += 2 + } + if s.table[index+1] <= s.ngram { + index += 1 + } + if s.table[index] > s.ngram { + index -= 1 + } + if index < 0 || s.table[index] != s.ngram { + return false + } + return true +} + +func (r *recognizerSingleByte) parseNgram(input []byte) int { + state := newNgramState(r.ngram) + for _, inChar := range input { + c := r.charMap[inChar] + if c != 0 { + state.AddByte(c) + } + } + state.AddByte(0x20) + rate := state.HitRate() + if rate > 0.33 { + return 98 + } + return int(rate * 300) +} + +var charMap_8859_1 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +} + +var ngrams_8859_1_en = [64]uint32{ + 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F, + 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74, + 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420, + 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320, +} + +var ngrams_8859_1_da = [64]uint32{ + 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620, + 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320, + 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520, + 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572, +} + +var ngrams_8859_1_de = [64]uint32{ + 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F, + 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220, + 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465, + 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572, +} + +var ngrams_8859_1_es = [64]uint32{ + 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, + 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C, + 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064, + 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20, +} + +var ngrams_8859_1_fr = [64]uint32{ + 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E, + 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20, + 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420, + 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220, +} + +var ngrams_8859_1_it = [64]uint32{ + 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073, + 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220, + 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20, + 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F, +} + +var ngrams_8859_1_nl = [64]uint32{ + 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665, + 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E, + 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F, + 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F, +} + +var ngrams_8859_1_no = [64]uint32{ + 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469, + 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474, + 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65, + 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572, +} + +var ngrams_8859_1_pt = [64]uint32{ + 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, + 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20, + 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065, + 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F, +} + +var ngrams_8859_1_sv = [64]uint32{ + 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469, + 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220, + 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20, + 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220, +} + +func newRecognizer_8859_1(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-1", + hasC1ByteCharset: "windows-1252", + language: language, + charMap: &charMap_8859_1, + ngram: ngram, + } +} + +func newRecognizer_8859_1_en() *recognizerSingleByte { + return newRecognizer_8859_1("en", &ngrams_8859_1_en) +} +func newRecognizer_8859_1_da() *recognizerSingleByte { + return newRecognizer_8859_1("da", &ngrams_8859_1_da) +} +func newRecognizer_8859_1_de() *recognizerSingleByte { + return newRecognizer_8859_1("de", &ngrams_8859_1_de) +} +func newRecognizer_8859_1_es() *recognizerSingleByte { + return newRecognizer_8859_1("es", &ngrams_8859_1_es) +} +func newRecognizer_8859_1_fr() *recognizerSingleByte { + return newRecognizer_8859_1("fr", &ngrams_8859_1_fr) +} +func newRecognizer_8859_1_it() *recognizerSingleByte { + return newRecognizer_8859_1("it", &ngrams_8859_1_it) +} +func newRecognizer_8859_1_nl() *recognizerSingleByte { + return newRecognizer_8859_1("nl", &ngrams_8859_1_nl) +} +func newRecognizer_8859_1_no() *recognizerSingleByte { + return newRecognizer_8859_1("no", &ngrams_8859_1_no) +} +func newRecognizer_8859_1_pt() *recognizerSingleByte { + return newRecognizer_8859_1("pt", &ngrams_8859_1_pt) +} +func newRecognizer_8859_1_sv() *recognizerSingleByte { + return newRecognizer_8859_1("sv", &ngrams_8859_1_sv) +} + +var charMap_8859_2 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20, + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, + 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7, + 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, +} + +var ngrams_8859_2_cs = [64]uint32{ + 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F, + 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465, + 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865, + 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564, +} + +var ngrams_8859_2_hu = [64]uint32{ + 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69, + 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20, + 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061, + 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320, +} + +var ngrams_8859_2_pl = [64]uint32{ + 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779, + 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20, + 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769, + 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720, +} + +var ngrams_8859_2_ro = [64]uint32{ + 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69, + 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070, + 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72, + 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20, +} + +func newRecognizer_8859_2(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-2", + hasC1ByteCharset: "windows-1250", + language: language, + charMap: &charMap_8859_2, + ngram: ngram, + } +} + +func newRecognizer_8859_2_cs() *recognizerSingleByte { + return newRecognizer_8859_1("cs", &ngrams_8859_2_cs) +} +func newRecognizer_8859_2_hu() *recognizerSingleByte { + return newRecognizer_8859_1("hu", &ngrams_8859_2_hu) +} +func newRecognizer_8859_2_pl() *recognizerSingleByte { + return newRecognizer_8859_1("pl", &ngrams_8859_2_pl) +} +func newRecognizer_8859_2_ro() *recognizerSingleByte { + return newRecognizer_8859_1("ro", &ngrams_8859_2_ro) +} + +var charMap_8859_5 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, +} + +var ngrams_8859_5_ru = [64]uint32{ + 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE, + 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD, + 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, + 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520, +} + +func newRecognizer_8859_5(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-5", + language: language, + charMap: &charMap_8859_5, + ngram: ngram, + } +} + +func newRecognizer_8859_5_ru() *recognizerSingleByte { + return newRecognizer_8859_5("ru", &ngrams_8859_5_ru) +} + +var charMap_8859_6 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, +} + +var ngrams_8859_6_ar = [64]uint32{ + 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8, + 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1, + 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, + 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620, +} + +func newRecognizer_8859_6(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-6", + language: language, + charMap: &charMap_8859_6, + ngram: ngram, + } +} + +func newRecognizer_8859_6_ar() *recognizerSingleByte { + return newRecognizer_8859_6("ar", &ngrams_8859_6_ar) +} + +var charMap_8859_7 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20, + 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE, + 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, +} + +var ngrams_8859_7_el = [64]uint32{ + 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7, + 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120, + 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, + 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20, +} + +func newRecognizer_8859_7(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-7", + hasC1ByteCharset: "windows-1253", + language: language, + charMap: &charMap_8859_7, + ngram: ngram, + } +} + +func newRecognizer_8859_7_el() *recognizerSingleByte { + return newRecognizer_8859_7("el", &ngrams_8859_7_el) +} + +var charMap_8859_8 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20, +} + +var ngrams_8859_8_I_he = [64]uint32{ + 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0, + 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4, + 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, + 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9, +} + +var ngrams_8859_8_he = [64]uint32{ + 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0, + 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC, + 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, + 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9, +} + +func newRecognizer_8859_8(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-8", + hasC1ByteCharset: "windows-1255", + language: language, + charMap: &charMap_8859_8, + ngram: ngram, + } +} + +func newRecognizer_8859_8_I_he() *recognizerSingleByte { + r := newRecognizer_8859_8("he", &ngrams_8859_8_I_he) + r.charset = "ISO-8859-8-I" + return r +} + +func newRecognizer_8859_8_he() *recognizerSingleByte { + return newRecognizer_8859_8("he", &ngrams_8859_8_he) +} + +var charMap_8859_9 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +} + +var ngrams_8859_9_tr = [64]uint32{ + 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961, + 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062, + 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, + 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD, +} + +func newRecognizer_8859_9(language string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "ISO-8859-9", + hasC1ByteCharset: "windows-1254", + language: language, + charMap: &charMap_8859_9, + ngram: ngram, + } +} + +func newRecognizer_8859_9_tr() *recognizerSingleByte { + return newRecognizer_8859_9("tr", &ngrams_8859_9_tr) +} + +var charMap_windows_1256 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, + 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20, + 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF, +} + +var ngrams_windows_1256 = [64]uint32{ + 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8, + 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD, + 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, + 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420, +} + +func newRecognizer_windows_1256() *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "windows-1256", + language: "ar", + charMap: &charMap_windows_1256, + ngram: &ngrams_windows_1256, + } +} + +var charMap_windows_1251 = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, + 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, + 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20, + 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF, + 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20, + 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, + 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, + 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, + 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, +} + +var ngrams_windows_1251 = [64]uint32{ + 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE, + 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED, + 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, + 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520, +} + +func newRecognizer_windows_1251() *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "windows-1251", + language: "ar", + charMap: &charMap_windows_1251, + ngram: &ngrams_windows_1251, + } +} + +var charMap_KOI8_R = [256]byte{ + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, +} + +var ngrams_KOI8_R = [64]uint32{ + 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1, + 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE, + 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, + 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF, +} + +func newRecognizer_KOI8_R() *recognizerSingleByte { + return &recognizerSingleByte{ + charset: "KOI8-R", + language: "ru", + charMap: &charMap_KOI8_R, + ngram: &ngrams_KOI8_R, + } +} + +var charMap_IBM424_he = [256]byte{ + /* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ + /* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40, + /* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, +} + +var ngrams_IBM424_he_rtl = [64]uint32{ + 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, + 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, + 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, + 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069, +} + +var ngrams_IBM424_he_ltr = [64]uint32{ + 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141, + 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054, + 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, + 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651, +} + +func newRecognizer_IBM424_he(charset string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: charset, + language: "he", + charMap: &charMap_IBM424_he, + ngram: ngram, + } +} + +func newRecognizer_IBM424_he_rtl() *recognizerSingleByte { + return newRecognizer_IBM424_he("IBM424_rtl", &ngrams_IBM424_he_rtl) +} + +func newRecognizer_IBM424_he_ltr() *recognizerSingleByte { + return newRecognizer_IBM424_he("IBM424_ltr", &ngrams_IBM424_he_ltr) +} + +var charMap_IBM420_ar = [256]byte{ + /* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ + /* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + /* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + /* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, + /* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, + /* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, + /* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF, + /* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, + /* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF, + /* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40, +} + +var ngrams_IBM420_ar_rtl = [64]uint32{ + 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158, + 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB, + 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, + 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40, +} + +var ngrams_IBM420_ar_ltr = [64]uint32{ + 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, + 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD, + 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, + 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156, +} + +func newRecognizer_IBM420_ar(charset string, ngram *[64]uint32) *recognizerSingleByte { + return &recognizerSingleByte{ + charset: charset, + language: "ar", + charMap: &charMap_IBM420_ar, + ngram: ngram, + } +} + +func newRecognizer_IBM420_ar_rtl() *recognizerSingleByte { + return newRecognizer_IBM420_ar("IBM420_rtl", &ngrams_IBM420_ar_rtl) +} + +func newRecognizer_IBM420_ar_ltr() *recognizerSingleByte { + return newRecognizer_IBM420_ar("IBM420_ltr", &ngrams_IBM420_ar_ltr) +} diff --git a/vendor/github.com/saintfish/chardet/unicode.go b/vendor/github.com/saintfish/chardet/unicode.go new file mode 100644 index 00000000..6f9fa9e6 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/unicode.go @@ -0,0 +1,103 @@ +package chardet + +import ( + "bytes" +) + +var ( + utf16beBom = []byte{0xFE, 0xFF} + utf16leBom = []byte{0xFF, 0xFE} + utf32beBom = []byte{0x00, 0x00, 0xFE, 0xFF} + utf32leBom = []byte{0xFF, 0xFE, 0x00, 0x00} +) + +type recognizerUtf16be struct { +} + +func newRecognizer_utf16be() *recognizerUtf16be { + return &recognizerUtf16be{} +} + +func (*recognizerUtf16be) Match(input *recognizerInput) (output recognizerOutput) { + output = recognizerOutput{ + Charset: "UTF-16BE", + } + if bytes.HasPrefix(input.raw, utf16beBom) { + output.Confidence = 100 + } + return +} + +type recognizerUtf16le struct { +} + +func newRecognizer_utf16le() *recognizerUtf16le { + return &recognizerUtf16le{} +} + +func (*recognizerUtf16le) Match(input *recognizerInput) (output recognizerOutput) { + output = recognizerOutput{ + Charset: "UTF-16LE", + } + if bytes.HasPrefix(input.raw, utf16leBom) && !bytes.HasPrefix(input.raw, utf32leBom) { + output.Confidence = 100 + } + return +} + +type recognizerUtf32 struct { + name string + bom []byte + decodeChar func(input []byte) uint32 +} + +func decodeUtf32be(input []byte) uint32 { + return uint32(input[0])<<24 | uint32(input[1])<<16 | uint32(input[2])<<8 | uint32(input[3]) +} + +func decodeUtf32le(input []byte) uint32 { + return uint32(input[3])<<24 | uint32(input[2])<<16 | uint32(input[1])<<8 | uint32(input[0]) +} + +func newRecognizer_utf32be() *recognizerUtf32 { + return &recognizerUtf32{ + "UTF-32BE", + utf32beBom, + decodeUtf32be, + } +} + +func newRecognizer_utf32le() *recognizerUtf32 { + return &recognizerUtf32{ + "UTF-32LE", + utf32leBom, + decodeUtf32le, + } +} + +func (r *recognizerUtf32) Match(input *recognizerInput) (output recognizerOutput) { + output = recognizerOutput{ + Charset: r.name, + } + hasBom := bytes.HasPrefix(input.raw, r.bom) + var numValid, numInvalid uint32 + for b := input.raw; len(b) >= 4; b = b[4:] { + if c := r.decodeChar(b); c >= 0x10FFFF || (c >= 0xD800 && c <= 0xDFFF) { + numInvalid++ + } else { + numValid++ + } + } + if hasBom && numInvalid == 0 { + output.Confidence = 100 + } else if hasBom && numValid > numInvalid*10 { + output.Confidence = 80 + } else if numValid > 3 && numInvalid == 0 { + output.Confidence = 100 + } else if numValid > 0 && numInvalid == 0 { + output.Confidence = 80 + } else if numValid > numInvalid*10 { + output.Confidence = 25 + } + return +} diff --git a/vendor/github.com/saintfish/chardet/utf8.go b/vendor/github.com/saintfish/chardet/utf8.go new file mode 100644 index 00000000..ae036ad9 --- /dev/null +++ b/vendor/github.com/saintfish/chardet/utf8.go @@ -0,0 +1,71 @@ +package chardet + +import ( + "bytes" +) + +var utf8Bom = []byte{0xEF, 0xBB, 0xBF} + +type recognizerUtf8 struct { +} + +func newRecognizer_utf8() *recognizerUtf8 { + return &recognizerUtf8{} +} + +func (*recognizerUtf8) Match(input *recognizerInput) (output recognizerOutput) { + output = recognizerOutput{ + Charset: "UTF-8", + } + hasBom := bytes.HasPrefix(input.raw, utf8Bom) + inputLen := len(input.raw) + var numValid, numInvalid uint32 + var trailBytes uint8 + for i := 0; i < inputLen; i++ { + c := input.raw[i] + if c&0x80 == 0 { + continue + } + if c&0xE0 == 0xC0 { + trailBytes = 1 + } else if c&0xF0 == 0xE0 { + trailBytes = 2 + } else if c&0xF8 == 0xF0 { + trailBytes = 3 + } else { + numInvalid++ + if numInvalid > 5 { + break + } + trailBytes = 0 + } + + for i++; i < inputLen; i++ { + c = input.raw[i] + if c&0xC0 != 0x80 { + numInvalid++ + break + } + if trailBytes--; trailBytes == 0 { + numValid++ + break + } + } + } + + if hasBom && numInvalid == 0 { + output.Confidence = 100 + } else if hasBom && numValid > numInvalid*10 { + output.Confidence = 80 + } else if numValid > 3 && numInvalid == 0 { + output.Confidence = 100 + } else if numValid > 0 && numInvalid == 0 { + output.Confidence = 80 + } else if numValid == 0 && numInvalid == 0 { + // Plain ASCII + output.Confidence = 10 + } else if numValid > numInvalid*10 { + output.Confidence = 25 + } + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f786bfbf..6d730133 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -84,7 +84,7 @@ github.com/anchore/stereoscope/pkg/image/oci github.com/anchore/stereoscope/pkg/image/sif github.com/anchore/stereoscope/pkg/tree github.com/anchore/stereoscope/pkg/tree/node -# github.com/anchore/syft v0.87.1 +# github.com/anchore/syft v0.88.0 ## explicit; go 1.19 github.com/anchore/syft/internal github.com/anchore/syft/internal/bus @@ -392,7 +392,7 @@ github.com/google/go-containerregistry/pkg/v1/types ## explicit; go 1.12 github.com/google/licensecheck github.com/google/licensecheck/internal/match -# github.com/google/uuid v1.3.0 +# github.com/google/uuid v1.3.1 ## explicit github.com/google/uuid # github.com/hashicorp/errwrap v1.1.0 @@ -417,9 +417,12 @@ github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1 # github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 ## explicit github.com/jbenet/go-context/io -# github.com/jinzhu/copier v0.3.5 +# github.com/jinzhu/copier v0.4.0 ## explicit; go 1.13 github.com/jinzhu/copier +# github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 +## explicit; go 1.21 +github.com/kastenhq/goversion/version # github.com/kevinburke/ssh_config v1.2.0 ## explicit github.com/kevinburke/ssh_config @@ -517,6 +520,9 @@ github.com/rivo/uniseg ## explicit; go 1.15 github.com/saferwall/pe github.com/saferwall/pe/log +# github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d +## explicit +github.com/saintfish/chardet # github.com/sassoftware/go-rpmutils v0.2.0 ## explicit; go 1.13 github.com/sassoftware/go-rpmutils