-
Notifications
You must be signed in to change notification settings - Fork 0
/
token.go
146 lines (124 loc) · 3.38 KB
/
token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
package habari
import (
"github.com/google/uuid"
)
type tokenCategory = string
const (
tokenCatUnknown tokenCategory = "unknown"
tokenCatDelimiter tokenCategory = "delimiter"
tokenCatSeparator tokenCategory = "separator"
tokenCatKnown tokenCategory = "known"
tokenCatParts tokenCategory = "parts"
tokenCatOpeningBracket tokenCategory = "openingBracket"
tokenCatClosingBracket tokenCategory = "closingBracket"
)
type tokenKind = string
const (
tokenKindUnknown tokenKind = ""
tokenKindCharacter tokenKind = "character"
tokenKindWord tokenKind = "word"
tokenKindNumber tokenKind = "number"
tokenKindNumberLike tokenKind = "numberLike"
tokenKindOrdinalNumber tokenKind = "ordinalNumber"
tokenKindCrc32 tokenKind = "crc32"
tokenKindPossibleVideoRes tokenKind = "possibleVideoRes"
tokenKindYear tokenKind = "year"
)
type token struct {
UUID string
Value string
Category tokenCategory
Kind tokenKind
IdentifiedKeywordCategory keywordCategory
IdentifiedKeywordKind keywordKind
Enclosed bool
Parts []*token
MetadataCategory metadataCategory
}
func newToken(value string) *token {
return &token{
UUID: uuid.NewString(),
Value: value,
Category: tokenCatUnknown,
Kind: tokenKindUnknown,
IdentifiedKeywordCategory: keywordCatNone,
IdentifiedKeywordKind: keywordKindStandalone,
Enclosed: false,
Parts: nil,
MetadataCategory: 0,
}
}
// setMetadataCategory will update the token's MetadataCategory
// and update its Category to tokenCatKnown if the metadataCategory is not metadataUnknown.
func (t *token) setMetadataCategory(mk metadataCategory) {
if t == nil {
return
}
t.MetadataCategory = mk
t.Category = tokenCatKnown
if mk == metadataUnknown {
t.Category = tokenCatUnknown
}
}
func (t *token) setValue(s string) {
if t == nil {
return
}
t.Value = s
}
// setCategory will update the token's Category to the specified tokenCategory value.
// If the token is nil, the function will return without making any changes.
//
// Example:
//
// tkn := &token{}
// tkn.setCategory(tokenCatKnown)
// fmt.Println(tkn.Category) // Output: tokenCatKnown
func (t *token) setCategory(c tokenCategory) {
if t == nil {
return
}
t.Category = c
}
// setIdentifiedKeywordCategory updates the `IdentifiedKeywordCategory` of a token
// with the provided keyword category.
//
// Example:
//
// tkn := &token{}
// tkn.setIdentifiedKeywordCategory(keywordCatSeasonPrefix)
// fmt.Println(tkn.IdentifiedKeywordCategory) // Output: keywordCatSeasonPrefix
func (t *token) setIdentifiedKeywordCategory(c keywordCategory, k keywordKind) {
if t == nil {
return
}
t.IdentifiedKeywordCategory = c
t.IdentifiedKeywordKind = k
}
func (t *token) setKind(k tokenKind) {
if t == nil {
return
}
t.Kind = k
}
func (t *token) setParts(p []*token) {
if t == nil {
return
}
t.Category = tokenCatParts
t.Parts = p
}
func (t *token) setEnclosed(v bool) {
if t == nil {
return
}
t.Enclosed = v
}
func (t *token) getParts() (tokenParts []*token, found bool) {
if t.Parts == nil {
found = false
}
tokenParts = t.Parts
found = true
return
}