From 017196175086270d7da2d7080d2b04b70a0ffbdd Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Thu, 19 Oct 2023 08:10:14 +0000 Subject: [PATCH 01/10] feat: add attributs saved test --- pkg/xixo/parser_test.go | 72 ++++++++++++++++++++++++++++++++ test/data/exemple.xml | 7 ++++ test/data/exemple_expected.jsonl | 1 + 3 files changed, 80 insertions(+) create mode 100644 test/data/exemple.xml create mode 100644 test/data/exemple_expected.jsonl diff --git a/pkg/xixo/parser_test.go b/pkg/xixo/parser_test.go index 2505006..6822dc7 100644 --- a/pkg/xixo/parser_test.go +++ b/pkg/xixo/parser_test.go @@ -120,3 +120,75 @@ func TestModifyElementWrappedWithTextWithCallback(t *testing.T) { t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) } } + +func TestAttributsShouldSavedAfterParser(t *testing.T) { + t.Parallel() + // Fichier XML en entrée + inputXML := ` + + Hello + ` + + // Lisez les résultats du canal et construisez le XML résultant + var resultXMLBuffer bytes.Buffer + + // Créez un bufio.Reader à partir du XML en entrée + reader := bytes.NewBufferString(inputXML) + + // Créez une nouvelle instance du parser XML avec la fonction de rappel et xPath + parser := xixo.NewXMLParser(reader, &resultXMLBuffer).EnableXpath() + parser.RegisterCallback("name", modifyElement1Content) + // Créez un canal pour collecter les résultats du parser + err := parser.Stream() + assert.Nil(t, err) + + // Résultat XML attendu avec le contenu modifié et attributs restés + expectedResultXML := ` + + ContenuModifie + ` + + // Vérifiez si le résultat XML correspond à l'attendu + resultXML := resultXMLBuffer.String() + + if resultXML != expectedResultXML { + t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) + } +} + +// func TestTagWithSlashShouldSaved(t *testing.T) { +// t.Parallel() +// // Fichier XML en entrée +// inputXML := ` +// +// Hello +// Hello +// ` + +// // Lisez les résultats du canal et construisez le XML résultant +// var resultXMLBuffer bytes.Buffer + +// // Créez un bufio.Reader à partir du XML en entrée +// reader := bytes.NewBufferString(inputXML) + +// // Créez une nouvelle instance du parser XML avec la fonction de rappel et xPath +// parser := xixo.NewXMLParser(reader, &resultXMLBuffer).EnableXpath() +// parser.RegisterCallback("name", modifyElement1Content) +// // Créez un canal pour collecter les résultats du parser +// err := parser.Stream() +// assert.Nil(t, err) + +// // Résultat XML attendu avec le contenu modifié et attributs restés +// expectedResultXML := ` +// +// ContenuModifie +// Hello +// ` + +// // Vérifiez si le résultat XML correspond à l'attendu +// resultXML := resultXMLBuffer.String() + +// if resultXML != expectedResultXML { +// t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) +// } +// } diff --git a/test/data/exemple.xml b/test/data/exemple.xml new file mode 100644 index 0000000..7fbe7ba --- /dev/null +++ b/test/data/exemple.xml @@ -0,0 +1,7 @@ + + + + Doe John + 12345 + + diff --git a/test/data/exemple_expected.jsonl b/test/data/exemple_expected.jsonl new file mode 100644 index 0000000..21c4c1a --- /dev/null +++ b/test/data/exemple_expected.jsonl @@ -0,0 +1 @@ +{"@location":"New York", "name":"Doe John", "name@gender":"male", "age":"23", "account_number":12345} From 518fa070399019d9a6a0397f8cbe911f45521313 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Fri, 20 Oct 2023 13:30:40 +0000 Subject: [PATCH 02/10] feat: add NewXMLElement and AddAtrribut --- pkg/xixo/calback_test.go | 25 ++++++++++++++++++++++++ pkg/xixo/callback.go | 41 ++++++++++++++++++++++++++++++++++++++-- pkg/xixo/element.go | 26 +++++++++++++++++++++++++ pkg/xixo/element_test.go | 41 ++++++++++++++++++++++++++++++++++++++++ pkg/xixo/parser_test.go | 41 ++-------------------------------------- 5 files changed, 133 insertions(+), 41 deletions(-) diff --git a/pkg/xixo/calback_test.go b/pkg/xixo/calback_test.go index 083a0e8..48028b6 100644 --- a/pkg/xixo/calback_test.go +++ b/pkg/xixo/calback_test.go @@ -77,3 +77,28 @@ func TestBadJsonCallback(t *testing.T) { assert.NotNil(t, err) } + +func TestMapCallbackWithAttributs(t *testing.T) { + t.Parallel() + + element1 := createTree() + //nolint + assert.Equal(t, "\n Hello world !\n Contenu2 \n", element1.String()) + + editedElement1, err := xixo.XMLElementToMapCallback(mapCallbackAttributs)(element1) + assert.Nil(t, err) + + text := editedElement1.FirstChild().InnerText + + assert.Equal(t, "newChildContent", text) + + //nolint + assert.Equal(t, "\n newChildContent\n Contenu2 \n", editedElement1.String()) +} + +func mapCallbackAttributs(dict map[string]string) (map[string]string, error) { + dict["element1@age"] = "50" + dict["element1"] = "newChildContent" + + return dict, nil +} diff --git a/pkg/xixo/callback.go b/pkg/xixo/callback.go index 7d9125e..962f751 100644 --- a/pkg/xixo/callback.go +++ b/pkg/xixo/callback.go @@ -1,6 +1,9 @@ package xixo -import "encoding/json" +import ( + "encoding/json" + "regexp" +) type Callback func(*XMLElement) (*XMLElement, error) @@ -8,10 +11,15 @@ type CallbackMap func(map[string]string) (map[string]string, error) type CallbackJSON func(string) (string, error) +type Attributs struct { + Attr string + AttrVal string +} + func XMLElementToMapCallback(callback CallbackMap) Callback { result := func(xmlElement *XMLElement) (*XMLElement, error) { dict := map[string]string{} - + var AttributsList map[string][]Attributs for name, child := range xmlElement.Childs { dict[name] = child[0].InnerText } @@ -26,7 +34,36 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { return nil, err } + // check dict[name] include "@" + re := regexp.MustCompile("@") + for key, value := range dict { + if re.MatchString(key) { + // if include, use regexp to get element:before@ ,attr:after@ + parts := re.Split(key, 2) + tagName := parts[0] + newAttribut := Attributs{Attr: parts[1], AttrVal: value} + if existingElement, ok := AttributsList[tagName]; ok { + // if key already in attributs + existingElement = append(existingElement, newAttribut) + AttributsList[tagName] = existingElement + } else { + // if key not in attributs yet + AttributsList[tagName] = []Attributs{newAttribut} + } + } + } + for _, child := range children { + // if attrlist, ok := AttributsList[child.Name]; ok { + // if child.Attrs == nil { + + // } + // child.InnerText = value + // } + // if xmlElement.Attrs == nil + + // creat one + // apprend {} to Attrs if value, ok := dict[child.Name]; ok { child.InnerText = value } diff --git a/pkg/xixo/element.go b/pkg/xixo/element.go index d0eed38..a79b1dc 100644 --- a/pkg/xixo/element.go +++ b/pkg/xixo/element.go @@ -35,6 +35,10 @@ func (n *XMLElement) SelectElement(exp string) (*XMLElement, error) { } func (n *XMLElement) FirstChild() *XMLElement { + if n.childs == nil { + return nil + } + if len(n.childs) > 0 { return n.childs[0] } @@ -107,3 +111,25 @@ func (n *XMLElement) String() string { xmlChilds, n.Name) } + +func (n *XMLElement) AddAttribut(name string, value string) { + if n.Attrs == nil { + n.Attrs = make(map[string]string) + } + n.Attrs[name] = value +} + +func NewXMLElement(name string, attrs map[string]string) *XMLElement { + return &XMLElement{ + Name: name, + Attrs: attrs, + InnerText: "", + Childs: map[string][]XMLElement{}, + Err: nil, + childs: []*XMLElement{}, + parent: nil, + attrs: []*xmlAttr{}, + localName: "", + prefix: "", + } +} diff --git a/pkg/xixo/element_test.go b/pkg/xixo/element_test.go index c75978b..64e7a8b 100644 --- a/pkg/xixo/element_test.go +++ b/pkg/xixo/element_test.go @@ -87,3 +87,44 @@ func TestElementStringShouldReturnXMLWithSameOrder(t *testing.T) { assert.Equal(t, rootXML, root.String()) } + +func TestCreatNewXMLElement(t *testing.T) { + t.Parallel() + + rootXML := ` + + ` + + var root *xixo.XMLElement + name := "root" + attrs := map[string]string{} + parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() + root = xixo.NewXMLElement(name, attrs) + err := parser.Stream() + assert.Nil(t, err) + expected := `` + + assert.Equal(t, expected, root.String()) +} + +func TestAddAttributsShouldSaved(t *testing.T) { + t.Parallel() + + rootXML := ` + + ` + + var root *xixo.XMLElement + + name := "root" + attrs := map[string]string{} + parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() + root = xixo.NewXMLElement(name, attrs) + + root.AddAttribut("foo", "bar") + err := parser.Stream() + assert.Nil(t, err) + expected := `` + + assert.Equal(t, expected, root.String()) +} diff --git a/pkg/xixo/parser_test.go b/pkg/xixo/parser_test.go index 6822dc7..063615c 100644 --- a/pkg/xixo/parser_test.go +++ b/pkg/xixo/parser_test.go @@ -125,7 +125,7 @@ func TestAttributsShouldSavedAfterParser(t *testing.T) { t.Parallel() // Fichier XML en entrée inputXML := ` - + Hello ` @@ -144,7 +144,7 @@ func TestAttributsShouldSavedAfterParser(t *testing.T) { // Résultat XML attendu avec le contenu modifié et attributs restés expectedResultXML := ` - + ContenuModifie ` @@ -155,40 +155,3 @@ func TestAttributsShouldSavedAfterParser(t *testing.T) { t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) } } - -// func TestTagWithSlashShouldSaved(t *testing.T) { -// t.Parallel() -// // Fichier XML en entrée -// inputXML := ` -// -// Hello -// Hello -// ` - -// // Lisez les résultats du canal et construisez le XML résultant -// var resultXMLBuffer bytes.Buffer - -// // Créez un bufio.Reader à partir du XML en entrée -// reader := bytes.NewBufferString(inputXML) - -// // Créez une nouvelle instance du parser XML avec la fonction de rappel et xPath -// parser := xixo.NewXMLParser(reader, &resultXMLBuffer).EnableXpath() -// parser.RegisterCallback("name", modifyElement1Content) -// // Créez un canal pour collecter les résultats du parser -// err := parser.Stream() -// assert.Nil(t, err) - -// // Résultat XML attendu avec le contenu modifié et attributs restés -// expectedResultXML := ` -// -// ContenuModifie -// Hello -// ` - -// // Vérifiez si le résultat XML correspond à l'attendu -// resultXML := resultXMLBuffer.String() - -// if resultXML != expectedResultXML { -// t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) -// } -// } From ca2c513bc156e92744a9538e2a002104d837fa24 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Fri, 20 Oct 2023 14:24:00 +0000 Subject: [PATCH 03/10] feat: retirer les arguments de constructor NewXMLElement --- pkg/xixo/element.go | 6 +++--- pkg/xixo/element_test.go | 20 ++++++-------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/pkg/xixo/element.go b/pkg/xixo/element.go index a79b1dc..af4e8ef 100644 --- a/pkg/xixo/element.go +++ b/pkg/xixo/element.go @@ -119,10 +119,10 @@ func (n *XMLElement) AddAttribut(name string, value string) { n.Attrs[name] = value } -func NewXMLElement(name string, attrs map[string]string) *XMLElement { +func NewXMLElement() *XMLElement { return &XMLElement{ - Name: name, - Attrs: attrs, + Name: "", + Attrs: map[string]string{}, InnerText: "", Childs: map[string][]XMLElement{}, Err: nil, diff --git a/pkg/xixo/element_test.go b/pkg/xixo/element_test.go index 64e7a8b..55dd860 100644 --- a/pkg/xixo/element_test.go +++ b/pkg/xixo/element_test.go @@ -97,9 +97,9 @@ func TestCreatNewXMLElement(t *testing.T) { var root *xixo.XMLElement name := "root" - attrs := map[string]string{} parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() - root = xixo.NewXMLElement(name, attrs) + root = xixo.NewXMLElement() + root.Name = name err := parser.Stream() assert.Nil(t, err) expected := `` @@ -110,21 +110,13 @@ func TestCreatNewXMLElement(t *testing.T) { func TestAddAttributsShouldSaved(t *testing.T) { t.Parallel() - rootXML := ` - - ` - var root *xixo.XMLElement name := "root" - attrs := map[string]string{} - parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() - root = xixo.NewXMLElement(name, attrs) + root = xixo.NewXMLElement() + root.Name = name root.AddAttribut("foo", "bar") - err := parser.Stream() - assert.Nil(t, err) - expected := `` - - assert.Equal(t, expected, root.String()) + expected := map[string]string{"foo": "bar"} + assert.Equal(t, root.Attrs, expected) } From f5193e6076e6905108efc18b8f562957e5abb4f6 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Fri, 20 Oct 2023 14:46:07 +0000 Subject: [PATCH 04/10] feat: add TestAddAttributsShouldInOutputWithString --- pkg/xixo/element.go | 4 ++-- pkg/xixo/element_test.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/xixo/element.go b/pkg/xixo/element.go index af4e8ef..591690f 100644 --- a/pkg/xixo/element.go +++ b/pkg/xixo/element.go @@ -99,8 +99,8 @@ func (n *XMLElement) String() string { attributes := n.Name + " " - for _, attr := range n.attrs { - attributes += fmt.Sprintf("%s=\"%s\" ", attr.name, attr.value) + for key, value := range n.Attrs { + attributes += fmt.Sprintf("%s=\"%s\" ", key, value) } attributes = strings.Trim(attributes, " ") diff --git a/pkg/xixo/element_test.go b/pkg/xixo/element_test.go index 55dd860..93239f6 100644 --- a/pkg/xixo/element_test.go +++ b/pkg/xixo/element_test.go @@ -120,3 +120,17 @@ func TestAddAttributsShouldSaved(t *testing.T) { expected := map[string]string{"foo": "bar"} assert.Equal(t, root.Attrs, expected) } + +func TestAddAttributsShouldInOutputWithString(t *testing.T) { + t.Parallel() + + var root *xixo.XMLElement + name := "root" + root = xixo.NewXMLElement() + root.Name = name + root.InnerText = "Hello" + root.AddAttribut("foo", "bar") + + expected := "Hello" + assert.Equal(t, expected, root.String()) +} From ba282b13c895fe203408a6ef59e5078c9c16a280 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Mon, 30 Oct 2023 11:00:57 +0000 Subject: [PATCH 05/10] feat: add addAttributs function --- pkg/xixo/calback_test.go | 6 ++-- pkg/xixo/callback.go | 76 ++++++++++++++++++++++++---------------- pkg/xixo/element_test.go | 67 +++++++++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 36 deletions(-) diff --git a/pkg/xixo/calback_test.go b/pkg/xixo/calback_test.go index 48028b6..113f1a2 100644 --- a/pkg/xixo/calback_test.go +++ b/pkg/xixo/calback_test.go @@ -81,9 +81,9 @@ func TestBadJsonCallback(t *testing.T) { func TestMapCallbackWithAttributs(t *testing.T) { t.Parallel() - element1 := createTree() + element1 := createTreeWithAttribut() //nolint - assert.Equal(t, "\n Hello world !\n Contenu2 \n", element1.String()) + assert.Equal(t, "\n Hello world !\n Contenu2 \n", element1.String()) editedElement1, err := xixo.XMLElementToMapCallback(mapCallbackAttributs)(element1) assert.Nil(t, err) @@ -93,7 +93,7 @@ func TestMapCallbackWithAttributs(t *testing.T) { assert.Equal(t, "newChildContent", text) //nolint - assert.Equal(t, "\n newChildContent\n Contenu2 \n", editedElement1.String()) + assert.Equal(t, "\n newChildContent\n Contenu2 \n", editedElement1.String()) } func mapCallbackAttributs(dict map[string]string) (map[string]string, error) { diff --git a/pkg/xixo/callback.go b/pkg/xixo/callback.go index 962f751..5c2ceba 100644 --- a/pkg/xixo/callback.go +++ b/pkg/xixo/callback.go @@ -2,7 +2,7 @@ package xixo import ( "encoding/json" - "regexp" + "strings" ) type Callback func(*XMLElement) (*XMLElement, error) @@ -12,14 +12,13 @@ type CallbackMap func(map[string]string) (map[string]string, error) type CallbackJSON func(string) (string, error) type Attributs struct { - Attr string - AttrVal string + Name string + Value string } func XMLElementToMapCallback(callback CallbackMap) Callback { result := func(xmlElement *XMLElement) (*XMLElement, error) { dict := map[string]string{} - var AttributsList map[string][]Attributs for name, child := range xmlElement.Childs { dict[name] = child[0].InnerText } @@ -34,39 +33,20 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { return nil, err } - // check dict[name] include "@" - re := regexp.MustCompile("@") - for key, value := range dict { - if re.MatchString(key) { - // if include, use regexp to get element:before@ ,attr:after@ - parts := re.Split(key, 2) - tagName := parts[0] - newAttribut := Attributs{Attr: parts[1], AttrVal: value} - if existingElement, ok := AttributsList[tagName]; ok { - // if key already in attributs - existingElement = append(existingElement, newAttribut) - AttributsList[tagName] = existingElement - } else { - // if key not in attributs yet - AttributsList[tagName] = []Attributs{newAttribut} - } - } - } + AttributsList := exctratAttributs(dict, xmlElement.Name) for _, child := range children { - // if attrlist, ok := AttributsList[child.Name]; ok { - // if child.Attrs == nil { - // } - // child.InnerText = value - // } - // if xmlElement.Attrs == nil - - // creat one - // apprend {} to Attrs if value, ok := dict[child.Name]; ok { child.InnerText = value } + + if attributes, ok := AttributsList[child.Name]; ok { + for _, attr := range attributes { + child.AddAttribut(attr.Name, attr.Value) + } + } + } return xmlElement, nil @@ -75,6 +55,40 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { return result } +func exctratAttributs(dict map[string]string, parentName string) map[string][]Attributs { + AttributsList := make(map[string][]Attributs) + // check dict[name] include "@" + for key, value := range dict { + parts := strings.SplitN(key, "@", 2) + // if include, use split to get element:before@ ,attr:after@ + if len(parts) == 1 { + tagName := parentName + newAttribut := Attributs{Name: parts[0], Value: value} + + if existingElement, ok := AttributsList[tagName]; ok { + existingElement = append(existingElement, newAttribut) + AttributsList[tagName] = existingElement + } else { + AttributsList[tagName] = []Attributs{newAttribut} + } + } else if len(parts) == 2 { + + tagName := parts[0] + newAttribut := Attributs{Name: parts[1], Value: value} + + // if key already in attributs + if existingElement, ok := AttributsList[tagName]; ok { + + existingElement = append(existingElement, newAttribut) + AttributsList[tagName] = existingElement + } else { + AttributsList[tagName] = []Attributs{newAttribut} + } + } + } + return AttributsList +} + func XMLElementToJSONCallback(callback CallbackJSON) Callback { resultCallback := func(dict map[string]string) (map[string]string, error) { source, err := json.Marshal(dict) diff --git a/pkg/xixo/element_test.go b/pkg/xixo/element_test.go index 93239f6..8ed0392 100644 --- a/pkg/xixo/element_test.go +++ b/pkg/xixo/element_test.go @@ -33,6 +33,54 @@ func createTree() *xixo.XMLElement { return root } +func createTreeWithAttribut() *xixo.XMLElement { + rootXML := ` + + Hello world ! + Contenu2 + ` + + var root *xixo.XMLElement + + parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() + parser.RegisterCallback("root", func(x *xixo.XMLElement) (*xixo.XMLElement, error) { + root = x + + return x, nil + }) + + err := parser.Stream() + if err != nil { + return nil + } + + return root +} + +func createTreeWithAttributParent() *xixo.XMLElement { + rootXML := ` + + Hello world ! + Contenu2 + ` + + var root *xixo.XMLElement + + parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() + parser.RegisterCallback("root", func(x *xixo.XMLElement) (*xixo.XMLElement, error) { + root = x + + return x, nil + }) + + err := parser.Stream() + if err != nil { + return nil + } + + return root +} + func TestElementStringShouldReturnXML(t *testing.T) { t.Parallel() @@ -125,12 +173,27 @@ func TestAddAttributsShouldInOutputWithString(t *testing.T) { t.Parallel() var root *xixo.XMLElement - name := "root" root = xixo.NewXMLElement() - root.Name = name + root.Name = "root" + root.InnerText = "Hello" + root.AddAttribut("foo", "bar") + + expected := "Hello" + assert.Equal(t, expected, root.String()) +} + +func TestEditAttributsShouldInOutputWithString(t *testing.T) { + t.Parallel() + + var root *xixo.XMLElement + root = xixo.NewXMLElement() + root.Name = "root" root.InnerText = "Hello" root.AddAttribut("foo", "bar") expected := "Hello" assert.Equal(t, expected, root.String()) + root.AddAttribut("foo", "bas") + expected = "Hello" + assert.Equal(t, expected, root.String()) } From 135405166919233ae819bce3c9b5f8beee8354c0 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Mon, 30 Oct 2023 15:16:11 +0000 Subject: [PATCH 06/10] feat: ajouter modification d'attribute de balise parent --- pkg/xixo/calback_test.go | 27 +++++++++++++++++++++++++++ pkg/xixo/callback.go | 36 ++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pkg/xixo/calback_test.go b/pkg/xixo/calback_test.go index 113f1a2..7ceb537 100644 --- a/pkg/xixo/calback_test.go +++ b/pkg/xixo/calback_test.go @@ -102,3 +102,30 @@ func mapCallbackAttributs(dict map[string]string) (map[string]string, error) { return dict, nil } + +func TestMapCallbackWithAttributsParentAndChilds(t *testing.T) { + t.Parallel() + + element1 := createTreeWithAttributParent() + //nolint + assert.Equal(t, "\n Hello world !\n Contenu2 \n", element1.String()) + + editedElement1, err := xixo.XMLElementToMapCallback(mapCallbackAttributsWithParent)(element1) + assert.Nil(t, err) + + text := editedElement1.FirstChild().InnerText + + assert.Equal(t, "newChildContent", text) + + //nolint + assert.Equal(t, "\n newChildContent\n Contenu2 \n", editedElement1.String()) +} + +func mapCallbackAttributsWithParent(dict map[string]string) (map[string]string, error) { + dict["@type"] = "bar" + dict["element1@age"] = "50" + dict["element1"] = "newChildContent" + dict["element2@age"] = "25" + + return dict, nil +} diff --git a/pkg/xixo/callback.go b/pkg/xixo/callback.go index 5c2ceba..fd2e0a8 100644 --- a/pkg/xixo/callback.go +++ b/pkg/xixo/callback.go @@ -27,13 +27,18 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { if err != nil { return nil, err } - + parentAttributs := extractAttributesParent(dict) + if parentAttributs != nil { + for _, attr := range parentAttributs { + xmlElement.AddAttribut(attr.Name, attr.Value) + } + } children, err := xmlElement.SelectElements("child::*") if err != nil { return nil, err } - AttributsList := exctratAttributs(dict, xmlElement.Name) + AttributsList := exctratAttributsChild(dict) for _, child := range children { @@ -55,23 +60,13 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { return result } -func exctratAttributs(dict map[string]string, parentName string) map[string][]Attributs { +func exctratAttributsChild(dict map[string]string) map[string][]Attributs { AttributsList := make(map[string][]Attributs) // check dict[name] include "@" for key, value := range dict { parts := strings.SplitN(key, "@", 2) // if include, use split to get element:before@ ,attr:after@ - if len(parts) == 1 { - tagName := parentName - newAttribut := Attributs{Name: parts[0], Value: value} - - if existingElement, ok := AttributsList[tagName]; ok { - existingElement = append(existingElement, newAttribut) - AttributsList[tagName] = existingElement - } else { - AttributsList[tagName] = []Attributs{newAttribut} - } - } else if len(parts) == 2 { + if len(parts) == 2 { tagName := parts[0] newAttribut := Attributs{Name: parts[1], Value: value} @@ -89,6 +84,19 @@ func exctratAttributs(dict map[string]string, parentName string) map[string][]At return AttributsList } +func extractAttributesParent(dict map[string]string) []Attributs { + AttributesMap := []Attributs{} + for key, value := range dict { + if strings.HasPrefix(key, "@") { + attributeKey := key[1:] + attribute := Attributs{Name: attributeKey, Value: value} + AttributesMap = append(AttributesMap, attribute) + } + } + + return AttributesMap +} + func XMLElementToJSONCallback(callback CallbackJSON) Callback { resultCallback := func(dict map[string]string) (map[string]string, error) { source, err := json.Marshal(dict) From 34986dbf4d234a0df9b7089ac14ae0b2b1a0ff19 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Tue, 31 Oct 2023 12:38:37 +0000 Subject: [PATCH 07/10] feat: refactor partie attributs --- pkg/xixo/calback_test.go | 10 ++++--- pkg/xixo/callback.go | 60 ++++++++++++++++++++-------------------- pkg/xixo/element.go | 5 +++- pkg/xixo/element_test.go | 30 ++++++++++++-------- pkg/xixo/parser_test.go | 40 ++++++++++++++++++++++++++- 5 files changed, 97 insertions(+), 48 deletions(-) diff --git a/pkg/xixo/calback_test.go b/pkg/xixo/calback_test.go index 7ceb537..81a079a 100644 --- a/pkg/xixo/calback_test.go +++ b/pkg/xixo/calback_test.go @@ -8,8 +8,10 @@ import ( "github.com/youen/xixo/pkg/xixo" ) +const newChildContent = "newChildContent" + func mapCallback(dict map[string]string) (map[string]string, error) { - dict["element1"] = "newChildContent" + dict["element1"] = newChildContent return dict, nil } @@ -41,7 +43,7 @@ func jsonCallback(source string) (string, error) { return "", err } - dict["element1"] = "newChildContent" + dict["element1"] = newChildContent result, err := json.Marshal(dict) @@ -98,7 +100,7 @@ func TestMapCallbackWithAttributs(t *testing.T) { func mapCallbackAttributs(dict map[string]string) (map[string]string, error) { dict["element1@age"] = "50" - dict["element1"] = "newChildContent" + dict["element1"] = newChildContent return dict, nil } @@ -124,7 +126,7 @@ func TestMapCallbackWithAttributsParentAndChilds(t *testing.T) { func mapCallbackAttributsWithParent(dict map[string]string) (map[string]string, error) { dict["@type"] = "bar" dict["element1@age"] = "50" - dict["element1"] = "newChildContent" + dict["element1"] = newChildContent dict["element2@age"] = "25" return dict, nil diff --git a/pkg/xixo/callback.go b/pkg/xixo/callback.go index fd2e0a8..a62fba9 100644 --- a/pkg/xixo/callback.go +++ b/pkg/xixo/callback.go @@ -11,11 +11,13 @@ type CallbackMap func(map[string]string) (map[string]string, error) type CallbackJSON func(string) (string, error) -type Attributs struct { +type Attribute struct { Name string Value string } +// XMLElementToMapCallback transforms an XML element into a map, applies a callback function, +// adds parent attributes, and updates child elements. func XMLElementToMapCallback(callback CallbackMap) Callback { result := func(xmlElement *XMLElement) (*XMLElement, error) { dict := map[string]string{} @@ -27,31 +29,29 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { if err != nil { return nil, err } - parentAttributs := extractAttributesParent(dict) - if parentAttributs != nil { - for _, attr := range parentAttributs { - xmlElement.AddAttribut(attr.Name, attr.Value) - } + // Extract parent attributes and add them to the XML element. + parentAttributes := extractParentAttributes(dict) + for _, attr := range parentAttributes { + xmlElement.AddAttribute(attr.Name, attr.Value) } + children, err := xmlElement.SelectElements("child::*") if err != nil { return nil, err } - - AttributsList := exctratAttributsChild(dict) + // Select child elements and update their text content and attributes. + childAttributes := extractChildAttributes(dict) for _, child := range children { - if value, ok := dict[child.Name]; ok { child.InnerText = value } - if attributes, ok := AttributsList[child.Name]; ok { + if attributes, ok := childAttributes[child.Name]; ok { for _, attr := range attributes { - child.AddAttribut(attr.Name, attr.Value) + child.AddAttribute(attr.Name, attr.Value) } } - } return xmlElement, nil @@ -60,41 +60,41 @@ func XMLElementToMapCallback(callback CallbackMap) Callback { return result } -func exctratAttributsChild(dict map[string]string) map[string][]Attributs { - AttributsList := make(map[string][]Attributs) +// extractChildAttributes extracts child attributes from the dictionary. +func extractChildAttributes(dict map[string]string) map[string][]Attribute { + childAttributes := make(map[string][]Attribute) // check dict[name] include "@" for key, value := range dict { parts := strings.SplitN(key, "@", 2) - // if include, use split to get element:before@ ,attr:after@ - if len(parts) == 2 { + if len(parts) == 2 { tagName := parts[0] - newAttribut := Attributs{Name: parts[1], Value: value} - - // if key already in attributs - if existingElement, ok := AttributsList[tagName]; ok { - - existingElement = append(existingElement, newAttribut) - AttributsList[tagName] = existingElement + newAttribut := Attribute{Name: parts[1], Value: value} + // if key already in attributes + if existingElement, ok := childAttributes[tagName]; ok { + childAttributes[tagName] = append(existingElement, newAttribut) } else { - AttributsList[tagName] = []Attributs{newAttribut} + childAttributes[tagName] = []Attribute{newAttribut} } } } - return AttributsList + + return childAttributes } -func extractAttributesParent(dict map[string]string) []Attributs { - AttributesMap := []Attributs{} +// extractParentAttributes extracts parent attributes from the dictionary. +func extractParentAttributes(dict map[string]string) []Attribute { + parentAttributes := []Attribute{} + for key, value := range dict { if strings.HasPrefix(key, "@") { attributeKey := key[1:] - attribute := Attributs{Name: attributeKey, Value: value} - AttributesMap = append(AttributesMap, attribute) + attribute := Attribute{Name: attributeKey, Value: value} + parentAttributes = append(parentAttributes, attribute) } } - return AttributesMap + return parentAttributes } func XMLElementToJSONCallback(callback CallbackJSON) Callback { diff --git a/pkg/xixo/element.go b/pkg/xixo/element.go index 591690f..2ed5f0e 100644 --- a/pkg/xixo/element.go +++ b/pkg/xixo/element.go @@ -8,6 +8,7 @@ import ( type XMLElement struct { Name string Attrs map[string]string + AttrKeys []string InnerText string Childs map[string][]XMLElement Err error @@ -112,10 +113,11 @@ func (n *XMLElement) String() string { n.Name) } -func (n *XMLElement) AddAttribut(name string, value string) { +func (n *XMLElement) AddAttribute(name string, value string) { if n.Attrs == nil { n.Attrs = make(map[string]string) } + n.Attrs[name] = value } @@ -123,6 +125,7 @@ func NewXMLElement() *XMLElement { return &XMLElement{ Name: "", Attrs: map[string]string{}, + AttrKeys: make([]string, 0), InnerText: "", Childs: map[string][]XMLElement{}, Err: nil, diff --git a/pkg/xixo/element_test.go b/pkg/xixo/element_test.go index 8ed0392..a5c9670 100644 --- a/pkg/xixo/element_test.go +++ b/pkg/xixo/element_test.go @@ -9,6 +9,8 @@ import ( "github.com/youen/xixo/pkg/xixo" ) +const parentTag = "root" + func createTree() *xixo.XMLElement { rootXML := ` @@ -144,12 +146,15 @@ func TestCreatNewXMLElement(t *testing.T) { ` var root *xixo.XMLElement - name := "root" + + name := parentTag parser := xixo.NewXMLParser(bytes.NewBufferString(rootXML), io.Discard).EnableXpath() root = xixo.NewXMLElement() root.Name = name err := parser.Stream() + assert.Nil(t, err) + expected := `` assert.Equal(t, expected, root.String()) @@ -160,23 +165,24 @@ func TestAddAttributsShouldSaved(t *testing.T) { var root *xixo.XMLElement - name := "root" + name := parentTag root = xixo.NewXMLElement() root.Name = name - root.AddAttribut("foo", "bar") + root.AddAttribute("foo", "bar") + expected := map[string]string{"foo": "bar"} + assert.Equal(t, root.Attrs, expected) } func TestAddAttributsShouldInOutputWithString(t *testing.T) { t.Parallel() - var root *xixo.XMLElement - root = xixo.NewXMLElement() - root.Name = "root" + root := xixo.NewXMLElement() + root.Name = parentTag root.InnerText = "Hello" - root.AddAttribut("foo", "bar") + root.AddAttribute("foo", "bar") expected := "Hello" assert.Equal(t, expected, root.String()) @@ -185,15 +191,15 @@ func TestAddAttributsShouldInOutputWithString(t *testing.T) { func TestEditAttributsShouldInOutputWithString(t *testing.T) { t.Parallel() - var root *xixo.XMLElement - root = xixo.NewXMLElement() - root.Name = "root" + root := xixo.NewXMLElement() + root.Name = parentTag root.InnerText = "Hello" - root.AddAttribut("foo", "bar") + root.AddAttribute("foo", "bar") expected := "Hello" assert.Equal(t, expected, root.String()) - root.AddAttribut("foo", "bas") + root.AddAttribute("foo", "bas") + expected = "Hello" assert.Equal(t, expected, root.String()) } diff --git a/pkg/xixo/parser_test.go b/pkg/xixo/parser_test.go index 063615c..56db271 100644 --- a/pkg/xixo/parser_test.go +++ b/pkg/xixo/parser_test.go @@ -142,7 +142,7 @@ func TestAttributsShouldSavedAfterParser(t *testing.T) { err := parser.Stream() assert.Nil(t, err) - // Résultat XML attendu avec le contenu modifié et attributs restés + // Résultat XML attendu avec le contenu modifié et attributes restés expectedResultXML := ` ContenuModifie @@ -155,3 +155,41 @@ func TestAttributsShouldSavedAfterParser(t *testing.T) { t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) } } + +func TestModifyAttributsWithMapCallback(t *testing.T) { + t.Parallel() + // Fichier XML en entrée + inputXML := ` + + Hello world! + Contenu2 ! + ` + + // Lisez les résultats du canal et construisez le XML résultant + var resultXMLBuffer bytes.Buffer + + // Créez un bufio.Reader à partir du XML en entrée + reader := bytes.NewBufferString(inputXML) + + // Créez une nouvelle instance du parser XML avec la fonction de rappel + parser := xixo.NewXMLParser(reader, &resultXMLBuffer).EnableXpath() + parser.RegisterMapCallback("root", mapCallbackAttributsWithParent) + + // Créez un canal pour collecter les résultats du parser + err := parser.Stream() + assert.Nil(t, err) + + // Résultat XML attendu avec le contenu modifié + expectedResultXML := ` + + newChildContent + Contenu2 ! +` + + // Vérifiez si le résultat XML correspond à l'attendu + resultXML := resultXMLBuffer.String() + + if resultXML != expectedResultXML { + t.Errorf("Le résultat XML ne correspond pas à l'attendu.\nAttendu:\n%s\nObtenu:\n%s", expectedResultXML, resultXML) + } +} From 93da3f39d3a02cd92cf2eff06a11f3c7c1f92763 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Tue, 31 Oct 2023 13:43:10 +0000 Subject: [PATCH 08/10] feat: add AttrKeys to keep order of attributes --- pkg/xixo/element.go | 12 ++++++++---- pkg/xixo/parser.go | 13 ++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/xixo/element.go b/pkg/xixo/element.go index 2ed5f0e..9ab4b23 100644 --- a/pkg/xixo/element.go +++ b/pkg/xixo/element.go @@ -99,9 +99,8 @@ func (n *XMLElement) String() string { } attributes := n.Name + " " - - for key, value := range n.Attrs { - attributes += fmt.Sprintf("%s=\"%s\" ", key, value) + for _, key := range n.AttrKeys { + attributes += fmt.Sprintf("%s=\"%s\" ", key, n.Attrs[key]) } attributes = strings.Trim(attributes, " ") @@ -117,7 +116,12 @@ func (n *XMLElement) AddAttribute(name string, value string) { if n.Attrs == nil { n.Attrs = make(map[string]string) } - + // if name don't exsite in Attrs yet + if _, ok := n.Attrs[name]; !ok { + // Add un key in slice to keep the order of attributes + n.AttrKeys = append(n.AttrKeys, name) + } + // change the value of attribute n.Attrs[name] = value } diff --git a/pkg/xixo/parser.go b/pkg/xixo/parser.go index 622fe09..b606ab9 100644 --- a/pkg/xixo/parser.go +++ b/pkg/xixo/parser.go @@ -467,12 +467,7 @@ search_close_tag: if x.isWS(cur) { continue } - if cur == '=' { - if result.Attrs == nil { - result.Attrs = map[string]string{} - } - cur, err = x.readByte() if err != nil { @@ -488,10 +483,10 @@ search_close_tag: if err != nil { return nil, false, x.defaultError() } - result.Attrs[attr] = attrVal - if x.xpathEnabled { - result.attrs = append(result.attrs, &xmlAttr{name: attr, value: attrVal}) - } + result.AddAttribute(attr, attrVal) + // if x.xpathEnabled { + // result.attrs = append(result.attrs, &xmlAttr{name: attr, value: attrVal}) + // } x.scratch.reset() continue From 8c17e74c2274004b7bf705c64a61126fb536bdb6 Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Fri, 3 Nov 2023 10:48:28 +0000 Subject: [PATCH 09/10] =?UTF-8?q?feat:=20suppression=20de=20fonctionnalit?= =?UTF-8?q?=C3=A9=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/data/exemple.xml | 7 ------- test/data/exemple_expected.jsonl | 1 - 2 files changed, 8 deletions(-) delete mode 100644 test/data/exemple.xml delete mode 100644 test/data/exemple_expected.jsonl diff --git a/test/data/exemple.xml b/test/data/exemple.xml deleted file mode 100644 index 7fbe7ba..0000000 --- a/test/data/exemple.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - Doe John - 12345 - - diff --git a/test/data/exemple_expected.jsonl b/test/data/exemple_expected.jsonl deleted file mode 100644 index 21c4c1a..0000000 --- a/test/data/exemple_expected.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"@location":"New York", "name":"Doe John", "name@gender":"male", "age":"23", "account_number":12345} From 806907f5dbe23ad1d3ef6938ee197f44313cb13e Mon Sep 17 00:00:00 2001 From: "jianchao.ma" Date: Fri, 3 Nov 2023 13:55:54 +0000 Subject: [PATCH 10/10] =?UTF-8?q?feat:=20suppression=20de=20fonctionnalit?= =?UTF-8?q?=C3=A9=20de=20command=20and=20test=20venom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 2 +- Makefile | 4 -- README.md | 83 ++++++++-------------- build.yml | 22 +----- cmd/xixo/main.go | 137 ------------------------------------- go.mod | 5 +- go.sum | 8 --- pkg/xixo/driver.go | 51 ++------------ pkg/xixo/driver_test.go | 21 +----- pkg/xixo/process.go | 77 --------------------- pkg/xixo/process_test.go | 61 ----------------- test/suites/01-run-cli.yml | 77 --------------------- 13 files changed, 39 insertions(+), 513 deletions(-) delete mode 100644 cmd/xixo/main.go delete mode 100644 pkg/xixo/process.go delete mode 100644 pkg/xixo/process_test.go delete mode 100644 test/suites/01-run-cli.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d5f9b2..5131225 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: docker-compose exec -T -u root -w /workspace ci make init - uses: FranzDiebold/github-env-vars-action@v2 - - name: Run CI # up to test-int (info → refresh → lint → test → release → test-int) + - name: Run CI # up to release (info → refresh → lint → test → release) run: | docker-compose exec \ -T \ @@ -45,4 +45,4 @@ jobs: -w /workspace \ -e PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/home/vscode/go/bin:/workspace/bin \ ci \ - neon -props "{tag: ${CI_ACTION_REF_NAME}, MODULE: github.com/${CI_REPOSITORY,,}, BY: ${CI_ACTOR}}" test-int + neon -props "{tag: ${CI_ACTION_REF_NAME}, MODULE: github.com/${CI_REPOSITORY,,}, BY: ${CI_ACTOR}}" release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d8ab58..187cce4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,4 +24,4 @@ Details are available on [the official Visual Studio documentation](https://code - [ ] The code you added/modified **compile** (run the `neon compile` command) - [ ] The code you added/modified **is linted** (run the `neon lint` command) - [ ] The code you added/modified **is covered by unit tests** and **all tests are passing** (run the `neon test` command) -- [ ] The features you added/modified or the bugs you fixed **are covered by integration tests** and **all integration tests are passing** (run the `neon test-int` command) + diff --git a/Makefile b/Makefile index 97fe8cc..9f8fd81 100644 --- a/Makefile +++ b/Makefile @@ -60,10 +60,6 @@ release-%: warning release: warning @neon release -.PHONY: test-int -test-int: warning - @neon test-int - .PHONY: publish-% publish-%: warning @neon -props "{buildpaths: ["cmd/$*"]}" publish diff --git a/README.md b/README.md index 6d44691..2e0d042 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,57 @@ # XIXO - XML Input XML Output -**XIXO** is a versatile command-line tool designed for seamless XML input and XML output operations. It empowers you to manipulate XML content effortlessly, offering a range of features for both basic and advanced operations. This README provides a comprehensive guide to **XIXO**, including installation instructions, usage examples, and acknowledgments to the open-source community. +**XIXO** is a Go library that allows you to parse XML files and extract custom attributes. It empowers you to manipulate XML content effortlessly, offering a range of features for both basic and advanced operations. This README provides a comprehensive guide to **XIXO**, including installation instructions, usage examples, and acknowledgments to the open-source community. ## Installation -To install xixo, follow these steps: +To install xixo, you can use go get: -- Download the right release for your operating system and architecture from the xixo releases page. - -- Extract the xixo binary from the downloaded archive. - -- Move the xixo binary to a directory that is included in your system's PATH. This step is essential to make xixo accessible from any location in your terminal. - -Note: If you are not sure which directory to use, you can typically place it in /usr/local/bin or ~/bin (for Linux/macOS) or C:\Windows\System32 (for Windows). You may need administrator privileges to move the binary to some directories. - -- Verify the installation by running the following command in your terminal: - -```bash -$ xixo --version ``` - -If installed correctly, it should display the version of xixo. +go get github.com/CGI-FR/XIXO +``` Now, you can start using xixo to edit XML files with ease. ## Example -### Input XML +To use **XIXO**, you need to create a Parser object with the path of the XML file to parse and the name of element, here is a emexple of XML file: (same exemple in Unit testing **TestMapCallbackWithAttributsParentAndChilds()** in callback_test.go ) ```xml - - - a - z - - - b - - - c - - -``` - -### Command - -```shell -$ xixo --subscribers foo="tee debug.jsonl | jq --unbuffered -c '.bar |= ascii_upcase' " < test/data/foo_bar_baz.xml - - - A - z - - - B - - - c - + + Hello world ! + Contenu2 ``` ### Process Description -1. **Initialization**: **xixo** begins by parsing the input XML file in a streaming manner. It identifies the structure of the XML and locates elements that match the subscriber criteria (`foo` elements in this case). +1. **Initialization**: **xixo** begins by parsing the input XML file in a streaming manner. It identifies the structure of the XML and locates elements that match the begin element name. (`root` elements in this case). -2. **Subscriber Script Execution**: The subscriber script (`tee debug.jsonl | jq --unbuffered -c '.bar |= ascii_upcase'`) is executed once at the beginning of the parsing process, as indicated. It's important to note that the script is not called separately for each `foo` element but rather only once as the input is piped from **xixo**. The script performs the following steps: +2. **Transform XML to Go map** As the XML parsing progresses, **xixo** reads the XML file and creat a element tree of root element. In exemple will be: - - It starts by using `tee` to write matched elements (e.g., `az`) as JSON lines, such as `{"bar":"a","baz":"z"}`, to the `debug.jsonl` file. - - It then uses `jq` to apply the transformation to the `bar` element within the JSON content. The `ascii_upcase` function is used to convert the text within the `bar` element to uppercase. The `--unbuffered` flag ensures that **jq** processes the input line by line. - - The script generates modified JSON lines, such as `{"bar":"A","baz":"z"}` and `{"bar":"B"}`, and writes them to the standard output. +```go +{"@type":"foo","element1":"Hello world !","element1@age":"22","element1":"male","element2":"Contenu2 "}. +``` -3. **Merging JSON Output**: As the XML parsing progresses, **xixo** reads the JSON lines from the script's standard output. Each line of JSON data corresponds to a `foo` element in the XML. **xixo** combines this JSON data with the current matching XML element to produce the updated XML structure with the transformations applied. The modified XML is then emitted as output in a streaming manner. +3. **Edite element value and attribute value with callback**: For modify the data, we need give callback function a map with the element name as key, new data as value. In exemple we give a map like this: -4. **Final Output**: The final XML output, with the transformations applied to the `bar` elements within the `foo` elements. +```go +{"@type":"bar","element1@age":"50","element1":"newChildContent","element2@age":"25"}. +``` -### Key Points +4. **Final Output**: The final XML output will be: -- **Subscriber Script Execution**: The subscriber script is executed only once at the beginning of the parsing process and then stopped at the end. It processes the XML elements as they match the criteria and serializes them into JSON lines, which are then merged with the corresponding XML elements to produce the modified XML output. +```xml + + newChildContent + Contenu2 + +``` + +### Key Points -- **Performance Optimization**: **xixo** optimizes performance by not calling the subscriber script for each `foo` element separately but rather processing the input in a stream and merging the results efficiently. +- **Performance Optimization**: **xixo** optimizes performance by not calling the subscriber script for each `root` element separately but rather processing the input in a stream and merging the results efficiently. This detailed process demonstrates how **xixo** processes XML files in a streaming and efficient manner, applying custom transformations to specific elements using subscribers. diff --git a/build.yml b/build.yml index d9de12d..06c6383 100644 --- a/build.yml +++ b/build.yml @@ -9,7 +9,6 @@ expose: "lint", "test", "release", - "test-int", "publish", "docker", "docker-tag", @@ -93,8 +92,6 @@ targets: print -P "%B%F{blue}lint%f%b Examine source code and report suspicious constructs [info->refresh]" print -P "%B%F{blue}test%f%b Run all tests with coverage [info->refresh->lint]" print -P "%B%F{blue}release%f%b Compile binary files for production [info->refresh->lint->test]" - print -P "%B%F{blue}test-int%f%b Run all integration tests [info->refresh->lint->test->release]" - print -P "%B%F{blue}publish%f%b Publish tagged binary to Github [info->refresh->lint->test->release->test-int]" print -P "%B%F{blue}docker%f%b Build docker images [info]" print -P "%B%F{blue}docker-tag%f%b Tag docker images [info->docker]" print -P "%B%F{blue}docker-push%f%b Publish docker images to Dockerhub [info->docker->docker-tag]" @@ -108,7 +105,6 @@ targets: print -P "→ promote" print -P "→ info ┰ docker → docker-tag → docker-push" print -P " ┖ refresh ┰ compile → license" - print -P " ┖ lint → test → release → test-int → publish" info: doc: "Print build informations" @@ -259,27 +255,11 @@ targets: - ldflags = ldflags + " -s -w" # Omit the DWARF symbol table. Omit the symbol table and debug information. - call: compile - # to debug a single test use : - # neon -props '{testsuites: ""}' test-int-debug - test-int-debug: - doc: "Run all integration tests" - depends: ["info", "compile"] - steps: - - delete: venom*.log - - $: venom run test/suites/={testsuites} - - test-int: - doc: "Run all integration tests" - depends: ["info", "refresh", "lint", "test", "bench", "release"] - steps: - - delete: venom*.log - - $: venom run test/suites/={testsuites} - # run "neon -props '{buildpaths: ["path/to/main/package1","path/to/main/package2"]}' publish" to publish specific targets # example : neon -props '{buildpaths: ["cmd/cli"]}' publish publish: doc: "Publish tagged binaries to Github" - depends: ["info", "refresh", "lint", "test", "release", "test-int"] + depends: ["info", "refresh", "lint", "test", "release"] steps: - $: git clean -f -d - if: len(buildpaths) == 0 diff --git a/cmd/xixo/main.go b/cmd/xixo/main.go deleted file mode 100644 index f4413da..0000000 --- a/cmd/xixo/main.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - "strings" - - "github.com/mattn/go-isatty" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - "github.com/youen/xixo/pkg/xixo" -) - -//nolint:gochecknoglobals -var ( - name string // provisioned by ldflags - version string // provisioned by ldflags - commit string // provisioned by ldflags - buildDate string // provisioned by ldflags - builtBy string // provisioned by ldflags - - verbosity string - jsonlog bool - debug bool - colormode string - - subscribers map[string]string -) - -func main() { - cobra.OnInitialize(initLog) - - rootCmd := &cobra.Command{ //nolint:exhaustruct - Use: fmt.Sprintf("%v real-data-file.jsonl", name), - Short: "Masked Input Metrics Output", - Long: `XIXO is a purpose-built tool designed for edit XML file with stream process like PIMO`, - Example: ` - to apply "jq -c '.bar |= ascii_upcase'" shell on all text of 'bar' elements of 'foo' entity: - $ echo 'a' | xixo --subscribers foo="jq -c '.bar |= ascii_upcase'" - A - `, - Version: fmt.Sprintf(`%v (commit=%v date=%v by=%v) -Copyright (C) 2021 CGI France -License GPLv3: GNU GPL version 3 . -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDate, builtBy), - PersistentPreRun: func(cmd *cobra.Command, args []string) { - log.Info(). - Str("verbosity", verbosity). - Bool("log-json", jsonlog). - Bool("debug", debug). - Str("color", colormode). - Msg("start XIXO") - }, - Args: cobra.ExactArgs(0), - - Run: func(cmd *cobra.Command, args []string) { - if err := run(cmd); err != nil { - log.Fatal().Err(err).Msg("end XIXO") - } - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - log.Info().Int("return", 0).Msg("end XIXO") - }, - } - - rootCmd.PersistentFlags().StringVarP(&verbosity, "verbosity", "v", "warn", - "set level of log verbosity : none (0), error (1), warn (2), info (3), debug (4), trace (5)") - rootCmd.PersistentFlags().BoolVar(&jsonlog, "log-json", false, "output logs in JSON format") - rootCmd.PersistentFlags().StringVar(&colormode, "color", "auto", "use colors in log outputs : yes, no or auto") - rootCmd.PersistentFlags().StringToStringVar( - &subscribers, "subscribers", map[string]string{}, - "subscribers shell for matching elements", - ) - - if err := rootCmd.Execute(); err != nil { - log.Err(err).Msg("error when executing command") - os.Exit(1) - } -} - -func run(_ *cobra.Command) error { - driver := xixo.NewShellDriver(os.Stdin, os.Stdout, subscribers) - - err := driver.Stream() - if err != nil { - log.Err(err).Msg("Error during processing") - - return err - } - - return nil -} - -func initLog() { - color := false - - switch strings.ToLower(colormode) { - case "auto": - if isatty.IsTerminal(os.Stdout.Fd()) && runtime.GOOS != "windows" { - color = true - } - case "yes", "true", "1", "on", "enable": - color = true - } - - if jsonlog { - log.Logger = zerolog.New(os.Stderr) - } else { - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, NoColor: !color}) //nolint:exhaustruct - } - - if debug { - log.Logger = log.Logger.With().Caller().Logger() - } - - setVerbosity() -} - -func setVerbosity() { - switch verbosity { - case "trace", "5": - zerolog.SetGlobalLevel(zerolog.TraceLevel) - case "debug", "4": - zerolog.SetGlobalLevel(zerolog.DebugLevel) - case "info", "3": - zerolog.SetGlobalLevel(zerolog.InfoLevel) - case "warn", "2": - zerolog.SetGlobalLevel(zerolog.WarnLevel) - case "error", "1": - zerolog.SetGlobalLevel(zerolog.ErrorLevel) - default: - zerolog.SetGlobalLevel(zerolog.Disabled) - } -} diff --git a/go.mod b/go.mod index eee7cfe..0a4e3d1 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,7 @@ module github.com/youen/xixo go 1.21 require ( - github.com/mattn/go-isatty v0.0.14 github.com/rs/zerolog v1.28.0 - github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/tamerh/xpath v1.0.0 ) @@ -13,10 +11,9 @@ require ( require ( github.com/antchfx/xpath v1.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 45f2bdf..ca6bd53 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,9 @@ github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -17,11 +14,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tamerh/xpath v1.0.0 h1:NccMES/Ej8slPCFDff73Kf6V1xu9hdbuKf2RyDsxf5Q= diff --git a/pkg/xixo/driver.go b/pkg/xixo/driver.go index f2416eb..9c4984e 100644 --- a/pkg/xixo/driver.go +++ b/pkg/xixo/driver.go @@ -4,52 +4,13 @@ import ( "io" ) -// ShellDriver represents a driver that processes XML using shell commands. -type ShellDriver struct { - parser *XMLParser - processes []*Process -} - -// NewShellDriver creates a new ShellDriver instance with the given reader, writer, and callbacks. -func NewShellDriver(reader io.Reader, writer io.Writer, callbacks map[string]string) ShellDriver { - // Create a new XML parser with XPath enabled and initialize processes. - parser := NewXMLParser(reader, writer).EnableXpath() - processes := []*Process{} - - // Iterate through the callbacks and create a process for each element. - for elementName, shell := range callbacks { - process := NewProcess(shell) - // Register a JSON callback for the element and add the process to the list. - parser.RegisterJSONCallback(elementName, process.Callback()) - processes = append(processes, process) - } - - // Return the ShellDriver with the parser and processes. - return ShellDriver{parser: parser, processes: processes} -} - -// Stream processes the XML using the registered processes and returns any error encountered. -func (d ShellDriver) Stream() error { - // Start each process in parallel. - for _, process := range d.processes { - if err := process.Start(); err != nil { - return err - } - } - - // Stream the XML using the parser and return any error encountered. - err := d.parser.Stream() - - return err -} - -// FuncDriver represents a driver that processes XML using callback functions. -type FuncDriver struct { +// Driver represents a driver that processes XML using callback functions. +type Driver struct { parser *XMLParser } -// NewFuncDriver creates a new FuncDriver instance with the given reader, writer, and callbacks. -func NewFuncDriver(reader io.Reader, writer io.Writer, callbacks map[string]CallbackMap) FuncDriver { +// NewDriver creates a new FuncDriver instance with the given reader, writer, and callbacks. +func NewDriver(reader io.Reader, writer io.Writer, callbacks map[string]CallbackMap) Driver { // Create a new XML parser with XPath enabled. parser := NewXMLParser(reader, writer).EnableXpath() @@ -59,11 +20,11 @@ func NewFuncDriver(reader io.Reader, writer io.Writer, callbacks map[string]Call } // Return the FuncDriver with the parser. - return FuncDriver{parser: parser} + return Driver{parser: parser} } // Stream processes the XML using registered callback functions and returns any error encountered. -func (d FuncDriver) Stream() error { +func (d Driver) Stream() error { // Stream the XML using the parser and return any error encountered. err := d.parser.Stream() diff --git a/pkg/xixo/driver_test.go b/pkg/xixo/driver_test.go index 9a31aef..7261840 100644 --- a/pkg/xixo/driver_test.go +++ b/pkg/xixo/driver_test.go @@ -8,25 +8,6 @@ import ( "github.com/youen/xixo/pkg/xixo" ) -// TestDriverSimpleEdit tests the ShellDriver by creating a reader, writer, and a map of subscribers, -// then it processes the XML and asserts the expected output. -func TestDriverSimpleEdit(t *testing.T) { - t.Parallel() - - // Create a reader with an XML string, an empty writer, and a map of subscribers. - reader := bytes.NewBufferString("innerTexta") - writer := bytes.Buffer{} - subscribers := map[string]string{"root": "tr a b"} - driver := xixo.NewShellDriver(reader, &writer, subscribers) - - // Stream the XML using the driver and assert the expected output. - err := driver.Stream() - assert.Nil(t, err) - - expected := "\n innerTextb\n" - assert.Equal(t, expected, writer.String()) -} - // TestFuncDriverEdit tests the FuncDriver by creating a reader, writer, and a callback function, // then it processes the XML and asserts the expected output and that the callback was called. func TestFuncDriverEdit(t *testing.T) { @@ -44,7 +25,7 @@ func TestFuncDriverEdit(t *testing.T) { } subscribers := map[string]xixo.CallbackMap{"root": callback} - driver := xixo.NewFuncDriver(reader, &writer, subscribers) + driver := xixo.NewDriver(reader, &writer, subscribers) // Stream the XML using the driver, assert the expected output, and check if the callback was called. err := driver.Stream() diff --git a/pkg/xixo/process.go b/pkg/xixo/process.go deleted file mode 100644 index b17ebea..0000000 --- a/pkg/xixo/process.go +++ /dev/null @@ -1,77 +0,0 @@ -package xixo - -import ( - "bufio" - "fmt" - "io" - "os/exec" - - "github.com/rs/zerolog/log" -) - -type Process struct { - command string - cmd *exec.Cmd - stdout io.ReadCloser - stdin io.WriteCloser - scanner *bufio.Scanner -} - -func NewProcess(command string) *Process { - return &Process{command: command} -} - -func (p *Process) Start() error { - var err error - - //nolint: gosec - p.cmd = exec.Command("/bin/sh", "-c", p.command) - - p.stdout, err = p.cmd.StdoutPipe() - if err != nil { - return err - } - - p.stdin, err = p.cmd.StdinPipe() - if err != nil { - return err - } - - p.scanner = bufio.NewScanner(p.stdout) - - go func(p *Process) { - //nolint - p.cmd.Run() - }(p) - - return nil -} - -func (p *Process) Stop() error { - errStdin := p.stdin.Close() - - if errStdin != nil { - return fmt.Errorf("can't close pipe with process: %w", errStdin) - } - - return nil -} - -func (p *Process) Callback() CallbackJSON { - return func(s string) (string, error) { - log.Debug().Str("json", s).Msg("request edit json") - - _, err := p.stdin.Write([]byte(s + "\n")) - if err != nil { - return "", err - } - - if !p.scanner.Scan() { - return "", fmt.Errorf("command doesn't return line") - } - - log.Debug().Str("read", p.scanner.Text()).Msg("reading from process") - - return p.scanner.Text(), nil - } -} diff --git a/pkg/xixo/process_test.go b/pkg/xixo/process_test.go deleted file mode 100644 index bb29419..0000000 --- a/pkg/xixo/process_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package xixo_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/youen/xixo/pkg/xixo" -) - -func TestProcessCallbackWriteParamToStdinAndReturnStdout(t *testing.T) { - t.Parallel() - - process := xixo.NewProcess("tr 1 2") - - err := process.Start() - assert.Nil(t, err) - - result, err := process.Callback()("element1") - assert.Nil(t, err) - - err = process.Stop() - assert.Nil(t, err) - - assert.Equal(t, "element2", result) -} - -func TestProcessCallbackWriteParamToStdinAndReturnStdoutLimitTo2(t *testing.T) { - t.Parallel() - - process := xixo.NewProcess("head -n 2 | tr 1 2") - - err := process.Start() - assert.Nil(t, err) - - for i := 0; i < 2; i++ { - result, err := process.Callback()("element1") - assert.Nil(t, err) - assert.Equal(t, "element2", result) - } - - _, err = process.Callback()("element1") - assert.NotNil(t, err) - - err = process.Stop() - assert.Nil(t, err) -} - -func TestProcessCallbackFailedToStart(t *testing.T) { - t.Parallel() - - process := xixo.NewProcess("false") - - err := process.Start() - assert.Nil(t, err) - - _, err = process.Callback()("element1") - assert.NotNil(t, err) - - err = process.Stop() - assert.Nil(t, err) -} diff --git a/test/suites/01-run-cli.yml b/test/suites/01-run-cli.yml deleted file mode 100644 index 2fee56c..0000000 --- a/test/suites/01-run-cli.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Venom Test Suite definition -# Check Venom documentation for more information : https://github.com/ovh/venom -name: run xixo -testcases: - - name: no arguments - steps: - - script: xixo - assertions: - - result.code ShouldEqual 0 - - - name: simple inputs - steps: - - script: echo 'innerText' | xixo - assertions: - - result.code ShouldEqual 0 - - result.systemout ShouldEqual innerText - - - name: simple inputs and subscribers - steps: - - script: echo 'innerTexta' | xixo --subscribers root='tee /tmp/debug | tr a b | tee /tmp/debugout ' - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - result.systemout ShouldContainSubstring innerTextb - - - name: copy users - steps: - - script: cat ../data/users.xml | xixo > ../workspace/users.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - script: diff ../data/users.xml ../workspace/users.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - result.systemout ShouldBeEmpty - - - name: copy users with attributs - steps: - - script: cat ../data/users_with_attributs.xml | xixo > ../workspace/users_with_attributs.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - script: diff ../data/users_with_attributs.xml ../workspace/users_with_attributs.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - result.systemout ShouldBeEmpty - - - name: foo bar example - steps: - - script: cat ../data/foo_bar_baz.xml | xixo --subscribers foo="tee ../workspace/debug_foo_bar_baz_input.jsonl | jq --unbuffered -c '.bar |= ascii_upcase' " > ../workspace/foo_bar_baz.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - script: diff ../data/foo_bar_baz_expected.xml ../workspace/foo_bar_baz.xml - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - result.systemout ShouldBeEmpty - - script: diff ../data/debug_foo_bar_baz_input_expected.jsonl ../workspace/debug_foo_bar_baz_input.jsonl - assertions: - - result.code ShouldEqual 0 - - result.systemerr ShouldBeEmpty - - result.systemout ShouldBeEmpty - - # - name: copy stream content of user entities in users.xml - # steps: - # - script: cat ../data/users.xml | xixo --subscribers user=cat > ../workspace/users.xml - # assertions: - # - result.code ShouldEqual 0 - # - result.systemerr ShouldBeEmpty - # - script: diff ../data/users.xml ../workspace/users.xml - # assertions: - # - result.code ShouldEqual 0 - # - result.systemerr ShouldBeEmpty - # - result.systemout ShouldBeEmpty