diff --git a/.github/workflows/go-ci-build.yaml b/.github/workflows/go-ci-build.yaml new file mode 100644 index 0000000..6fdd15e --- /dev/null +++ b/.github/workflows/go-ci-build.yaml @@ -0,0 +1,29 @@ +name: Go CI Build + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + go-version: ["1.20"] + + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Check out code + uses: actions/checkout@v2 + + - name: Run cibuild script + run: ./script/cibuild diff --git a/.gitignore b/.gitignore index 3b735ec..30ffce3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ # Go workspace file go.work + +covprofile +/bin +.vscode diff --git a/README.md b/README.md index 04bfe9a..9b47fb9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,9 @@ # statcapn -metrics collection but easy + +**Alpha Status** + +metrics collection but easy. Tired of working with environments with no metrics collection setup, limited install rights, in hostile environments and especially at the frequency, duration, and level of aggregation you want? Now you can use StatCapn the easy to install metrics capture utility written in go. Originally conceived to capture usage stats out of a K8s pod with no monitoring or metrics collection setup I decided to use it + +## License + +Apache License, Version 2.0 diff --git a/changelog.MD b/changelog.MD new file mode 100644 index 0000000..6b3bec2 --- /dev/null +++ b/changelog.MD @@ -0,0 +1,10 @@ +# statcapn changelog + +All notable changes to this project will be documented in this file. + +## [0.1.0] - 2023-07-03 + +### Added + +- initial release +- support for Linux, Mac (Darwin), and Windows all on arm64 and amd64 platforms \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c10a93a --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/rsvihladremio/statcapn + +go 1.20 + +require github.com/shirou/gopsutil/v3 v3.23.5 + +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/stretchr/testify v1.8.4 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/sys v0.8.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..67b93c7 --- /dev/null +++ b/go.sum @@ -0,0 +1,46 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= +github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/license_header.txt b/license_header.txt new file mode 100644 index 0000000..55047d2 --- /dev/null +++ b/license_header.txt @@ -0,0 +1,13 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. diff --git a/main.go b/main.go new file mode 100644 index 0000000..3e5359a --- /dev/null +++ b/main.go @@ -0,0 +1,49 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// main is the entry point for the statcapn cli +package main + +import ( + "errors" + "flag" + "log" + + "github.com/rsvihladremio/statcapn/pkg" +) + +func main() { + args := ArgParse() + if err := pkg.SystemMetrics(args); err != nil { + log.Fatalf("unable to collect %v", errors.Unwrap(err)) + } +} + +func ArgParse() pkg.Args { + var intervalSeconds int + var durationSeconds int + var outFile string + + flag.IntVar(&intervalSeconds, "i", 1, "number of seconds between execution of collection") + flag.IntVar(&durationSeconds, "d", 60, "number of seconds for duration of all collection") + flag.Parse() + if flag.NArg() > 0 { + outFile = flag.Arg(0) + } + return pkg.Args{ + IntervalSeconds: intervalSeconds, + DurationSeconds: durationSeconds, + OutFile: outFile, + } +} diff --git a/pkg/gopsmetrics/gopsmetrics.go b/pkg/gopsmetrics/gopsmetrics.go new file mode 100644 index 0000000..19da6ef --- /dev/null +++ b/pkg/gopsmetrics/gopsmetrics.go @@ -0,0 +1,87 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// gopsmetrics implements metrics collection using gopsutil +package gopsmetrics + +import ( + "github.com/rsvihladremio/statcapn/pkg/metrics" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v3/mem" +) + +type Collector struct { +} + +func (g *Collector) IOCounters() (map[string]metrics.IOCountersStat, error) { + counters, err := disk.IOCounters() + if err != nil { + return make(map[string]metrics.IOCountersStat), err + } + return g.mapIOCounters(counters), nil +} + +func (g *Collector) mapIOCounters(counters map[string]disk.IOCountersStat) map[string]metrics.IOCountersStat { + result := make(map[string]metrics.IOCountersStat) + for k, v := range counters { + result[k] = metrics.IOCountersStat{ + ReadBytes: v.ReadBytes, + WriteBytes: v.WriteBytes, + IoTime: v.IoTime, + WeightedIO: v.WeightedIO, + Name: v.Name, + } + } + return result +} + +func (g *Collector) Times() (metrics.TimesStat, error) { + c, err := cpu.Times(false) + if err != nil { + return metrics.TimesStat{}, err + } + first := c[0] + return g.mapTimes(first), nil +} + +func (g *Collector) mapTimes(cpuTime cpu.TimesStat) metrics.TimesStat { + return metrics.TimesStat{ + User: cpuTime.User, + System: cpuTime.System, + Idle: cpuTime.Idle, + Nice: cpuTime.Nice, + Iowait: cpuTime.Iowait, + Irq: cpuTime.Irq, + Softirq: cpuTime.Softirq, + Steal: cpuTime.Steal, + Guest: cpuTime.Guest, + GuestNice: cpuTime.GuestNice, + } +} + +func (g *Collector) VirtualMemory() (*metrics.VirtualMemoryStat, error) { + virt, err := mem.VirtualMemory() + if err != nil { + return &metrics.VirtualMemoryStat{}, err + } + return g.mapVirtualMemory(virt), nil +} + +func (g Collector) mapVirtualMemory(virt *mem.VirtualMemoryStat) *metrics.VirtualMemoryStat { + return &metrics.VirtualMemoryStat{ + Available: virt.Available, + Cached: virt.Cached, + } +} diff --git a/pkg/gopsmetrics/gopsmetrics_test.go b/pkg/gopsmetrics/gopsmetrics_test.go new file mode 100644 index 0000000..c688fdf --- /dev/null +++ b/pkg/gopsmetrics/gopsmetrics_test.go @@ -0,0 +1,161 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// gopsmetrics implements metrics collection using gopsutil +package gopsmetrics + +import ( + "testing" + + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v3/mem" +) + +func TestIOCounters(t *testing.T) { + g := &Collector{} + _, err := g.IOCounters() + if err != nil { + t.Fatal(err) + } +} + +func TestIOCountersMapping(t *testing.T) { + + g := &Collector{} + result := make(map[string]disk.IOCountersStat) + result["Disk1"] = disk.IOCountersStat{ + ReadBytes: 100, + WriteBytes: 200, + IoTime: 300, + WeightedIO: 400, + Name: "Disk1", + } + counters := g.mapIOCounters(result) + for k, v := range counters { + e := result[k] + if v.ReadBytes != e.ReadBytes { + t.Errorf("IOCounters did not perform mapping correctly on read bytes '%v'", v.ReadBytes) + } + if v.WriteBytes != e.WriteBytes { + t.Errorf("IOCounters did not perform mapping correctly on write bytes '%v'", v.WriteBytes) + } + + if v.IoTime != e.IoTime { + t.Errorf("IOCounters did not perform mapping correctly on iotime '%v'", v.IoTime) + } + + if v.WeightedIO != e.WeightedIO { + t.Errorf("IOCounters did not perform mapping correctly on weightedIO '%v'", v.WeightedIO) + } + + if v.Name != e.Name { + t.Errorf("IOCounters did not perform mapping correctly on name '%v'", v.Name) + } + } +} + +func TestTimes(t *testing.T) { + g := &Collector{} + _, err := g.Times() + + if err != nil { + t.Errorf("Times method failed: %v", err) + } + +} + +func TestTimesMap(t *testing.T) { + g := &Collector{} + expected := cpu.TimesStat{ + User: 1.0, + System: 2.0, + Idle: 3.0, + Nice: 4.0, + Iowait: 5.0, + Irq: 6.0, + Softirq: 7.0, + Steal: 8.0, + Guest: 9.0, + GuestNice: 10.0, + } + times := g.mapTimes(cpu.TimesStat{}) + + // Check that all fields are not their zero values + if times.User == expected.User { + t.Errorf("Times did not perform mapping correctly on system %v", expected.User) + } + + if times.System == expected.System { + t.Errorf("Times did not perform mapping correctly on system %v", expected.System) + } + + if times.Idle == expected.Idle { + t.Errorf("Times did not perform mapping correctly on idle %v", expected.Idle) + } + + if times.Nice == expected.Nice { + t.Errorf("Times did not perform mapping correctly on nice %v", expected.Nice) + } + + if times.Iowait == expected.Iowait { + t.Errorf("Times did not perform mapping correctly on iowait %v", expected.Iowait) + } + + if times.Irq == expected.Irq { + t.Errorf("Times did not perform mapping correctly on irq %v", expected.Irq) + } + + if times.Softirq == expected.Softirq { + t.Errorf("Times did not perform mapping correctly on soft irq %v", expected.Softirq) + } + + if times.Steal == expected.Steal { + t.Errorf("Times did not perform mapping correctly on steal %v", expected.Steal) + } + + if times.Guest == expected.Guest { + t.Errorf("Times did not perform mapping correctly on guest %v", expected.Guest) + } + + if times.GuestNice == expected.GuestNice { + t.Errorf("Times did not perform mapping correctly on guest nice %v", expected.GuestNice) + } +} + +func TestVirtualMemory(t *testing.T) { + g := &Collector{} + _, err := g.VirtualMemory() + + if err != nil { + t.Errorf("VirtualMemory method failed: %v", err) + } + +} + +func TestVirtualMemoryMapping(t *testing.T) { + g := &Collector{} + e := &mem.VirtualMemoryStat{ + Cached: 100, + Available: 200, + } + mem := g.mapVirtualMemory(e) + if mem.Available != e.Available { + t.Errorf("VirtualMemory did not perform mapping correctly on mem available %v", mem.Available) + } + + if mem.Cached != e.Cached { + t.Errorf("VirtualMemory did not perform mapping correctly on mem cached %v", mem.Cached) + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..7bd24ca --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,51 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// metrics provides the interface to implement metrics +package metrics + +type Collector interface { + IOCounters() (map[string]IOCountersStat, error) + Times() (TimesStat, error) + VirtualMemory() (*VirtualMemoryStat, error) +} + +type VirtualMemoryStat struct { + // RAM available for programs to allocate + // + // This value is computed from the kernel specific values. + Available uint64 + Cached uint64 +} + +type TimesStat struct { + User float64 + System float64 + Idle float64 + Nice float64 + Iowait float64 + Irq float64 + Softirq float64 + Steal float64 + Guest float64 + GuestNice float64 +} + +type IOCountersStat struct { + ReadBytes uint64 + WriteBytes uint64 + IoTime uint64 + WeightedIO uint64 + Name string +} diff --git a/pkg/sysmetrics.go b/pkg/sysmetrics.go new file mode 100644 index 0000000..61a201c --- /dev/null +++ b/pkg/sysmetrics.go @@ -0,0 +1,324 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package pkg is where we build all of the code that is shared +package pkg + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path" + "strings" + "time" + + "github.com/rsvihladremio/statcapn/pkg/gopsmetrics" + "github.com/rsvihladremio/statcapn/pkg/metrics" +) + +type Args struct { + IntervalSeconds int + DurationSeconds int + OutFile string +} + +// SystemMetricsRow represents a row of system metrics data. +type SystemMetricsRow struct { + CollectionTimeStamp time.Time `json:"collectionTimestamp"` + UserCPUPercent float64 `json:"userCPUPercent"` + SystemCPUPercent float64 `json:"systmeCPUPercent"` + IdleCPUPercent float64 `json:"idleCPUPercent"` + NiceCPUPercent float64 `json:"niceCPUPercent"` + IOWaitCPUPercent float64 `json:"ioWaitCPUPercent"` + IRQCPUPercent float64 `json:"irqCPUPercent"` + SoftIRQCPUPercent float64 `json:"softIRQCPUPercent"` + StealCPUPercent float64 `json:"stealCPUPercent"` + GuestCPUPercent float64 `json:"guestCPUPercent"` + GuestNiceCPUPercent float64 `json:"guestCPUNicePercent"` + QueueDepth float64 `json:"queueDepth"` + DiskLatency float64 `json:"diskLatency"` + ReadBytes int64 `json:"readBytes"` + WriteBytes int64 `json:"writeBytes"` + FreeRAMMB int64 `json:"freeRAMMB"` + CachedRAMMB int64 `json:"cachedRAMMB"` +} + +// CollectionParams includes all the necessary parameters to complete a collection +type CollectionParams struct { + IntervalSeconds int + DurationSeconds int + RowWriter func(SystemMetricsRow) error +} + +func CollectSystemMetrics(params CollectionParams, sleeper func(time.Duration), metrics metrics.Collector) error { + if params.DurationSeconds < 1 { + return fmt.Errorf("duration must be at least 1 second %v", params.DurationSeconds) + } + if params.IntervalSeconds < 1 { + return fmt.Errorf("interval must be at least 1 second %v", params.IntervalSeconds) + } + interval := time.Second * time.Duration(params.IntervalSeconds) + iterations := params.DurationSeconds / params.IntervalSeconds + if iterations < 1 { + return fmt.Errorf("interval of %v cannot be greater than the duration of %v", params.IntervalSeconds, params.DurationSeconds) + } + + prevDiskIO, err := metrics.IOCounters() + if err != nil { + return err + } + prevCPUTimes, err := metrics.Times() + if err != nil { + return err + } + for i := 0; i < iterations; i++ { + // Sleep + sleeper(interval) + + // CPU Times + cpuTimes, err := metrics.Times() + if err != nil { + return err + } + + // Memory + memoryInfo, err := metrics.VirtualMemory() + if err != nil { + return err + } + + // Disk I/O + diskIO, err := metrics.IOCounters() + if err != nil { + return err + } + + var weightedIOTime, totalIOs uint64 + var readBytes, writeBytes float64 + for i, io := range diskIO { + p := prevDiskIO[i] + weightedIOTime += io.WeightedIO - p.WeightedIO + totalIOs += io.IoTime - p.IoTime + + if prev, ok := prevDiskIO[io.Name]; ok { + readBytes += float64(io.ReadBytes - prev.ReadBytes) + writeBytes += float64(io.WriteBytes - prev.WriteBytes) + } + } + prevDiskIO = diskIO + total := getTotalTime(cpuTimes, prevCPUTimes) + var queueDepth float64 + var diskLatency float64 + if weightedIOTime > 0 { + queueDepth = round(float64(weightedIOTime) / 1000) + diskLatency = round(float64(weightedIOTime) / float64(totalIOs)) + } + + row := SystemMetricsRow{} + row.CollectionTimeStamp = time.Now() + user := cpuTimes.User - prevCPUTimes.User + if user > 0 { + row.UserCPUPercent = round((user / total) * 100) + } + system := cpuTimes.System - prevCPUTimes.System + if system > 0 { + row.SystemCPUPercent = round((system / total) * 100) + } + idle := cpuTimes.Idle - prevCPUTimes.Idle + if idle > 0 { + row.IdleCPUPercent = round((idle / total) * 100) + } + nice := cpuTimes.Nice - prevCPUTimes.Nice + if nice > 0 { + row.NiceCPUPercent = round((nice / total) * 100) + } + iowait := cpuTimes.Iowait - prevCPUTimes.Iowait + if iowait > 0 { + row.IOWaitCPUPercent = round((iowait / total) * 100) + } + + irq := cpuTimes.Irq - prevCPUTimes.Irq + if irq > 0 { + row.IRQCPUPercent = round((irq / total) * 100) + } + + softIRQ := cpuTimes.Softirq - prevCPUTimes.Softirq + if softIRQ > 0 { + row.SoftIRQCPUPercent = round((softIRQ / total) * 100) + } + steal := cpuTimes.Steal - prevCPUTimes.Steal + if steal > 0 { + row.StealCPUPercent = round((steal / total) * 100) + } + + guestCPU := cpuTimes.Guest - prevCPUTimes.Guest + if guestCPU > 0 { + row.GuestCPUPercent = round((guestCPU / total) * 100) + } + guestCPUNice := cpuTimes.GuestNice - prevCPUTimes.GuestNice + if guestCPUNice > 0 { + row.GuestNiceCPUPercent = round((guestCPUNice / total) * 100) + } + + prevCPUTimes = cpuTimes + row.DiskLatency = diskLatency + row.QueueDepth = queueDepth + + var memoryFreeMB float64 + if memoryInfo.Available > 0 { + memoryFreeMB = round(float64(memoryInfo.Available) / (1024 * 1024)) + } + row.FreeRAMMB = int64(memoryFreeMB) + + var memoryCachedMB float64 + if memoryCachedMB > 0 { + memoryCachedMB = round(float64(memoryInfo.Cached) / (1024 * 1024)) + } + row.CachedRAMMB = int64(memoryCachedMB) + + if err := params.RowWriter(row); err != nil { + return err + } + } + return nil +} + +func SystemMetrics(args Args) error { + var w io.Writer + var rowWriter func(SystemMetricsRow) error + var cleanup func() error + outputFile := args.OutFile + + if strings.HasSuffix(outputFile, ".json") { + f, err := os.Create(path.Clean(outputFile)) + if err != nil { + return fmt.Errorf("unable to create file %v due to error '%w'", outputFile, err) + } + w = f + // we manually close this so we do not care that we are not handling the error + defer f.Close() + + bufWriter := bufio.NewWriter(w) + cleanup = func() error { + if err := bufWriter.Flush(); err != nil { + return fmt.Errorf("unable to flush metrics file %v due to error %w", outputFile, err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("unable to close metrics file %v due to error %w", outputFile, err) + } + return nil + } + //write json file + rowWriter = func(row SystemMetricsRow) error { + str, err := json.Marshal(&row) + if err != nil { + return fmt.Errorf("unable to marshal row %#v due to error %w", row, err) + } + txt := fmt.Sprintf("%v\n", string(str)) + _, err = bufWriter.Write([]byte(txt)) + if err != nil { + return fmt.Errorf("unable to write to json file due to error %w", err) + } + return nil + } + if err != nil { + return fmt.Errorf("unable to write metrics file %v due to error %w", outputFile, err) + } + } else { + if outputFile == "" { + cleanup = func() error { return nil } + w = os.Stdout + } else { + f, err := os.Create(path.Clean(outputFile)) + if err != nil { + return fmt.Errorf("unable to create file %v due to error '%w'", outputFile, err) + } + cleanup = func() error { + if err := f.Close(); err != nil { + return fmt.Errorf("unable to close metrics file %v due to error %w", outputFile, err) + } + return nil + } + w = f + // we don't care as this is just an emergency cleanup we manually call "cleanup" which closes the file anyway + defer f.Close() + } + + //write metrics.txt file + template := "%25s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s\t%10s" + floatTemplate := "%.2f" + percentTemplate := "%.2f%%" + txtHeader := fmt.Sprintf(template, "Timestamp", "usr %%", "sys %%", "iowait %%", "other %%", "idl %%", "Queue", "Latency (ms)", "Read (MB/s)", "Write (MB/s)", "Free Mem (GB)") + if _, err := fmt.Fprintln(w, txtHeader); err != nil { + return fmt.Errorf("unable to write metrics file %v due to error %w", outputFile, err) + } + rowWriter = func(row SystemMetricsRow) error { + otherCPU := row.NiceCPUPercent + row.IRQCPUPercent + row.SoftIRQCPUPercent + row.StealCPUPercent + row.GuestCPUPercent + row.GuestNiceCPUPercent + var readBytesMB, writeBytesMB, freeRAMGB float64 + if row.ReadBytes > 0 { + readBytesMB = float64(row.ReadBytes) / (1024 * 1024) + } + if row.WriteBytes > 0 { + writeBytesMB = float64(row.WriteBytes) / (1024 * 1024) + } + if row.FreeRAMMB > 0 { + freeRAMGB = float64(row.FreeRAMMB) / 1024.0 + } + rowString := fmt.Sprintf(template, + row.CollectionTimeStamp.Format(time.RFC3339), + fmt.Sprintf(percentTemplate, row.UserCPUPercent), + fmt.Sprintf(percentTemplate, row.SystemCPUPercent), + fmt.Sprintf(percentTemplate, row.IOWaitCPUPercent), + fmt.Sprintf(percentTemplate, otherCPU), + fmt.Sprintf(percentTemplate, row.IdleCPUPercent), + fmt.Sprintf(floatTemplate, row.QueueDepth), + fmt.Sprintf(floatTemplate, row.DiskLatency), + fmt.Sprintf(floatTemplate, readBytesMB), + fmt.Sprintf(floatTemplate, writeBytesMB), + fmt.Sprintf(floatTemplate, freeRAMGB)) + if _, err := fmt.Fprintln(w, rowString); err != nil { + return fmt.Errorf("unable to write metrics file %v due to error %w", outputFile, err) + } + return nil + } + } + params := CollectionParams{ + DurationSeconds: args.DurationSeconds, + IntervalSeconds: args.IntervalSeconds, + RowWriter: rowWriter, + } + + if err := CollectSystemMetrics(params, func(d time.Duration) { + time.Sleep(d) + }, &gopsmetrics.Collector{}); err != nil { + return fmt.Errorf("unable to collect system metrics with error %v", err) + } + + return cleanup() +} + +func round(num float64) float64 { + factor := math.Pow(10, float64(2)) + return math.Round(num*factor) / factor +} +func getTotalTime(c metrics.TimesStat, p metrics.TimesStat) float64 { + current := c.User + c.System + c.Idle + c.Nice + c.Iowait + c.Irq + + c.Softirq + c.Steal + c.Guest + c.GuestNice + prev := p.User + p.System + p.Idle + p.Nice + p.Iowait + p.Irq + + p.Softirq + p.Steal + p.Guest + p.GuestNice + return current - prev +} diff --git a/pkg/sysmetrics_test.go b/pkg/sysmetrics_test.go new file mode 100644 index 0000000..4842d45 --- /dev/null +++ b/pkg/sysmetrics_test.go @@ -0,0 +1,404 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package pkg is where we build all of the code that is shared +package pkg + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/rsvihladremio/statcapn/pkg/metrics" +) + +type MockCollector struct { + ioCounters []map[string]metrics.IOCountersStat + times []metrics.TimesStat + virtualMemory []*metrics.VirtualMemoryStat + counter int +} + +func (m *MockCollector) IOCounters() (map[string]metrics.IOCountersStat, error) { + return m.ioCounters[m.counter], nil +} + +func (m *MockCollector) Times() (metrics.TimesStat, error) { + return m.times[m.counter], nil +} + +func (m *MockCollector) VirtualMemory() (*metrics.VirtualMemoryStat, error) { + return m.virtualMemory[m.counter], nil +} + +func (m *MockCollector) Tick() { + m.counter++ +} + +func TestCollectSystemMetricsForEachInterval(t *testing.T) { + called := 0 + sleeper := func(d time.Duration) { + called++ + } + var rows []SystemMetricsRow + writer := func(row SystemMetricsRow) error { + rows = append(rows, row) + return nil + } + params := CollectionParams{ + IntervalSeconds: 1, + DurationSeconds: 60, + RowWriter: writer, + } + ioCounters := make(map[string]metrics.IOCountersStat) + ioCounters["Disk1"] = metrics.IOCountersStat{ + Name: "Disk1", + } + mockCollector := MockCollector{ + ioCounters: []map[string]metrics.IOCountersStat{ioCounters}, + virtualMemory: []*metrics.VirtualMemoryStat{{}}, + times: []metrics.TimesStat{{}}, + } + err := CollectSystemMetrics(params, sleeper, &mockCollector) + if err != nil { + t.Fatalf("unable to collect metrics: %v", err) + } + + if called != 60 { + t.Errorf("expected 60 iterations but got %v", called) + } + + if len(rows) != 60 { + t.Errorf("expected 60 rows but got %v", len(rows)) + } +} + +func TestCollectSystemMetricsWithNoDuration(t *testing.T) { + called := 0 + sleeper := func(d time.Duration) { + called++ + } + var rows []SystemMetricsRow + writer := func(row SystemMetricsRow) error { + rows = append(rows, row) + return nil + } + params := CollectionParams{ + IntervalSeconds: 1, + DurationSeconds: 0, + RowWriter: writer, + } + ioCounters := make(map[string]metrics.IOCountersStat) + ioCounters["Disk1"] = metrics.IOCountersStat{ + Name: "Disk1", + } + mockCollector := MockCollector{ + ioCounters: []map[string]metrics.IOCountersStat{ioCounters}, + virtualMemory: []*metrics.VirtualMemoryStat{{}}, + times: []metrics.TimesStat{{}}, + } + err := CollectSystemMetrics(params, sleeper, &mockCollector) + if err == nil { + t.Fatal("expected an error collecting") + } + + if called != 0 { + t.Errorf("expected 0 iterations but got %v", called) + } + + if len(rows) != 0 { + t.Errorf("expected 0 rows but got %v", len(rows)) + } +} + +func TestCalculatesCPUUsageCorrectly(t *testing.T) { + + var rows []SystemMetricsRow + writer := func(row SystemMetricsRow) error { + rows = append(rows, row) + return nil + } + params := CollectionParams{ + IntervalSeconds: 1, + DurationSeconds: 2, + RowWriter: writer, + } + ioCounters := make(map[string]metrics.IOCountersStat) + ioCounters["Disk1"] = metrics.IOCountersStat{ + Name: "Disk1", + } + ioCounters2 := make(map[string]metrics.IOCountersStat) + ioCounters2["Disk1"] = metrics.IOCountersStat{ + Name: "Disk1", + } + ioCounters3 := make(map[string]metrics.IOCountersStat) + ioCounters3["Disk1"] = metrics.IOCountersStat{ + Name: "Disk1", + } + mockCollector := &MockCollector{ + ioCounters: []map[string]metrics.IOCountersStat{ioCounters, ioCounters2, ioCounters3}, + virtualMemory: []*metrics.VirtualMemoryStat{{}, {}, {}}, + times: []metrics.TimesStat{ + { + User: 19.0, + System: 8.0, + Idle: 45.0, + Nice: 7.0, + Iowait: 6.0, + Irq: 5.0, + Softirq: 4.0, + Steal: 3.0, + Guest: 2.0, + GuestNice: 1.0, + }, + { + User: 19.0, + System: 8.0, + Idle: 145.0, + Nice: 7.0, + Iowait: 6.0, + Irq: 5.0, + Softirq: 4.0, + Steal: 3.0, + Guest: 2.0, + GuestNice: 1.0, + }, + { + User: 29.0, + System: 17.0, + Idle: 196.0, + Nice: 15.0, + Iowait: 13.0, + Irq: 10.0, + Softirq: 8.0, + Steal: 6.0, + Guest: 4.0, + GuestNice: 2.0, + }, + }, + } + + err := CollectSystemMetrics(params, func(d time.Duration) { + mockCollector.Tick() + }, mockCollector) + if err != nil { + t.Fatalf("unable to collect metrics: %v", err) + } + + if len(rows) != 2 { + t.Errorf("expected 2 rows but got %v", len(rows)) + } + perc := "%.2f" + expected := "0.00" + actual := rows[0].UserCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("user cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].SystemCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("system cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].StealCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("steal cpu %v but expected %v", actual, expected) + } + + expected = "100.00" + actual = rows[0].IdleCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("idle cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].GuestCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("guest cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].GuestNiceCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("guest nice cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].NiceCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("nice cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].IRQCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].SoftIRQCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } + + expected = "0.00" + actual = rows[0].IOWaitCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } + + //second row should have some measurements + + expected = "10.00" + actual = rows[1].UserCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("user cpu %v but expected %v", actual, expected) + } + + expected = "9.00" + actual = rows[1].SystemCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("system cpu %v but expected %v", actual, expected) + } + + expected = "3.00" + actual = rows[1].StealCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("steal cpu %v but expected %v", actual, expected) + } + + expected = "51.00" + actual = rows[1].IdleCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("idle cpu %v but expected %v", actual, expected) + } + + expected = "2.00" + actual = rows[1].GuestCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("guest cpu %v but expected %v", actual, expected) + } + + expected = "1.00" + actual = rows[1].GuestNiceCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("guest nice cpu %v but expected %v", actual, expected) + } + + expected = "8.00" + actual = rows[1].NiceCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("nice cpu %v but expected %v", actual, expected) + } + + expected = "5.00" + actual = rows[1].IRQCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } + + expected = "4.00" + actual = rows[1].SoftIRQCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } + + expected = "7.00" + actual = rows[1].IOWaitCPUPercent + if fmt.Sprintf(perc, actual) != expected { + t.Errorf("irq cpu %v but expected %v", actual, expected) + } +} + +func TestSystemMetricsIntegrationWithJson(t *testing.T) { + outFile := filepath.Join(t.TempDir(), "metrics.json") + args := Args{ + IntervalSeconds: 1, + DurationSeconds: 2, + OutFile: outFile, + } + err := SystemMetrics(args) + if err != nil { + t.Fatal(err) + } + + //verify each row is readable as json + f, err := os.Open(outFile) + if err != nil { + t.Fatal(err) + } + scanner := bufio.NewScanner(f) + counter := 0 + for scanner.Scan() { + counter++ + line := scanner.Text() + obj := SystemMetricsRow{} + err := json.Unmarshal([]byte(strings.TrimSpace(line)), &obj) + if err != nil { + t.Errorf("error unmarshalling line %v with error %v", line, err) + } + } + if counter != 2 { + t.Errorf("expected 2 iterations but got %v", counter) + } +} + +func TestSystemMetricsIntegrationWithTxt(t *testing.T) { + outFile := filepath.Join(t.TempDir(), "metrics.txt") + args := Args{ + IntervalSeconds: 1, + DurationSeconds: 2, + OutFile: outFile, + } + err := SystemMetrics(args) + if err != nil { + t.Fatal(err) + } + + //verify each has the right count of values + f, err := os.Open(outFile) + if err != nil { + t.Fatal(err) + } + scanner := bufio.NewScanner(f) + rowCounter := 0 + header := "" + for scanner.Scan() { + line := scanner.Text() + if header == "" { + header = line + continue + } + rowCounter++ + //should have 10 tabs + tabs := strings.Count(line, "\t") + if tabs != 10 { + t.Errorf("expected 10 tabs for line %v but had %v", line, tabs) + } + } + expectedHeader := " Timestamp usr %% sys %% iowait %% other %% idl %% Queue Latency (ms) Read (MB/s) Write (MB/s) Free Mem (GB)" + if header != expectedHeader { + t.Errorf("expected header\n%q\nbut got\n%q", expectedHeader, header) + } + if rowCounter != 2 { + t.Errorf("expected 2 rows but got %v", rowCounter) + } +} diff --git a/pkg/versions/versions.go b/pkg/versions/versions.go new file mode 100644 index 0000000..66312f6 --- /dev/null +++ b/pkg/versions/versions.go @@ -0,0 +1,27 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// versions contains all the global variables for setting version +package versions + +var version = "dev" +var gitSha = "unknown" + +func GetVersion() string { + return version +} + +func GetGitSha() string { + return gitSha +} diff --git a/pkg/versions/versions_test.go b/pkg/versions/versions_test.go new file mode 100644 index 0000000..d6c4f2e --- /dev/null +++ b/pkg/versions/versions_test.go @@ -0,0 +1,32 @@ +// Copyright 2023 Dremio Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// versions contains all the global variables for setting version +package versions + +import "testing" + +func TestVersionIsWhatIsSet(t *testing.T) { + version = "my_version" + if version != GetVersion() { + t.Errorf("incorrect version %v instead of %v", GetVersion(), version) + } +} + +func TestGitShaIsWhatIsSet(t *testing.T) { + gitSha = "my_version" + if gitSha != GetGitSha() { + t.Errorf("incorrect gitsha %v instead of %v", GetGitSha(), gitSha) + } +} diff --git a/script/audit b/script/audit new file mode 100755 index 0000000..8fe9e70 --- /dev/null +++ b/script/audit @@ -0,0 +1,10 @@ +#!/bin/sh + +# script/audit: runs gosec against the mod file to find security issues +# + +set -e + +cd "$(dirname "$0")/.." + +gosec ./... diff --git a/script/audit.ps1 b/script/audit.ps1 new file mode 100644 index 0000000..e6679a9 --- /dev/null +++ b/script/audit.ps1 @@ -0,0 +1,10 @@ +# script\audit.ps1: runs gosec against the mod file to find security issues + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Write-Output "Running gosec..." +Invoke-Expression "gosec ./..." diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..48e99ae --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,37 @@ +#!/bin/sh + +# script/bootstrap: Resolve all dependencies that the application requires to +# run. + +set -e + +cd "$(dirname "$0")/.." + +echo "Checking if license-header-checker is installed" +date "+%H:%M:%S" + +if ! type "license-header-checker" > /dev/null; then + echo "license-header-checker not found installing.." + date "+%H:%M:%S" + go install github.com/lluissm/license-header-checker/cmd/license-header-checker@latest +fi + +echo "Checking if golangci-lint is installed…" +date "+%H:%M:%S" + +if ! type "golangci-lint" > /dev/null; then + echo "golaningci-lint not found installing…" + date "+%H:%M:%S" + + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 +fi + +echo "Checking if gosec is installed…" +date "+%H:%M:%S" + +if ! type "gosec" > /dev/null; then + echo "gosec not found installing…" + date "+%H:%M:%S" + curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.16.0 +fi + diff --git a/script/bootstrap.ps1 b/script/bootstrap.ps1 new file mode 100644 index 0000000..94bf7af --- /dev/null +++ b/script/bootstrap.ps1 @@ -0,0 +1,34 @@ +# script\bootstrap.ps1: Resolve all dependencies that the application requires to run. + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Write-Output "Checking if license-header-checker is installed..." +Get-Date -Format "HH:mm:ss" + +if (-not (Get-Command "license-header-checker" -ErrorAction SilentlyContinue)) { + Write-Output "license-header-checker not found, installing..." + Get-Date -Format "HH:mm:ss" + go install github.com/lluissm/license-header-checker/cmd/license-header-checker@latest +} + +Write-Output "Checking if golangci-lint is installed..." +Get-Date -Format "HH:mm:ss" + +if (-not (Get-Command "golangci-lint" -ErrorAction SilentlyContinue)) { + Write-Output "golangci-lint not found, installing..." + Get-Date -Format "HH:mm:ss" + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 +} + +Write-Output "Checking if gosec is installed..." +Get-Date -Format "HH:mm:ss" + +if (-not (Get-Command "gosec" -ErrorAction SilentlyContinue)) { + Write-Output "gosec not found, installing..." + Get-Date -Format "HH:mm:ss" + go install github.com/securego/gosec/v2/cmd/gosec@v2.16.0 +} diff --git a/script/build b/script/build new file mode 100755 index 0000000..b52c811 --- /dev/null +++ b/script/build @@ -0,0 +1,44 @@ +#!/bin/sh + +# script/build: build binary +set -e + +cd "$(dirname "$0")/.." + +./script/clean + +echo "Checking if zip is installed…" +date "+%H:%M:%S" + +if ! type "zip" > /dev/null; then + echo "zip not found installing…" + date "+%H:%M:%S" + OS="`uname`" + case $OS in + 'Linux') + if type "apt" > /dev/null; then + echo "detected debian based system installing via apt" + sudo apt install zip + fi + if type "yum" > /dev/null; then + echo "detected rpm based system installing via yum" + sudo yum install zip + fi + ;; + 'Darwin') + if type "brew" > /dev/null; then + brew install zip + else + echo "install homebrew to have zip https://brew.sh/" + exit 1 + fi + ;; + *) ;; + esac +fi + +# this is also set in script/release and is a copy paste +GIT_SHA=`git rev-parse --short HEAD` +VERSION=`git rev-parse --abbrev-ref HEAD` +LDFLAGS="-X github.com/dremio/statcapn/pkg/versions.gitSha=$GIT_SHA -X github.com/dremio/statcapn/pkg/versions.version=$VERSION" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn diff --git a/script/build.ps1 b/script/build.ps1 new file mode 100644 index 0000000..b5dfcb0 --- /dev/null +++ b/script/build.ps1 @@ -0,0 +1,17 @@ +# script\build.ps1: Script to build the binary + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +.\script\clean.ps1 + +# Get Git SHA and Version +$GIT_SHA = git rev-parse --short HEAD +$VERSION = git rev-parse --abbrev-ref HEAD +$LDFLAGS = "-X github.com/dremio/statcapn/pkg/versions.gitSha=$GIT_SHA -X github.com/dremio/statcapn/pkg/versions.version=$VERSION" + +# Build again and copy default-ddc.yaml +go build -ldflags "$LDFLAGS" -o .\bin\statcapn.exe \ No newline at end of file diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..0886cc3 --- /dev/null +++ b/script/cibuild @@ -0,0 +1,38 @@ +#!/bin/sh + +# script/cibuild: Setup environment for CI to run tests. This is primarily +# designed to run on the continuous integration server. + +set -e + +cd "$(dirname "$0")/.." + +echo "Validating if all dependencies are fullfilled…" +date "+%H:%M:%S" +./script/bootstrap + +echo "Tests started at…" +date "+%H:%M:%S" + +./script/test + +echo "Coverage file started at…" +date "+%H:%M:%S" + +./script/cover + +echo "Linting started at…" +date "+%H:%M:%S" + +./script/lint + +echo "Linting finished at…" +date "+%H:%M:%S" + +echo "Audit started at" +date "+%H:%M:%S" + +./script/audit + +echo "Audit finished at…" +date "+%H:%M:%S" diff --git a/script/clean b/script/clean new file mode 100755 index 0000000..bed98c4 --- /dev/null +++ b/script/clean @@ -0,0 +1,8 @@ +#!/bin/sh + +# script/clean: Remove bin folder +set -e + +cd "$(dirname "$0")/.." + +rm -fr ./bin diff --git a/script/clean.ps1 b/script/clean.ps1 new file mode 100644 index 0000000..3664100 --- /dev/null +++ b/script/clean.ps1 @@ -0,0 +1,9 @@ +# script\clean.ps1: Remove bin folder + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Remove-Item -Path .\bin -Recurse -Force diff --git a/script/cover b/script/cover new file mode 100755 index 0000000..225014e --- /dev/null +++ b/script/cover @@ -0,0 +1,11 @@ +#!/bin/sh + +# script/cover: Run the coverage + +#set -e + +cd "$(dirname "$0")/.." + +[ -z "$DEBUG" ] || set -x + +go tool cover -func=covprofile diff --git a/script/cover-html b/script/cover-html new file mode 100755 index 0000000..f5c2b32 --- /dev/null +++ b/script/cover-html @@ -0,0 +1,3 @@ +#!/bin/bash + +go tool cover -html=covprofile \ No newline at end of file diff --git a/script/cover-html.ps1 b/script/cover-html.ps1 new file mode 100644 index 0000000..fe4247c --- /dev/null +++ b/script/cover-html.ps1 @@ -0,0 +1,10 @@ +# script\cover.ps1: Run go tool cover and open the coverage report in a web browser + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Write-Output "Running go tool cover..." +go tool cover -html=covprofile diff --git a/script/cover.ps1 b/script/cover.ps1 new file mode 100644 index 0000000..b06180e --- /dev/null +++ b/script/cover.ps1 @@ -0,0 +1,13 @@ +# script\cover.ps1: Run the coverage + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +if ($env:DEBUG) { + $DebugPreference = "Continue" +} + +go tool cover -func=covprofile \ No newline at end of file diff --git a/script/install b/script/install new file mode 100755 index 0000000..b10bcdc --- /dev/null +++ b/script/install @@ -0,0 +1,36 @@ +#!/bin/bash + +# script/install: Script to install from source, eventually when there +# are releases this will switch to latest release + + +DOWNLOAD="" +OS="" +ARCH=`uname -m` +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + OS="linux" +elif [[ "$OSTYPE" == "darwin"* ]]; then + OS="darwin" +elif [[ "$OSTYPE" == "cygwin" ]]; then + OS="linux" +elif [[ "$OSTYPE" == "msys" ]]; then + OS="linux" +else + echo script for os type $OSTYPE is unsupported, build from sournce instead + exit 1 +fi + +if [[ "$ARCH" == "x86_64" ]]; then + ARCH="amd64" +fi + +DOWNLOAD=statcapn-$OS-$ARCH.zip +curl -o $DOWNLOAD -L "https://github.com/dremio/statcapn/releases/latest/download/$DOWNLOAD" +unzip $DOWNLOAD + +sudo mkdir -p /usr/local/share/statcapn +sudo mv ./bin/* /usr/local/share/statcapn/ +sudo ln -s /usr/local/share/statcapn/statcapn /usr/local/bin + +rm $DOWNLOAD +rm -fr statcapn-$OS-$ARCH diff --git a/script/lint b/script/lint new file mode 100755 index 0000000..d2b4606 --- /dev/null +++ b/script/lint @@ -0,0 +1,15 @@ +#!/bin/sh + +# script/lint: Run gofmt and golangci-lint run + +set -e + +cd "$(dirname "$0")/.." + +go fmt ./... + +echo "executing golangci-lint run" +golangci-lint run -E exportloopref,revive,gofmt -D structcheck + +echo "executing license-header-checker" +license-header-checker license_header.txt . go diff --git a/script/lint.ps1 b/script/lint.ps1 new file mode 100644 index 0000000..9054e4b --- /dev/null +++ b/script/lint.ps1 @@ -0,0 +1,13 @@ +# script/lint.ps1: Run gofmt and golangci-lint run + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Write-Output "Running gofmt..." +go fmt ./... + +Write-Output "Executing golangci-lint run" +golangci-lint run -E exportloopref,revive,gofmt -D structcheck \ No newline at end of file diff --git a/script/release b/script/release new file mode 100755 index 0000000..b84d86a --- /dev/null +++ b/script/release @@ -0,0 +1,56 @@ +#!/bin/sh + +# script/release: build binaries in all supported platforms and upload them with the gh client + +set -e + +cd "$(dirname "$0")/.." + +echo "Checking if gh is installed…" +date "+%H:%M:%S" + +if ! type "gh" > /dev/null; then + echo "gh not found installing…" + date "+%H:%M:%S" + OS="`uname`" + case $OS in + 'Linux') + if type "apt" > /dev/null; then + echo "detected debian based system installing via apt" + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update -y + sudo apt install gh -y + fi + if type "yum" > /dev/null; then + echo "detected rpm based system installing via yum" + sudo dnf install 'dnf-command(config-manager)' -y + sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + sudo dnf install gh -y + fi + ;; + 'WindowsNT') + echo "install gh and try again https://github.com/cli/cli/releases" + exit 1 + ;; + 'Darwin') + if type "brew" > /dev/null; then + brew install gh + else + echo "install homebrew to have zip https://brew.sh/" + exit 1 + fi + ;; + *) ;; + esac +fi + +# this is also set in script/build-release and is a copy paste +GIT_SHA=`git rev-parse --short HEAD` +VERSION=$1 + +./script/release-build $VERSION + + +gh release create $VERSION --title $VERSION -F changelog.md ./bin/statcapn-windows-arm64.zip ./bin/statcapn-windows-amd64.zip ./bin/statcapn-darwin-arm64.zip ./bin/statcapn-darwin-amd64.zip ./bin/statcapn-linux-arm64.zip ./bin/statcapn-linux-amd64.zip + diff --git a/script/release-build b/script/release-build new file mode 100755 index 0000000..6936525 --- /dev/null +++ b/script/release-build @@ -0,0 +1,71 @@ +#!/bin/sh + +# script/release-build: build binaries in all supported platforms + +echo "Checking if zip is installed…" +date "+%H:%M:%S" + +if ! type "zip" > /dev/null; then + echo "zip not found installing…" + date "+%H:%M:%S" + OS="`uname`" + case $OS in + 'Linux') + if type "apt" > /dev/null; then + echo "detected debian based system installing via apt" + sudo apt install zip + fi + if type "yum" > /dev/null; then + echo "detected rpm based system installing via yum" + sudo yum install zip + fi + ;; + 'WindowsNT') + echo "install gnu zip and try again http://gnuwin32.sourceforge.net/packages/zip.htm" + exit 1 + ;; + 'Darwin') + if type "brew" > /dev/null; then + brew install zip + else + echo "install homebrew to have zip https://brew.sh/" + exit 1 + fi + ;; + *) ;; + esac +fi +# this is also set in script/build and is a copy paste +GIT_SHA=`git rev-parse --short HEAD` +VERSION=$1 +LDFLAGS="-X github.com/dremio/statcapn/pkg/versions.gitSha=$GIT_SHA -X github.com/dremio/statcapn/pkg/versions.version=$VERSION" + +echo "Cleaning bin folder…" +date "+%H:%M:%S" +./script/clean + + +echo "Building linux-amd64…" +date "+%H:%M:%S" +GOOS=linux GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn +zip ./bin/statcapn-linux-amd64.zip ./bin/statcapn +echo "Building linux-arm64…" +date "+%H:%M:%S" +GOOS=linux GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn +zip ./bin/statcapn-linux-arm64.zip ./bin/statcapn +echo "Building darwin-os-x-amd64…" +date "+%H:%M:%S" +GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn +zip ./bin/statcapn-darwin-amd64.zip ./bin/statcapn +echo "Building darwin-os-x-arm64…" +date "+%H:%M:%S" +GOOS=darwin GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn +zip ./bin/statcapn-darwin-arm64.zip ./bin/statcapn +echo "Building windows-amd64…" +date "+%H:%M:%S" +GOOS=windows GOARCH=amd64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn.exe +zip ./bin/statcapn-windows-amd64.zip ./bin/statcapn.exe +echo "Building windows-arm64…" +date "+%H:%M:%S" +GOOS=windows GOARCH=arm64 go build -ldflags "$LDFLAGS" -o ./bin/statcapn.exe +zip ./bin/statcapn-windows-arm64.zip ./bin/statcapn.exe \ No newline at end of file diff --git a/script/release-build.ps1 b/script/release-build.ps1 new file mode 100644 index 0000000..45b75bb --- /dev/null +++ b/script/release-build.ps1 @@ -0,0 +1,56 @@ +# Script to build binaries in all supported platforms + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +# Get Git SHA and Version +$GIT_SHA = git rev-parse --short HEAD +$VERSION = $args[0] +$LDFLAGS = "-X github.com/dremio/statcapn/pkg/versions.gitSha=$GIT_SHA -X github.com/dremio/statcapn/pkg/versions.version=$VERSION" + +Write-Output "Cleaning bin folder…" +Get-Date -Format "HH:mm:ss" +.\script\clean + +Write-Output "Building linux-amd64…" +Get-Date -Format "HH:mm:ss" +$env:GOOS="linux" +$env:GOARCH="amd64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn +Compress-Archive -Path ./bin/statcapn -DestinationPath ./bin/statcapn-linux-amd64.zip + +Write-Output "Building linux-arm64…" +Get-Date -Format "HH:mm:ss" +$env:GOARCH="arm64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn +Compress-Archive -Path ./bin/statcapn -DestinationPath ./bin/statcapn-linux-arm64.zip + +Write-Output "Building darwin-os-x-amd64…" +Get-Date -Format "HH:mm:ss" +$env:GOOS="darwin" +$env:GOARCH="amd64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn +Compress-Archive -Path ./bin/statcapn -DestinationPath ./bin/statcapn-darwin-amd64.zip + +Write-Output "Building darwin-os-x-arm64…" +Get-Date -Format "HH:mm:ss" +$env:GOARCH="arm64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn +Compress-Archive -Path ./bin/statcapn -DestinationPath ./bin/statcapn-darwin-arm64.zip + +Write-Output "Building windows-amd64…" +Get-Date -Format "HH:mm:ss" +$env:GOOS="windows" +$env:GOARCH="amd64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn.exe +Compress-Archive -Path ./bin/statcapn.exe -DestinationPath ./bin/statcapn-windows-amd64.zip + +Write-Output "Building windows-arm64…" +Get-Date -Format "HH:mm:ss" +$env:GOOS="windows" +$env:GOARCH="arm64" +go build -ldflags "$LDFLAGS" -o ./bin/statcapn.exe +Compress-Archive -Path ./bin/statcapn.exe -DestinationPath ./bin/statcapn-windows-arm64.zip \ No newline at end of file diff --git a/script/release.ps1 b/script/release.ps1 new file mode 100644 index 0000000..db108ef --- /dev/null +++ b/script/release.ps1 @@ -0,0 +1,27 @@ +# Script to build binaries in all supported platforms and upload them with the gh client + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +# Get Version +$VERSION = $args[0] + +# Check if gh is installed +if (-Not (Get-Command "gh" -ErrorAction SilentlyContinue)) { + Write-Output "gh not found. Please install gh and try again https://github.com/cli/cli/releases" + Exit 1 +} +# Get Git SHA and Version +$GIT_SHA = git rev-parse --short HEAD +$VERSION = $args[0] + +Write-Output "Running release-build script…" +Get-Date -Format "HH:mm:ss" +.\script\release-build.ps1 $VERSION + +# Run gh release command +gh release create $VERSION --title $VERSION -F changelog.md ./bin/ddc-windows-arm64.zip ./bin/ddc-windows-amd64.zip ./bin/ddc-darwin-arm64.zip ./bin/ddc-darwin-amd64.zip ./bin/ddc-linux-arm64.zip ./bin/ddc-linux-amd64.zip + diff --git a/script/test b/script/test new file mode 100755 index 0000000..463a009 --- /dev/null +++ b/script/test @@ -0,0 +1,14 @@ +#!/bin/sh + +# script/test: Run test suite for application. + +#set -e + +cd "$(dirname "$0")/.." + +[ -z "$DEBUG" ] || set -x + +./script/clean +./script/build + +go test -race -covermode atomic -coverprofile=covprofile ./... diff --git a/script/test.ps1 b/script/test.ps1 new file mode 100644 index 0000000..5bffc0f --- /dev/null +++ b/script/test.ps1 @@ -0,0 +1,13 @@ +# script\test.ps1: Run test suite for application. + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +if ($env:DEBUG) { + $DebugPreference = "Continue" +} + +go test -covermode atomic -coverprofile=covprofile ./... \ No newline at end of file diff --git a/script/update b/script/update new file mode 100755 index 0000000..4678078 --- /dev/null +++ b/script/update @@ -0,0 +1,15 @@ +#!/bin/sh + +# script/update: Update application to run for its current checkout. + +set -e + +cd "$(dirname "$0")/.." + +echo "==> Running bootstrap…" + +script/bootstrap + +echo "==> Cleaning bin folder…" + +./script/clean \ No newline at end of file diff --git a/script/update.ps1 b/script/update.ps1 new file mode 100644 index 0000000..d92977c --- /dev/null +++ b/script/update.ps1 @@ -0,0 +1,15 @@ +# script\update.ps1: Update application to run for its current checkout. + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Change working directory to script's grandparents directory +Set-Location -Path (Get-Item (Split-Path -Parent $MyInvocation.MyCommand.Definition)).Parent.FullName + +Write-Output "==> Running bootstrap..." + +.\script\bootstrap + +Write-Output "==> Cleaning bin folder..." + +.\script\clean \ No newline at end of file