From bfc58c132e2c2deb06e7b657147e3f3647e3db7c Mon Sep 17 00:00:00 2001 From: Hussam Date: Mon, 11 Mar 2024 15:36:30 -0500 Subject: [PATCH] Enable Pprof --- .gitignore | 3 ++- cmd/utils/flags.go | 58 +++++++++++++++++++++++++++++++++++++++++ go.mod | 6 +++-- go.sum | 11 ++++++++ pprof_tool/memsimple.sh | 36 +++++++++++++++++++++++++ pprof_tool/pprof.sh | 32 +++++++++++++++++++++++ 6 files changed, 143 insertions(+), 3 deletions(-) create mode 100755 pprof_tool/memsimple.sh create mode 100755 pprof_tool/pprof.sh diff --git a/.gitignore b/.gitignore index 11eb4dd62c..857e05b1a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ build/bin/ nodelogs/ +traces/ /data/ */config.toml */private.key .vscode/ -.idea* \ No newline at end of file +.idea* diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 558b7e1652..04fa5c0a3f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1,14 +1,18 @@ package utils import ( + "bytes" "encoding/json" "errors" "fmt" "math" "math/big" + "net/http" + _ "net/http/pprof" "os" "path/filepath" "regexp" + "runtime" godebug "runtime/debug" "strconv" "strings" @@ -86,6 +90,7 @@ var NodeFlags = []Flag{ UnlockedAccountFlag, PasswordFileFlag, VMEnableDebugFlag, + PprofFlag, InsecureUnlockAllowedFlag, GpoBlocksFlag, GpoPercentileFlag, @@ -465,6 +470,12 @@ var ( Usage: "Record information useful for VM and contract debugging" + generateEnvDoc(c_NodeFlagPrefix+"vmdebug"), } + PprofFlag = Flag{ + Name: "pprof", + Value: false, + Usage: "Enable the pprof HTTP server", + } + InsecureUnlockAllowedFlag = Flag{ Name: c_NodeFlagPrefix + "allow-insecure-unlock", Value: false, @@ -1212,10 +1223,51 @@ func CheckExclusive(args ...interface{}) { } } +func EnablePprof(nodeLocation common.Location) { + runtime.SetBlockProfileRate(1) + runtime.SetMutexProfileFraction(1) + var port string + myContext := nodeLocation + switch { + case bytes.Equal(myContext, []byte{}): // PRIME + port = "8081" + case bytes.Equal(myContext, []byte{0}): // Region 0 + port = "8090" + // case bytes.Equal(myContext, []byte{1}): // Region 1 + // port = "8100" + // case bytes.Equal(myContext, []byte{2}): // Region 2 + // port = "8110" + case bytes.Equal(myContext, []byte{0, 0}): // Zone 0-0 + port = "8091" + // case bytes.Equal(myContext, []byte{0, 1}): // Zone 0-1 + // port = "8092" + // case bytes.Equal(myContext, []byte{0, 2}): // Zone 0-2 + // port = "8093" + // case bytes.Equal(myContext, []byte{1, 0}): // Zone 1-0 + // port = "8101" + // case bytes.Equal(myContext, []byte{1, 1}): // Zone 1-1 + // port = "8102" + // case bytes.Equal(myContext, []byte{1, 2}): // Zone 1-2 + // port = "8103" + // case bytes.Equal(myContext, []byte{2, 0}): // Zone 2-0 + // port = "8111" + // case bytes.Equal(myContext, []byte{2, 1}): // Zone 2-1 + // port = "8112" + // case bytes.Equal(myContext, []byte{2, 2}): // Zone 2-2 + // port = "8113" + default: + port = "8085" + } + go func() { + log.Global.Print(http.ListenAndServe("localhost:"+port, nil)) + }() +} + // SetQuaiConfig applies quai-related command line flags to the config. func SetQuaiConfig(stack *node.Node, cfg *quaiconfig.Config, slicesRunning []common.Location, nodeLocation common.Location, logger *log.Logger) { cfg.NodeLocation = nodeLocation cfg.SlicesRunning = slicesRunning + // only set etherbase if its a zone chain if len(nodeLocation) == 2 { setEtherbase(cfg) @@ -1242,6 +1294,12 @@ func SetQuaiConfig(stack *node.Node, cfg *quaiconfig.Config, slicesRunning []com // set the gas limit ceil setGasLimitCeil(cfg) + // if cfg.PprofFlag { + if viper.IsSet(PprofFlag.Name) { + log.Global.Info("Starting pprof server") + EnablePprof(nodeLocation) + } + // Cap the cache allowance and tune the garbage collector mem, err := gopsutil.VirtualMemory() if err == nil { diff --git a/go.mod b/go.mod index 953d6f9476..de36ea5639 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chzyer/readline v1.5.1 // indirect github.com/cockroachdb/errors v1.8.1 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect @@ -78,12 +79,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect + github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huin/goupnp v1.3.0 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20240205174729-1f824a1a9b87 // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect @@ -186,5 +188,5 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.17.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.18.0 ) diff --git a/go.sum b/go.sum index b4cf8adf14..bcd135fb91 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,12 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -296,6 +300,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -339,6 +345,8 @@ github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFck github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20240205174729-1f824a1a9b87 h1:IG+58MeF0K/wSXknV4FMOABt6B4VZCguRDxYSCJqWb4= +github.com/ianlancetaylor/demangle v0.0.0-20240205174729-1f824a1a9b87/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -975,6 +983,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -984,6 +993,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pprof_tool/memsimple.sh b/pprof_tool/memsimple.sh new file mode 100755 index 0000000000..f1e91a5130 --- /dev/null +++ b/pprof_tool/memsimple.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +convert_to_kilobytes() { + value=$1 + case $value in + *K) printf "%.0f" ${value%K} ;; + *M) printf "%.0f" $(echo "${value%M} * 1024" | bc) ;; + *G) printf "%.0f" $(echo "${value%G} * 1024 * 1024" | bc) ;; + *) printf "%.0f" $(echo "$value / 1024" | bc) ;; + esac +} + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 pid" + exit 1 +fi + +pid=$1 +start_time=$(date +%s) + +# Print headers +echo "Elapsed_Time,PID,RSS,Stack_VSZ,Stack_Resident,Stack_Dirty,Stack_Swapped,VM_Allocate_VSZ,VM_Allocate_Resident,VM_Allocate_Dirty,VM_Allocate_Swapped" + +while true; do + if ps -p $pid > /dev/null; then + elapsed_time=$(($(date +%s) - start_time)) + + rss=$(ps -p $pid -o rss=) # ps prints rss in KB by default + + echo "$elapsed_time,$pid,$rss" + else + echo "Process $pid not found." + exit 1 + fi +sleep 1 +done diff --git a/pprof_tool/pprof.sh b/pprof_tool/pprof.sh new file mode 100755 index 0000000000..bccfaef4eb --- /dev/null +++ b/pprof_tool/pprof.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Exit if no arguments were provided +if [ $# -eq 0 ]; then + echo "No arguments provided" + exit 1 +fi + +# Get the folder name from the argument +folder=$1 + +# Array of ports and corresponding names +# ports=("9001" "9002" "9003" "9004" "9100" "9101" "9102" "9120" "9121" "9122" "9140" "9141" "1942") +# names=("PRIME" "Region_0" "Region_1" "Region_2" "Zone_0-0" "Zone_0-1" "Zone_0-2" "Zone_1-0" "Zone_1-1" "Zone_1-2" "Zone_2-0" "Zone_2-1" "Zone_2-2") + +# Array of ports and corresponding names +ports=("8081" "8090" "8091") +names=("PRIME" "Region_0" "Zone_0-0") + +# Create the new folder in traces +mkdir -p "traces/$folder" + +# Loop over the ports +for i in ${!ports[@]} +do + port=${ports[$i]} + name=${names[$i]} + + # Run the go tool pprof command for each port + echo "Running pprof for port $port" + curl http://localhost:$port/debug/pprof/goroutine -o "traces/$folder/$name"_"$port"_goroutine.pb.gz +done