From 57eb69bf02d12a9e7cd42911e6b89e4ee0bd01cb Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 7 Dec 2023 14:21:33 +0545 Subject: [PATCH 1/3] feat: azure activity logs --- go.mod | 25 ++++----- go.sum | 47 ++++++++-------- hack/generate-schemas/go.mod | 6 +-- hack/generate-schemas/go.sum | 12 ++--- scrapers/azure/azure.go | 102 ++++++++++++++++++++++++++++++++++- 5 files changed, 148 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 87e2a605..0c6ba10d 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,18 @@ module github.com/flanksource/config-db go 1.20 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.1.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager v1.0.0 @@ -52,7 +53,6 @@ require ( github.com/hashicorp/go-getter v1.7.3 github.com/henvic/httpretty v0.1.2 github.com/hexops/gotextdiff v1.0.3 - github.com/itchyny/gojq v0.12.13 github.com/jackc/pgx/v5 v5.4.3 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 @@ -80,8 +80,8 @@ require ( ariga.io/atlas v0.14.2 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/AlekSi/pointer v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf // indirect github.com/agext/levenshtein v1.2.3 // indirect @@ -97,13 +97,14 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/inflect v0.19.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/hashicorp/hcl/v2 v2.18.1 // indirect + github.com/itchyny/gojq v0.12.13 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -243,13 +244,13 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.146.0 // indirect diff --git a/go.sum b/go.sum index 3e1ba310..884e62d4 100644 --- a/go.sum +++ b/go.sum @@ -610,14 +610,14 @@ github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor v1.1.0 h1:sp/3PYuecBd75CE8gYYgr+nQKhcl2fU5tB4YmfrJBFc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/advisor/armadvisor v1.1.0/go.mod h1:YoItTrIa+aGP3yYBH62yX1ot4L0g0HTvoY7eFOo3Pu0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice v1.0.0 h1:kRX8I0dWAcpW6Vq0m90CgV+qw4O1vXodgwrhoPr1RWs= @@ -631,13 +631,16 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontai github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.1.0 h1:8iR6OLffWWorFdzL2JFCab5xpD8VKEE2DUBBl+HNTDY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.1.0/go.mod h1:copqlcjMWc/wgQ1N2fzsJFQxDdqKGg1EQt8T5wJMOGE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0/go.mod h1:jj6P8ybImR+5topJ+eH6fgcemSFBmU6/6bFF8KkwuDI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.1.0 h1:rR8ZW79lE/ppfXTfiYSnMFv5EzmVuY4pfZWIkscIJ64= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.1.0/go.mod h1:y2zXtLSMM/X5Mfawq0lOftpWn3f4V6OCsRdINsvWBPI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.0 h1:yV3wcPPLQ+SLqJmgCs/wXKLxZkswMV4wCdNlG5XY4bQ= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.0/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 h1:LcJtQjCXJUm1s7JpUHZvu+bpgURhCatxVNbGADXniX0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0/go.mod h1:+OgGVo0Httq7N5oayfvaLQ/Jq+2gJdqfp++Hyyl7Tws= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.1.0 h1:pYhaMoTHP/zYIJGDA1sWsfyTDjdglaoYjIFMOEcL+/U= @@ -652,8 +655,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0= -github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -997,8 +1000,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -1652,8 +1655,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1791,8 +1794,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1949,8 +1952,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1963,8 +1966,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1982,8 +1985,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index 0bb023c2..d333d1a1 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -51,9 +51,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index a29f47b4..c2ad67db 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -134,8 +134,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -143,12 +143,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index bc3b3cf4..15c2d9be 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -12,21 +13,51 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager" + "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" commonutils "github.com/flanksource/commons/utils" "github.com/flanksource/duty/models" + "github.com/flanksource/commons/utils" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" ) -const ConfigTypePrefix = "Azure::" +const ( + activityLogTimespan = time.Hour * -1 + + ConfigTypePrefix = "Azure::" +) + +var activityLogFilter = strings.Join([]string{ + "authorization", + // "claims", + "correlationId", + "description", + "eventDataId", + "eventName", + "eventTimestamp", + "httpRequest", + "level", + "operationId", + "operationName", + "properties", + "resourceGroupName", + "resourceId", + "resourceProviderName", + "resourceType", // not present in the docs, but is necessary + "status", + "submissionTimestamp", + "subscriptionId", + "subStatus", +}, ",") var defaultExcludes = []v1.Filter{ {JSONPath: "$..etag"}, // Remove etags from the config json as they produce unecessary changes. @@ -111,6 +142,7 @@ func (azure Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { results = append(results, azure.fetchNetworkSecurityGroups()...) results = append(results, azure.fetchPublicIPAddresses()...) results = append(results, azure.fetchAdvisorAnalysis()...) + results = append(results, azure.fetchActivityLogs()...) } // Establish relationship of all resources to the corresponding subscription & resource group @@ -155,6 +187,74 @@ func (azure Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { return results } +// azureActivityExcludeOperations is a list of activity logs we can avoid +// as they do not mutate any resources. +var azureActivityExcludeOperations = []string{ + "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action", + "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action", +} + +func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { + logger.Debugf("fetching activity logs for subscription %s", azure.config.SubscriptionID) + + var results v1.ScrapeResults + + clientFactory, err := armmonitor.NewClientFactory(azure.config.SubscriptionID, azure.cred, nil) + if err != nil { + return append(results, v1.ScrapeResult{Error: fmt.Errorf("failed to initiate arm monitor client: %w", err)}) + } + + filter := fmt.Sprintf("eventTimestamp ge '%s'", time.Now().UTC().Add(activityLogTimespan).Format(time.RFC3339)) + pager := clientFactory.NewActivityLogsClient().NewListPager(filter, &armmonitor.ActivityLogsClientListOptions{Select: &activityLogFilter}) + for pager.More() { + page, err := pager.NextPage(azure.ctx) + if err != nil { + return append(results, v1.ScrapeResult{Error: fmt.Errorf("failed to read activity logs next page: %w", err)}) + } + logger.Tracef("Fetched %d activity logs", len(page.Value)) + + for _, v := range page.Value { + if collections.Contains(azureActivityExcludeOperations, utils.Deref(v.OperationName.Value)) { + continue + } + + change := v1.ChangeResult{ + ChangeType: utils.Deref(v.OperationName.Value), + CreatedAt: v.EventTimestamp, + Details: v1.NewJSON(*v), + ExternalChangeID: utils.Deref(v.EventDataID), + ExternalID: strings.ToLower(utils.Deref(v.ResourceID)), + ConfigType: getARMType(v.ResourceType.Value), + Severity: string(getSeverityFromReason(v)), + Source: ConfigTypePrefix + "ActivityLog", + Summary: utils.Deref(v.OperationName.LocalizedValue), + } + results = append(results, v1.ScrapeResult{Changes: []v1.ChangeResult{change}}) + } + } + + return results +} + +func getSeverityFromReason(v *armmonitor.EventData) models.Severity { + switch utils.Deref(v.Level) { + case "Critical": + return models.SeverityCritical + + case "Error": + return models.SeverityHigh + + case "Warning": + return models.SeverityMedium + + case "Verbose": + return models.SeverityLow + + default: + return models.SeverityInfo + } +} + // fetchDatabases gets all databases in a subscription. func (azure Scraper) fetchDatabases() v1.ScrapeResults { logger.Debugf("fetching databases for subscription %s", azure.config.SubscriptionID) From ad7bf54f5af9eb9f9459bccba619f55ca3fd5802 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 5 Dec 2023 18:41:27 +0545 Subject: [PATCH 2/3] feat: merge correlated events into one change record * aggregate timestamps of the events --- api/v1/interface.go | 3 +++ db/update.go | 20 ++++++++++++------ scrapers/azure/azure.go | 47 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/api/v1/interface.go b/api/v1/interface.go index 8df3f1d1..7262da96 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -78,6 +78,9 @@ type ChangeResult struct { CreatedBy *string `json:"created_by"` CreatedAt *time.Time `json:"created_at"` Details map[string]interface{} `json:"details"` + + // UpdateExisting indicates whether to update an existing change + UpdateExisting bool `json:"update_existing"` } func (r ChangeResult) AsMap() map[string]any { diff --git a/db/update.go b/db/update.go index 9c3dc17b..a205816f 100644 --- a/db/update.go +++ b/db/update.go @@ -138,19 +138,19 @@ func updateCI(ctx api.ScrapeContext, ci models.ConfigItem) error { } func updateChange(ctx api.ScrapeContext, result *v1.ScrapeResult) error { - for _, change := range result.Changes { - if change.Action == v1.Ignore { + for _, changeResult := range result.Changes { + if changeResult.Action == v1.Ignore { continue } - if change.Action == v1.Delete { - if err := deleteChangeHandler(ctx, change); err != nil { + if changeResult.Action == v1.Delete { + if err := deleteChangeHandler(ctx, changeResult); err != nil { return err } continue } - change := models.NewConfigChangeFromV1(*result, change) + change := models.NewConfigChangeFromV1(*result, changeResult) if change.CreatedBy != nil { person, err := FindPersonByEmail(ctx, ptr.ToString(change.CreatedBy)) @@ -174,8 +174,14 @@ func updateChange(ctx api.ScrapeContext, result *v1.ScrapeResult) error { change.ConfigID = *id - if err := db.Create(change).Error; err != nil { - return err + if changeResult.UpdateExisting { + if err := db.Save(change).Error; err != nil { + return err + } + } else { + if err := db.Create(change).Error; err != nil { + return err + } } } diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index 15c2d9be..01298a2f 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -194,6 +194,13 @@ var azureActivityExcludeOperations = []string{ "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action", } +// activityChangeRecord holds together the change result generated from an activity log +// along with the original activity log event. +type activityChangeRecord struct { + Result v1.ChangeResult + EventData *armmonitor.EventData +} + func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { logger.Debugf("fetching activity logs for subscription %s", azure.config.SubscriptionID) @@ -204,6 +211,10 @@ func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { return append(results, v1.ScrapeResult{Error: fmt.Errorf("failed to initiate arm monitor client: %w", err)}) } + // corelatedActivities keeps together events that belong to the same "uber operation" + // https://learn.microsoft.com/en-us/rest/api/monitor/activity-logs/list?view=rest-monitor-2015-04-01&tabs=HTTP#eventdata + var corelatedActivities = map[string][]activityChangeRecord{} + filter := fmt.Sprintf("eventTimestamp ge '%s'", time.Now().UTC().Add(activityLogTimespan).Format(time.RFC3339)) pager := clientFactory.NewActivityLogsClient().NewListPager(filter, &armmonitor.ActivityLogsClientListOptions{Select: &activityLogFilter}) for pager.More() { @@ -222,15 +233,47 @@ func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { ChangeType: utils.Deref(v.OperationName.Value), CreatedAt: v.EventTimestamp, Details: v1.NewJSON(*v), - ExternalChangeID: utils.Deref(v.EventDataID), + ExternalChangeID: utils.Deref(v.CorrelationID), ExternalID: strings.ToLower(utils.Deref(v.ResourceID)), ConfigType: getARMType(v.ResourceType.Value), Severity: string(getSeverityFromReason(v)), Source: ConfigTypePrefix + "ActivityLog", Summary: utils.Deref(v.OperationName.LocalizedValue), } - results = append(results, v1.ScrapeResult{Changes: []v1.ChangeResult{change}}) + corelatedActivities[utils.Deref(v.CorrelationID)] = append(corelatedActivities[utils.Deref(v.CorrelationID)], activityChangeRecord{ + Result: change, + EventData: v, + }) } + + // Stop the pager because there's a bug in the SDK itself + // error: failed to read activity logs next page: invalid semicolon separator in query + // TODO: + break + } + + // For the correlated activities, we merge some fields and aggregate the timestamps into a single change record + for _, changeRecords := range corelatedActivities { + change := changeRecords[0].Result // Use the latest event as the base + change.UpdateExisting = true // New events might have arrived, so we want to update the existing change in db + + statusTimestamps := map[string]*time.Time{} + for i := range changeRecords { + change.CreatedAt = changeRecords[i].EventData.EventTimestamp // Use the oldest event's timestamp + + status := utils.Deref(changeRecords[i].EventData.Status.Value) + if status == "" { + continue + } + statusTimestamps[strings.ToLower(status)] = changeRecords[i].EventData.EventTimestamp + + if _, ok := change.Details["httpRequest"]; !ok { + change.Details["httpRequest"] = changeRecords[i].EventData + } + } + + change.Details["timestamps"] = statusTimestamps + results = append(results, v1.ScrapeResult{Changes: []v1.ChangeResult{change}}) } return results From fe47ed717c6135bf7203076924432f4a869a3a85 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 6 Dec 2023 09:32:48 +0545 Subject: [PATCH 3/3] feat: add activity log exclusions --- api/v1/azure.go | 21 +++++++---- api/v1/zz_generated.deepcopy.go | 25 +++++++++++++ ...configs.flanksource.com_scrapeconfigs.yaml | 10 ++++++ config/schemas/config_azure.schema.json | 2 +- config/schemas/scrape_config.schema.json | 2 +- scrapers/azure/azure.go | 36 +++++++++++-------- 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/api/v1/azure.go b/api/v1/azure.go index b9877338..34853d54 100644 --- a/api/v1/azure.go +++ b/api/v1/azure.go @@ -15,10 +15,19 @@ type AzureDevops struct { type Azure struct { BaseScraper `json:",inline"` - ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` - SubscriptionID string `yaml:"subscriptionID" json:"subscriptionID"` - Organisation string `yaml:"organisation" json:"organisation"` - ClientID types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` - ClientSecret types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` - TenantID string `yaml:"tenantID" json:"tenantID"` + ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` + SubscriptionID string `yaml:"subscriptionID" json:"subscriptionID"` + Organisation string `yaml:"organisation" json:"organisation"` + ClientID types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` + ClientSecret types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` + TenantID string `yaml:"tenantID" json:"tenantID"` + Exclusions *AzureExclusions `yaml:"exclusions,omitempty" json:"exclusions,omitempty"` +} + +type AzureExclusions struct { + // ActivityLogs is a list of operations to exclude from activity logs. + // Example: + // "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action" + // "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action" + ActivityLogs []string `yaml:"activityLogs,omitempty" json:"activityLogs,omitempty"` } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index d15ea09b..737b4555 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -104,6 +104,11 @@ func (in *Azure) DeepCopyInto(out *Azure) { in.BaseScraper.DeepCopyInto(&out.BaseScraper) in.ClientID.DeepCopyInto(&out.ClientID) in.ClientSecret.DeepCopyInto(&out.ClientSecret) + if in.Exclusions != nil { + in, out := &in.Exclusions, &out.Exclusions + *out = new(AzureExclusions) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Azure. @@ -143,6 +148,26 @@ func (in *AzureDevops) DeepCopy() *AzureDevops { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureExclusions) DeepCopyInto(out *AzureExclusions) { + *out = *in + if in.ActivityLogs != nil { + in, out := &in.ActivityLogs, &out.ActivityLogs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureExclusions. +func (in *AzureExclusions) DeepCopy() *AzureExclusions { + if in == nil { + return nil + } + out := new(AzureExclusions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BaseScraper) DeepCopyInto(out *BaseScraper) { *out = *in diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index 7740ccd5..078465f2 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -321,6 +321,16 @@ spec: items: type: string type: array + exclusions: + properties: + activityLogs: + description: 'ActivityLogs is a list of operations to exclude + from activity logs. Example: "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action" + "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action"' + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties diff --git a/config/schemas/config_azure.schema.json b/config/schemas/config_azure.schema.json index 2ffb2cfc..0c92e1b9 100644 --- a/config/schemas/config_azure.schema.json +++ b/config/schemas/config_azure.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure","definitions":{"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure","definitions":{"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index 5643a016..8e3f5775 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"items":{"type":"string"},"type":"array"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEvent"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEvent":{"properties":{"exclusions":{"items":{"type":"string"},"type":"array"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"items":{"type":"string"},"type":"array"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEvent"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEvent":{"properties":{"exclusions":{"items":{"type":"string"},"type":"array"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index 01298a2f..5c1d2289 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -22,20 +23,21 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager" "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" - commonutils "github.com/flanksource/commons/utils" + "github.com/flanksource/commons/utils" "github.com/flanksource/duty/models" - "github.com/flanksource/commons/utils" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" ) const ( - activityLogTimespan = time.Hour * -1 - - ConfigTypePrefix = "Azure::" + ConfigTypePrefix = "Azure::" + defaultActivityLogMaxage = time.Hour * 24 * 7 ) +// activityLogLastRecordTime keeps track of the time of the last activity log per subscription. +var activityLogLastRecordTime = sync.Map{} + var activityLogFilter = strings.Join([]string{ "authorization", // "claims", @@ -187,13 +189,6 @@ func (azure Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { return results } -// azureActivityExcludeOperations is a list of activity logs we can avoid -// as they do not mutate any resources. -var azureActivityExcludeOperations = []string{ - "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action", - "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action", -} - // activityChangeRecord holds together the change result generated from an activity log // along with the original activity log event. type activityChangeRecord struct { @@ -215,7 +210,12 @@ func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { // https://learn.microsoft.com/en-us/rest/api/monitor/activity-logs/list?view=rest-monitor-2015-04-01&tabs=HTTP#eventdata var corelatedActivities = map[string][]activityChangeRecord{} - filter := fmt.Sprintf("eventTimestamp ge '%s'", time.Now().UTC().Add(activityLogTimespan).Format(time.RFC3339)) + var recordSince = time.Now().Add(-defaultActivityLogMaxage) + if v, ok := activityLogLastRecordTime.Load(azure.config.SubscriptionID); ok { + recordSince = v.(time.Time) + } + + filter := fmt.Sprintf("eventTimestamp ge '%s'", recordSince.Format(time.RFC3339)) pager := clientFactory.NewActivityLogsClient().NewListPager(filter, &armmonitor.ActivityLogsClientListOptions{Select: &activityLogFilter}) for pager.More() { page, err := pager.NextPage(azure.ctx) @@ -225,7 +225,11 @@ func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { logger.Tracef("Fetched %d activity logs", len(page.Value)) for _, v := range page.Value { - if collections.Contains(azureActivityExcludeOperations, utils.Deref(v.OperationName.Value)) { + if recordSince.Before(utils.Deref(v.EventTimestamp)) { + recordSince = *v.EventTimestamp + } + + if azure.config.Exclusions != nil && collections.Contains(azure.config.Exclusions.ActivityLogs, utils.Deref(v.OperationName.Value)) { continue } @@ -252,6 +256,8 @@ func (azure Scraper) fetchActivityLogs() v1.ScrapeResults { break } + activityLogLastRecordTime.Store(azure.config.SubscriptionID, recordSince) + // For the correlated activities, we merge some fields and aggregate the timestamps into a single change record for _, changeRecords := range corelatedActivities { change := changeRecords[0].Result // Use the latest event as the base @@ -570,7 +576,7 @@ func (azure Scraper) fetchSubscriptions() v1.ScrapeResults { Name: *v.DisplayName, Config: v, ConfigClass: "Subscription", - Type: getARMType(commonutils.Ptr("Subscription")), + Type: getARMType(utils.Ptr("Subscription")), }) } }