Skip to content
This repository has been archived by the owner on Aug 22, 2024. It is now read-only.

Commit

Permalink
Don't use bytes for values of pg_stat_statements' blocks hits and blo…
Browse files Browse the repository at this point in the history 
…ck dirtied. (#39)

Don't use bytes for values of pg_stat_statements' blocks hits and block dirtied.
Adjust metrics names and descriptions accordingly.
Merge WAL FPI into WAL bytes and show it as labeled metric.
Introduce wal_all_bytes metric.
  • Loading branch information
@lesovsky
lesovsky authored Jun 1, 2022
1 parent 7698a9a commit f7da3e2
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 96 deletions.
48 changes: 25 additions & 23 deletions internal/collector/postgres_statements.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type postgresStatementsCollector struct {
tempRead typedDesc
tempWritten typedDesc
walRecords typedDesc
walFPI typedDesc
walAllBytes typedDesc
walBytes typedDesc
}

Expand Down Expand Up @@ -92,49 +92,49 @@ func NewPostgresStatementsCollector(constLabels labels, settings model.Collector
settings.Filters,
),
sharedHit: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "shared_hit_bytes_total", "Total number of bytes found in shared buffers by the statement.", 0},
descOpts{"postgres", "statements", "shared_buffers_hit_total", "Total number of blocks have been found in shared buffers by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
sharedRead: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "shared_read_bytes_total", "Total number of bytes read from disk or OS page cache when reading from shared buffers by the statement.", 0},
descOpts{"postgres", "statements", "shared_buffers_read_bytes_total", "Total number of bytes read from disk or OS page cache by the statement when block not found in shared buffers.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
sharedDirtied: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "shared_dirtied_bytes_total", "Total number of bytes dirtied in shared buffers by the statement.", 0},
descOpts{"postgres", "statements", "shared_buffers_dirtied_total", "Total number of blocks have been dirtied in shared buffers by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
sharedWritten: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "shared_written_bytes_total", "Total number of bytes written to shared buffers by the statement.", 0},
descOpts{"postgres", "statements", "shared_buffers_written_bytes_total", "Total number of bytes written from shared buffers to disk by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
localHit: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "local_hit_bytes_total", "Total number of bytes found in local buffers by the statement.", 0},
descOpts{"postgres", "statements", "local_buffers_hit_total", "Total number of blocks have been found in local buffers by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
localRead: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "local_read_bytes_total", "Total number of bytes read from disk or OS page cache when reading from local buffers by the statement.", 0},
descOpts{"postgres", "statements", "local_buffers_read_bytes_total", "Total number of bytes read from disk or OS page cache by the statement when block not found in local buffers.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
localDirtied: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "local_dirtied_bytes_total", "Total number of bytes dirtied in local buffers by the statement.", 0},
descOpts{"postgres", "statements", "local_buffers_dirtied_total", "Total number of blocks have been dirtied in local buffers by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
localWritten: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "local_written_bytes_total", "Total number of bytes written to local buffers by the statement.", 0},
descOpts{"postgres", "statements", "local_buffers_written_bytes_total", "Total number of bytes written from local buffers to disk by the statement.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
Expand All @@ -157,16 +157,16 @@ func NewPostgresStatementsCollector(constLabels labels, settings model.Collector
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
walFPI: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "wal_fpi_bytes_total", "Total number of WAL full-page images generated by the statement.", 0},
walAllBytes: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "wal_bytes_all_total", "Total number of WAL generated by the statement, in bytes.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
settings.Filters,
),
walBytes: newBuiltinTypedDesc(
descOpts{"postgres", "statements", "wal_bytes_total", "Total number of WAL bytes (not including FPI) generated by the statement.", 0},
descOpts{"postgres", "statements", "wal_bytes_total", "Total number of WAL bytes generated by the statement, by type.", 0},
prometheus.CounterValue,
[]string{"user", "database", "queryid"}, constLabels,
[]string{"user", "database", "queryid", "wal"}, constLabels,
settings.Filters,
),
}, nil
Expand Down Expand Up @@ -240,25 +240,25 @@ func (c *postgresStatementsCollector) Update(config Config, ch chan<- prometheus
ch <- c.times.newConstMetric(stat.blkWriteTime, stat.user, stat.database, stat.queryid, "iowrite")
}
if stat.sharedBlksHit > 0 {
ch <- c.sharedHit.newConstMetric(stat.sharedBlksHit*blockSize, stat.user, stat.database, stat.queryid)
ch <- c.sharedHit.newConstMetric(stat.sharedBlksHit, stat.user, stat.database, stat.queryid)
}
if stat.sharedBlksRead > 0 {
ch <- c.sharedRead.newConstMetric(stat.sharedBlksRead*blockSize, stat.user, stat.database, stat.queryid)
}
if stat.sharedBlksDirtied > 0 {
ch <- c.sharedDirtied.newConstMetric(stat.sharedBlksDirtied*blockSize, stat.user, stat.database, stat.queryid)
ch <- c.sharedDirtied.newConstMetric(stat.sharedBlksDirtied, stat.user, stat.database, stat.queryid)
}
if stat.sharedBlksWritten > 0 {
ch <- c.sharedWritten.newConstMetric(stat.sharedBlksWritten*blockSize, stat.user, stat.database, stat.queryid)
}
if stat.localBlksHit > 0 {
ch <- c.localHit.newConstMetric(stat.localBlksHit*blockSize, stat.user, stat.database, stat.queryid)
ch <- c.localHit.newConstMetric(stat.localBlksHit, stat.user, stat.database, stat.queryid)
}
if stat.localBlksRead > 0 {
ch <- c.localRead.newConstMetric(stat.localBlksRead*blockSize, stat.user, stat.database, stat.queryid)
}
if stat.localBlksDirtied > 0 {
ch <- c.localDirtied.newConstMetric(stat.localBlksDirtied*blockSize, stat.user, stat.database, stat.queryid)
ch <- c.localDirtied.newConstMetric(stat.localBlksDirtied, stat.user, stat.database, stat.queryid)
}
if stat.localBlksWritten > 0 {
ch <- c.localWritten.newConstMetric(stat.localBlksWritten*blockSize, stat.user, stat.database, stat.queryid)
Expand All @@ -270,13 +270,15 @@ func (c *postgresStatementsCollector) Update(config Config, ch chan<- prometheus
ch <- c.tempWritten.newConstMetric(stat.tempBlksWritten*blockSize, stat.user, stat.database, stat.queryid)
}
if stat.walRecords > 0 {
// WAL records
ch <- c.walRecords.newConstMetric(stat.walRecords, stat.user, stat.database, stat.queryid)
}
if stat.walFPI > 0 {
ch <- c.walFPI.newConstMetric(stat.walFPI*blockSize, stat.user, stat.database, stat.queryid)
}
if stat.walBytes > 0 {
ch <- c.walBytes.newConstMetric(stat.walBytes, stat.user, stat.database, stat.queryid)

// WAL total bytes
ch <- c.walAllBytes.newConstMetric((stat.walFPI*blockSize)+stat.walBytes, stat.user, stat.database, stat.queryid)

// WAL bytes by type (regular of fpi)
ch <- c.walBytes.newConstMetric(stat.walFPI*blockSize, stat.user, stat.database, stat.queryid, "fpi")
ch <- c.walBytes.newConstMetric(stat.walBytes, stat.user, stat.database, stat.queryid, "regular")
}
}

Expand Down
82 changes: 9 additions & 73 deletions internal/collector/postgres_statements_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,18 @@ func TestPostgresStatementsCollector_Update(t *testing.T) {
"postgres_statements_time_seconds_all_total",
},
optional: []string{
"postgres_statements_shared_hit_bytes_total",
"postgres_statements_shared_read_bytes_total",
"postgres_statements_shared_dirtied_bytes_total",
"postgres_statements_shared_written_bytes_total",
"postgres_statements_local_hit_bytes_total",
"postgres_statements_local_read_bytes_total",
"postgres_statements_local_dirtied_bytes_total",
"postgres_statements_local_written_bytes_total",
"postgres_statements_shared_buffers_hit_total",
"postgres_statements_shared_buffers_read_bytes_total",
"postgres_statements_shared_buffers_dirtied_total",
"postgres_statements_shared_buffers_written_bytes_total",
"postgres_statements_local_buffers_hit_total",
"postgres_statements_local_buffers_read_bytes_total",
"postgres_statements_local_buffers_dirtied_total",
"postgres_statements_local_buffers_written_bytes_total",
"postgres_statements_temp_read_bytes_total",
"postgres_statements_temp_written_bytes_total",
"postgres_statements_wal_records_total",
"postgres_statements_wal_fpi_bytes_total",
"postgres_statements_wal_bytes_all_total",
"postgres_statements_wal_bytes_total",
},
collector: NewPostgresStatementsCollector,
Expand Down Expand Up @@ -151,70 +151,6 @@ func Test_parsePostgresStatementsStats(t *testing.T) {
},
},
},
//{
// // in this testcase, stats of first two rows should be grouped because of similar queries.
// name: "query normalization",
// res: &model.PGResult{
// Nrows: 1,
// Ncols: 22,
// Colnames: []pgproto3.FieldDescription{
// {Name: []byte("database")}, {Name: []byte("user")}, {Name: []byte("query")},
// {Name: []byte("calls")}, {Name: []byte("rows")},
// {Name: []byte("total_exec_time")}, {Name: []byte("total_plan_time")}, {Name: []byte("blk_read_time")}, {Name: []byte("blk_write_time")},
// {Name: []byte("shared_blks_hit")}, {Name: []byte("shared_blks_read")}, {Name: []byte("shared_blks_dirtied")}, {Name: []byte("shared_blks_written")},
// {Name: []byte("local_blks_hit")}, {Name: []byte("local_blks_read")}, {Name: []byte("local_blks_dirtied")}, {Name: []byte("local_blks_written")},
// {Name: []byte("temp_blks_read")}, {Name: []byte("temp_blks_written")}, {Name: []byte("wal_records")}, {Name: []byte("wal_fpi")},
// {Name: []byte("wal_bytes")},
// },
// Rows: [][]sql.NullString{
// {
// {String: "testdb", Valid: true}, {String: "testuser", Valid: true}, {String: "SELECT 123", Valid: true},
// {String: "1000", Valid: true}, {String: "2000", Valid: true},
// {String: "30000", Valid: true}, {String: "100", Valid: true}, {String: "6000", Valid: true}, {String: "4000", Valid: true},
// {String: "100", Valid: true}, {String: "110", Valid: true}, {String: "120", Valid: true}, {String: "130", Valid: true},
// {String: "500", Valid: true}, {String: "510", Valid: true}, {String: "520", Valid: true}, {String: "530", Valid: true},
// {String: "700", Valid: true}, {String: "710", Valid: true}, {String: "720", Valid: true}, {String: "730", Valid: true},
// {String: "740", Valid: true},
// },
// {
// {String: "testdb", Valid: true}, {String: "testuser", Valid: true}, {String: "SELECT 456", Valid: true},
// {String: "1000", Valid: true}, {String: "2000", Valid: true},
// {String: "30000", Valid: true}, {String: "200", Valid: true}, {String: "6000", Valid: true}, {String: "4000", Valid: true},
// {String: "100", Valid: true}, {String: "110", Valid: true}, {String: "120", Valid: true}, {String: "130", Valid: true},
// {String: "500", Valid: true}, {String: "510", Valid: true}, {String: "520", Valid: true}, {String: "530", Valid: true},
// {String: "700", Valid: true}, {String: "710", Valid: true}, {String: "720", Valid: true}, {String: "730", Valid: true},
// {String: "740", Valid: true},
// },
// {
// {String: "testdb", Valid: true}, {String: "testuser", Valid: true}, {String: "SELECT 'whatever'", Valid: true},
// {String: "1000", Valid: true}, {String: "2000", Valid: true},
// {String: "30000", Valid: true}, {String: "300", Valid: true}, {String: "6000", Valid: true}, {String: "4000", Valid: true},
// {String: "100", Valid: true}, {String: "110", Valid: true}, {String: "120", Valid: true}, {String: "130", Valid: true},
// {String: "500", Valid: true}, {String: "510", Valid: true}, {String: "520", Valid: true}, {String: "530", Valid: true},
// {String: "700", Valid: true}, {String: "710", Valid: true}, {String: "720", Valid: true}, {String: "730", Valid: true},
// {String: "740", Valid: true},
// },
// },
// },
// want: map[string]postgresStatementStat{
// "testdb/testuser/095f2345f262d090a83ff1ac64ca8c76": {
// database: "testdb", user: "testuser", md5hash: "095f2345f262d090a83ff1ac64ca8c76", query: "SELECT ?",
// calls: 2000, rows: 4000,
// totalExecTime: 60000, totalPlanTime: 300, blkReadTime: 12000, blkWriteTime: 8000,
// sharedBlksHit: 200, sharedBlksRead: 220, sharedBlksDirtied: 240, sharedBlksWritten: 260,
// localBlksHit: 1000, localBlksRead: 1020, localBlksDirtied: 1040, localBlksWritten: 1060,
// tempBlksRead: 1400, tempBlksWritten: 1420, walRecords: 1440, walFPI: 1460, walBytes: 1480,
// },
// "testdb/testuser/6fc7663c0674ba2b5e0239d56eddf235": {
// database: "testdb", user: "testuser", md5hash: "6fc7663c0674ba2b5e0239d56eddf235", query: "SELECT '?'",
// calls: 1000, rows: 2000,
// totalExecTime: 30000, totalPlanTime: 300, blkReadTime: 6000, blkWriteTime: 4000,
// sharedBlksHit: 100, sharedBlksRead: 110, sharedBlksDirtied: 120, sharedBlksWritten: 130,
// localBlksHit: 500, localBlksRead: 510, localBlksDirtied: 520, localBlksWritten: 530,
// tempBlksRead: 700, tempBlksWritten: 710, walRecords: 720, walFPI: 730, walBytes: 740,
// },
// },
//},
}

for _, tc := range testCases {
Expand Down

0 comments on commit f7da3e2

Please sign in to comment.