Skip to content

Commit 1b93e0f

Browse files
MartyEwingsMartyEwings
authored
Merge pull request #175 from m0dular/SUP-2115-postgres_bloat
(SUP-2115) Add postgres bloat size and percents
2 parents e480186 + 42ee263 commit 1b93e0f

File tree

2 files changed

+165
-2
lines changed

2 files changed

+165
-2
lines changed

‎files/metrics_tidy

100755100644
File mode changed.

‎files/psql_metrics

Lines changed: 165 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,9 @@ SELECT json_object_agg(
417417
'toast_blks_read', si.toast_blks_read,
418418
'toast_blks_hit', si.toast_blks_hit,
419419
'tidx_blks_read', si.tidx_blks_read,
420-
'tidx_blks_hit', si.tidx_blks_hit
420+
'tidx_blks_hit', si.tidx_blks_hit,
421+
'bloat_size', bloat_size,
422+
'bloat_percent', bloat_pct
421423
)
422424
)
423425
FROM
@@ -428,6 +430,63 @@ JOIN
428430
pg_catalog.pg_stat_all_tables AS s ON c.oid = s.relid
429431
JOIN
430432
pg_catalog.pg_statio_all_tables AS si ON c.oid = si.relid
433+
JOIN
434+
(
435+
SELECT
436+
tblid,
437+
CASE WHEN tblpages - est_tblpages_ff > 0
438+
THEN (tblpages-est_tblpages_ff)*bs
439+
ELSE 0
440+
END AS bloat_size,
441+
CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0
442+
THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float
443+
ELSE 0
444+
END AS bloat_pct
445+
-- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO)
446+
FROM (
447+
SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages,
448+
ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff,
449+
tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na
450+
-- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO)
451+
FROM (
452+
SELECT
453+
( 4 + tpl_hdr_size + tpl_data_size + (2*ma)
454+
- CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END
455+
- CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END
456+
) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages,
457+
toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na
458+
-- , tpl_hdr_size, tpl_data_size
459+
FROM (
460+
SELECT
461+
tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples,
462+
tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages,
463+
coalesce(toast.reltuples, 0) AS toasttuples,
464+
coalesce(substring(
465+
array_to_string(tbl.reloptions, ' ')
466+
FROM 'fillfactor=([0-9]+)')::smallint, 100) AS fillfactor,
467+
current_setting('block_size')::numeric AS bs,
468+
CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma,
469+
24 AS page_hdr,
470+
23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END
471+
+ CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size,
472+
sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size,
473+
bool_or(att.atttypid = 'pg_catalog.name'::regtype)
474+
OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na
475+
FROM pg_attribute AS att
476+
JOIN pg_class AS tbl ON att.attrelid = tbl.oid
477+
JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace
478+
LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname
479+
AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname
480+
LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid
481+
WHERE NOT att.attisdropped
482+
AND tbl.relkind in ('r','m')
483+
GROUP BY 1,2,3,4,5,6,7,8,9,10
484+
ORDER BY 2,3
485+
) AS s
486+
) AS s2
487+
) AS s3
488+
) a on
489+
a.tblid = s.relid
431490
WHERE
432491
n.nspname NOT IN ('pg_catalog', 'information_schema')
433492
AND c.relkind = 'r'
@@ -485,7 +544,9 @@ SELECT json_object_agg(
485544
'idx_tup_read', idx_tup_read,
486545
'idx_tup_fetch', idx_tup_fetch,
487546
'idx_blks_read', idx_blks_read,
488-
'idx_blks_hit', idx_blks_hit
547+
'idx_blks_hit', idx_blks_hit,
548+
'bloat_size', bloat_size,
549+
'bloat_percent', bloat_pct
489550
)
490551
)
491552
FROM
@@ -496,6 +557,108 @@ JOIN
496557
pg_catalog.pg_stat_all_indexes AS s ON c.oid = s.indexrelid
497558
JOIN
498559
pg_catalog.pg_statio_all_indexes AS si ON c.oid = si.indexrelid
560+
JOIN
561+
(
562+
SELECT nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size,
563+
bs*(relpages-est_pages)::bigint AS extra_size,
564+
100 * (relpages-est_pages)::float / relpages AS extra_pct,
565+
fillfactor,
566+
idxoid,
567+
CASE WHEN relpages > est_pages_ff
568+
THEN bs*(relpages-est_pages_ff)
569+
ELSE 0
570+
END AS bloat_size,
571+
100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct,
572+
is_na
573+
-- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO)
574+
FROM (
575+
SELECT coalesce(1 +
576+
ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth)
577+
) AS est_pages,
578+
coalesce(1 +
579+
ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0
580+
) AS est_pages_ff,
581+
bs, nspname, tblname, idxname, idxoid, relpages, fillfactor, is_na
582+
-- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO)
583+
FROM (
584+
SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor,
585+
( index_tuple_hdr_bm +
586+
maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN
587+
WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign
588+
ELSE index_tuple_hdr_bm%maxalign
589+
END
590+
+ nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN
591+
WHEN nulldatawidth = 0 THEN 0
592+
WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign
593+
ELSE nulldatawidth::integer%maxalign
594+
END
595+
)::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na
596+
-- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO)
597+
FROM (
598+
SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages,
599+
i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs,
600+
CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?)
601+
WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8
602+
ELSE 4
603+
END AS maxalign,
604+
/* per page header, fixed size: 20 for 7.X, 24 for others */
605+
24 AS pagehdr,
606+
/* per page btree opaque data */
607+
16 AS pageopqdata,
608+
/* per tuple header: add IndexAttributeBitMapData if some cols are null-able */
609+
CASE WHEN max(coalesce(s.null_frac,0)) = 0
610+
THEN 8 -- IndexTupleData size
611+
ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8)
612+
END AS index_tuple_hdr_bm,
613+
/* data len: we remove null values save space using it fractionnal part from stats */
614+
sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth,
615+
max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na
616+
FROM (
617+
SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor,
618+
coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid,
619+
CASE WHEN a1.attnum IS NULL
620+
THEN ic.idxname
621+
ELSE ct.relname
622+
END AS attrelname
623+
FROM (
624+
SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey,
625+
pg_catalog.generate_series(1,indnatts) AS attpos
626+
FROM (
627+
SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid,
628+
i.indexrelid AS idxoid,
629+
coalesce(substring(
630+
array_to_string(ci.reloptions, ' ')
631+
from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor,
632+
i.indnatts,
633+
pg_catalog.string_to_array(pg_catalog.textin(
634+
pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey
635+
FROM pg_catalog.pg_index i
636+
JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid
637+
WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree')
638+
AND ci.relpages > 0
639+
) AS idx_data
640+
) AS ic
641+
JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid
642+
LEFT JOIN pg_catalog.pg_attribute a1 ON
643+
ic.indkey[ic.attpos] <> 0
644+
AND a1.attrelid = ic.tbloid
645+
AND a1.attnum = ic.indkey[ic.attpos]
646+
LEFT JOIN pg_catalog.pg_attribute a2 ON
647+
ic.indkey[ic.attpos] = 0
648+
AND a2.attrelid = ic.idxoid
649+
AND a2.attnum = ic.attpos
650+
) i
651+
JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace
652+
JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname
653+
AND s.tablename = i.attrelname
654+
AND s.attname = i.attname
655+
GROUP BY 1,2,3,4,5,6,7,8,9,10,11
656+
) AS rows_data_stats
657+
) AS rows_hdr_pdg_stats
658+
) AS relation_stats
659+
ORDER BY nspname, tblname, idxname
660+
) a on
661+
a.idxoid = s.indexrelid
499662
WHERE
500663
n.nspname NOT IN ('pg_catalog', 'information_schema')
501664
AND c.relkind = 'i'

0 commit comments

Comments
 (0)