Gobblin metrics pushing to pushgateway are generating tens of GB per day. On 2025-06-27, prometheus1005 ran out of disk space because of the volume of these logs.
prometheus-pushgateway
{ "ts": "2025-06-27T00:00:37.679Z", "caller": "diskmetricstore.go:166", "level": "info", "msg": "metric families inconsistent help strings", "err": "Metric families have inconsistent help strings. The latter will have priority. This is bad. Fix your pushed metrics!", "new": { "name": "gobblin_kafka_extracted_records_expected_count", "help": "Number of records expected to be extracted. This is expected high watermark - actual low watermark ", "type": "GAUGE", "metric": [ [ { "label": { "name": "instance", "value": "" } }, { "label": { "name": "job", "value": "webrequest" } }, { "label": { "name": "kafka_partition", "value": "20" } }, { "label": { "name": "kafka_topic", "value": "webrequest_text" } }, { "label": { "name": "reporter_type", "value": "EVENT" } }, { "gauge": { "value": "1.0591e+06" } } ] ] }, "old": { "name": "gobblin_kafka_extracted_records_expected_count", "help": "Number of records expected to be extracted. This is expected high watermark - actual low watermark", "type": "GAUGE", "metric": [ [ { "label": { "name": "instance", "value": "" } }, { "label": { "name": "job", "value": "eventlogging_legacy" } }, { "label": { "name": "kafka_partition", "value": "0" } }, { "label": { "name": "kafka_topic", "value": "eventlogging_MediaWikiPingback" } }, { "label": { "name": "reporter_type", "value": "EVENT" } }, { "gauge": { "value": "27224" } } ], [ { "label": { "name": "instance", "value": "" } }, { "label": { "name": "job", "value": "webrequest_frontend" } }, { "label": { "name": "kafka_partition", "value": "255" } }, { "label": { "name": "kafka_topic", "value": "webrequest_frontend_text" } }, { "label": { "name": "reporter_type", "value": "EVENT" } }, { "gauge": { "value": "886341" } } ] ] } }