promdb2pcp: new tool to import Prometheus node_exporter metrics

qa/1687

@@ -0,0 +1,115 @@
+#!/bin/sh
+# PCP QA Test No. 1687
+#
+# promdb2pcp basic conversion exerciser
+#
+# Copyright (c) 2026 Red Hat.  All Rights Reserved.
+#
+
+if [ $# -eq 0 ]
+then
+    seq=`basename $0`
+    echo "QA output created by $seq"
+else
+    # use $seq from caller, unless not set
+    [ -n "$seq" ] || seq=`basename $0`
+    echo "QA output created by `basename $0` $*"
+fi
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+. ./common.python
+
+$python -c 'from pcp import pmi' 2>/dev/null
+test $? -eq 0 || _notrun "Python pcp pmi module is not installed"
+
+if which promdb2pcp >/dev/null 2>&1
+then
+    promdb2pcp=promdb2pcp
+elif [ -f "$here/../src/promdb2pcp/promdb2pcp.py" ]
+then
+    promdb2pcp="$python $here/../src/promdb2pcp/promdb2pcp.py"
+else
+    _notrun "promdb2pcp not installed"
+fi
+
+_cleanup()
+{
+    cd $here
+    $sudo rm -rf $tmp $tmp.*
+}
+
+status=0	# success is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_filter()
+{
+    _filter_pmdumplog \
+    | sed \
+	-e "s@$tmp@TMP@g" \
+    # end
+}
+
+# real QA test starts here
+
+echo "=== basic conversion ==="
+rm -f $tmp.0 $tmp.index $tmp.meta
+$promdb2pcp -a $tmp promdb2pcp
+if [ $? -ne 0 ]
+then
+    echo "promdb2pcp failed"
+    status=1
+    exit
+fi
+
+echo "=== archive check ==="
+pmlogcheck $tmp
+
+echo
+echo "=== metric descriptors ==="
+pminfo -d -a $tmp \
+    mem.physmem \
+    mem.freemem \
+    kernel.all.load \
+    kernel.all.load15 \
+    kernel.percpu.cpu.user \
+    kernel.percpu.cpu.idle \
+    disk.dev.read_bytes \
+    disk.dev.write_bytes \
+    network.interface.in.bytes \
+    network.interface.out.bytes
+
+echo
+echo "=== PMID structure (item IDs reset per cluster) ==="
+pmdumplog -d $tmp 2>&1 \
+| grep 'PMID:' \
+| sed -e 's/ (.*//' \
+| sort -t. -k2,2n -k3,3n
+
+echo
+echo "=== CPU metrics share same instance domain ==="
+pminfo -d -a $tmp kernel.percpu.cpu.user kernel.percpu.cpu.idle \
+| grep InDom \
+| sort -u
+
+echo
+echo "=== CPU instance names ==="
+pmprobe -I -a $tmp kernel.percpu.cpu.user
+
+echo
+echo "=== metric values (first timestamp) ==="
+echo "--- mem.physmem (should be MemTotal/1024) ---"
+pminfo -f -a $tmp -O +0 mem.physmem 2>&1
+
+echo "--- kernel.all.load ---"
+pminfo -f -a $tmp -O +0 kernel.all.load 2>&1
+
+echo
+echo "=== hostname from metadata ==="
+pmdumplog -l $tmp 2>&1 | grep 'Performance metrics from host testhost.example.com'
+
+# success, all done
+exit

qa/1687.out

@@ -0,0 +1,82 @@
+QA output created by 1687
+=== basic conversion ===
+=== archive check ===
+
+=== metric descriptors ===
+
+mem.physmem
+    Data Type: 64-bit unsigned int  InDom: PM_INDOM_NULL 0xffffffff
+    Semantics: instant  Units: Kbyte
+
+mem.freemem
+    Data Type: 64-bit unsigned int  InDom: PM_INDOM_NULL 0xffffffff
+    Semantics: instant  Units: Kbyte
+
+kernel.all.load
+    Data Type: float  InDom: PM_INDOM_NULL 0xffffffff
+    Semantics: instant  Units: none
+
+kernel.all.load15
+    Data Type: float  InDom: PM_INDOM_NULL 0xffffffff
+    Semantics: instant  Units: none
+
+kernel.percpu.cpu.user
+    Data Type: 64-bit unsigned int  InDom: 510.0 0x7f800000
+    Semantics: counter  Units: millisec
+
+kernel.percpu.cpu.idle
+    Data Type: 64-bit unsigned int  InDom: 510.0 0x7f800000
+    Semantics: counter  Units: millisec
+
+disk.dev.read_bytes
+    Data Type: 64-bit unsigned int  InDom: 510.1 0x7f800001
+    Semantics: counter  Units: Kbyte
+
+disk.dev.write_bytes
+    Data Type: 64-bit unsigned int  InDom: 510.2 0x7f800002
+    Semantics: counter  Units: Kbyte
+
+network.interface.in.bytes
+    Data Type: 64-bit unsigned int  InDom: 510.3 0x7f800003
+    Semantics: counter  Units: byte
+
+network.interface.out.bytes
+    Data Type: 64-bit unsigned int  InDom: 510.4 0x7f800004
+    Semantics: counter  Units: byte
+
+=== PMID structure (item IDs reset per cluster) ===
+PMID: 510.0.0
+PMID: 510.0.1
+PMID: 510.1.0
+PMID: 510.1.1
+PMID: 510.2.0
+PMID: 510.2.1
+PMID: 510.2.2
+PMID: 510.2.3
+PMID: 510.2.4
+PMID: 510.2.5
+PMID: 510.2.6
+PMID: 510.2.7
+PMID: 510.3.0
+PMID: 510.3.1
+PMID: 510.4.0
+PMID: 510.4.1
+
+=== CPU metrics share same instance domain ===
+    Data Type: 64-bit unsigned int  InDom: 510.0 0x7f800000
+
+=== CPU instance names ===
+kernel.percpu.cpu.user 2 "cpu0" "cpu1"
+
+=== metric values (first timestamp) ===
+--- mem.physmem (should be MemTotal/1024) ---
+
+mem.physmem
+    value 8192000
+--- kernel.all.load ---
+
+kernel.all.load
+    value 0.5
+
+=== hostname from metadata ===
+Performance metrics from host testhost.example.com

qa/1699

@@ -0,0 +1,108 @@
+#!/bin/sh
+# PCP QA Test No. 1699
+#
+# promdb2pcp edge case exerciser - malformed data, cpu-N labels,
+# bad metadata, missing files
+#
+# Copyright (c) 2026 Red Hat.  All Rights Reserved.
+#
+
+if [ $# -eq 0 ]
+then
+    seq=`basename $0`
+    echo "QA output created by $seq"
+else
+    # use $seq from caller, unless not set
+    [ -n "$seq" ] || seq=`basename $0`
+    echo "QA output created by `basename $0` $*"
+fi
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+. ./common.python
+
+$python -c 'from pcp import pmi' 2>/dev/null
+test $? -eq 0 || _notrun "Python pcp pmi module is not installed"
+
+if which promdb2pcp >/dev/null 2>&1
+then
+    promdb2pcp=promdb2pcp
+elif [ -f "$here/../src/promdb2pcp/promdb2pcp.py" ]
+then
+    promdb2pcp="$python $here/../src/promdb2pcp/promdb2pcp.py"
+else
+    _notrun "promdb2pcp not installed"
+fi
+
+_cleanup()
+{
+    cd $here
+    $sudo rm -rf $tmp $tmp.*
+}
+
+status=0	# success is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_filter()
+{
+    sed \
+	-e "s@$tmp@TMP@g" \
+    # end
+}
+
+# real QA test starts here
+
+echo "=== malformed values and bad metadata ==="
+rm -f $tmp.0 $tmp.index $tmp.meta
+$promdb2pcp -v -a $tmp promdb2pcp/baddata 2>&1 | _filter
+if [ -f $tmp.0 ]
+then
+    echo "archive created successfully despite bad data"
+
+    echo
+    echo "=== archive check ==="
+    pmlogcheck $tmp
+
+    echo
+    echo "=== mem.physmem (malformed timestamp skipped) ==="
+    pminfo -f -a $tmp mem.physmem 2>&1
+
+    echo
+    echo "=== mem.freemem (non-numeric value skipped) ==="
+    pminfo -f -a $tmp mem.freemem 2>&1
+
+    echo
+    echo "=== CPU instances from cpu-N labels ==="
+    pmprobe -I -a $tmp kernel.percpu.cpu.user
+
+    echo
+    echo "=== CPU metrics share same indom ==="
+    pminfo -d -a $tmp kernel.percpu.cpu.user kernel.percpu.cpu.idle \
+    | grep InDom \
+    | sort -u
+else
+    echo "promdb2pcp failed to produce an archive"
+    status=1
+fi
+
+echo
+echo "=== non-existent data directory ==="
+rm -f $tmp.0 $tmp.index $tmp.meta
+$promdb2pcp -a $tmp /no/such/directory >$tmp.out 2>&1
+echo "exit status: $?"
+_filter <$tmp.out
+
+echo
+echo "=== empty data directory ==="
+mkdir -p $tmp.emptydir
+rm -f $tmp.0 $tmp.index $tmp.meta
+$promdb2pcp -a $tmp $tmp.emptydir >$tmp.out 2>&1
+echo "exit status: $?"
+_filter <$tmp.out
+rmdir $tmp.emptydir 2>/dev/null
+
+# success, all done
+exit

qa/1699.out

@@ -0,0 +1,42 @@
+QA output created by 1699
+=== malformed values and bad metadata ===
+Warning: failed to parse promdb2pcp/baddata/metadata.json, using defaults
+mem.physmem (2 samples)
+mem.freemem (2 samples)
+kernel.percpu.cpu.user (2 CPUs)
+kernel.percpu.cpu.nice (2 CPUs)
+kernel.percpu.cpu.system (2 CPUs)
+kernel.percpu.cpu.idle (2 CPUs)
+kernel.percpu.cpu.iowait (2 CPUs)
+kernel.percpu.cpu.irq (2 CPUs)
+kernel.percpu.cpu.softirq (2 CPUs)
+kernel.percpu.cpu.steal (2 CPUs)
+Writing 3 timestamps (10 metrics)
+Wrote 3 timestamp records to TMP
+archive created successfully despite bad data
+
+=== archive check ===
+
+=== mem.physmem (malformed timestamp skipped) ===
+
+mem.physmem
+    value 8192000
+
+=== mem.freemem (non-numeric value skipped) ===
+
+mem.freemem
+    value 3998343
+
+=== CPU instances from cpu-N labels ===
+kernel.percpu.cpu.user 2 "cpu0" "cpu1"
+
+=== CPU metrics share same indom ===
+    Data Type: 64-bit unsigned int  InDom: 510.0 0x7f800000
+
+=== non-existent data directory ===
+exit status: 1
+Not a directory: /no/such/directory
+
+=== empty data directory ===
+exit status: 1
+No metric data found in TMP.emptydir

qa/group

@@ -2237,6 +2237,7 @@ suse
 1684 python derive local
 1685 dbpmda python pmda.chrony local
 1686 pmda.chrony python local
+1687 promdb2pcp pmimport local python libpcp_import
 1688 pmieconf local
 1689 pmproxy libpcp_web local
 1690 pmseries local
@@ -2248,6 +2249,7 @@ suse
 1696 pmproxy valgrind local
 1697 pmproxy valgrind local pmjson
 1698 logutil local pmlogger_daily
+1699 promdb2pcp pmimport local python libpcp_import
 1700 pmda.bpftrace local python pmjson
 1701 pmda.bpftrace local python
 1702 pmda.bpftrace local python

qa/promdb2pcp/baddata/cpu.json

@@ -0,0 +1,6 @@
+{"status":"success","data":{"resultType":"matrix","result":[
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"cpu-0","mode":"user"},"values":[[1700000000,"100.5"],[1700000030,"110.2"],[1700000060,"120.8"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"cpu-0","mode":"idle"},"values":[[1700000000,"800.0"],[1700000030,"825.5"],[1700000060,"850.3"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"cpu-1","mode":"user"},"values":[[1700000000,"90.3"],[1700000030,"95.1"],[1700000060,"101.4"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"cpu-1","mode":"idle"},"values":[[1700000000,"810.5"],[1700000030,"835.0"],[1700000060,"860.2"]]}
+]}}

qa/promdb2pcp/baddata/memory.json

@@ -0,0 +1,4 @@
+{"status":"success","data":{"resultType":"matrix","result":[
+  {"metric":{"__name__":"node_memory_MemTotal_bytes"},"values":[[1700000000,"8388608000"],["bad_timestamp","1234"],[1700000060,"8388608000"]]},
+  {"metric":{"__name__":"node_memory_MemFree_bytes"},"values":[[1700000000,"not_a_number"],[1700000030,"4094304000"],[1700000060,"4094304000"]]}
+]}}

qa/promdb2pcp/baddata/metadata.json

@@ -0,0 +1 @@
+{this is not valid json}

qa/promdb2pcp/cpu.json

@@ -0,0 +1,8 @@
+{"status":"success","data":{"resultType":"matrix","result":[
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"0","mode":"user"},"values":[[1700000000,"100.5"],[1700000030,"110.2"],[1700000060,"120.8"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"0","mode":"system"},"values":[[1700000000,"50.1"],[1700000030,"55.3"],[1700000060,"60.7"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"0","mode":"idle"},"values":[[1700000000,"800.0"],[1700000030,"825.5"],[1700000060,"850.3"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"1","mode":"user"},"values":[[1700000000,"90.3"],[1700000030,"95.1"],[1700000060,"101.4"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"1","mode":"system"},"values":[[1700000000,"40.2"],[1700000030,"44.8"],[1700000060,"49.1"]]},
+  {"metric":{"__name__":"node_cpu_seconds_total","cpu":"1","mode":"idle"},"values":[[1700000000,"810.5"],[1700000030,"835.0"],[1700000060,"860.2"]]}
+]}}

qa/promdb2pcp/disk.json

@@ -0,0 +1,4 @@
+{"status":"success","data":{"resultType":"matrix","result":[
+  {"metric":{"__name__":"node_disk_read_bytes_total","device":"sda"},"values":[[1700000000,"1048576000"],[1700000030,"1148576000"],[1700000060,"1248576000"]]},
+  {"metric":{"__name__":"node_disk_written_bytes_total","device":"sda"},"values":[[1700000000,"524288000"],[1700000030,"624288000"],[1700000060,"724288000"]]}
+]}}

qa/promdb2pcp/loadavg.json

@@ -0,0 +1,4 @@
+{"status":"success","data":{"resultType":"matrix","result":[
+  {"metric":{"__name__":"node_load1"},"values":[[1700000000,"0.5"],[1700000030,"1.2"],[1700000060,"0.8"]]},
+  {"metric":{"__name__":"node_load15"},"values":[[1700000000,"0.3"],[1700000030,"0.4"],[1700000060,"0.35"]]}
+]}}