diff --git a/build/rpm/pcp.spec.in b/build/rpm/pcp.spec.in index 5fa1061e024..afbeca458b9 100755 --- a/build/rpm/pcp.spec.in +++ b/build/rpm/pcp.spec.in @@ -133,6 +133,7 @@ BuildRequires: libpfm-devel >= 4.4 BuildRequires: ncurses-devel BuildRequires: readline-devel +BuildRequires: sqlite-devel >= 3.9.0 # Utilities used indirectly e.g. by scripts we install Requires: bash xz zstd gawk sed grep coreutils diffutils findutils diff --git a/configure b/configure index 64dd8841ba3..6be0ffbad31 100755 --- a/configure +++ b/configure @@ -761,6 +761,10 @@ ncurses_CFLAGS HAVE_DELAYACCT libnl3_LIBS libnl3_CFLAGS +lib_for_sqlite3 +HAVE_SQLITE3 +sqlite3_LIBS +sqlite3_CFLAGS lib_for_inih HAVE_LIBINIH HAVE_LIBUV @@ -1096,6 +1100,8 @@ libdrm_amdgpu_CFLAGS libdrm_amdgpu_LIBS libuv_CFLAGS libuv_LIBS +sqlite3_CFLAGS +sqlite3_LIBS libnl3_CFLAGS libnl3_LIBS ncurses_CFLAGS @@ -1865,6 +1871,10 @@ Some influential environment variables: libuv_CFLAGS C compiler flags for libuv, overriding pkg-config libuv_LIBS linker flags for libuv, overriding pkg-config + sqlite3_CFLAGS + C compiler flags for sqlite3, overriding pkg-config + sqlite3_LIBS + linker flags for sqlite3, overriding pkg-config libnl3_CFLAGS C compiler flags for libnl3, overriding pkg-config libnl3_LIBS linker flags for libnl3, overriding pkg-config @@ -2176,6 +2186,50 @@ fi } # ac_fn_c_try_cpp +# ac_fn_c_try_run LINENO +# ---------------------- +# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that +# executables *can* be run. +ac_fn_c_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf "%s\n" "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf "%s\n" "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; } +then : + ac_retval=0 +else case e in #( + e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 + printf "%s\n" "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status ;; +esac +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_run + # ac_fn_c_check_func LINENO FUNC VAR # ---------------------------------- # Tests whether FUNC exists, setting the cache variable VAR accordingly @@ -2298,50 +2352,6 @@ printf "%s\n" "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_type - -# ac_fn_c_try_run LINENO -# ---------------------- -# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that -# executables *can* be run. -ac_fn_c_try_run () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' - { { case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 - (eval "$ac_try") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; } -then : - ac_retval=0 -else case e in #( - e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 - printf "%s\n" "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=$ac_status ;; -esac -fi - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_run ac_configure_args_raw= for ac_arg do @@ -12283,6 +12293,202 @@ HAVE_LIBINIH=$have_libinih +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3 >= 3.9.0" >&5 +printf %s "checking for sqlite3 >= 3.9.0... " >&6; } + +if test -n "$sqlite3_CFLAGS"; then + pkg_cv_sqlite3_CFLAGS="$sqlite3_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"sqlite3 >= 3.9.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "sqlite3 >= 3.9.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_sqlite3_CFLAGS=`$PKG_CONFIG --cflags "sqlite3 >= 3.9.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$sqlite3_LIBS"; then + pkg_cv_sqlite3_LIBS="$sqlite3_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"sqlite3 >= 3.9.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "sqlite3 >= 3.9.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_sqlite3_LIBS=`$PKG_CONFIG --libs "sqlite3 >= 3.9.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + sqlite3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.9.0" 2>&1` + else + sqlite3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.9.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$sqlite3_PKG_ERRORS" >&5 + + have_sqlite3=false +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_sqlite3=false +else + sqlite3_CFLAGS=$pkg_cv_sqlite3_CFLAGS + sqlite3_LIBS=$pkg_cv_sqlite3_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_sqlite3=true +fi +if test $have_sqlite3 = false; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_open in -lsqlite3" >&5 +printf %s "checking for sqlite3_open in -lsqlite3... " >&6; } +if test ${ac_cv_lib_sqlite3_sqlite3_open+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_open (void); +int +main (void) +{ +return sqlite3_open (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_sqlite3_sqlite3_open=yes +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_open=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_open" >&5 +printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_open" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_open" = xyes +then : + have_sqlite3=true + lib_for_sqlite3="-lsqlite3" + +else case e in #( + e) have_sqlite3=false + lib_for_sqlite3="" + + ;; +esac +fi + +else + lib_for_sqlite3="$sqlite3_LIBS" +fi +if test $have_sqlite3 = true; then + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$sqlite3_CFLAGS $CPPFLAGS" + save_LIBS="$LIBS" + LIBS="$lib_for_sqlite3 $LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for SQLite3 >= 3.9.0 with FTS5 support" >&5 +printf %s "checking for SQLite3 >= 3.9.0 with FTS5 support... " >&6; } + if test "$cross_compiling" = yes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: assuming yes (cross-compiling)" >&5 +printf "%s\n" "assuming yes (cross-compiling)" >&6; } +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#if SQLITE_VERSION_NUMBER < 3009000 +#error "SQLite >= 3.9.0 required for FTS5" +#endif + +int +main (void) +{ + +sqlite3 *db; +int sts = 1; + +if (sqlite3_open(":memory:", &db) == SQLITE_OK) + sts = (sqlite3_exec(db, "CREATE VIRTUAL TABLE t USING fts5(c)", + 0, 0, 0) == SQLITE_OK) ? 0 : 1; +sqlite3_close(db); +return sts; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_sqlite3=false + lib_for_sqlite3="" + ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + + LIBS="$save_LIBS" + CPPFLAGS="$save_CPPFLAGS" +fi +if test $have_sqlite3 = false; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built" >&5 +printf "%s\n" "$as_me: WARNING: SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built" >&2;} +fi +HAVE_SQLITE3=$have_sqlite3 + + + + pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libnl-3.0" >&5 printf %s "checking for libnl-3.0... " >&6; } diff --git a/configure.ac b/configure.ac index 307754212d2..da95e7f8f78 100644 --- a/configure.ac +++ b/configure.ac @@ -1996,6 +1996,56 @@ AC_CHECK_LIB(inih, ini_parse, AC_SUBST(HAVE_LIBINIH, [$have_libinih]) AC_SUBST(lib_for_inih) +PKG_CHECK_MODULES([sqlite3], [sqlite3 >= 3.9.0], [have_sqlite3=true], [have_sqlite3=false]) +if test $have_sqlite3 = false; then + AC_CHECK_LIB(sqlite3, sqlite3_open, + [ have_sqlite3=true + lib_for_sqlite3="-lsqlite3" + ], + [ have_sqlite3=false + lib_for_sqlite3="" + ] + ) +else + lib_for_sqlite3="$sqlite3_LIBS" +fi +dnl Verify SQLite >= 3.9.0 and that the linked library provides FTS5 +if test $have_sqlite3 = true; then + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$sqlite3_CFLAGS $CPPFLAGS" + save_LIBS="$LIBS" + LIBS="$lib_for_sqlite3 $LIBS" + AC_MSG_CHECKING([for SQLite3 >= 3.9.0 with FTS5 support]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([[ +#include +#if SQLITE_VERSION_NUMBER < 3009000 +#error "SQLite >= 3.9.0 required for FTS5" +#endif + ]], [[ +sqlite3 *db; +int sts = 1; + +if (sqlite3_open(":memory:", &db) == SQLITE_OK) + sts = (sqlite3_exec(db, "CREATE VIRTUAL TABLE t USING fts5(c)", + 0, 0, 0) == SQLITE_OK) ? 0 : 1; +sqlite3_close(db); +return sts; + ]])], + [AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) + have_sqlite3=false + lib_for_sqlite3="" + ], + [AC_MSG_RESULT([assuming yes (cross-compiling)])]) + LIBS="$save_LIBS" + CPPFLAGS="$save_CPPFLAGS" +fi +if test $have_sqlite3 = false; then + AC_MSG_WARN([SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built]) +fi +AC_SUBST(HAVE_SQLITE3, [$have_sqlite3]) +AC_SUBST(lib_for_sqlite3) + PKG_CHECK_MODULES([libnl3], [libnl-3.0], [have_delayacct=true], [have_delayacct=false]) AC_SUBST(HAVE_DELAYACCT, [$have_delayacct]) diff --git a/man/man1/pmsearch.1 b/man/man1/pmsearch.1 index d20f05c48b1..05a69cea5ff 100644 --- a/man/man1/pmsearch.1 +++ b/man/man1/pmsearch.1 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020,2024 Red Hat. +.\" Copyright (c) 2020,2024,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -21,8 +21,6 @@ [\fB\-CdinqsStTV?\fR] [\fB\-c\fR \fIconfig\fR] [\f3\-D\f1 \f2debug\f1] -[\fB\-h\fR \fIhost\fR] -[\fB\-p\fR \fIport\fR] [\fB\-N\fR \fInumber\fR] [\fB\-O\fR \fIoffset\fR] [\fIquery\fR | \fIindom\fR] @@ -41,38 +39,15 @@ performs full text search queries to find metrics using names and help text from metrics, instance domains and instances. .PP -It makes use of capabilities of the Performance Co-Pilot (PCP) -.BR pmproxy (1) -service, the Valkey distributed key-value store and associated -ValkeySearch module. +It uses a local SQLite FTS5 full-text search index. +A base index is built at package install time containing +help text for all PCP metrics from the source tree. +Each night, +.BR pmsearch_daily (1) +copies the base index and extends it with instance names +(disk devices, network interfaces, etc.) from the running system. .PP -Note that in order to use these services, it is -.B mandatory -that -.B pmproxy -is communicating with a Valkey key-value server -that has the -.I valkey-search.so -module loaded. -When configured to do so, -.B pmproxy -will then automatically index PCP metric names, instance names, -metric and instance domain help text into the ValkeySearch store, -from PCP archives that it discovers locally. -Refer to -.BR pmlogger (1) -and -.BR pmlogger_daily (1) -for further details. -.PP -By default -.B pmsearch -communicates with a local key-value server -however the \fB\-h\fR and \fB\-p\fR options can be used to -specify an alternate Valket instance. -If this instance is a node of a Valkey cluster, all -other instances in the cluster will be discovered -and used automatically. +No external services (such as Redis or Valkey) are required. .SH OPTIONS .TP 5 \fB\-c\fR \fIconfig\fR, \fB\-\-config\fR=\fIconfig\fR @@ -85,22 +60,15 @@ No highlighting in results text. .TP \fB\-d\fR, \fB\-\-docid\fR Report document ID of each search result. -Documents are identified using a unique SHA-1 hash which -is always displayed in a 40-hexdigit human readable form. -.TP -\fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR -Connect to the key-value server at -.IR host , -rather than the one the localhost. .TP \fB\-i\fR, \fB\-\-info\fR -Report search engine internal metrics. +Report search index metrics (document count). .TP \fB\-n\fR, \fB\-\-indom\fR Perform an .IR instance .IR domain -related entities search +related entities search. .TP \fB\-N\fR \fIN\fR, \fB\-\-number\fR=\fIN\fR Return @@ -110,23 +78,17 @@ search results at most. \fB\-O\fR \fIN\fR, \fB\-\-offset\fR=\fIN\fR Paginated results from given offset. .TP -\fB\-p\fR \fIport\fR, \fB\-\-port\fR=\fIport\fR -Connect to the key-value server at -.IR port , -rather than the default -.BR 6379 . -.TP \fB\-q\fR, \fB\-\-query\fR Perform a general text search (default). .TP \fB\-s\fR, \fB\-\-suggest\fR -Perform a name suggestion search +Perform a name suggestion search. .TP \fB\-S\fR, \fB\-\-score\fR Report score (rank) of each result. .TP \fB\-T\fR, \fB\-\-total\fR -Report total number of of search results. +Report total number of search results. .TP \fB\-t\fR, \fB\-\-times\fR Report elapsed search execution time. @@ -152,6 +114,16 @@ Name: \fBkernel\f1.percpu.cpu.\fBidle\fR InDom: 60.0 One line: percpu \fBidle\fR CPU time metric from /proc/stat .ESAMPLE +.SH FILES +.TP +.I $PCP_VAR_DIR/lib/pmsearch.index +Nightly search index (SQLite FTS5 database), rebuilt by +.BR pmsearch_daily (1). +This is a copy of the base index extended with runtime instance data. +.TP +.I $PCP_SHARE_DIR/lib/pmsearch.index +Build-time base index containing all metrics from the PCP source tree. +Used as a fallback when the nightly index is not yet available. .SH PCP ENVIRONMENT Environment variables with the prefix \fBPCP_\fP are used to parameterize the file and directory names used by PCP. @@ -179,11 +151,12 @@ option to obtain a list of the available debugging options and their meaning. .SH SEE ALSO .BR PCPIntro (1), +.BR pmsearch_daily (1), .BR pmproxy (1), .BR pmlogger (1) and .BR pmlogger_daily (1). .\" control lines for scripts/man-spell -.\" +ok+ SHA Valkey ValkeySearch hexdigit idletime +.\" +ok+ idletime .\" +ok+ stat {from /proc/stat} diff --git a/man/man1/pmsearch_daily.1 b/man/man1/pmsearch_daily.1 new file mode 100644 index 00000000000..5775e4eb73d --- /dev/null +++ b/man/man1/pmsearch_daily.1 @@ -0,0 +1,120 @@ +'\"macro stdmacro +.\" +.\" Copyright (c) 2026 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PMSEARCH_DAILY 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pmsearch_daily\f1 \- rebuild the PCP full-text search index +.SH SYNOPSIS +\fBpmsearch_daily\fR +[\fB\-NV?\fR] +[\fB\-a\fR \fIarchive\fR] +[\fB\-o\fR \fIindex\fR] +.SH DESCRIPTION +.B pmsearch_daily +rebuilds the runtime full-text search index used by +.BR pmsearch (1) +and the +.BR pmproxy (1) +.I /search +REST API endpoint. +.PP +Its primary purpose is to capture +.B "instance names" +(disk devices, network interfaces, filesystems, etc.) +that are only known on a running system. +It queries the running +.BR pmcd (1) +(or a PCP archive when +.B \-a +is specified) +for all metric names, help text, and instance domain members, +then builds a SQLite FTS5 search index using +.BR newhelp (1). +The resulting index file is written to +.I $PCP_VAR_DIR/lib/pmsearch.index +by default. +.PP +Each night, the script copies the base index shipped with the +PCP package and extends the copy with instance data from the +running system. +This ensures the nightly index always starts fresh from the +complete set of metric help text. +.PP +This command is typically invoked nightly via a systemd timer +or cron job. +.PP +If +.B pmcd +is not running or no metrics are found, the command exits +without modifying any existing index file. +The base index at +.I $PCP_SHARE_DIR/lib/pmsearch.index +remains available as a fallback. +.SH OPTIONS +.TP 5 +\fB\-a\fR \fIarchive\fR, \fB\-\-archive\fR=\fIarchive\fR +Use +.I archive +as the source of metric metadata and instance data instead of +querying the running +.BR pmcd (1). +This is useful for testing or for building an index on a system +where +.B pmcd +is not running. +.TP +\fB\-N\fR, \fB\-\-showme\fR +Dry-run mode. +Show what would be done without writing any files. +.TP +\fB\-o\fR \fIindex\fR, \fB\-\-output\fR=\fIindex\fR +Write the index to +.IR index +instead of the default +.IR $PCP_VAR_DIR/lib/pmsearch.index . +.TP +\fB\-V\fR, \fB\-\-verbose\fR +Verbose diagnostics. +Report the number of metrics extracted and the output path. +.TP +\fB\-?\fR, \fB\-\-help\fR +Display usage message and exit. +.SH FILES +.TP 5 +.I $PCP_VAR_DIR/lib/pmsearch.index +Default output path for the runtime search index. +This is a copy of the base index extended with instance data. +When present, it is preferred over the base index. +.TP +.I $PCP_SHARE_DIR/lib/pmsearch.index +Base search index shipped with the PCP package, +built at package build time from help text source files. +Copied as the starting point for each nightly rebuild; +used directly as a fallback when the runtime index does not exist. +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.SH SEE ALSO +.BR PCPIntro (1), +.BR newhelp (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmsearch (1) +and +.BR pmproxy (1). diff --git a/man/man3/pmsearchinfo.3 b/man/man3/pmsearchinfo.3 index 4c27b1f1c15..c7f5b7ce360 100644 --- a/man/man3/pmsearchinfo.3 +++ b/man/man3/pmsearchinfo.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -33,13 +33,14 @@ cc ... \-lpcp_web .ad .ft 1 .SH DESCRIPTION -Diagnostic metrics related to the Performance Co-Pilot (PCP) fast, scalable and full-text capable search services. +Provides diagnostic information about the Performance Co-Pilot (PCP) +full-text search index. .PP -Currently, as all full-text related services are using single ValkeySearch index, diagnostic metrics are requestable by passing in +Diagnostic metrics are requestable by passing in .IR key with a value of .BR `text' . -The results will passed to +The results (such as document count) will be passed to the .B on_metrics callback provided by .B settings. @@ -55,5 +56,5 @@ On failure a negative PMAPI error code is returned. and .BR PMWEBAPI (3). .\" control lines for scripts/man-spell -.\" +ok+ ValkeySearch on_metrics {from callback} +.\" +ok+ on_metrics {from callback} .\" +ok+ sds {from typedef char *sds in pcp/sds.h} diff --git a/man/man3/pmsearchsetup.3 b/man/man3/pmsearchsetup.3 index 7d24aafb501..859ceec5865 100644 --- a/man/man3/pmsearchsetup.3 +++ b/man/man3/pmsearchsetup.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -52,28 +52,45 @@ cc ... \-lpcp_web .SH DESCRIPTION The .B pmSearchSetup -and related API functions prepare an application for accessing the fast and scalable metric, instance and instance domain, full-text capable, searching functionality of the Performance Co-Pilot (PCP). +and related API functions prepare an application for accessing the +full-text search functionality of the Performance Co-Pilot (PCP). .PP -This functionality is provided trough asynchronous APIs, which function in event-driven fashion. The interface described here prepare a given search +The search engine uses a local SQLite FTS5 full-text index. +A build-time base index at +.I $PCP_SHARE_DIR/lib/pmsearch.index +provides help text for all metrics shipped with PCP. +Each night, +.BR pmsearch_daily (1) +copies the base index to +.I $PCP_VAR_DIR/lib/pmsearch.index +and extends it with instance names from the running system. +When the nightly index exists it is used; +otherwise the base index is used as a fallback. +.PP +The interfaces described here prepare a given search .IR module -which associates callback routines with certain asynchronous events that occur as part of servicing search requests. +which associates callback routines with events that occur +as part of servicing search requests. .PP As a general pattern, all interfaces in these APIs that need to invoke .I callbacks provided by the calling program will take an opaque (void * pointer) .IR arg parameter. -This pointer will be passed through unchanged and is typically used to access a data structure maintaining state within the calling program. +This pointer will be passed through unchanged and is typically used to +access a data structure maintaining state within the calling program. .PP -Where asynchronous setup is required by +On completion of .B pmSearchSetup -its completion will be indicated through use of the +the .I on_setup -callback, which is part of the passed in +callback will be invoked. +This callback is part of the passed in .I pmSearchModule -structure. This structure also provides for custom diagnostics handling, through the +structure, which also provides for custom diagnostics handling through the .I on_info -callback. These are self-explanatory, see +callback. +See .I for exact calling conventions. .PP @@ -103,15 +120,16 @@ failure (negative PMAPI code) of the operation. .PP The helper functions .B pmSearchSetSlots -(key-value server), +and .B pmSearchSetEventLoop -(libuv), +are retained for API compatibility but have no effect. .B pmSearchSetConfiguration (configuration file) and .B pmSearchSetMetricRegistry (MMV instrumentation) -interfaces provide a mechanism for passing in state for each of the associated subsystems. +provide a mechanism for passing in state for each of the +associated subsystems. .PP Finally, a call to .B pmSearchClose @@ -128,10 +146,20 @@ utilities are the primary users of the interfaces, providing REST API and comman .SH DIAGNOSTICS Where these functions return a status code, this is always zero on success. On failure a negative PMAPI error code is returned. +.SH FILES +.TP +.I $PCP_VAR_DIR/lib/pmsearch.index +Nightly search index (SQLite FTS5 database), a copy of the base +index extended with runtime instance data by +.BR pmsearch_daily (1). +.TP +.I $PCP_SHARE_DIR/lib/pmsearch.index +Build-time base index shipped with the PCP package. .SH SEE ALSO +.BR pmsearch (1), +.BR pmsearch_daily (1), .BR pmproxy (1), .BR pmlogger (1), -.BR pmsearch (1), .BR mmv_stats_registry (3), .BR pmSearchInfo (3), .BR pmSearchTextInDom (3), @@ -144,4 +172,4 @@ and .\" control lines for scripts/man-spell .\" +ok+ on_text_result on_metrics on_setup on_done on_info {from callbacks} .\" +ok+ pmwebapi {from include } -.\" +ok+ libuv dict {from struct dict} +.\" +ok+ dict {from struct dict} diff --git a/man/man3/pmsearchtextsuggest.3 b/man/man3/pmsearchtextsuggest.3 index 1a4c8427e8c..7c10a5e0722 100644 --- a/man/man3/pmsearchtextsuggest.3 +++ b/man/man3/pmsearchtextsuggest.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -42,11 +42,12 @@ Query string that will be used to search. \fIcount\fR Limits number of results. Defaults to 10. .PP -Words with length less than 2 are omitted. -.PP -This search matches terms inside entity name that either have +This search performs prefix matching on entity names using +FTS5 prefix indexes. +Terms in .B query -prefix or are within Levenshtein distance of 1, with former having slightly higher priority than the latter. +are appended with a wildcard so that partial names +match (e.g.\& \fBker\fP matches \fBkernel\fP). .PP Each entity matched by the .B request @@ -68,4 +69,4 @@ and .\" control lines for scripts/man-spell .\" +ok+ on_text_result {from callback} -.\" +ok+ Levenshtein {from Levenshtein distance} +.\" +ok+ ker {from example prefix} diff --git a/qa/1861 b/qa/1861 index 9c4f00264de..43f756b2296 100755 --- a/qa/1861 +++ b/qa/1861 @@ -1,34 +1,26 @@ #!/bin/sh # PCP QA Test No. 1861 -# Test reserved by nathans on Wed 5 Aug 2026 09:23:40 AEST -# [what am I here for?] +# Exercise inserting metrics from a PMDA without build-time help files +# (e.g. Python PMDAs) into the pmsearch SQLite FTS5 index using the +# @ M format that pmsearch_daily generates at runtime. # -# Copyright (c) 2026 [who are you?]. All Rights Reserved. +# Copyright (c) 2026 Red Hat. All Rights Reserved. # -if [ $# -eq 0 ] -then - seq=`basename $0` - echo "QA output created by $seq" -else - # use $seq from caller, unless not set - [ -n "$seq" ] || seq=`basename $0` - echo "QA output created by `basename $0` $*" -fi +seq=`basename $0` +echo "QA output created by $seq" # get standard environment, filters and checks . ./common.product . ./common.filter . ./common.check -do_valgrind=false -if [ "$1" = "--valgrind" ] -then - _check_valgrind - do_valgrind=true -fi - -# test for-some-thing || _notrun No support for some-thing +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" _cleanup() { @@ -36,34 +28,81 @@ _cleanup() $sudo rm -rf $tmp $tmp.* } -status=0 # success is the default! +status=1 # failure is the default! trap "_cleanup; exit \$status" 0 1 2 3 15 -_filter() -{ - sed \ - -e "s@$tmp@TMP@g" \ - -e "s@/privateTMP@TMP@g" \ - # end -} - # real QA test starts here -if $do_valgrind -then - _run_valgrind [--save-output] ...your test goes here... -else - ...your test goes here... 2>&1 -fi \ -| _filter +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" -# if really bad error -status=1 -exit +# --- Step 1: Build base index from sample PMDA help files --- +mkdir -p $tmp.share/lib +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Base index built" + +# --- Step 2: Copy base to nightly location (as pmsearch_daily does) --- +mkdir -p $tmp.var/lib +cp $tmp.share/lib/pmsearch.index $tmp.var/lib/pmsearch.index +echo "Nightly index initialized from base" + +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share + +# --- Step 3: Verify Python PMDA metrics are NOT in base index --- +echo +echo "=== Verify Python PMDA metrics absent from base index ===" +pmsearch -C "qa_python" + +# --- Step 4: Create synthetic @ M entries mirroring Python PMDA patterns --- +# Python PMDAs define help text via add_metric() in code, not help files. +# pmsearch_daily scrapes running PMDAs with pminfo -tT and generates +# @ M nameindomoneline entries for newhelp -S. +# +# Pattern (a): one-line help only (bpftrace style - most common) +# Pattern (b): both one-line and long help (openmetrics style) +# Pattern (c): metric with an indom (bpftrace per-script style) +# +cat > $tmp.helptext <<'EOF' +@ M qa_python.control.status status of the test python agent +@ M qa_python.control.debug debug flag to enable verbose log messages +Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 +@ M qa_python.data.counter 99.0 count of events processed by python agent +@ M qa_python.data.value 99.0 instantaneous value from python agent +EOF + +# --- Step 5: Insert Python PMDA metrics into nightly index --- +echo +echo "=== Inserting Python PMDA metrics ===" +newhelp -S -o $tmp.var/lib/pmsearch.index $tmp.helptext 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Python PMDA metrics inserted" + +# --- Step 6: Verify Python PMDA metrics now searchable --- +echo +echo "=== Search for Python PMDA metrics by name ===" +pmsearch -C "qa_python" + +echo +echo "=== Search for Python PMDA metric by one-line help ===" +pmsearch -C "verbose log messages" + +echo +echo "=== Search for Python PMDA metric with indom ===" +pmsearch -C "events processed" + +echo +echo "=== Suggest search for Python PMDA metrics ===" +pmsearch -s qa_python -# optional stuff if your test has verbose output to help resolve problems -#echo -#echo "If failure, check $seq_full" +# --- Step 7: Verify original sample PMDA metrics still present --- +echo +echo "=== Verify sample PMDA metrics still present ===" +pmsearch -C "trended" # success, all done +status=0 exit diff --git a/qa/1861.out b/qa/1861.out index c0537127120..92729d52bc0 100644 --- a/qa/1861.out +++ b/qa/1861.out @@ -1,2 +1,86 @@ QA output created by 1861 -[skeleton from qa/new, replace me] +Base index built +Nightly index initialized from base + +=== Verify Python PMDA metrics absent from base index === +0 search results + +=== Inserting Python PMDA metrics === +Python PMDA metrics inserted + +=== Search for Python PMDA metrics by name === +Type: metric +Name: qa_python.data.value +InDom: 99.0 +One line: instantaneous value from python agent + +Type: metric +Name: qa_python.control.status +InDom: none +One line: status of the test python agent + +Type: metric +Name: qa_python.data.counter +InDom: 99.0 +One line: count of events processed by python agent + +Type: metric +Name: qa_python.control.debug +InDom: none +One line: debug flag to enable verbose log messages +Help: Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 + +=== Search for Python PMDA metric by one-line help === +Type: metric +Name: qa_python.control.debug +InDom: none +One line: debug flag to enable verbose log messages +Help: Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 + +=== Search for Python PMDA metric with indom === +Type: metric +Name: qa_python.data.counter +InDom: 99.0 +One line: count of events processed by python agent + +=== Suggest search for Python PMDA metrics === +Name: qa_python.data.value + +Name: qa_python.control.status + +Name: qa_python.data.counter + +Name: qa_python.control.debug + +=== Verify sample PMDA metrics still present === +Type: metric +Name: sample.drift +InDom: none +One line: A random trended metric +Help: This metric returns a random value (expected mean is approximately 200), +subject to a trending pattern such that the sequence is mainly monotonic, +with a change in direction after on average 4 consecutive samples. + +Use pmStore() to modify the instantaneous value, which becomes the new +expected mean. + +Type: metric +Name: sample.colour +InDom: 29.1 +One line: Metrics with a "saw-tooth" trend over time +Help: This metric has 3 instances, designated "red", "green" and "blue". + +The value of the metric is monotonic increasing in the range N to +N+100, then back to N. The different instances have different N +values, namely 100 (red), 200 (green) and 300 (blue). + +The underlying counter starts at 0 and is incremented once +for each pmFetch() to this metric and/or sample.mirage and/or +sample.mirage_longlong. + +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). diff --git a/qa/1871 b/qa/1871 index 7916b86c365..9946690f6b4 100755 --- a/qa/1871 +++ b/qa/1871 @@ -1,8 +1,10 @@ #!/bin/sh # PCP QA Test No. 1871 -# Exercise pmsearch functionality. +# Exercise pmsearch functionality using local SQLite FTS5 index. +# Tests the copy-and-extend strategy: build a base index from help +# files, copy it, then append instance data from an archive. # -# Copyright (c) 2020 Red Hat. All Rights Reserved. +# Copyright (c) 2020,2026 Red Hat. All Rights Reserved. # seq=`basename $0` @@ -13,13 +15,17 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_series -_check_search -_check_key_server_version_offline +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" +[ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ + _notrun "pmsearch_daily not installed" _cleanup() { - [ -n "$port" ] && $keys_cli $port shutdown cd $here $sudo rm -rf $tmp $tmp.* } @@ -27,76 +33,85 @@ _cleanup() status=1 # failure is the default! trap "_cleanup; exit \$status" 0 1 2 3 15 -# real QA test starts here -echo "=== Start test key server ===" -key_server_port=`_find_free_port` -key_server_name=`_find_key_server_name` -key_server_path=`_find_key_server_modules` -key_server_search=`_find_key_server_search` -keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" -# workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -$sudo chmod +x $keysearch -# module is only accessible by the associated key server user account -$sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & -port="-p $key_server_port" -_check_key_server_ping $key_server_port -echo +_filter_info() +{ + sed \ + -e 's/: [0-9][0-9]*/: INTEGER/g' \ + #end +} -echo "=== Load an empty index ===" -pmsearch $port no results +_filter_extras() +{ + sed \ + -e 's/\(Score:\) [0-9][0-9]*\.[0-9][0-9]*/\1 N.NN/g' \ + -e 's/\([0-9][0-9]* hit in\) [0-9][0-9]*\.[0-9][0-9]* seconds/\1 N.NNN seconds/g' \ + -e 's/\([0-9][0-9]* hits in\) [0-9][0-9]*\.[0-9][0-9]* seconds/\1 N.NNN seconds/g' \ + #end +} -echo "=== Load deterministic search content ===" -_filter_load() +_filter_docid() { sed \ - -e "s,$here,PATH,g" \ + -e 's/^ID: [0-9][0-9]*:/ID: N:/g' \ #end } -pmseries $port --load $here/archives/sample-labels | _filter_load + +# real QA test starts here + +archive="$here/archives/sample-labels" +[ -f "$archive.0" ] || _notrun "sample-labels archive not found" +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" + +# Mirror the real copy-and-extend strategy: +# 1. Build base index from PMDA help files (like package install) +# 2. Copy base to nightly location (like pmsearch_daily.sh does) +# 3. Append instance data from archive to the nightly copy + +# --- Base index (like $PCP_SHARE_DIR/lib/pmsearch.index) --- +mkdir -p $tmp.share/lib +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Base index built" + +# --- Nightly index (like $PCP_VAR_DIR/lib/pmsearch.index) --- +# Use pmsearch_daily to copy base and append instance data from archive +mkdir -p $tmp.var/lib +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share +$PCP_BINADM_DIR/pmsearch_daily -a $archive +echo "Nightly index built (base + instances)" +echo + +# PCP_VAR_DIR and PCP_SHARE_DIR already exported above for pmsearch_daily + +echo "=== Empty search ===" +pmsearch -C xyznonexistent search_terms="99 random interesting result" for term in $search_terms do echo "=== Search on $term ===" - pmsearch $port -C $term + pmsearch -C $term done echo "=== Search engine info ===" -_filter_info() -{ - sed \ - -e 's/: [0-9][0-9]*\.[0-9][0-9]*/: DOUBLE/g' \ - -e 's/: [0-9][0-9]*/: INTEGER/g' \ - #end -} -pmsearch $port -i -C | _filter_info +pmsearch -C -i | _filter_info +echo echo "=== Extra reporting - timing, score, hits, doc IDs ===" -_filter_extras() -{ - # need to filter out score, since it seems the search module indexes - # records asynchronously and therefore there may be a delay before - # "score" "stabilizes" until all records are processed - this may - # take longer then the adding of the records themselves - sed \ - -e 's/\(Score:\) 0\.[0-9][0-9]*/\1 0.XX/g' \ - -e 's/\([0-9][0-9]* hit in\) 0\.[0-9][0-9]* seconds/\1 0.X seconds/g' \ - -e 's/\([0-9][0-9]* hits in\) 0\.[0-9][0-9]* seconds/\1 0.Y seconds/g' \ - #end -} -pmsearch $port -dStT 99 | _filter_extras +pmsearch -dStT 99 | _filter_extras | _filter_docid +echo echo "=== Suggestion search 1 ===" -pmsearch $port -s sample | _filter_extras +pmsearch -s sample | _filter_extras echo "=== Suggestion search 2 ===" -pmsearch $port -s blue | _filter_extras +pmsearch -s blue | _filter_extras echo "=== Indom search ===" -pmsearch $port -n 29.3 | _filter_extras - +pmsearch -n 29.3 | _filter_extras -cat $tmp.keys >> $seq_full # success, all done status=0 diff --git a/qa/1871.out b/qa/1871.out index dd5514bbc59..a1314a65297 100644 --- a/qa/1871.out +++ b/qa/1871.out @@ -1,12 +1,9 @@ QA output created by 1871 -=== Start test key server === -PING -PONG +Base index built +Nightly index built (base + instances) -=== Load an empty index === +=== Empty search === 0 search results -=== Load deterministic search content === -pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === Search on 99 === Type: indom Name: 29.3 @@ -24,30 +21,132 @@ Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" +Type: indom +Name: 29.6 +InDom: 29.6 +One line: Instance domain "dodgey" for sample PMDA. +Help: Instances for an unreliable instance domain, where the instances can +under certain conditions (random or controlled) return holes, errors, +and other interesting scenarios for testing. A sample.dodgey.control +metric store controls this instance domain. + Type: metric -Name: sample.mirage +Name: sample.drift +InDom: none +One line: A random trended metric +Help: This metric returns a random value (expected mean is approximately 200), +subject to a trending pattern such that the sequence is mainly monotonic, +with a change in direction after on average 4 consecutive samples. + +Use pmStore() to modify the instantaneous value, which becomes the new +expected mean. + +Type: indom +Name: 29.12 +InDom: 29.12 +One line: Instance domain "procs" for sample PMDA +Help: Simulated and small instance domain that mimics the behaviour of the +process instance domain from the proc PMDA. + +The first 5 instances are fixed. Then next 15 instances are variable +such that each time an associated metric is fetched: +- if an instance is currently defined, it may disappear with + probability 0.075 +- for each instance that is not defined, a new instance may + appear with probability 0.075 + +The internal instance identifiers mimic PIDs and are monotonically +increasing up to 1000, when they wrap (duplicates are obviously +avoided). + +The external instance names are the instance number, space, then +a random "executable" name. + +Type: metric +Name: sample.byte_rate +InDom: none +One line: instantaneous bytes/second +Help: random value in the range (0,1023), so avg value is 512 bytes/second + +Type: metric +Name: sample.kbyte_rate +InDom: none +One line: instantaneous Kbytes/second +Help: random value in the range (0,1023), so avg value is 512 Kbytes/second + +Type: metric +Name: sample.byte_rate_perhour +InDom: none +One line: instantaneous bytes/hour +Help: random value in the range (0,1023), so avg value is 512 bytes/hour + +Type: metric +Name: sample.dodgey.value +InDom: 29.6 +One line: 5 unreliable instances +Help: The metric is a set of 5 instantaneous values, drawn at random from the +range 0 to 100. The number of instances "visible" is controlled by +sample.dodgey.control. + +Type: metric +Name: sample.scramble.bin +InDom: 29.9 +One line: Several constant instances, instances scrambled +Help: Like sample.bin, except +1. instances are missing with probability 0.33 +2. order of the instances from pmFetch is random + +Designed to help testing instance matching between pmFetch calls +for PCP clients. + +Type: metric +Name: sample.dodgey.control +InDom: none +One line: control values retured for sample.dodgey.value +Help: If sample.dodgey.control is <= 0, then this is returned as the "numval" +component in the pmResult (0 => no values available, less than 0 => +various errors). + +If sample.dodgey.control is between 1 and 5 (inclusive), then this many +of the values will be "visible". The values will be selected in order +from the underlying 5 instances. + +If sample.dodgey.control is > 5, then at random times (between 1 and +sample.dodgey.control fetches of the metric), the number of instances +available is changed according to the following probabilities ... + 0.9 some number of instances in the range 0 to 5, selected at random + from the underlying 5 instances. + 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION) +=== Search on interesting === +Type: indom +Name: 29.6 +InDom: 29.6 +One line: Instance domain "dodgey" for sample PMDA. +Help: Instances for an unreliable instance domain, where the instances can +under certain conditions (random or controlled) return holes, errors, +and other interesting scenarios for testing. A sample.dodgey.control +metric store controls this instance domain. + +Type: metric +Name: sample.mirage_longlong InDom: 29.3 One line: Simple saw-tooth rate, but instances come and go -Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution +Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution over time. Different instances of the metric have different baselines -for the saw-tooth, but all have an max-to-min range of 100. - -What makes this metric interesting is that instances come and go although -not more often than once every 10 seconds by default. Use pmstore to -change sample.controller.mirage and the frequency of instance domain -changes can be varied. +for the saw-tooth, but all have an max-to-min range of 100,000,000. -Instance 0 is always present, but the other instances 1 thru 49 come -and go in a cyclic pattern with a large random component influencing -when each instance appears and disappears. +What makes this metric interesting is that instances come and go (not more +often than once every 10 seconds however). Instance 0 is always present, +but the other instances 1 thru 49 come and go in a cyclic pattern with +a large random component influencing when each instance appears and +disappears. -The underlying counter starts at 0 and is incremented once -for each pmFetch() to this metric and/or sample.colour and/or -sample.mirage_longlong. +The underlying counter starts at 0 and is incremented once for each +pmFetch() to this metric and/or sample.mirage and/or sample.colour. Use pmStore() to modify the underlying counter (independent of which instance or instances are used). -=== Search on interesting === + Type: metric Name: sample.mirage InDom: 29.3 @@ -77,28 +176,41 @@ Name: sample.rapid InDom: none One line: count very quickly Help: Base counter increments by 8*10^7 per fetch. Result is 10 x base counter. + +Type: metric +Name: sample.not_ready +InDom: none +One line: interval (in seconds) during which PMDA does not respond to PDUs +Help: Store a positive number of seconds as the value of this metric. The +following PDU received will result in the following sequence of events: + 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd + 2. sleep for the given number of seconds + 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd +If everything went as planned, sample.not_ready returns to 0, otherwise it +has a negative error code as the value. + +Type: metric +Name: sample.not_ready_msec +InDom: none +One line: interval (in milliseconds) during which PMDA does not respond to PDUs +Help: Store a positive number of milliseconds as the value of this metric. The +following PDU received will result in the following sequence of events: + 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd + 2. sleep for the given number of milliseconds + 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd +If everything went as planned, sample.not_ready returns to 0, otherwise it +has a negative error code as the value. === Search engine info === -RediSearch statistics: +Search index statistics: Documents: INTEGER Terms: INTEGER Records: INTEGER -- Average records per doc: DOUBLE -- Average bytes per record: DOUBLE -- Inverted Index - Size: DOUBLE MB - Capacity: DOUBLE MB - Overhead: DOUBLE -- Skip Index - Size: DOUBLE MB -- Score Index - Size: DOUBLE MB -- Average offsets per term: DOUBLE -- Average offset bits per record: DOUBLE + === Extra reporting - timing, score, hits, doc IDs === -1 hit in 0.X seconds +1 hit in N.NNN seconds -ID: 6697a5ee42032f3e914b3475c07a16605d3c8b7a -Score: 0.XX +ID: N:29.3 +Score: N.NN Type: indom Name: 29.3 InDom: 29.3 @@ -106,34 +218,49 @@ One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" + === Suggestion search 1 === -Name: sample.mirage +Name: sample.string.null -Name: sample.rapid +Name: sample.aggregate.null + +Name: sample.bad.novalues + +Name: sample.long.ten -Name: sample.colour +Name: sample.long.hundred + +Name: sample.long.million + +Name: sample.long.bin + +Name: sample.ulong.bin + +Name: sample.longlong.one + +Name: sample.longlong.ten === Suggestion search 2 === Name: blue === Indom search === -Type: indom -Name: 29.3 +Type: metric +Name: sample.mirage_longlong InDom: 29.3 -One line: Instance domain "mirage" for sample PMDA -Help: Random number of instances, that change with time. Instance "m-00" (0) -is always present, while the others are numbered 1 .. 49 and named "m-01" -.. "m-99" +One line: Simple saw-tooth rate, but instances come and go +Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution +over time. Different instances of the metric have different baselines +for the saw-tooth, but all have an max-to-min range of 100,000,000. -Type: instance -Name: m-04 -InDom: 29.3 +What makes this metric interesting is that instances come and go (not more +often than once every 10 seconds however). Instance 0 is always present, +but the other instances 1 thru 49 come and go in a cyclic pattern with +a large random component influencing when each instance appears and +disappears. -Type: instance -Name: m-01 -InDom: 29.3 +The underlying counter starts at 0 and is incremented once for each +pmFetch() to this metric and/or sample.mirage and/or sample.colour. -Type: instance -Name: m-00 -InDom: 29.3 +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). Type: metric Name: sample.mirage @@ -158,3 +285,23 @@ sample.mirage_longlong. Use pmStore() to modify the underlying counter (independent of which instance or instances are used). + +Type: indom +Name: 29.3 +InDom: 29.3 +One line: Instance domain "mirage" for sample PMDA +Help: Random number of instances, that change with time. Instance "m-00" (0) +is always present, while the others are numbered 1 .. 49 and named "m-01" +.. "m-99" + +Type: instance +Name: m-00 +InDom: 29.3 + +Type: instance +Name: m-01 +InDom: 29.3 + +Type: instance +Name: m-04 +InDom: 29.3 diff --git a/qa/1872 b/qa/1872 index adac854962e..925894146cb 100755 --- a/qa/1872 +++ b/qa/1872 @@ -1,8 +1,9 @@ #!/bin/sh # PCP QA Test No. 1872 # Exercise pmsearch REST API endpoints using curl(1). +# Uses a local SQLite FTS5 search index (no Redis/Valkey). # -# Copyright (c) 2020 Red Hat. All Rights Reserved. +# Copyright (c) 2020,2026 Red Hat. All Rights Reserved. # seq=`basename $0` @@ -12,142 +13,91 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check -. ./common.keys -_check_series -_check_search -_check_key_server_version_offline +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" +[ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ + _notrun "pmsearch_daily not installed" +which pmproxy >/dev/null 2>&1 || \ + _notrun "pmproxy not installed" +which curl >/dev/null 2>&1 || \ + _notrun "curl not installed" +which pmjson >/dev/null 2>&1 || \ + _notrun "pmjson not installed" _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$port" ] && $keys_cli $port shutdown - if $need_restore - then - need_restore=false - _restore_config $PCP_SYSCONF_DIR/pmproxy - _restore_config $PCP_SYSCONF_DIR/pmseries - fi $sudo rm -rf $tmp $tmp.* } status=1 # failure is the default! signal=$PCP_BINADM_DIR/pmsignal - -userid=`id -u` username=`id -u -n` -hostname=`hostname` -machineid=`_machine_id` -domainname=`_domain_name` -need_restore=false trap "_cleanup; exit \$status" 0 1 2 3 15 -_filter_source() +_filter_elapsed() { sed \ - -e "s,$here,PATH,g" \ - -e "s,$hostname,QAHOST,g" \ + -e 's/\(\"elapsed\":\) [0-9][0-9]*\.[0-9][0-9]*/\1 XXX/g' \ #end } -_filter_archive() +_filter_info() { - pmjson | \ sed \ - -e "s,bozo-laptop,QALOGHOST,g" \ + -e 's/\(\"docs\":\) [0-9][0-9]*/\1 NNN/g' \ + -e 's/\(\"terms\":\) [0-9][0-9]*/\1 NNN/g' \ + -e 's/\(\"records\":\) [0-9][0-9]*/\1 NNN/g' \ #end } # real QA test starts here -_save_config $PCP_SYSCONF_DIR/pmproxy -_save_config $PCP_SYSCONF_DIR/pmseries -$sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* -need_restore=true - -echo "=== Start test key server ===" -key_server_port=`_find_free_port` -key_server_name=`_find_key_server_name` -key_server_path=`_find_key_server_modules` -key_server_search=`_find_key_server_search` -keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" -# workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -$sudo chmod +x $keysearch -# search module is only accessible by the key server user account -$sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & -_check_key_server_ping $key_server_port -port="-p $key_server_port" + +archive="$here/archives/sample-labels" +[ -f "$archive.0" ] || _notrun "sample-labels archive not found" +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" + +# Build a search index using the copy-and-extend strategy: +# base index from help file, then pmsearch_daily copies and appends instances. + +echo "=== Build search index ===" +mkdir -p $tmp.share/lib $tmp.var/lib +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share +$PCP_BINADM_DIR/pmsearch_daily -a $archive +echo "Index built" echo +# Configure and start pmproxy with search enabled cat > $tmp.conf <>$seq_full -proxyopts="-p $proxyport -r $key_server_port -c $tmp.conf" # -Dseries,http,af -pmproxy -f -U $username -x $seq_full -l $tmp.pmproxy.log $proxyopts & +pmproxy -f -U $username -x $seq_full -l $tmp.pmproxy.log \ + -p $proxyport -c $tmp.conf & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec -# have to wait until everything is properly indexed in the key server -pmsleep 10s - -$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq_full -$keys_cli $port keys pcp:text >> $seq_full -cat $tmp.pmproxy.log >> $seq_full - echo "=== /search/text - query for '99' 'random' 'interesting' 'result' ===" search_terms="99 random interesting result" for term in $search_terms @@ -170,7 +120,7 @@ echo "=== /search/text - query param is required ===" url="http://localhost:$proxyport/search/text" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/text - highlighting is supported on 'name', 'oneline' and 'helptext' by using 'highlight' param ===" @@ -189,7 +139,6 @@ echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full curl --get --silent "$url" | tee -a $seq_full | pmjson | _filter_elapsed -# url query returns 2 results echo "=== /search/text - pagination is supported using cursor params 'limit' and 'offset'" url="http://localhost:$proxyport/search/text?query=random&limit=1" @@ -297,7 +246,7 @@ echo "=== /search/suggest - query param is required ===" url="http://localhost:$proxyport/search/suggest" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/suggest - no match response ===" @@ -330,7 +279,7 @@ echo "=== /search/indom - query param is required ===" url="http://localhost:$proxyport/search/indom" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/indom - pagination is supported using cursor params 'limit' and 'offset'" @@ -352,7 +301,6 @@ echo "$url" >> $seq_full curl --get --silent "$url" | tee -a $seq_full | pmjson | _filter_info -cat $tmp.keys >> $seq_full cat $tmp.pmproxy.log >> $seq_full # success, all done diff --git a/qa/1872.out b/qa/1872.out index c2bd4ca51f9..90c70ff4f55 100644 --- a/qa/1872.out +++ b/qa/1872.out @@ -1,10 +1,7 @@ QA output created by 1872 -=== Start test key server === -PING -PONG +=== Build search index === +Index built -=== Load deterministic search content === -pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/text - query for '99' 'random' 'interesting' 'result' === == verify request - http://localhost:PORT/search/text?query=99 { @@ -24,7 +21,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, "limit": 10, @@ -37,21 +34,86 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { - "name": "sample.mirage", + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.drift", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "indom": "29.6", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "sample.scramble.bin", + "type": "metric", + "indom": "29.9", + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" } ] } == verify request - http://localhost:PORT/search/text?query=interesting { - "total": 1, + "total": 3, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.mirage_longlong", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, { "name": "sample.mirage", "type": "metric", @@ -63,7 +125,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=result { - "total": 1, + "total": 3, "elapsed": XXX, "offset": 0, "limit": 10, @@ -71,9 +133,20 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels { "name": "sample.rapid", "type": "metric", - "indom": "none", "oneline": "count very quickly", "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + }, + { + "name": "sample.not_ready", + "type": "metric", + "oneline": "interval (in seconds) during which PMDA does not respond to PDUs", + "helptext": "Store a positive number of seconds as the value of this metric. The\nfollowing PDU received will result in the following sequence of events:\n 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd\n 2. sleep for the given number of seconds\n 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd\nIf everything went as planned, sample.not_ready returns to 0, otherwise it\nhas a negative error code as the value." + }, + { + "name": "sample.not_ready_msec", + "type": "metric", + "oneline": "interval (in milliseconds) during which PMDA does not respond to PDUs", + "helptext": "Store a positive number of milliseconds as the value of this metric. The\nfollowing PDU received will result in the following sequence of events:\n 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd\n 2. sleep for the given number of milliseconds\n 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd\nIf everything went as planned, sample.not_ready returns to 0, otherwise it\nhas a negative error code as the value." } ] } @@ -92,11 +165,39 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/text - highlighting is supported on 'name', 'oneline' and 'helptext' by using 'highlight' param === == verify request - http://localhost:PORT/search/text?query=domain&highlight=name,oneline,helptext { - "total": 2, + "total": 24, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ + { + "name": "29.5", + "type": "indom", + "indom": "29.5", + "oneline": "Instance domain \"hordes\" for sample PMDA.", + "helptext": "A relatively large instance domain, with hundreds of numeric instances." + }, + { + "name": "29.11", + "type": "indom", + "indom": "29.11", + "oneline": "Instance domain \"ghosts\" for sample PMDA.", + "helptext": "Instances are the names of some famous ghosts." + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "29.10", + "type": "indom", + "indom": "29.10", + "oneline": "Instance domain \"events\" for sample PMDA.", + "helptext": "Instances \"fungus\" and \"bogus\" for testing event metrics indoms." + }, { "name": "29.1", "type": "indom", @@ -105,41 +206,76 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "29.3", + "name": "29.8", "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + "indom": "29.8", + "oneline": "Instance domain \"many\" for sample PMDA.", + "helptext": "A varable size set of instances controlled by sample.many.count" + }, + { + "name": "29.7", + "type": "indom", + "indom": "29.7", + "oneline": "Instance domain \"dynamic\" for sample PMDA.", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists." + }, + { + "name": "29.4", + "type": "indom", + "indom": "29.4", + "oneline": "Instance domain \"family\" for sample PMDA.", + "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" + }, + { + "name": "29.2", + "type": "indom", + "indom": "29.2", + "oneline": "Instance domain \"bin\" for sample PMDA", + "helptext": "Universally 9 instances numbered 100 .. 900 in steps of 100, and named\n\"bin-100\" .. \"bin-900\"" + }, + { + "name": "29.9", + "type": "indom", + "indom": "29.9", + "oneline": "Instance domain \"scramble\" for sample PMDA.", + "helptext": "Several constant instances, instances reordered on successive pmFetch\ncalls, and sometimes (probability 0.33) some instances are missing." } ] } == verify request - http://localhost:PORT/search/text?query=sample&highlight=name { - "total": 5, + "total": 211, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ { - "name": "sample.mirage", - "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "name": "29.8", + "type": "indom", + "indom": "29.8", + "oneline": "Instance domain \"many\" for sample PMDA.", + "helptext": "A varable size set of instances controlled by sample.many.count" }, { - "name": "sample.colour", - "type": "metric", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "name": "29.7", + "type": "indom", + "indom": "29.7", + "oneline": "Instance domain \"dynamic\" for sample PMDA.", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists." }, { - "name": "sample.rapid", - "type": "metric", - "indom": "none", - "oneline": "count very quickly", - "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + "name": "29.11", + "type": "indom", + "indom": "29.11", + "oneline": "Instance domain \"ghosts\" for sample PMDA.", + "helptext": "Instances are the names of some famous ghosts." + }, + { + "name": "29.10", + "type": "indom", + "indom": "29.10", + "oneline": "Instance domain \"events\" for sample PMDA.", + "helptext": "Instances \"fungus\" and \"bogus\" for testing event metrics indoms." }, { "name": "29.1", @@ -149,48 +285,121 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "29.3", + "name": "29.5", "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + "indom": "29.5", + "oneline": "Instance domain \"hordes\" for sample PMDA.", + "helptext": "A relatively large instance domain, with hundreds of numeric instances." + }, + { + "name": "29.4", + "type": "indom", + "indom": "29.4", + "oneline": "Instance domain \"family\" for sample PMDA.", + "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" + }, + { + "name": "29.2", + "type": "indom", + "indom": "29.2", + "oneline": "Instance domain \"bin\" for sample PMDA", + "helptext": "Universally 9 instances numbered 100 .. 900 in steps of 100, and named\n\"bin-100\" .. \"bin-900\"" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "29.9", + "type": "indom", + "indom": "29.9", + "oneline": "Instance domain \"scramble\" for sample PMDA.", + "helptext": "Several constant instances, instances reordered on successive pmFetch\ncalls, and sometimes (probability 0.33) some instances are missing." } ] } == verify request - http://localhost:PORT/search/text?query=counter&highlight=oneline,helptext { - "total": 3, + "total": 44, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ { - "name": "sample.colour", + "name": "sample.step_counter", "type": "metric", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "A step function (counter)", + "helptext": "This metric changes magnitude every 30 seconds, between a base value and\n3 times the base value.\n\nThe metric has \"counter\" semantics. See also sample.step.\n\nUse pmStore() to modify the base value." }, { - "name": "sample.mirage", + "name": "sample.byte_ctr", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "counter byte counter", + "helptext": "value increments randomly in the range (0,1023) bytes per fetch" }, { - "name": "sample.rapid", + "name": "sample.dynamic.counter", "type": "metric", - "indom": "none", - "oneline": "count very quickly", - "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + "indom": "29.7", + "oneline": "counter metric with dynamic indom", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists.\nEach line in this file is\n\tinternal_id external_id\n\nThis metric increments each time this instance has been seen when scanning\nthe dynamic.indom file, and resets to zero each time the instance appears." + }, + { + "name": "sample.long.bin_ctr", + "type": "metric", + "indom": "29.2", + "oneline": "like sample.bin but type 32, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.ulong.bin_ctr", + "type": "metric", + "indom": "29.2", + "oneline": "like sample.bin but type U32, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.longlong.bin_ctr", + "type": "metric", + "indom": "29.2", + "oneline": "like sample.bin but type 64, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.float.bin_ctr", + "type": "metric", + "indom": "29.2", + "oneline": "like sample.bin but type FLOAT, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.double.bin_ctr", + "type": "metric", + "indom": "29.2", + "oneline": "like sample.bin but type DOUBLE, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.kbyte_ctr", + "type": "metric", + "oneline": "counter Kbytes/second", + "helptext": "value increments randomly in the range (0,1023) Kbytes per fetch" + }, + { + "name": "sample.const_rate.value", + "type": "metric", + "oneline": "constant rate counter", + "helptext": "A counter that changes with constant rate between fetches.\n\nThe rate is set by storing the desired rate (counts per second)\ninto sample.const_rate.gradient" } ] } === /search/text - pagination is supported using cursor params 'limit' and 'offset' == verify request - http://localhost:PORT/search/text?query=random&limit=1 { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, "limit": 1, @@ -206,17 +415,17 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random&limit=1&offset=1 { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 1, "limit": 1, "results": [ { - "name": "sample.mirage", - "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." } ] } @@ -316,7 +525,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random&return=name,type,indom { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, "limit": 10, @@ -327,20 +536,106 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "indom": "29.3" }, { - "name": "sample.mirage", + "name": "29.6", + "type": "indom", + "indom": "29.6" + }, + { + "name": "sample.drift", + "type": "metric" + }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12" + }, + { + "name": "sample.byte_rate", + "type": "metric" + }, + { + "name": "sample.kbyte_rate", + "type": "metric" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric" + }, + { + "name": "sample.dodgey.value", "type": "metric", - "indom": "29.3" + "indom": "29.6" + }, + { + "name": "sample.scramble.bin", + "type": "metric", + "indom": "29.9" + }, + { + "name": "sample.dodgey.control", + "type": "metric" } ] } === /search/text - filtering based on entity type (metric, instance, indom) using 'type' param === == verify request - http://localhost:PORT/search/text?query=random&type=metric { - "total": 1, + "total": 9, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ + { + "name": "sample.drift", + "type": "metric", + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "indom": "29.6", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "sample.scramble.bin", + "type": "metric", + "indom": "29.9", + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" + }, + { + "name": "sample.mirage_longlong", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, { "name": "sample.mirage", "type": "metric", @@ -352,7 +647,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random&type=indom { - "total": 1, + "total": 3, "elapsed": XXX, "offset": 0, "limit": 10, @@ -363,6 +658,20 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "indom": "29.3", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } @@ -376,7 +685,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random&type=metric,indom { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, "limit": 10, @@ -389,32 +698,97 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { - "name": "sample.mirage", + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.drift", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "indom": "29.6", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "sample.scramble.bin", + "type": "metric", + "indom": "29.9", + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" } ] } === /search/suggest - generic request === == verify request - http://localhost:PORT/search/suggest?query=sam [ - "sample.mirage", - "sample.colour", - "sample.rapid" + "sample.string.null", + "sample.aggregate.null", + "sample.bad.novalues", + "sample.long.ten", + "sample.long.hundred", + "sample.long.million", + "sample.long.bin", + "sample.ulong.bin", + "sample.longlong.one", + "sample.longlong.ten" ] == verify request - http://localhost:PORT/search/suggest?query=sample.r [ - "sample.mirage", + "sample.const_rate.gradient", + "sample.event.no_indom_records", + "sample.byte_rate", + "sample.kbyte_rate", "sample.rapid", - "sample.colour" + "sample.byte_rate_perhour", + "sample.proc.reset", + "sample.recv_pdu", + "sample.const_rate.value", + "sample.percontext.recv_pdu" ] === /search/suggest - limit is supported == verify request - http://localhost:PORT/search/suggest?query=sam&limit=2 [ - "sample.mirage", - "sample.colour" + "sample.string.null", + "sample.aggregate.null" ] === /search/suggest - query param is required === == verify request - http://localhost:PORT/search/suggest @@ -425,25 +799,24 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/indom - generic request === == verify request - http://localhost:PORT/search/indom?query=29.1 { - "total": 5, + "total": 6, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ { - "name": "blue", - "type": "instance", - "indom": "29.1" - }, - { - "name": "green", - "type": "instance", - "indom": "29.1" + "name": "sample.darkness", + "type": "metric", + "indom": "29.1", + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." }, { - "name": "red", - "type": "instance", - "indom": "29.1" + "name": "sample.colour", + "type": "metric", + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { "name": "29.1", @@ -453,21 +826,43 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "sample.colour", - "type": "metric", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "name": "red", + "type": "instance", + "indom": "29.1" + }, + { + "name": "green", + "type": "instance", + "indom": "29.1" + }, + { + "name": "blue", + "type": "instance", + "indom": "29.1" } ] } == verify request - http://localhost:PORT/search/indom?query=29.3 { - "total": 5, + "total": 6, "elapsed": XXX, "offset": 0, "limit": 10, "results": [ + { + "name": "sample.mirage_longlong", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, + { + "name": "sample.mirage", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, { "name": "29.3", "type": "indom", @@ -476,7 +871,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { - "name": "m-04", + "name": "m-00", "type": "instance", "indom": "29.3" }, @@ -486,16 +881,9 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "indom": "29.3" }, { - "name": "m-00", + "name": "m-04", "type": "instance", "indom": "29.3" - }, - { - "name": "sample.mirage", - "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } ] } @@ -514,57 +902,52 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/indom - pagination is supported using cursor params 'limit' and 'offset' == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=0 { - "total": 5, + "total": 6, "elapsed": XXX, "offset": 0, "limit": 2, "results": [ { - "name": "blue", - "type": "instance", - "indom": "29.1" + "name": "sample.darkness", + "type": "metric", + "indom": "29.1", + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." }, { - "name": "green", - "type": "instance", - "indom": "29.1" + "name": "sample.colour", + "type": "metric", + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } ] } == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=2 { - "total": 5, + "total": 6, "elapsed": XXX, "offset": 2, "limit": 2, "results": [ - { - "name": "red", - "type": "instance", - "indom": "29.1" - }, { "name": "29.1", "type": "indom", "indom": "29.1", "oneline": "Instance domain \"colour\" for sample PMDA", "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + }, + { + "name": "red", + "type": "instance", + "indom": "29.1" } ] } === /search/info === == verify request - http://localhost:PORT/search/info { - "docs": 18, - "terms": 174, - "records": XXX, - "records_per_doc_avg": XXX, - "bytes_per_record_avg": XXX, - "inverted_sz_mb": 0.00, - "inverted_cap_mb": 0.00, - "inverted_cap_ovh": 0.00, - "skip_index_size_mb": 0.00, - "score_index_size_mb": 0.00, - "offsets_per_term_avg": XXX, - "offset_bits_per_record_avg": 8.00 + "docs": NNN, + "terms": NNN, + "records": NNN } diff --git a/qa/admin/other-packages/manifest b/qa/admin/other-packages/manifest index 88262e3f66e..ebfbe28d290 100644 --- a/qa/admin/other-packages/manifest +++ b/qa/admin/other-packages/manifest @@ -1018,6 +1018,17 @@ S_pkg? /usr/lib/*/libsqlite3.so [database/sqlite-3 (QA optional)] slackpkg? /usr/lib*/libsqlite3.so [sqlite (QA optional)] pacman? /usr/lib/libsqlite3.so [core/sqlite (QA optional)] brew? /usr/lib/libsqlite3.dylib [sqlite (QA optional)] +# -- libsqlite3 headers for pmsearch FTS5 index +dpkg? /usr/include/sqlite3.h [libsqlite3-dev] +rpm? /usr/include/sqlite3.h [sqlite-devel or sqlite3-devel] +emerge? /usr/include/sqlite3.h [dev-db/sqlite] +pkgin? /usr/pkg/include/sqlite3.h [sqlite3] +pkg_add? /usr/local/include/sqlite3.h [sqlite3] +F_pkg? /usr/local/include/sqlite3.h [sqlite3] +S_pkg? /usr/include/sqlite3.h [database/sqlite-3] +slackpkg? /usr/include/sqlite3.h [sqlite] +pacman? /usr/include/sqlite3.h [core/sqlite] +brew? /usr/local/include/sqlite3.h [sqlite] # -- libicui18n for certutil dpkg? /usr/lib/*/libicui18n.so.* [libicu[0-9][0-9]* (QA optional)] rpm? /usr/lib*/libicui18n.so [libicu-devel (QA optional)] diff --git a/qa/admin/package-lists/AmazonLinux+2023+aarch64 b/qa/admin/package-lists/AmazonLinux+2023+aarch64 index 3bfb97d4768..290f544fc03 100644 --- a/qa/admin/package-lists/AmazonLinux+2023+aarch64 +++ b/qa/admin/package-lists/AmazonLinux+2023+aarch64 @@ -105,6 +105,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/AmazonLinux+2023+x86_64 b/qa/admin/package-lists/AmazonLinux+2023+x86_64 index 94649929248..e876c81e9c9 100644 --- a/qa/admin/package-lists/AmazonLinux+2023+x86_64 +++ b/qa/admin/package-lists/AmazonLinux+2023+x86_64 @@ -108,6 +108,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+8+x86_64 b/qa/admin/package-lists/CentOS+8+x86_64 index 9765fbfb3a4..f0c7a106c4e 100644 --- a/qa/admin/package-lists/CentOS+8+x86_64 +++ b/qa/admin/package-lists/CentOS+8+x86_64 @@ -114,6 +114,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat systemd-devel diff --git a/qa/admin/package-lists/CentOS+Stream10+x86_64 b/qa/admin/package-lists/CentOS+Stream10+x86_64 index c5cbba5aa38..18cd1118879 100644 --- a/qa/admin/package-lists/CentOS+Stream10+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream10+x86_64 @@ -126,6 +126,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+Stream8+x86_64 b/qa/admin/package-lists/CentOS+Stream8+x86_64 index e662358a1cc..6c6226613ae 100644 --- a/qa/admin/package-lists/CentOS+Stream8+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream8+x86_64 @@ -121,6 +121,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+Stream9+x86_64 b/qa/admin/package-lists/CentOS+Stream9+x86_64 index 652169939b9..638d7b4fce4 100644 --- a/qa/admin/package-lists/CentOS+Stream9+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream9+x86_64 @@ -128,6 +128,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Debian+11+x86_64 b/qa/admin/package-lists/Debian+11+x86_64 index c0589faa99a..d8b20f5de96 100644 --- a/qa/admin/package-lists/Debian+11+x86_64 +++ b/qa/admin/package-lists/Debian+11+x86_64 @@ -72,6 +72,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+aarch64 b/qa/admin/package-lists/Debian+12+aarch64 index ee56a676b80..9f16aac4233 100644 --- a/qa/admin/package-lists/Debian+12+aarch64 +++ b/qa/admin/package-lists/Debian+12+aarch64 @@ -71,6 +71,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+i686 b/qa/admin/package-lists/Debian+12+i686 index 30daf7bdae3..cd3058914b2 100644 --- a/qa/admin/package-lists/Debian+12+i686 +++ b/qa/admin/package-lists/Debian+12+i686 @@ -74,6 +74,7 @@ libspreadsheet-read-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+x86_64 b/qa/admin/package-lists/Debian+12+x86_64 index 8ff6f199545..79ff8ea7453 100644 --- a/qa/admin/package-lists/Debian+12+x86_64 +++ b/qa/admin/package-lists/Debian+12+x86_64 @@ -77,6 +77,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+13+i686 b/qa/admin/package-lists/Debian+13+i686 index c976f4c821d..198206b0f90 100644 --- a/qa/admin/package-lists/Debian+13+i686 +++ b/qa/admin/package-lists/Debian+13+i686 @@ -73,6 +73,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+13+x86_64 b/qa/admin/package-lists/Debian+13+x86_64 index fb8c5a73429..9435e6be9b1 100644 --- a/qa/admin/package-lists/Debian+13+x86_64 +++ b/qa/admin/package-lists/Debian+13+x86_64 @@ -74,6 +74,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+14+x86_64 b/qa/admin/package-lists/Debian+14+x86_64 index 9464ac98b51..7e88e8eb3ef 100644 --- a/qa/admin/package-lists/Debian+14+x86_64 +++ b/qa/admin/package-lists/Debian+14+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Fedora+42+aarch64 b/qa/admin/package-lists/Fedora+42+aarch64 index c4a2a1b389a..1438d3654ac 100644 --- a/qa/admin/package-lists/Fedora+42+aarch64 +++ b/qa/admin/package-lists/Fedora+42+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+42+x86_64 b/qa/admin/package-lists/Fedora+42+x86_64 index 2c76c948c02..37a64060e1c 100644 --- a/qa/admin/package-lists/Fedora+42+x86_64 +++ b/qa/admin/package-lists/Fedora+42+x86_64 @@ -132,6 +132,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+43+aarch64 b/qa/admin/package-lists/Fedora+43+aarch64 index e96d484ed2c..5c6d144e2e2 100644 --- a/qa/admin/package-lists/Fedora+43+aarch64 +++ b/qa/admin/package-lists/Fedora+43+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+43+x86_64 b/qa/admin/package-lists/Fedora+43+x86_64 index e4b5219ee59..160e1d878e7 100644 --- a/qa/admin/package-lists/Fedora+43+x86_64 +++ b/qa/admin/package-lists/Fedora+43+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+44+aarch64 b/qa/admin/package-lists/Fedora+44+aarch64 index 36c69166c18..351e89a30be 100644 --- a/qa/admin/package-lists/Fedora+44+aarch64 +++ b/qa/admin/package-lists/Fedora+44+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+44+x86_64 b/qa/admin/package-lists/Fedora+44+x86_64 index f1648cd0cfd..f749ca745ed 100644 --- a/qa/admin/package-lists/Fedora+44+x86_64 +++ b/qa/admin/package-lists/Fedora+44+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+45+aarch64 b/qa/admin/package-lists/Fedora+45+aarch64 index 0f8936901d3..0bc98dcae68 100644 --- a/qa/admin/package-lists/Fedora+45+aarch64 +++ b/qa/admin/package-lists/Fedora+45+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+45+x86_64 b/qa/admin/package-lists/Fedora+45+x86_64 index 1578325f3cc..a115773aa91 100644 --- a/qa/admin/package-lists/Fedora+45+x86_64 +++ b/qa/admin/package-lists/Fedora+45+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/MX+23.6+x86_64 b/qa/admin/package-lists/MX+23.6+x86_64 index 797804a9cb2..12b34c621e1 100644 --- a/qa/admin/package-lists/MX+23.6+x86_64 +++ b/qa/admin/package-lists/MX+23.6+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/RHEL+10+x86_64 b/qa/admin/package-lists/RHEL+10+x86_64 index fe5f4f47a22..0a3a65717a2 100644 --- a/qa/admin/package-lists/RHEL+10+x86_64 +++ b/qa/admin/package-lists/RHEL+10+x86_64 @@ -122,6 +122,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat sudo diff --git a/qa/admin/package-lists/RHEL+8+x86_64 b/qa/admin/package-lists/RHEL+8+x86_64 index fa2e3db4d75..390d30e219d 100644 --- a/qa/admin/package-lists/RHEL+8+x86_64 +++ b/qa/admin/package-lists/RHEL+8+x86_64 @@ -119,6 +119,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/RHEL+9+x86_64 b/qa/admin/package-lists/RHEL+9+x86_64 index 3d40352cd6f..dc78842e947 100644 --- a/qa/admin/package-lists/RHEL+9+x86_64 +++ b/qa/admin/package-lists/RHEL+9+x86_64 @@ -125,6 +125,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat sudo diff --git a/qa/admin/package-lists/Ubuntu+18.04+i686 b/qa/admin/package-lists/Ubuntu+18.04+i686 index 7eb6111f0c5..04b624c6216 100644 --- a/qa/admin/package-lists/Ubuntu+18.04+i686 +++ b/qa/admin/package-lists/Ubuntu+18.04+i686 @@ -70,6 +70,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+18.04+x86_64 b/qa/admin/package-lists/Ubuntu+18.04+x86_64 index 4516770b7c3..59fa19cf16e 100644 --- a/qa/admin/package-lists/Ubuntu+18.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+18.04+x86_64 @@ -68,6 +68,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+20.04+x86_64 b/qa/admin/package-lists/Ubuntu+20.04+x86_64 index 2bab5e76ae7..b125b081521 100644 --- a/qa/admin/package-lists/Ubuntu+20.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+20.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+22.04+x86_64 b/qa/admin/package-lists/Ubuntu+22.04+x86_64 index b9f4cd3a9b0..68a38fbe520 100644 --- a/qa/admin/package-lists/Ubuntu+22.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+22.04+x86_64 @@ -76,6 +76,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+24.04+any b/qa/admin/package-lists/Ubuntu+24.04+any index 4ef4c87d85a..cbd1edc98aa 100644 --- a/qa/admin/package-lists/Ubuntu+24.04+any +++ b/qa/admin/package-lists/Ubuntu+24.04+any @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+24.04+x86_64 b/qa/admin/package-lists/Ubuntu+24.04+x86_64 index bfa71e37d1b..ee9686d4dbf 100644 --- a/qa/admin/package-lists/Ubuntu+24.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+24.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+26.04+any b/qa/admin/package-lists/Ubuntu+26.04+any index 0fec7a41ccf..82aaaaa1034 100644 --- a/qa/admin/package-lists/Ubuntu+26.04+any +++ b/qa/admin/package-lists/Ubuntu+26.04+any @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+26.04+x86_64 b/qa/admin/package-lists/Ubuntu+26.04+x86_64 index 87c80d499a9..08afca7c26d 100644 --- a/qa/admin/package-lists/Ubuntu+26.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+26.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/openSUSE+15.6+x86_64 b/qa/admin/package-lists/openSUSE+15.6+x86_64 index 22b4016ab0b..25873f5c9a6 100644 --- a/qa/admin/package-lists/openSUSE+15.6+x86_64 +++ b/qa/admin/package-lists/openSUSE+15.6+x86_64 @@ -119,6 +119,7 @@ sensors setools-console smartmontools socat +sqlite3-devel sudo sysstat systemd diff --git a/qa/admin/package-lists/openSUSE+16.0+x86_64 b/qa/admin/package-lists/openSUSE+16.0+x86_64 index b11b78028d9..3e7d0ab4e3b 100644 --- a/qa/admin/package-lists/openSUSE+16.0+x86_64 +++ b/qa/admin/package-lists/openSUSE+16.0+x86_64 @@ -118,6 +118,7 @@ sensors setools-console smartmontools socat +sqlite3-devel sudo sysstat systemd diff --git a/qa/group b/qa/group index ecdfcb8245c..74451468130 100644 --- a/qa/group +++ b/qa/group @@ -2344,14 +2344,14 @@ suse 1854 pmlogger pmimport local 1855 pmda.rabbitmq local 1856 dstat python local derive -1861:reserved pmsearch local +1861 pmsearch help local 1862 other local 1863 logutil pmlogger_daily local 1864 logutil pmlogger_daily local 1869 logutil pmlogger_daily local 1870 guidellm2pcp python pmimport libpcp_import local 1871 pmsearch local -1872 pmproxy local pmjson +1872 pmsearch pmproxy help local pmjson 1874 pmseries pmproxy local pmjson 1875 pmseries pmproxy libpcp_web local 1876:retired pmcd secure local diff --git a/src/include/builddefs.in b/src/include/builddefs.in index d88c24ec1db..0e229964059 100644 --- a/src/include/builddefs.in +++ b/src/include/builddefs.in @@ -136,6 +136,7 @@ INVISIBILITY = @INVISIBILITY@ # hide shared library symbols LZMACFLAGS = @lzma_CFLAGS@ LIBUVCFLAGS = @libuv_CFLAGS@ +SQLITE3CFLAGS = @sqlite3_CFLAGS@ OPENSSLCFLAGS = @openssl_CFLAGS@ SASLCFLAGS = @libsasl2_CFLAGS@ CMOCKACFLAGS = @cmocka_CFLAGS@ @@ -316,6 +317,8 @@ HAVE_LIBUV = @HAVE_LIBUV@ LIB_FOR_LIBUV = @libuv_LIBS@ HAVE_LIBINIH = @HAVE_LIBINIH@ LIB_FOR_INIH = @lib_for_inih@ +HAVE_SQLITE3 = @HAVE_SQLITE3@ +LIB_FOR_SQLITE3 = @lib_for_sqlite3@ HAVE_LIBELF = @HAVE_LIBELF@ LIB_FOR_LIBELF = @libelf_LIBS@ HAVE_NCURSES = @HAVE_NCURSES@ diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index d032e521b75..d902946eac6 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -57,7 +57,8 @@ ifneq "$(HAVE_LIBINIH)" "true" LCFLAGS += -Iinih endif -LCFLAGS += $(C99_CFLAGS) -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ +LCFLAGS += $(C99_CFLAGS) \ + -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/include \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/src @@ -69,6 +70,13 @@ else CFILES += nodiscover.c nologgroup.c nowebgroup.c notimer.c endif +ifeq "$(HAVE_SQLITE3)" "true" +LCFLAGS += $(SQLITE3CFLAGS) +LLDLIBS += $(LIB_FOR_SQLITE3) +else +CFILES := $(subst search.c,nosearch.c,$(CFILES)) +endif + ifeq "$(HAVE_LIBINIH)" "true" LLDLIBS += $(LIB_FOR_INIH) endif diff --git a/src/libpcp_web/src/nosearch.c b/src/libpcp_web/src/nosearch.c new file mode 100644 index 00000000000..357d19c9441 --- /dev/null +++ b/src/libpcp_web/src/nosearch.c @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Stub routines for builds without SQLite FTS5 support. + */ +#include "pmwebapi.h" +#include "search.h" +#include "discover.h" + +const char * +pmSearchTextTypeStr(pmSearchTextType type) +{ + (void)type; + return "unknown"; +} + +int +pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) +{ + (void)settings; (void)key; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchSetSlots(pmSearchModule *module, void *slots) +{ + (void)module; (void)slots; + return 0; +} + +int +pmSearchSetConfiguration(pmSearchModule *module, struct dict *config) +{ + (void)module; (void)config; + return 0; +} + +int +pmSearchSetEventLoop(pmSearchModule *module, void *events) +{ + (void)module; (void)events; + return 0; +} + +int +pmSearchSetMetricRegistry(pmSearchModule *module, struct mmv_registry *registry) +{ + (void)module; (void)registry; + return 0; +} + +int +pmSearchSetup(pmSearchModule *module, void *arg) +{ + (void)module; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchEnabled(void *arg) +{ + (void)arg; + return 0; +} + +void +pmSearchClose(pmSearchModule *module) +{ + (void)module; +} + +/* --- stubs for schema.c / keys.c compatibility --- */ + +extern void keys_slots_end_phase(void *); + +void +keysSearchInit(struct dict *config) +{ + (void)config; +} + +void +keysSearchClose(void) +{ +} + +void +keys_load_search_schema(void *arg) +{ + keys_slots_end_phase(arg); +} + +void +keys_search_text_add(struct keySlots *slots, pmSearchTextType type, + const char *name, const char *indom, + const char *oneline, const char *helptext, void *arg) +{ + (void)slots; (void)type; (void)name; (void)indom; + (void)oneline; (void)helptext; (void)arg; +} + +/* --- discover no-ops --- */ + +void +pmSearchDiscoverMetric(pmDiscoverEvent *event, + pmDesc *desc, int numnames, char **names, void *arg) +{ + (void)event; (void)desc; (void)numnames; (void)names; (void)arg; +} + +void +pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) +{ + (void)event; (void)in; (void)arg; +} + +void +pmSearchDiscoverText(pmDiscoverEvent *event, + int ident, int type, char *text, void *arg) +{ + (void)event; (void)ident; (void)type; (void)text; (void)arg; +} diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 6dfd78db984..143391f64ab 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2020-2022,2024 Red Hat. + * Copyright (c) 2026 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -10,39 +11,28 @@ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public * License for more details. + * + * SQLite FTS5 search backend for PCP metric/instance full-text search. */ -#include +#include #include "pmapi.h" -#include "pmda.h" -#include "schema.h" +#include "libpcp.h" #include "search.h" -#include "util.h" -#include "sha1.h" -static sds resultcount_str; -static sds DEFAULT_RESULTCOUNT; -static unsigned int resultcount; /* converted resultcount_str */ +/* Type values stored in the SQLite index — must match pmSearchTextType */ +#define SEARCH_DOC_METRIC PM_SEARCH_TYPE_METRIC +#define SEARCH_DOC_INDOM PM_SEARCH_TYPE_INDOM +#define SEARCH_DOC_INST PM_SEARCH_TYPE_INST -static void -initKeysSearchBaton(keysSearchBaton *baton, keySlots *slots, - pmSearchSettings *settings, void *userdata) -{ - initSeriesBatonMagic(baton, MAGIC_SEARCH); - baton->callbacks = &settings->callbacks; - baton->info = settings->module.on_info; - baton->slots = slots; - baton->module = &settings->module; - baton->userdata = userdata; - pmtimespecNow(&baton->started); -} +static int search_enabled; +static unsigned int default_resultcount = 10; -static void -doneKeysSearchBaton(keysSearchBaton *baton) -{ - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "doneKeysSearchBaton"); - memset(baton, 0, sizeof(keysSearchBaton)); - free(baton); -} +typedef struct searchModuleData { + sqlite3 *db; + struct dict *config; + unsigned int loaded; + unsigned int resultcount; +} searchModuleData; const char * pmSearchTextTypeStr(pmSearchTextType type) @@ -60,1066 +50,884 @@ pmSearchTextTypeStr(pmSearchTextType type) return "unknown"; } -static sds -keys_search_docid(const char *key, const char *type, const char *name) +static searchModuleData * +getSearchModuleData(pmSearchModule *module) { - unsigned char hash[20]; - sds docid = sdsempty(); - - docid = sdscatfmt(docid, "\"key\":\"%s\",\"type\":\"%s\",name:\"%s\"", - key, type, name); - pmwebapi_search_hash(hash, docid, sdslen(docid)); - return pmwebapi_hash_sds(docid, hash); + if (module->privdata == NULL) + module->privdata = calloc(1, sizeof(searchModuleData)); + return (searchModuleData *)module->privdata; } - /* - * This issue isn't fixed in version of search module we are using, - * https://github.com/RediSearch/RediSearch/issues/748 + * Build an FTS5 MATCH expression from the request. + * If infields are restricted, wrap the query in column filters. */ -static int -keys_search_is_stopword(sds s) +static sds +search_build_match(pmSearchTextRequest *request) { - size_t i; - static const char* const stopwords[] = { - "a", "is", "the", "an", "and", "are", "as", "at", "be", "but", "by", "for", - "if", "in", "into", "it", "no", "not", "of", "on", "or", "such", "that", "their", - "then", "there", "these", "they", "this", "to", "was", "will", "with", - }; - - for (i = 0; i < sizeof(stopwords) / sizeof(stopwords[0]); i++) { - if (strcmp(s, stopwords[i]) == 0) - return 1; + sds match; + int restricted = 0; + + if (request->infields_name || request->infields_oneline || + request->infields_helptext) { + int count = request->infields_name + request->infields_oneline + + request->infields_helptext; + if (count < 3) + restricted = 1; } - return 0; + + if (restricted) { + sds cols = sdsempty(); + int first = 1; + + if (request->infields_name) { + cols = sdscat(cols, "name"); + first = 0; + } + if (request->infields_oneline) { + if (!first) + cols = sdscat(cols, " "); + cols = sdscat(cols, "oneline"); + first = 0; + } + if (request->infields_helptext) { + if (!first) + cols = sdscat(cols, " "); + cols = sdscat(cols, "helptext"); + } + match = sdscatfmt(sdsempty(), "{%S} : (%S)", cols, request->query); + sdsfree(cols); + } else { + match = sdsnew(request->query); + } + + return match; } /* - * Tokenizes text by delimiters described in search module Escaping.html doc + - * ('/' - this one giving troubles to prefix search), - * omits tokens of length greated then *min_length* (0 = don't omit any), optionally prepends *prefix* - * and appends *suffix* (NULL doesnt get prepended / appended), joins tokens by space back into new sds. - * Ideally we would just escape delimtiers with double backslash, - * unfortunately, this doesnt seem to work when searching for indoms and buch of other inconsitencies + * Build the type filter clause for SQL queries. + * Returns an sds string like " AND type IN (1,3)" or empty string. */ static sds -keys_search_text_prep(sds s, int min_length, char *prefix, char *suffix) +search_type_filter(pmSearchTextRequest *request) { - static const char *delimiters = ",.<>{}[]\"\':;!@#$%^&*()-+=~/"; - size_t len = sdslen(s); - size_t i, j; - sds result = sdsempty(); - sds formatted_result; - sds* tokens; - int token_count, non_digit_found; - - for (i = 0; i < len; i++) { - const char *is_found = strchr(delimiters, s[i]); - if (is_found != NULL) { - result = sdscat(result, " "); - } else { - result = sdscatlen(result, &s[i], 1); - } + sds filter = sdsempty(); + int any = request->type_metric + request->type_indom + + request->type_inst; + + if (any == 0 || any == 3) + return filter; + + filter = sdscat(filter, " AND type IN ("); + any = 0; + if (request->type_metric) { + filter = sdscatfmt(filter, "%i", SEARCH_DOC_METRIC); + any = 1; } - sdstrim(result, " "); - tokens = sdssplitlen(result, sdslen(result), " ", 1, &token_count); - if (tokens == NULL) { - /* Coverity: CID370640 */ - sdsfree(result); - return NULL; + if (request->type_indom) { + if (any) + filter = sdscat(filter, ","); + filter = sdscatfmt(filter, "%i", SEARCH_DOC_INDOM); + any = 1; } - formatted_result = sdsempty(); - for (i = 0; i < token_count; i++) { - size_t token_len = sdslen(tokens[i]); - if (min_length != 0 && token_len < min_length) - continue; - if (keys_search_is_stopword(tokens[i])) - continue; - if (prefix == NULL && suffix == NULL) - formatted_result = sdscatsds(formatted_result, tokens[i]); - else if (prefix == NULL) - formatted_result = sdscatfmt(formatted_result, "%S%s", tokens[i], suffix); - else if (suffix == NULL) - formatted_result = sdscatfmt(formatted_result, "%s%S", prefix, tokens[i]); - else { - // Monkey patch for numbers in fuzzy search, not fixed in OSS version of RediSearch - // https://github.com/RediSearch/RediSearch/issues/346 - // Lets just assume that if both prefix and suffix start with %, - // fuzzy search formatting is intended (OSS RediSearch doesn't support LD > 1 anyway) - if (prefix[0] == '%' && suffix[0] == '%') { - non_digit_found = 0; - for (j = 0; j < token_len; j++) { - if (tokens[i][j] < '0' || tokens[i][j] > '9') { - non_digit_found = 1; - } - } - if (!non_digit_found) { - goto outer_loop; - } - } - formatted_result = sdscatfmt(formatted_result, "%s%S%s", prefix, tokens[i], suffix); - } - if (i != token_count - 1) { - formatted_result = sdscat(formatted_result, " "); - } - outer_loop: - continue; + if (request->type_inst) { + if (any) + filter = sdscat(filter, ","); + filter = sdscatfmt(filter, "%i", SEARCH_DOC_INST); } - sdsfree(result); - sdsfreesplitres(tokens, token_count); - sdstrim(formatted_result, " "); - return formatted_result; + filter = sdscat(filter, ")"); + return filter; } -static void -keys_search_text_add_callback( - keyClusterAsyncContext *c, void *r, void *arg) +/* + * Build a docid string from rowid and name. + */ +static sds +search_make_docid(sqlite3_int64 rowid, const char *name) { - seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesGetContext *context = &baton->pmapi; - respReply *reply = r; + char buf[32]; - checkStatusReplyOK(baton->info, baton->userdata, c, reply, - "%s: %s", FT_ADD, "search text add"); - doneSeriesGetContext(context, "keys_search_text_add_callback"); + pmsprintf(buf, sizeof(buf), "%lld", (long long)rowid); + return sdscatfmt(sdsempty(), "%s:%s", buf, name ? name : ""); } -void -keys_search_text_add(keySlots *slots, pmSearchTextType type, - const char *name, const char *indom, - const char *oneline, const char *helptext, void *arg) +/* Grow the hits array, doubling capacity (starting at 64). Returns 0 or -ENOMEM. */ +static int +search_hits_grow(pmSearchTextResult **hits, int *nhits, int *maxhits) { - seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesGetContext *context = &baton->pmapi; - unsigned int length; - const char *typestr = pmSearchTextTypeStr(type); - char buffer[8]; - sds cmd, key, docid; - - seriesBatonCheckMagic(baton, MAGIC_LOAD, "keys_search_text_add"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s %s\n", "keys_search_text_add", typestr, name); - - seriesBatonReference(context, "keys_search_text_add"); - - /* - * FT.ADD pcp:text 1.0 - * REPLACE PARTIAL - * PAYLOAD - * FIELDS NAME TYPE - * [INDOM ] [ONELINE ] [HELPTEXT ] - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - length = 4 + 2 + 2 + 5; - if (indom && *indom != '\0') - length += 2; - if (oneline && *oneline != '\0') - length += 2; - if (helptext && *helptext != '\0') - length += 2; - cmd = resp_command(length); - - cmd = resp_param_str(cmd, FT_ADD, FT_ADD_LEN); - cmd = resp_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - docid = keys_search_docid(FT_TEXT_KEY, typestr, name); - cmd = resp_param_sds(cmd, docid); - sdsfree(docid); - cmd = resp_param_str(cmd, "1", 1); - - cmd = resp_param_str(cmd, FT_REPLACE, FT_REPLACE_LEN); - cmd = resp_param_str(cmd, FT_PARTIAL, FT_PARTIAL_LEN); - - length = pmsprintf(buffer, sizeof(buffer), "%u", type); - cmd = resp_param_str(cmd, FT_PAYLOAD, FT_PAYLOAD_LEN); - cmd = resp_param_str(cmd, buffer, length); - - cmd = resp_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, name, strlen(name)); - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, typestr, strlen(typestr)); - if (indom && *indom != '\0') { - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = resp_param_str(cmd, indom, strlen(indom)); - } - if (oneline && *oneline != '\0') { - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = resp_param_str(cmd, oneline, strlen(oneline)); - } - if (helptext && *helptext != '\0') { - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = resp_param_str(cmd, helptext, strlen(helptext)); - } + int newmax = *maxhits ? *maxhits * 2 : 64; + pmSearchTextResult *tmp; - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_add_callback, arg); - sdsfree(cmd); + if (newmax < *maxhits) + return -ENOMEM; + tmp = realloc(*hits, (size_t)newmax * sizeof(pmSearchTextResult)); + if (tmp == NULL) + return -ENOMEM; + *hits = tmp; + *maxhits = newmax; + return 0; } -void -pmSearchDiscoverMetric(pmDiscoverEvent *event, - pmDesc *desc, int numnames, char **names, void *arg) +static void +search_hits_free(pmSearchTextResult *hits, int nhits) { - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = (seriesLoadBaton *)p->baton; - context_t *context = &baton->pmapi.context; - char *oneline = NULL, *helptext = NULL; - char buffer[64] = {0}; - pmID id = desc->pmid; - int i; - - if (pmDebugOptions.discovery || pmDebugOptions.search) { - for (i = 0; i < numnames; i++) - fprintf(stderr, "%s: [%d/%d] %s - %s\n", "pmSearchDiscoverMetric", - i + 1, numnames, pmIDStr_r(id, buffer, sizeof(buffer)), - names[i]); + int i; + + for (i = 0; i < nhits; i++) { + sdsfree(hits[i].docid); + sdsfree(hits[i].name); + sdsfree(hits[i].indom); + sdsfree(hits[i].oneline); + sdsfree(hits[i].helptext); } + free(hits); +} - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; +/* Count matching docs for a text query; returns 0 on failure. */ +static int +search_count_table(sqlite3 *db, sds match_expr, sds type_filter) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc, total = 0; - /* we have the metric name(s) and desc, has text been discovered yet? */ - pmUseContext(context->context); - pmLookupText(id, PM_TEXT_PMID | PM_TEXT_ONELINE, &oneline); - pmLookupText(id, PM_TEXT_PMID | PM_TEXT_HELP | PM_TEXT_DIRECT, &helptext); + sql = sdscatfmt(sdsempty(), + "SELECT count(*) FROM docs WHERE docs MATCH ?%S", + type_filter); - if (desc->indom != PM_INDOM_NULL) - pmInDomStr_r(desc->indom, buffer, sizeof(buffer)); + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_count_table: prepare: %s\n", + sqlite3_errmsg(db)); + return 0; + } + + sqlite3_bind_text(stmt, 1, match_expr, sdslen(match_expr), SQLITE_STATIC); - for (i = 0; i < numnames; i++) - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, - names[i], buffer, oneline, helptext, baton); + if (sqlite3_step(stmt) == SQLITE_ROW) + total = sqlite3_column_int(stmt, 0); - if (oneline) - free(oneline); - if (helptext) - free(helptext); + sqlite3_finalize(stmt); + return total; } -void -pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) -{ - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = p->baton; - pmInDom id = in->indom; - char *oneline = NULL, *helptext = NULL; - char buffer[64]; - int i; +/* Execute a text query against docs; appends results to the hits array. */ +static int +search_query_table(sqlite3 *db, pmSearchTextRequest *request, + sds match_expr, sds type_filter, + unsigned int limit, unsigned int offset, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdscatfmt(sdsempty(), + "SELECT rowid, name, oneline, helptext, type, indom," + " bm25(docs, 9.0, 4.0, 2.0)," + " highlight(docs, 0, '', '')," + " highlight(docs, 1, '', '')," + " highlight(docs, 2, '', '')" + " FROM docs WHERE docs MATCH ?%S" + " ORDER BY CASE WHEN type = %i" + " THEN bm25(docs, 9.0, 4.0, 2.0) * 2.0" + " ELSE bm25(docs, 9.0, 4.0, 2.0) END" + " LIMIT ? OFFSET ?", + type_filter, SEARCH_DOC_INDOM); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_query_table: prepare: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } + + sqlite3_bind_text(stmt, 1, match_expr, sdslen(match_expr), SQLITE_STATIC); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64)limit); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64)offset); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; + } - pmInDomStr_r(id, buffer, sizeof(buffer)); + memset(&result, 0, sizeof(result)); - if (pmDebugOptions.discovery || pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "pmSearchDiscoverInDom", buffer); + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + if (request->return_type) + result.type = sqlite3_column_int(stmt, 4); + result.score = -sqlite3_column_double(stmt, 6); + + if (request->return_name) { + if (request->highlight_name) + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 7)); + else + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + } + if (request->return_indom) { + const char *indom = (const char *)sqlite3_column_text(stmt, 5); + if (indom && *indom) + result.indom = sdsnew(indom); + } + if (request->return_oneline) { + if (request->highlight_oneline) + result.oneline = sdsnew((const char *)sqlite3_column_text(stmt, 8)); + else { + const char *ol = (const char *)sqlite3_column_text(stmt, 2); + if (ol && *ol) + result.oneline = sdsnew(ol); + } + } + if (request->return_helptext) { + if (request->highlight_helptext) + result.helptext = sdsnew((const char *)sqlite3_column_text(stmt, 9)); + else { + const char *ht = (const char *)sqlite3_column_text(stmt, 3); + if (ht && *ht) + result.helptext = sdsnew(ht); + } + } - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; + (*hits)[*nhits] = result; + (*nhits)++; + } - /* - * We have the indom and instances, has text been discovered yet? - * Not a problem if not as we use PARTIAL FT.ADD and subsequently - * we will find the text via pmSearchDiscoverText. - */ - pmUseContext(p->ctx); - pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_ONELINE, &oneline); - pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_HELP | PM_TEXT_DIRECT, &helptext); - - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, - buffer, buffer, oneline, helptext, baton); - for (i = 0; i < in->numinst; i++) { - if (in->namelist[i] == NULL) - continue; - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INST, - in->namelist[i], buffer, NULL, NULL, baton); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_query_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; } - if (oneline) - free(oneline); - if (helptext) - free(helptext); + return 0; } -void -pmSearchDiscoverText(pmDiscoverEvent *event, - int ident, int type, char *text, void *arg) -{ - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = p->baton; - char indom[64] = {0}, **metrics, *oneline, *helptext; - int i, count; - - if (pmDebugOptions.discovery || pmDebugOptions.search) - fprintf(stderr, "%s: ident=%u type=%u arg=" PRINTF_P_PFX "%p\n", - "pmSearchDiscoverText", ident, type, arg); +static void +search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) +{ + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + sds match, type_filter; + int nhits = 0, maxhits = 0, total, sts; + unsigned int count, offset, i; + double timer; - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; + pmtimespecNow(&started); - oneline = (type & PM_TEXT_ONELINE) ? text : NULL; - helptext = (type & PM_TEXT_HELP) ? text : NULL; - - if (type & PM_TEXT_PMID) { - pmUseContext(p->ctx); - if ((count = pmNameAll(ident, &metrics)) <= 0) - return; - for (i = 0; i < count; i++) - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, - metrics[i], NULL, oneline, helptext, baton); - free(metrics); - } else { /* PM_TEXT_INDOM */ - pmInDomStr_r(ident, indom, sizeof(indom)); - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, - indom, indom, oneline, helptext, baton); + if (!request->return_name && !request->return_indom && + !request->return_oneline && !request->return_helptext && + !request->return_type) { + request->return_name = 1; + request->return_indom = 1; + request->return_oneline = 1; + request->return_helptext = 1; + request->return_type = 1; } -} -static void -keys_search_info_callback( - keyClusterAsyncContext *c, void *r, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - respReply *reply = r; - respReply *child, *value; - pmSearchMetrics metrics = {0}; - int i; - sds msg; - - if (reply && reply->type == RESP_REPLY_ARRAY && reply->elements >= 30) { - for (i = 0; i < reply->elements-1; i++) { - value = reply->element[i+1]; - child = reply->element[i]; - if (child->type != RESP_REPLY_STRING && - value->type != RESP_REPLY_STRING) - continue; - else if (strcmp("num_docs", child->str) == 0) - metrics.docs = strtoull(value->str, NULL, 0); - else if (strcmp("num_terms", child->str) == 0) - metrics.terms = strtoull(value->str, NULL, 0); - else if (strcmp("num_records", child->str) == 0) - metrics.records = strtoull(value->str, NULL, 0); - else if (strcmp("inverted_sz_mb", child->str) == 0) - metrics.inverted_sz_mb = strtod(value->str, NULL); - else if (strcmp("inverted_cap_mb", child->str) == 0) - metrics.inverted_cap_mb = strtod(value->str, NULL); - else if (strcmp("inverted_cap_ovh", child->str) == 0) - metrics.inverted_cap_ovh = strtod(value->str, NULL); - else if (strcmp("offset_vectors_sz_mb", child->str) == 0) - metrics.offset_vectors_sz_mb = strtod(value->str, NULL); - else if (strcmp("skip_index_size_mb", child->str) == 0) - metrics.skip_index_size_mb = strtod(value->str, NULL); - else if (strcmp("score_index_size_mb", child->str) == 0) - metrics.score_index_size_mb = strtod(value->str, NULL); - else if (strcmp("records_per_doc_avg", child->str) == 0) - metrics.records_per_doc_avg = strtod(value->str, NULL); - else if (strcmp("bytes_per_record_avg", child->str) == 0) - metrics.bytes_per_record_avg = strtod(value->str, NULL); - else if (strcmp("offsets_per_term_avg", child->str) == 0) - metrics.offsets_per_term_avg = strtod(value->str, NULL); - else if (strcmp("offset_bits_per_record_avg", child->str) == 0) - metrics.offset_bits_per_record_avg = strtod(value->str, NULL); - } - baton->callbacks->on_metrics(&metrics, baton->userdata); - } else { - msg = NULL; - infofmt(msg, "expected array from %s (reply=%s)", - FT_INFO, resp_reply_type(reply)); - batoninfo(baton, PMLOG_RESPONSE, msg); - baton->error = -EPROTO; + offset = request->offset; + if (!request->count) + request->count = smd->resultcount; + count = request->count; + + match = search_build_match(request); + type_filter = search_type_filter(request); + + total = search_count_table(smd->db, match, type_filter); + + sts = search_query_table(smd->db, request, match, type_filter, + count, offset, &hits, &nhits, &maxhits); + + sdsfree(match); + sdsfree(type_filter); + + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; } - baton->callbacks->on_done(baton->error, baton->userdata); - doneKeysSearchBaton(baton); -} + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); -void -keys_search_info(keySlots *slots, sds pcpkey, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - sds cmd, key; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_info"); - seriesBatonCheckCount(baton, "keys_search_info"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: search key metrics\n", "keys_search_info"); - - seriesBatonReference(baton, "keys_search_info"); - - /* - * FT.INFO pcp: - */ - key = sdscatfmt(sdsempty(), "pcp:%S", pcpkey); - cmd = resp_command(2); - cmd = resp_param_str(cmd, FT_INFO, FT_INFO_LEN); - cmd = resp_param_sds(cmd, key); - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_info_callback, arg); - sdsfree(cmd); -} + for (i = 0; i < (unsigned int)nhits; i++) { + hits[i].total = total; + hits[i].count = i + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); + } -int -pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) -{ - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + search_hits_free(hits, nhits); - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_info(data->slots, key, baton); - return 0; + callbacks->on_done(sts, userdata); } -static void -extract_search_results(keysSearchBaton *baton, - unsigned int total, double timer, respReply *reply) -{ - pmSearchTextResult result; - respReply *docid, *score, *payload, *array; - int i, j; - - for (i = 1; i < reply->elements - 3; i += 4) { - docid = reply->element[i]; - score = reply->element[i+1]; - payload = reply->element[i+2]; - array = reply->element[i+3]; - if (payload->type != RESP_REPLY_STRING || - score->type != RESP_REPLY_STRING || - docid->type != RESP_REPLY_STRING || - array->type != RESP_REPLY_ARRAY) { - baton->error = -EPROTO; - break; +static int +search_suggest_table(sqlite3 *db, sds match, unsigned int limit, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdscatfmt(sdsempty(), + "SELECT rowid, name, type, bm25(docs, 9.0, 4.0, 2.0)" + " FROM docs WHERE docs MATCH ?" + " AND type IN (%i, %i)" + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)" + " LIMIT ?", + SEARCH_DOC_METRIC, SEARCH_DOC_INST); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_suggest_table: prepare: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } + + sqlite3_bind_text(stmt, 1, match, sdslen(match), SQLITE_STATIC); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64)limit); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; } memset(&result, 0, sizeof(result)); - result.total = total; - result.timer = timer; - result.count = (i / 2) + 1; - result.docid = sdsnewlen(docid->str, docid->len); - result.score = strtod(score->str, NULL); - - for (j = 0; j < array->elements; j += 2) { - respReply *field = array->element[j]; - respReply *value = array->element[j+1]; - - if (field->type != RESP_REPLY_STRING || - (value->type != RESP_REPLY_STRING && - value->type != RESP_REPLY_NIL)) { - baton->error = -EPROTO; - break; - } + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + result.type = sqlite3_column_int(stmt, 2); + result.score = -sqlite3_column_double(stmt, 3); - if (strcmp(field->str, FT_NAME) == 0) - result.name = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_INDOM) == 0) - result.indom = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_ONELINE) == 0) - result.oneline = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_HELPTEXT) == 0) - result.helptext = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_TYPE) == 0) - result.type = atoi(payload->str); - } - if (baton->error == 0) - baton->callbacks->on_text_result(&result, baton->userdata); - - sdsfree(result.docid); - sdsfree(result.name); - sdsfree(result.indom); - sdsfree(result.oneline); - sdsfree(result.helptext); + (*hits)[(*nhits)++] = result; } + + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_suggest_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } + return 0; } static void -keys_search_text_query_callback( - keyClusterAsyncContext *c, void *r, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - respReply *reply = r; - respReply *value; - struct timespec finished; - unsigned int total; +search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) +{ + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + sds match; + int nhits = 0, maxhits = 0, sts; + unsigned int count, i; double timer; - sds msg; - - if (LIKELY(reply && reply->type == RESP_REPLY_ARRAY)) { - value = reply->element[0]; - if (reply->elements == 1) /* no search results - done! */ - /* do nothing */; - else if (reply->elements < 2) /* expect total then results */ - baton->error = -EPROTO; - else if (value->type != RESP_REPLY_INTEGER) - baton->error = -EPROTO; - else { - pmtimespecNow(&finished); - timer = pmtimespecSub(&finished, &baton->started); - total = (unsigned int)value->integer; - extract_search_results(baton, total, timer, reply); + + pmtimespecNow(&started); + + { + sds query = request->query; + int len = sdslen(query); + int j, start; + + match = sdsnew("name : ("); + for (j = 0, start = 0; j <= len; j++) { + if (j == len || query[j] == '.') { + if (j > start) { + if (start > 0) + match = sdscat(match, " "); + match = sdscatlen(match, query + start, j - start); + } + start = j + 1; + } } - } else { - msg = NULL; - infofmt(msg, "expected array from %s (reply=%s)", - FT_SEARCH, resp_reply_type(reply)); - batoninfo(baton, PMLOG_RESPONSE, msg); - baton->error = -EPROTO; + match = sdscat(match, "*)"); } - baton->callbacks->on_done(baton->error, baton->userdata); - doneKeysSearchBaton(baton); -} + if (!request->count) + request->count = smd->resultcount; + count = request->count; -static void -keys_search_text_query(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - const char *typestr; - size_t length; - char buffer[64]; - sds cmd, key, query, base_query; - unsigned int types = 0, infields = 0, returns = 0, highlights = 0; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_query"); - seriesBatonCheckCount(baton, "keys_search_text_query"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_query", request->query); - - seriesBatonReference(baton, "keys_search_text_query"); - - types += request->type_metric; - types += request->type_indom; - types += request->type_inst; - - highlights += request->highlight_name; - highlights += request->highlight_oneline; - highlights += request->highlight_helptext; - - infields += request->infields_name; - infields += request->infields_oneline; - infields += request->infields_helptext; - if (infields == 0) { - infields = 3; /* defaults */ - request->infields_name = 1; - request->infields_oneline = 1; - request->infields_helptext = 1; + sts = search_suggest_table(smd->db, match, count, &hits, &nhits, &maxhits); + + sdsfree(match); + + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; } - returns += request->return_name; - returns += request->return_indom; - returns += request->return_oneline; - returns += request->return_helptext; - returns += request->return_type; - if (returns == 0) { - returns = 5; /* defaults */ - request->return_name = 1; - request->return_indom = 1; - request->return_oneline = 1; - request->return_helptext = 1; - request->return_type = 1; + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); + + for (i = 0; i < (unsigned int)nhits && i < count; i++) { + hits[i].total = nhits; + hits[i].count = i + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); } - /* - * FT.SEARCH pcp:text "query [@TYPE={ {?type separated by pipe} }]" WITHSCORES WITHPAYLOADS - * [INFIELDS {?field item count} {?field separated by space}] - * [RETURN {?return item count} {?return separated by space}] - * [HIGHLIGHT FIELDS {num} {field} ... ] - * SCORER BM25 - * LIMIT {?pagination offset} {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - length = 5; - if (infields) - length += 2 + infields; - if (returns) - length += 2 + returns; - if (highlights) - length += 3 + highlights; - length += 2 + 3; - - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - - query = sdscatlen(sdsempty(), "\'", 1); - if (types) { - query = sdscatlen(query, "@TYPE:{", 7); - if (request->type_metric) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_METRIC); - query = sdscat(query, typestr); + search_hits_free(hits, nhits); + + callbacks->on_done(sts, userdata); +} + +static int +search_indom_table(sqlite3 *db, const char *query, int querylen, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdsnew( + "SELECT d.rowid, d.name, d.oneline, d.helptext, d.type, d.indom" + " FROM indom_map m JOIN docs d ON d.rowid = m.docid" + " WHERE m.indom = ?" + " ORDER BY d.type"); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + /* fall back for indexes built without indom_map */ + sql = sdsnew( + "SELECT rowid, name, oneline, helptext, type, indom" + " FROM docs WHERE indom = ?" + " ORDER BY type"); + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_indom_table: prepare: %s\n", + sqlite3_errmsg(db)); + return -EIO; } - if (request->type_indom && (request->type_metric)) - query = sdscatlen(query, "|", 1); - if (request->type_indom) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_INDOM); - query = sdscat(query, typestr); + } + + sqlite3_bind_text(stmt, 1, query, querylen, SQLITE_STATIC); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; } - if (request->type_inst && (request->type_indom || request->type_metric)) - query = sdscatlen(query, "|", 1); - if (request->type_inst) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_INST); - query = sdscat(query, typestr); + + memset(&result, 0, sizeof(result)); + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + result.type = sqlite3_column_int(stmt, 4); + result.score = (result.type == SEARCH_DOC_INDOM) ? 2.0 : + (result.type == SEARCH_DOC_METRIC) ? 1.0 : 0.5; + + { + const char *ol = (const char *)sqlite3_column_text(stmt, 2); + if (ol && *ol) + result.oneline = sdsnew(ol); } - query = sdscatlen(query, "} ", 2); - } - base_query = keys_search_text_prep(request->query, 0, NULL, NULL); - query = sdscatfmt(query, "(%S)=>{$inorder:true}\'", base_query); - sdsfree(base_query); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - - if (infields) { - cmd = resp_param_str(cmd, FT_INFIELDS, FT_INFIELDS_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", infields); - cmd = resp_param_str(cmd, buffer, length); - if (request->infields_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->infields_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->infields_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + { + const char *ht = (const char *)sqlite3_column_text(stmt, 3); + if (ht && *ht) + result.helptext = sdsnew(ht); + } + { + const char *indom = (const char *)sqlite3_column_text(stmt, 5); + if (indom && *indom) + result.indom = sdsnew(indom); + } + + (*hits)[(*nhits)++] = result; } - if (returns) { - cmd = resp_param_str(cmd, FT_RETURN, FT_RETURN_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", returns); - cmd = resp_param_str(cmd, buffer, length); - if (request->return_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->return_indom) - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - if (request->return_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->return_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - if (request->return_type) - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_indom_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; } + return 0; +} + +static void +search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) +{ + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + int nhits = 0, maxhits = 0, sts; + unsigned int count, offset, i; + double timer; + + pmtimespecNow(&started); - if (highlights) { - cmd = resp_param_str(cmd, FT_HIGHLIGHT, FT_HIGHLIGHT_LEN); - cmd = resp_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", highlights); - cmd = resp_param_str(cmd, buffer, length); - if (request->highlight_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->highlight_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->highlight_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + sts = search_indom_table(smd->db, request->query, + sdslen(request->query), + &hits, &nhits, &maxhits); + + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; } - cmd = resp_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = resp_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = resp_param_str(cmd, buffer, length); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + offset = request->offset; + if (!request->count) + request->count = smd->resultcount; + count = request->count; + + for (i = offset; i < (unsigned int)nhits && (i - offset) < count; i++) { + hits[i].total = nhits; + hits[i].count = (i - offset) + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); } - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + search_hits_free(hits, nhits); + + callbacks->on_done(sts, userdata); } +/* --- public API --- */ + int -pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + searchModuleData *smd = (searchModuleData *)settings->module.privdata; + pmSearchMetrics metrics; + sqlite3_stmt *stmt = NULL; + int rc; - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_query(data->slots, request, baton); - return 0; -} + (void)key; -static void -keys_search_text_suggest(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - size_t length, prefix_length, fuzzy_length; - const char *prefix; - char buffer[64]; - sds cmd, key, query, - prefix_query, fuzzy_query; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_suggest"); - seriesBatonCheckCount(baton, "keys_search_text_suggest"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_suggest", request->query); - - seriesBatonReference(baton, "keys_search_text_suggest"); - - /* by default we cannot use prefix search with words of length less than 2 */ - prefix_query = keys_search_text_prep(request->query, 2, NULL, "*"); - fuzzy_query = keys_search_text_prep(request->query, 2, "%", "%"); - prefix_length = sdslen(prefix_query); - fuzzy_length = sdslen(fuzzy_query); - - query = sdsnewlen("\'", 1); - prefix = ""; - if (prefix_length || fuzzy_length) { - prefix = "("; - } - if (prefix_length) { - query = sdscatfmt(query, "%s@NAME:(%S)", prefix, prefix_query); - prefix = "|"; + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; } - if (fuzzy_length) { - query = sdscatfmt(query, "%s@NAME:(%S)=>{$weight:0.25;}", prefix, fuzzy_query); - prefix = "|"; - } - if (prefix_length || fuzzy_length) { - prefix = ")"; + + memset(&metrics, 0, sizeof(metrics)); + + rc = sqlite3_prepare_v2(smd->db, + "SELECT COUNT(*) FROM docs", -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) + metrics.docs = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); } - query = sdscatfmt( - query, - "%s @TYPE:{%s|%s}", - prefix, - pmSearchTextTypeStr(PM_SEARCH_TYPE_METRIC), - pmSearchTextTypeStr(PM_SEARCH_TYPE_INST) - ); - query = sdscat(query, "\'"); - - sdsfree(prefix_query); - sdsfree(fuzzy_query); - - /* - * FT.SEARCH pcp:text - * "(@NAME:({query}*)|@NAME:(%{query}%)=>{$weight:0.25;}) @TYPE={metric|instance}" - * WITHSCORES WITHPAYLOADS - * RETURN 1 NAME - * SCORER BM25 - * LIMIT 0 {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - - length = 13; // Resp array size - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - cmd = resp_param_str(cmd, FT_RETURN, FT_RETURN_LEN); - cmd = resp_param_str(cmd, "1", 1); - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = resp_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - cmd = resp_param_str(cmd, "0", 1); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + + rc = sqlite3_prepare_v2(smd->db, + "SELECT COUNT(*), COALESCE(SUM(cnt),0)" + " FROM docs_vocab", -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + metrics.terms = sqlite3_column_int64(stmt, 0); + metrics.records = sqlite3_column_int64(stmt, 1); + } + sqlite3_finalize(stmt); + } else if (pmDebugOptions.search) { + fprintf(stderr, "pmSearchInfo: docs_vocab: %s\n", + sqlite3_errmsg(smd->db)); } - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + settings->callbacks.on_metrics(&metrics, arg); + settings->callbacks.on_done(0, arg); + return 0; } int -pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_suggest(data->slots, request, baton); - return 0; -} - -static void -keys_search_text_indom(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - size_t length; - char buffer[64]; - sds cmd, key, query; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_indom"); - seriesBatonCheckCount(baton, "keys_search_text_indom"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_indom", request->query); - - seriesBatonReference(baton, "keys_search_text_indom"); - - query = sdscatfmt( - sdsnewlen("", 0), - "\'@INDOM:{%s}\'", - request->query - ); - - /* - * FT.SEARCH pcp:text - * "@INDOM:{{query}}" WITHSCORES WITHPAYLOADS - * SORTBY 2 TYPE ASC - *. LIMIT {?pagination offset} {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - - length = 12; // Resp array size - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - - cmd = resp_param_str(cmd, FT_SORTBY, FT_SORTBY_LEN); - cmd = resp_param_str(cmd, "2", 1); - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, FT_ASC, FT_ASC_LEN); - - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = resp_param_str(cmd, buffer, length); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + if (request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); + return 0; } - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + search_do_text_query(smd, request, &settings->callbacks, arg); + return 0; } int -pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_indom(data->slots, request, baton); + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + if (request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); + return 0; + } + + search_do_text_suggest(smd, request, &settings->callbacks, arg); return 0; } -static void -keys_search_schema_callback( - keyClusterAsyncContext *c, void *r, void *arg) +int +pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - keySlotsBaton *baton = (keySlotsBaton *)arg; - respReply *reply = r; - - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_search_schema_callback"); + searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (testReplyError(reply, RESP_EDROPINDEX)) { - // index already exists - baton->slots->search = 1; + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; } - else if (reply && reply->type == RESP_REPLY_STATUS && - (strcmp("OK", reply->str) == 0 || strcmp("QUEUED", reply->str) == 0)) { - // index created - baton->slots->search = 1; - } else { - // probably no search module installed, ignore silently - baton->slots->search = 0; + if (request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); + return 0; } - keys_slots_end_phase(baton); + search_do_text_indom(smd, request, &settings->callbacks, arg); + return 0; } -void -keys_load_search_schema(void *arg) -{ - keySlotsBaton *baton = (keySlotsBaton *)arg; - sds cmd, key; - - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_search_schema"); - - if (pmDebugOptions.search && pmDebugOptions.desperate) - fprintf(stderr, "%s: loading schema\n", "keys_search_schema"); - - seriesBatonReference(baton, "keys_load_search_schema"); - - /* - * FT.CREATE pcp:text SCHEMA - * type TAG SORTABLE - * name TEXT WEIGHT 9 SORTABLE - * indom TAG - * oneline TEXT WEIGHT 4 - * helptext TEXT WEIGHT 2 - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = resp_command(3 + 3 + 5 + 2 + 4 + 4); - - cmd = resp_param_str(cmd, FT_CREATE, FT_CREATE_LEN); - cmd = resp_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = resp_param_str(cmd, FT_SCHEMA, FT_SCHEMA_LEN); - - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, FT_TAG, FT_TAG_LEN); - cmd = resp_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "9", sizeof("9")-1); - cmd = resp_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = resp_param_str(cmd, FT_TAG, FT_TAG_LEN); - - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "4", sizeof("4")-1); - - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "2", sizeof("2")-1); - - sdsfree(key); - keySlotsRequestFirstNode(baton->slots, cmd, keys_search_schema_callback, arg); - sdsfree(cmd); -} +/* --- module setup / teardown --- */ int pmSearchSetSlots(pmSearchModule *module, void *slots) { - return pmSeriesSetSlots(module, slots); + (void)module; + (void)slots; + return 0; } int -pmSearchSetConfiguration(pmSearchModule *module, dict *config) +pmSearchSetConfiguration(pmSearchModule *module, struct dict *config) { - return pmSeriesSetConfiguration(module, config); + searchModuleData *smd = getSearchModuleData(module); + + if (smd == NULL) + return -ENOMEM; + smd->config = config; + return 0; } int pmSearchSetEventLoop(pmSearchModule *module, void *events) { - return pmSeriesSetEventLoop(module, events); + (void)module; + (void)events; + return 0; } int -pmSearchSetMetricRegistry(pmSearchModule *module, mmv_registry_t *registry) +pmSearchSetMetricRegistry(pmSearchModule *module, struct mmv_registry *registry) { - return pmSeriesSetMetricRegistry(module, registry); -} - -void -keysSearchInit(struct dict *config) -{ - sds option; - - if (!resultcount) { - if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) - resultcount_str = option; - else - resultcount_str = DEFAULT_RESULTCOUNT = sdsnew("10"); - resultcount = atoi(resultcount_str); - } -} - -void -keysSearchClose(void) -{ - if (DEFAULT_RESULTCOUNT) { - sdsfree(DEFAULT_RESULTCOUNT); - DEFAULT_RESULTCOUNT = NULL; - } + (void)module; + (void)registry; + return 0; } int pmSearchSetup(pmSearchModule *module, void *arg) { - seriesModuleData *data = getSeriesModuleData(module); + searchModuleData *smd = getSearchModuleData(module); + char nightly[MAXPATHLEN]; + char base[MAXPATHLEN]; sds option; - keySlotsFlags flags; + int rc; - if (data == NULL) + if (smd == NULL) return -ENOMEM; - /* create string map caches */ - keysGlobalsInit(data->config); + smd->resultcount = default_resultcount; - /* fast path for when key server has been setup already */ - if (data->slots) { - module->on_setup(arg); - data->shareslots = 1; - } else { - if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); // compat + if (smd->config) { + option = pmIniFileLookup(smd->config, "pmsearch", "enabled"); if (option && strcmp(option, "false") == 0) return -ENOTSUP; - /* establish an initial connection to key server instance(s) */ - flags = SLOTS_VERSION; + option = pmIniFileLookup(smd->config, "pmsearch", "result.count"); + if (option) { + char *endp; + unsigned long value; + + errno = 0; + value = strtoul(option, &endp, 10); + if (errno == 0 && *endp == '\0' && value > 0 && value <= UINT_MAX) + smd->resultcount = (unsigned int)value; + else + pmNotifyErr(LOG_WARNING, "ignoring invalid " + "pmsearch result.count \"%s\"", option); + } + } - option = pmIniFileLookup(data->config, "pmsearch", "enabled"); - if (option && strcmp(option, "true") == 0) - flags |= SLOTS_SEARCH; - else - return -ENOTSUP; + option = smd->config ? + pmIniFileLookup(smd->config, "pmsearch", "index.path") : NULL; + + if (option) { + rc = sqlite3_open_v2(option, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc != SQLITE_OK) { + pmNotifyErr(LOG_WARNING, + "cannot open pmsearch index %s: %s", + option, sqlite3_errmsg(smd->db)); + sqlite3_close(smd->db); + smd->db = NULL; + } + } else { + pmsprintf(nightly, sizeof(nightly), "%s/lib/pmsearch.index", + pmGetConfig("PCP_VAR_DIR")); + pmsprintf(base, sizeof(base), "%s/lib/pmsearch.index", + pmGetConfig("PCP_SHARE_DIR")); + + rc = sqlite3_open_v2(nightly, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc == SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: loaded nightly index %s\n", + nightly); + } else { + sqlite3_close(smd->db); + smd->db = NULL; + + rc = sqlite3_open_v2(base, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc == SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: loaded base index %s\n", + base); + } else { + pmNotifyErr(LOG_WARNING, + "no pmsearch index found " + "(tried %s and %s); run pmsearch_daily(1)", + nightly, base); + sqlite3_close(smd->db); + smd->db = NULL; + } + } + } - /* establish an initial connection to key server instance(s) */ - data->slots = &(keySlotsConnect( - data->config, flags, module->on_info, - module->on_setup, arg, data->events, arg))->slots; - data->shareslots = 0; + if (smd->db != NULL) { + smd->loaded = 1; + search_enabled = 1; + } else { + smd->loaded = 0; } + + if (module->on_setup) + module->on_setup(arg); return 0; } int pmSearchEnabled(void *arg) { - keySlots *slots = (keySlots *)arg; - - if (slots) - return slots->search > 0 ? 1 : 0; - return 0; + (void)arg; + return search_enabled; } void pmSearchClose(pmSearchModule *module) { - seriesModuleData *search = (seriesModuleData *)module->privdata; + searchModuleData *smd = (searchModuleData *)module->privdata; - if (search) { - if (search->slots && !search->shareslots) - keySlotsFree(search->slots); - memset(search, 0, sizeof(*search)); - free(search); + if (smd) { + if (smd->db) + sqlite3_close(smd->db); + memset(smd, 0, sizeof(*smd)); + free(smd); module->privdata = NULL; } + search_enabled = 0; +} + +/* --- stubs for schema.c / keys.c compatibility --- */ - keysGlobalsClose(); +extern void keys_slots_end_phase(void *); + +void +keysSearchInit(struct dict *config) +{ + sds option; + char *endp; + unsigned long value; + + if (config) { + if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) { + errno = 0; + value = strtoul(option, &endp, 10); + if (errno == 0 && *endp == '\0' && value > 0 && value <= UINT_MAX) + default_resultcount = (unsigned int)value; + else + pmNotifyErr(LOG_WARNING, "ignoring invalid " + "pmsearch result.count \"%s\"", option); + } + } +} + +void +keysSearchClose(void) +{ + default_resultcount = 10; +} + +void +keys_load_search_schema(void *arg) +{ + keys_slots_end_phase(arg); +} + +void +keys_search_text_add(struct keySlots *slots, pmSearchTextType type, + const char *name, const char *indom, + const char *oneline, const char *helptext, void *arg) +{ + (void)slots; (void)type; (void)name; (void)indom; + (void)oneline; (void)helptext; (void)arg; +} + +/* --- discover no-ops (declared in discover.h) --- */ + +void +pmSearchDiscoverMetric(pmDiscoverEvent *event, + pmDesc *desc, int numnames, char **names, void *arg) +{ + (void)event; (void)desc; (void)numnames; (void)names; (void)arg; +} + +void +pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) +{ + (void)event; (void)in; (void)arg; +} + +void +pmSearchDiscoverText(pmDiscoverEvent *event, + int ident, int type, char *text, void *arg) +{ + (void)event; (void)ident; (void)type; (void)text; (void)arg; } diff --git a/src/libpcp_web/src/search.h b/src/libpcp_web/src/search.h index b753f7c00b2..8790d8959be 100644 --- a/src/libpcp_web/src/search.h +++ b/src/libpcp_web/src/search.h @@ -1,6 +1,7 @@ /* * Copyright (c) 2020,2022 Red Hat. - * + * Copyright (c) 2026 Red Hat. + * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation; either version 2.1 of the License, or @@ -15,95 +16,22 @@ #define SEARCH_SCHEMA_H #include -#include -#include "keys.h" -#include "private.h" -#include "schema.h" -#include "slots.h" - -#define FT_TEXT_KEY "pcp:text" -#define FT_TEXT_KEY_LEN (sizeof(FT_TEXT_KEY)-1) +#include -#define FT_ADD "FT.ADD" -#define FT_ADD_LEN (sizeof(FT_ADD)-1) -#define FT_CREATE "FT.CREATE" -#define FT_CREATE_LEN (sizeof(FT_CREATE)-1) -#define FT_SEARCH "FT.SEARCH" -#define FT_SEARCH_LEN (sizeof(FT_SEARCH)-1) -#define FT_INFIELDS "INFIELDS" -#define FT_INFIELDS_LEN (sizeof(FT_INFIELDS)-1) -#define FT_INFO "FT.INFO" -#define FT_INFO_LEN (sizeof(FT_INFO)-1) -#define FT_ASC "ASC" -#define FT_ASC_LEN (sizeof(FT_ASC)-1) -#define FT_FIELDS "FIELDS" -#define FT_FIELDS_LEN (sizeof(FT_FIELDS)-1) -#define FT_HELPTEXT "HELPTEXT" -#define FT_HELPTEXT_LEN (sizeof(FT_HELPTEXT)-1) -#define FT_HIGHLIGHT "HIGHLIGHT" -#define FT_HIGHLIGHT_LEN (sizeof(FT_HIGHLIGHT)-1) -#define FT_INDOM "INDOM" -#define FT_INDOM_LEN (sizeof(FT_INDOM)-1) -#define FT_LIMIT "LIMIT" -#define FT_LIMIT_LEN (sizeof(FT_LIMIT)-1) -#define FT_NAME "NAME" -#define FT_NAME_LEN (sizeof(FT_NAME)-1) -#define FT_ONELINE "ONELINE" -#define FT_ONELINE_LEN (sizeof(FT_ONELINE)-1) -#define FT_PARTIAL "PARTIAL" -#define FT_PARTIAL_LEN (sizeof(FT_PARTIAL)-1) -#define FT_PAYLOAD "PAYLOAD" -#define FT_PAYLOAD_LEN (sizeof(FT_PAYLOAD)-1) -#define FT_RETURN "RETURN" -#define FT_RETURN_LEN (sizeof(FT_RETURN)-1) -#define FT_REPLACE "REPLACE" -#define FT_REPLACE_LEN (sizeof(FT_REPLACE)-1) -#define FT_SCHEMA "SCHEMA" -#define FT_SCHEMA_LEN (sizeof(FT_SCHEMA)-1) -#define FT_SCORE "SCORE" -#define FT_SCORE_LEN (sizeof(FT_SCORE)-1) -#define FT_SCORER "SCORER" -#define FT_SCORER_LEN (sizeof(FT_SCORER)-1) -#define FT_SCORER_BM25 "BM25" -#define FT_SCORER_BM25_LEN (sizeof(FT_SCORER_BM25)-1) -#define FT_SORTABLE "SORTABLE" -#define FT_SORTABLE_LEN (sizeof(FT_SORTABLE)-1) -#define FT_SORTBY "SORTBY" -#define FT_SORTBY_LEN (sizeof(FT_SORTBY)-1) -#define FT_TAG "TAG" -#define FT_TAG_LEN (sizeof(FT_TAG)-1) -#define FT_TEXT "TEXT" -#define FT_TEXT_LEN (sizeof(FT_TEXT)-1) -#define FT_TYPE "TYPE" -#define FT_TYPE_LEN (sizeof(FT_TYPE)-1) -#define FT_WEIGHT "WEIGHT" -#define FT_WEIGHT_LEN (sizeof(FT_WEIGHT)-1) -#define FT_WITHPAYLOADS "WITHPAYLOADS" -#define FT_WITHPAYLOADS_LEN (sizeof(FT_WITHPAYLOADS)-1) -#define FT_WITHSCORES "WITHSCORES" -#define FT_WITHSCORES_LEN (sizeof(FT_WITHSCORES)-1) +struct dict; +struct keySlots; +/* + * The functions below are retained only for link compatibility with + * schema.c and keys.c. They are implemented as no-ops now that the + * search index is built by newhelp(1) into a local SQLite FTS5 file. + * Do not add new callers. + */ extern void keysSearchInit(struct dict *); extern void keysSearchClose(void); + extern void keys_load_search_schema(void *); -extern void keys_search_text_add(keySlots *, pmSearchTextType, +extern void keys_search_text_add(struct keySlots *, pmSearchTextType, const char *, const char *, const char *, const char *, void *); -/* - * Asynchronous search baton structures - */ -typedef struct keysSearchBaton { - seriesBatonMagic header; /* MAGIC_SEARCH */ - - keySlots *slots; /* key server slots */ - pmSearchFlags flags; - int error; - void *module; - pmSearchCallBacks *callbacks; - pmLogInfoCallBack info; - struct timespec started; - void *userdata; - void *arg; -} keysSearchBaton; - #endif /* SEARCH_SCHEMA_H */ diff --git a/src/newhelp/GNUmakefile b/src/newhelp/GNUmakefile index c23a8034f5f..34e34c3ea38 100644 --- a/src/newhelp/GNUmakefile +++ b/src/newhelp/GNUmakefile @@ -18,8 +18,13 @@ include $(TOPDIR)/src/include/builddefs include $(TOPDIR)/src/libpcp/src/GNUlibrarydefs TARGETS = newhelp$(EXECSUFFIX) chkhelp$(EXECSUFFIX) -CFILES = newhelp.c chkhelp.c - +ifeq "$(HAVE_SQLITE3)" "true" +SEARCH_SQLITE = search_sqlite.c +LCFLAGS += $(SQLITE3CFLAGS) +else +SEARCH_SQLITE = search_sqlite_stub.c +endif +CFILES = newhelp.c chkhelp.c $(SEARCH_SQLITE) LLDFLAGS = $(WARN_OFF) LLDLIBS = $(PCP_PMDALIB) @@ -40,12 +45,12 @@ install: default include $(BUILDRULES) -newhelp$(EXECSUFFIX): newhelp.c - $(CCF) -o $@ $(LDFLAGS) newhelp.c $(LDLIBS) +newhelp$(EXECSUFFIX): newhelp.c $(SEARCH_SQLITE) search_sqlite.h + $(CCF) -o $@ $(LDFLAGS) newhelp.c $(SEARCH_SQLITE) $(LDLIBS) $(LIB_FOR_SQLITE3) $(LINKER_MAKERULE) -newhelp.static$(EXECSUFFIX): newhelp.c $(STATIC_LIBPCP) - $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c $(STATIC_LIBPCP) $(STATIC_LDLIBS) +newhelp.static$(EXECSUFFIX): newhelp.c $(SEARCH_SQLITE) search_sqlite.h $(STATIC_LIBPCP) + $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c $(SEARCH_SQLITE) $(STATIC_LIBPCP) $(STATIC_LDLIBS) $(LIB_FOR_SQLITE3) chkhelp$(EXECSUFFIX): chkhelp.o $(CCF) -o $@ $(LDFLAGS) chkhelp.o $(LDLIBS) diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index 997815bc093..8ab874b2112 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -26,18 +26,21 @@ #include #include "pmapi.h" #include "libpcp.h" +#include "search_sqlite.h" #define DEFAULT_HELP_VERSION 2 /* maximum bytes per line and bytes per entry */ -#define MAXLINE 128 -#define MAXENTRY 1024 +#define MAXLINE 1024 +#define MAXENTRY 2048 static int verbose; static int ln; static char *filename; static int status; static int version = DEFAULT_HELP_VERSION; +static int search_mode; /* -S: build search index */ +static char *pmnsfile = PM_NS_DEFAULT; static FILE *f; typedef struct { @@ -217,6 +220,319 @@ newentry(char *buf) } } +/* + * Resolve a symbolic PMDA name (e.g. "SAMPLE") to its numeric domain + * by reading $PCP_VAR_DIR/pmns/stdpmid. Returns -1 on failure. + */ +static int +lookup_domain(const char *pmda_name) +{ + char path[MAXPATHLEN]; + char var_dir[MAXPATHLEN]; + char line[256]; + char name[64]; + int domain; + FILE *fp; + + /* + * Read PCP_VAR_DIR directly from pcp.conf, not the environment, + * since QA tests may override PCP_VAR_DIR to a temp directory. + */ + { + const char *conf = getenv("PCP_CONF"); + if (conf == NULL) + conf = "/etc/pcp.conf"; + if ((fp = fopen(conf, "r")) == NULL) + return -1; + var_dir[0] = '\0'; + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "PCP_VAR_DIR=%1023s", var_dir) == 1) + break; + } + fclose(fp); + if (var_dir[0] == '\0') + return -1; + } + + pmsprintf(path, sizeof(path), "%s/pmns/stdpmid", var_dir); + if ((fp = fopen(path, "r")) == NULL) + return -1; + + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "#define %63s %d", name, &domain) == 2) { + if (strcmp(name, pmda_name) == 0) { + fclose(fp); + return domain; + } + } + } + fclose(fp); + return -1; +} + +static void +trim_trailing_newlines(char *str) +{ + int len; + + if (*str) { + len = (int)strlen(str) - 1; + while (len >= 0 && str[len] == '\n') + len--; + str[len + 1] = '\0'; + } +} + +/* + * Search index mode (-S): collect entries for the FTS5 search index. + * Unlike newentry() which writes .pag files, this just parses the + * name/oneline/helptext and passes them to the SQLite index builder. + */ +static void +newentry_search(char *buf) +{ + char *p; + char *name; + char *oneline; + char *helptext; + int type; + char indom_buf[64]; + pmID pmid; + pmDesc desc; + + /* skip leading white space */ + for (p = buf; isspace((int)*p); p++) + ; + + /* + * Instance entry: @I indom_idinstance name + * Followed by optional oneline and helptext on subsequent lines. + * Tab separates indom_id from instance name (names may contain spaces). + */ + if (*p == 'I' && isspace((int)*(p + 1))) { + char *indom_str, *inst_name; + + p += 2; + while (*p == ' ') + p++; + indom_str = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] instance entry missing tab separator\n", + pmGetProgname(), filename, ln); + if (!status) status = 1; + return; + } + inst_name = p; + while (*p && *p != '\n') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + /* remainder is oneline[\nhelptext] */ + oneline = p; + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + helptext = p; + trim_trailing_newlines(helptext); + + if (verbose) + fprintf(stderr, "instance %s [%s]\n", inst_name, indom_str); + + search_sqlite_add(inst_name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + indom_str, SEARCH_DOC_INST); + return; + } + + /* + * Metric with pre-resolved indom: @ M nameindomoneline + * Generated by pmsearch_daily to avoid needing PMNS/PMCD lookup. + */ + if (*p == 'M' && isspace((int)*(p + 1))) { + char *indom_str; + + p += 2; + while (*p == ' ') + p++; + name = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] metric entry missing first tab separator\n", + pmGetProgname(), filename, ln); + if (!status) status = 1; + return; + } + indom_str = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] metric entry missing second tab separator\n", + pmGetProgname(), filename, ln); + if (!status) status = 1; + return; + } + + oneline = p; + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + helptext = p; + trim_trailing_newlines(helptext); + + if (verbose) + fprintf(stderr, "%s [%s]\n", name, indom_str); + + search_sqlite_add(name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + (*indom_str != '\0') ? indom_str : NULL, + SEARCH_DOC_METRIC); + return; + } + + name = p; + + /* skip over metric name or indom spec */ + for ( ; *p != '\n' && !isspace((int)*p); p++) + ; + *p = '\0'; + p++; + + /* + * Determine type: indom entries have exactly one dot with only + * uppercase letters/digits before it and only digits after it. + * This matches both numeric (29.3) and symbolic (SAMPLE.3) forms. + * Names with multiple dots (SAMPLE.0.1000) or lowercase are metrics. + */ + { + char *dot = strchr(name, '.'); + char *cp; + int is_indom = 0; + + if (dot != NULL && strchr(dot + 1, '.') == NULL) { + is_indom = 1; + for (cp = name; cp < dot; cp++) { + if (!isupper((int)*cp) && !isdigit((int)*cp)) { + is_indom = 0; + break; + } + } + if (is_indom) { + for (cp = dot + 1; *cp != '\0'; cp++) { + if (!isdigit((int)*cp)) { + is_indom = 0; + break; + } + } + } + } + if (is_indom) { + type = SEARCH_DOC_INDOM; + if (isdigit((int)*name)) { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } else { + char pmda_name[64]; + int len = dot - name; + int domain; + + if (len < (int)sizeof(pmda_name)) { + memcpy(pmda_name, name, len); + pmda_name[len] = '\0'; + domain = lookup_domain(pmda_name); + if (domain >= 0) + pmsprintf(indom_buf, sizeof(indom_buf), + "%d.%s", domain, dot + 1); + else + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } else { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } + } + } else { + static int pmns_loaded = 0; + static int pmns_usable = 0; + + type = SEARCH_DOC_METRIC; + indom_buf[0] = '\0'; + if (!pmns_loaded) { + int sts; + + pmns_loaded = 1; + if ((sts = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) + fprintf(stderr, "%s: pmLoadASCIINameSpace(%s): %s\n", + pmGetProgname(), + pmnsfile ? pmnsfile : "default", pmErrStr(sts)); + else if ((sts = pmNewContext(PM_CONTEXT_HOST, "localhost")) < 0) + fprintf(stderr, "%s: pmNewContext(localhost): %s\n", + pmGetProgname(), pmErrStr(sts)); + else + pmns_usable = 1; + } + if (pmns_usable) { + const char *np = name; + + if (pmLookupName(1, &np, &pmid) >= 0 && + pmLookupDesc(pmid, &desc) >= 0 && + desc.indom != PM_INDOM_NULL) { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", + pmInDomStr(desc.indom)); + } + } + } + } + + /* skip whitespace to start of oneline */ + while (*p != '\n' && isspace((int)*p)) + p++; + oneline = p; + + /* find end of oneline */ + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + /* remainder is helptext */ + helptext = p; + trim_trailing_newlines(helptext); + + if (verbose) + fprintf(stderr, "%s\n", name); + + search_sqlite_add((type == SEARCH_DOC_INDOM && indom_buf[0] != '\0') ? + indom_buf : name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + (indom_buf[0] != '\0') ? indom_buf : NULL, + type); +} + static int idcomp(const void *a, const void *b) { @@ -248,13 +564,14 @@ static pmLongOptions longopts[] = { PMOPT_HELP, PMAPI_OPTIONS_HEADER("Output options"), { "output", 1, 'o', "FILE", "base name for output files" }, + { "search", 0, 'S', 0, "build search index (output file is a SQLite FTS5 database)" }, { "verbose", 0, 'V', 0, "verbose/diagnostic output" }, { "version", 1, 'v', "N", "deprecated (only version 2 format supported)" }, PMAPI_OPTIONS_END }; static pmOptions opts = { - .short_options = "D:n:o:Vv:?", + .short_options = "D:n:o:SVv:?", .long_options = longopts, .short_usage = "[options] [file ...]", }; @@ -266,7 +583,6 @@ main(int argc, char **argv) int c; int i; int skip; - char *pmnsfile = PM_NS_DEFAULT; char *fname = NULL; char pathname[MAXPATHLEN]; FILE *inf; @@ -295,6 +611,10 @@ main(int argc, char **argv) fname = opts.optarg; break; + case 'S': /* build search index */ + search_mode = 1; + break; + case 'V': /* more chit-chat */ verbose++; break; @@ -324,9 +644,22 @@ main(int argc, char **argv) exit(2); } - if ((n = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) { - fprintf(stderr, "%s: pmLoadASCIINameSpace(%s, 1): %s\n", pmGetProgname(), pmnsfile, pmErrStr(n)); - exit(2); + if (!search_mode) { + if ((n = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) { + fprintf(stderr, "%s: pmLoadASCIINameSpace(%s, 1): %s\n", + pmGetProgname(), pmnsfile, pmErrStr(n)); + exit(2); + } + } + + if (search_mode) { + if (fname == NULL) { + fprintf(stderr, "%s: -S mode requires -o to name the output file\n", + pmGetProgname()); + exit(2); + } + if (search_sqlite_open(fname) < 0) + exit(2); } do { @@ -341,7 +674,7 @@ main(int argc, char **argv) } else { if (fname == NULL) { - fprintf(stderr, + fprintf(stderr, "%s: need either a -o option or a filename " "argument to name the output file\n", pmGetProgname()); exit(2); @@ -350,7 +683,7 @@ main(int argc, char **argv) inf = stdin; } - if (version == 2 && f == NULL) { + if (!search_mode && version == 2 && f == NULL) { pmsprintf(pathname, sizeof(pathname), "%s.pag", fname); if ((f = fopen(pathname, "w")) == NULL) { fprintf(stderr, "%s: fopen(\"%s\", ...) failed: %s\n", @@ -380,7 +713,10 @@ main(int argc, char **argv) p--; *++p = '\n'; *++p = '\0'; - newentry(buf); + if (search_mode) + newentry_search(buf); + else + newentry(buf); } if (skip == -1) break; @@ -432,7 +768,10 @@ main(int argc, char **argv) opts.optind++; } while (opts.optind < argc); - if (f != NULL) { + if (search_mode) { + if (search_sqlite_close() < 0) + status = 2; + } else if (f != NULL) { fclose(f); /* do the directory index ... */ diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c new file mode 100644 index 00000000000..fa7618d16de --- /dev/null +++ b/src/newhelp/search_sqlite.c @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * Build a SQLite FTS5 search index from help text entries. + */ +#include +#include +#include "search_sqlite.h" + +static sqlite3 *search_db; +static sqlite3_stmt *search_insert; +static sqlite3_stmt *search_lookup; +static sqlite3_stmt *search_delete; +static sqlite3_stmt *search_indom_insert; +static sqlite3_stmt *search_indom_delete; + +int +search_sqlite_open(const char *path) +{ + int rc; + + rc = sqlite3_open_v2(path, &search_db, + SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: %s: %s\n", + path, sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE VIRTUAL TABLE IF NOT EXISTS docs USING fts5(" + " name," + " oneline," + " helptext," + " type UNINDEXED," + " indom UNINDEXED," + " tokenize='porter unicode61'," + " prefix='2,3'" + ");", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create table: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE TABLE IF NOT EXISTS indom_map(" + " docid INTEGER PRIMARY KEY," + " indom TEXT NOT NULL" + ");", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create indom_map: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE INDEX IF NOT EXISTS idx_indom_map" + " ON indom_map(indom);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create index: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, "BEGIN", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: begin: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "INSERT INTO docs(name, oneline, helptext, type, indom)" + " VALUES(?, ?, ?, ?, ?)", + -1, &search_insert, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare insert: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "SELECT rowid FROM docs WHERE name = ? AND type = ?", + -1, &search_lookup, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare lookup: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "DELETE FROM docs WHERE rowid = ?", + -1, &search_delete, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare delete: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "INSERT INTO indom_map(docid, indom) VALUES(?, ?)", + -1, &search_indom_insert, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare indom insert: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_delete); + search_delete = NULL; + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "DELETE FROM indom_map WHERE docid = ?", + -1, &search_indom_delete, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare indom delete: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_indom_insert); + search_indom_insert = NULL; + sqlite3_finalize(search_delete); + search_delete = NULL; + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + return 0; +} + +void +search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type) +{ + if (search_db == NULL || search_insert == NULL || name == NULL || + search_lookup == NULL || search_delete == NULL) + return; + + /* upsert: remove existing entry with same (name, type) if any */ + sqlite3_bind_text(search_lookup, 1, name, -1, SQLITE_STATIC); + sqlite3_bind_int(search_lookup, 2, type); + if (sqlite3_step(search_lookup) == SQLITE_ROW) { + sqlite3_int64 rowid = sqlite3_column_int64(search_lookup, 0); + sqlite3_bind_int64(search_indom_delete, 1, rowid); + sqlite3_step(search_indom_delete); + sqlite3_reset(search_indom_delete); + sqlite3_bind_int64(search_delete, 1, rowid); + if (sqlite3_step(search_delete) != SQLITE_DONE) { + fprintf(stderr, "search_sqlite_add: delete: %s\n", + sqlite3_errmsg(search_db)); + } + sqlite3_reset(search_delete); + } + sqlite3_reset(search_lookup); + + sqlite3_bind_text(search_insert, 1, name, -1, SQLITE_STATIC); + sqlite3_bind_text(search_insert, 2, oneline ? oneline : "", -1, SQLITE_STATIC); + sqlite3_bind_text(search_insert, 3, helptext ? helptext : "", -1, SQLITE_STATIC); + sqlite3_bind_int(search_insert, 4, type); + sqlite3_bind_text(search_insert, 5, indom ? indom : "", -1, SQLITE_STATIC); + + if (sqlite3_step(search_insert) != SQLITE_DONE) { + fprintf(stderr, "search_sqlite_add: %s: %s\n", + name, sqlite3_errmsg(search_db)); + } + + if (indom && *indom) { + sqlite3_int64 docid = sqlite3_last_insert_rowid(search_db); + sqlite3_bind_int64(search_indom_insert, 1, docid); + sqlite3_bind_text(search_indom_insert, 2, indom, -1, SQLITE_STATIC); + sqlite3_step(search_indom_insert); + sqlite3_reset(search_indom_insert); + } + + sqlite3_reset(search_insert); +} + +int +search_sqlite_close(void) +{ + int rc; + + if (search_db == NULL) + return -1; + + if (search_insert) { + sqlite3_finalize(search_insert); + search_insert = NULL; + } + if (search_lookup) { + sqlite3_finalize(search_lookup); + search_lookup = NULL; + } + if (search_delete) { + sqlite3_finalize(search_delete); + search_delete = NULL; + } + if (search_indom_insert) { + sqlite3_finalize(search_indom_insert); + search_indom_insert = NULL; + } + if (search_indom_delete) { + sqlite3_finalize(search_indom_delete); + search_indom_delete = NULL; + } + + /* optimize the FTS index before closing */ + rc = sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: optimize: %s\n", + sqlite3_errmsg(search_db)); + } + + rc = sqlite3_exec(search_db, "COMMIT", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: commit: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab" + " USING fts5vocab(docs, row)", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: create docs_vocab: %s\n", + sqlite3_errmsg(search_db)); + } + + sqlite3_close(search_db); + search_db = NULL; + return 0; +} diff --git a/src/newhelp/search_sqlite.h b/src/newhelp/search_sqlite.h new file mode 100644 index 00000000000..5791eefd710 --- /dev/null +++ b/src/newhelp/search_sqlite.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#ifndef SEARCH_SQLITE_H +#define SEARCH_SQLITE_H + +#define SEARCH_DOC_METRIC 1 +#define SEARCH_DOC_INDOM 2 +#define SEARCH_DOC_INST 3 + +extern int search_sqlite_open(const char *path); +extern void search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type); +extern int search_sqlite_close(void); + +#endif /* SEARCH_SQLITE_H */ diff --git a/src/newhelp/search_sqlite_stub.c b/src/newhelp/search_sqlite_stub.c new file mode 100644 index 00000000000..0566ca1ff76 --- /dev/null +++ b/src/newhelp/search_sqlite_stub.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * Stub routines when SQLite is not available. + */ +#include "search_sqlite.h" + +int +search_sqlite_open(const char *path) +{ + (void)path; + return -1; +} + +void +search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type) +{ + (void)name; (void)oneline; (void)helptext; (void)indom; (void)type; +} + +int +search_sqlite_close(void) +{ + return -1; +} diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index f23dc50dd0a..eab900b77ba 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -44,7 +44,7 @@ PYPMDAS = gluster zswap unbound haproxy \ uwsgi rocestat hdb rds chrony db2 SUBDIRS = $(CPMDAS) $(PLPMDAS) $(PYPMDAS) -LDIRT = local.conf pmcd.conf local.root +LDIRT = local.conf pmcd.conf local.root pmsearch.index ifneq ("$(wildcard $(TOPDIR)/src/include/pcp/config.h)","") HAVE_UNIX_DOMAIN_SOCKETS = $(shell grep 'define HAVE_STRUCT_SOCKADDR_UN' $(TOPDIR)/src/include/pcp/config.h >/dev/null && echo yes) @@ -94,5 +94,21 @@ install_pcp :: default_pcp $(INSTALL) -m 644 pmcd.conf $(PCP_PMCDCONF_PATH) $(INSTALL) -m 644 local.conf indom.conf $(PCP_ETC_DIR)/pcp $(INSTALL) -m 644 -t $(PCP_PMNS_DIR)/local.root local.root $(PCP_PMNSADM_DIR)/local.root +ifneq "$(CROSS_COMPILING)" "yes" +ifeq "$(HAVE_SQLITE3)" "true" + help_files=""; \ + for f in `find . -maxdepth 2 -name help -type f`; do \ + help_files="$$help_files $$f"; \ + done; \ + if [ -n "$$help_files" ]; then \ + $(TOPDIR)/src/newhelp/newhelp.static -S -o pmsearch.index $$help_files && \ + $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib && \ + $(INSTALL) -m 644 pmsearch.index $(PCP_SHARE_DIR)/lib/pmsearch.index; \ + rc=$$?; \ + rm -f pmsearch.index; \ + test $$rc -eq 0 || exit 1; \ + fi +endif +endif include $(BUILDRULES) diff --git a/src/pmproxy/src/search.c b/src/pmproxy/src/search.c index 4ff7de278b3..3826bc06dad 100644 --- a/src/pmproxy/src/search.c +++ b/src/pmproxy/src/search.c @@ -101,22 +101,8 @@ on_pmsearch_metrics(pmSearchMetrics *metrics, void *arg) baton->suffix = json_push_suffix(baton->suffix, JSON_FLAG_OBJECT); result = sdscatprintf(result, - "{\"docs\":%llu,\"terms\":%llu,\"records\":%llu," - "\"records_per_doc_avg\":%.2f," - "\"bytes_per_record_avg\":%.2f," - "\"inverted_sz_mb\":%.2f," - "\"inverted_cap_mb\":%.2f," - "\"inverted_cap_ovh\":%.2f," - "\"skip_index_size_mb\":%.2f," - "\"score_index_size_mb\":%.2f," - "\"offsets_per_term_avg\":%.2f," - "\"offset_bits_per_record_avg\":%.2f", - metrics->docs, metrics->terms, metrics->records, - metrics->records_per_doc_avg, metrics->bytes_per_record_avg, - metrics->inverted_sz_mb, metrics->inverted_cap_mb, - metrics->inverted_cap_ovh, metrics->skip_index_size_mb, - metrics->score_index_size_mb, metrics->offsets_per_term_avg, - metrics->offset_bits_per_record_avg); + "{\"docs\":%llu,\"terms\":%llu,\"records\":%llu", + metrics->docs, metrics->terms, metrics->records); http_set_buffer(client, result, HTTP_FLAG_JSON); http_transfer(client); diff --git a/src/pmsearch/GNUmakefile b/src/pmsearch/GNUmakefile index 21dad170274..038e0b30d1d 100644 --- a/src/pmsearch/GNUmakefile +++ b/src/pmsearch/GNUmakefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2017-2020 Red Hat. +# Copyright (c) 2017-2020,2026 Red Hat. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -15,23 +15,62 @@ TOPDIR = ../.. include $(TOPDIR)/src/include/builddefs +default: build-me + +ifeq "$(HAVE_SQLITE3)" "true" CFILES = pmsearch.c CMDTARGET = pmsearch$(EXECSUFFIX) LLDLIBS = $(PCP_WEBLIB) -LCFLAGS = $(LIBUVCFLAGS) +GENSCRIPT = crontab pmsearch_daily.service +LDIRT = $(GENSCRIPT) ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 endif -default: build-me +ifeq ($(TARGET_OS),linux) +CRONTAB_USER = $(PCP_USER) +CRONTAB_PATH = $(PCP_ETC_DIR)/cron.d/pcp-pmsearch +ifeq ($(ENABLE_SYSTEMD),true) +CRONTAB_INSTALL = $(PCP_SHARE_DIR)/cron.d/pcp-pmsearch +else +CRONTAB_INSTALL = $(CRONTAB_PATH) +endif +else +CRONTAB_USER = +CRONTAB_PATH = $(PCP_SYSCONF_DIR)/pmsearch/crontab +CRONTAB_INSTALL = $(CRONTAB_PATH) +endif -ifeq "$(HAVE_LIBUV)" "true" -build-me: $(CMDTARGET) +build-me: $(CMDTARGET) $(GENSCRIPT) install: default $(INSTALL) -m 755 $(CMDTARGET) $(PCP_BIN_DIR)/$(CMDTARGET) + $(INSTALL) -m 755 pmsearch_daily.sh $(PCP_BINADM_DIR)/pmsearch_daily $(INSTALL) -m 755 -d $(PCP_SYSCONF_DIR)/pmsearch $(INSTALL) -S $(PCP_SYSCONF_DIR)/pmproxy/pmproxy.conf $(PCP_SYSCONF_DIR)/pmsearch/pmsearch.conf +ifeq ($(ENABLE_SYSTEMD),true) + $(INSTALL) -m 644 pmsearch_daily.service $(PCP_SYSTEMDUNIT_DIR)/pmsearch_daily.service + $(INSTALL) -m 644 pmsearch_daily.timer $(PCP_SYSTEMDUNIT_DIR)/pmsearch_daily.timer +endif + $(INSTALL) -m 755 -d `dirname $(CRONTAB_INSTALL)` + $(INSTALL) -m 644 crontab $(CRONTAB_INSTALL) + +pmsearch_daily.service : pmsearch_daily.service.in + $(SED) <$< >$@ \ + -e 's;@CRONTAB_PATH@;'$(CRONTAB_PATH)';' \ + -e 's;@PCP_BINADM_DIR@;'$(PCP_BINADM_DIR)';' \ + -e 's;@PCP_VAR_DIR@;'$(PCP_VAR_DIR)';' \ + -e 's;@PCP_GROUP@;'$(PCP_GROUP)';' \ + -e 's;@PCP_USER@;'$(PCP_USER)';' \ + -e 's;@SD_SERVICE_TYPE@;'$(SD_SERVICE_TYPE)';' \ + # END + +crontab : crontab.in + $(SED) <$< >$@ \ + -e 's;@user@;'$(CRONTAB_USER)';' \ + -e 's;@path@;'$(PCP_BINADM_DIR)';' \ + # END + else build-me: install: diff --git a/src/pmsearch/crontab.in b/src/pmsearch/crontab.in new file mode 100644 index 00000000000..b75130b3ea3 --- /dev/null +++ b/src/pmsearch/crontab.in @@ -0,0 +1,5 @@ +# +# Performance Co-Pilot crontab entry for pmsearch index rebuild +# +# daily rebuild of the full-text search index +20 0 * * * @user@ @path@/pmsearch_daily diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index 2b553cd6278..5480104eb5d 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Red Hat. + * Copyright (c) 2020-2021,2026 Red Hat. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -13,7 +13,6 @@ */ #include "pmwebapi.h" -#include typedef enum search_flags { PMSEARCH_COLOUR = (1<<0), /* report in colour if possible */ @@ -32,7 +31,6 @@ typedef enum search_flags { #define ANSI_FG_CYAN "\x1b[36m" /* highlights and statistics */ typedef struct search_data { - uv_loop_t *loop; pmSearchSettings settings; pmSearchTextRequest request; search_flags flags; @@ -66,6 +64,7 @@ static void search_data_free(search_data *dp) { sdsfree(dp->request.query); + free(dp); } static void @@ -94,28 +93,10 @@ on_search_metrics(pmSearchMetrics *metrics, void *arg) on = off = ""; } - printf("RediSearch statistics:\n"); + printf("Search index statistics:\n"); printf(" Documents: %s%llu%s\n", on, metrics->docs, off); printf(" Terms: %s%llu%s\n", on, metrics->terms, off); printf(" Records: %s%llu%s\n", on, metrics->records, off); - printf("- Average records per doc: %s%.2f%s\n", - on, metrics->records_per_doc_avg, off); - printf("- Average bytes per record: %s%.2f%s\n", - on, metrics->bytes_per_record_avg, off); - printf("- Inverted Index\n"); - printf(" Size: %s%.2f MB%s\n", on, metrics->inverted_sz_mb, off); - printf(" Capacity: %s%.2f MB%s\n", on, metrics->inverted_cap_mb, off); - printf(" Overhead: %s%.2f%s\n", on, metrics->inverted_cap_ovh, off); - printf("- Skip Index\n"); - printf(" Size: %s%.2f MB%s\n", - on, metrics->skip_index_size_mb, off); - printf("- Score Index\n"); - printf(" Size: %s%.2f MB%s\n", - on, metrics->score_index_size_mb, off); - printf("- Average offsets per term: %s%.2f%s\n", - on, metrics->offsets_per_term_avg, off); - printf("- Average offset bits per record: %s%.2f%s\n", - on, metrics->offset_bits_per_record_avg, off); } /* @@ -126,7 +107,7 @@ static void printv(search_data *dp, const char *name, sds input) { const char *on, *off; - char *start, *end, *tmp; + char *start, *end, *tmp; sds value; if (input == NULL || input[0] == '\0') @@ -202,12 +183,17 @@ on_search_result(pmSearchTextResult *result, void *arg) printf("ID: %s\n", result->docid); if (dp->flags & PMSEARCH_SCORES) printf("Score: %.2f\n", result->score); - if (result->type != PM_SEARCH_TYPE_UNKNOWN) + if (result->type != PM_SEARCH_TYPE_UNKNOWN && + !(dp->flags & PMSEARCH_OPT_SUGGEST)) printf("Type: %s\n", pmSearchTextTypeStr(result->type)); if (result->name != NULL) printv(dp, "Name", result->name); - if (result->indom != NULL) - printv(dp, "InDom", result->indom); + if (!(dp->flags & PMSEARCH_OPT_SUGGEST)) { + if (result->indom != NULL && result->indom[0] != '\0') + printv(dp, "InDom", result->indom); + else if (result->type != PM_SEARCH_TYPE_UNKNOWN) + printv(dp, "InDom", "none"); + } if (result->oneline != NULL) printv(dp, "One line", result->oneline); if (result->helptext != NULL) @@ -218,21 +204,18 @@ static int pmsearch_overrides(int opt, pmOptions *opts) { switch (opt) { - case 'h': case 'n': case 'N': case 'O': case 'p': case 's': case 'S': case 't': case 'T': + case 'n': case 'N': case 'O': case 's': case 'S': case 't': case 'T': return 1; } return 0; } static pmLongOptions longopts[] = { - PMAPI_OPTIONS_HEADER("Connection Options"), - { "config", 1, 'c', "FILE", "configuration file path"}, - { "host", 1, 'h', "HOST", "connect to Redis using given host name" }, - { "port", 1, 'p', "PORT", "connect to Redis using given TCP/IP port" }, PMAPI_OPTIONS_HEADER("General Options"), + { "config", 1, 'c', "FILE", "configuration file path"}, { "no-colour", 0, 'C', 0, "no highlighting in results text" }, { "docid", 0, 'd', 0, "report document ID of each result" }, - { "info", 0, 'i', 0, "report search engine interal metrics" }, + { "info", 0, 'i', 0, "report search index metrics" }, { "indom", 0, 'n', 0, "perform an instance domain related entities search"}, { "number", 1, 'N', "N", "return N search results at most" }, { "offset", 1, 'O', "N", "paginated results from given offset" }, @@ -248,7 +231,7 @@ static pmLongOptions longopts[] = { }; static pmOptions opts = { - .short_options = "c:CdD:h:inN:O:qp:sStTvV?", + .short_options = "c:CdD:inN:O:qsStTvV?", .long_options = longopts, .short_usage = "[options] [query | indom]", .override = pmsearch_overrides, @@ -263,21 +246,21 @@ on_search_done(int sts, void *arg) if ((dp->flags & (PMSEARCH_OPT_QUERY | PMSEARCH_OPT_SUGGEST | PMSEARCH_OPT_INDOM)) && (dp->count == 0)) printf("0 search results\n"); - } else if (dp->flags & PMSEARCH_OPT_INFO) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchInfo", pmErrStr(sts)); - else if (dp->flags & PMSEARCH_OPT_QUERY) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextQuery", pmErrStr(sts)); - else if (dp->flags & PMSEARCH_OPT_SUGGEST) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextSuggest", pmErrStr(sts)); - else - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextIndom", pmErrStr(sts)); - - pmSearchClose(&dp->settings.module); - search_data_free(dp); + } else { + dp->status = 1; + if (dp->flags & PMSEARCH_OPT_INFO) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchInfo", pmErrStr(sts)); + else if (dp->flags & PMSEARCH_OPT_QUERY) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextQuery", pmErrStr(sts)); + else if (dp->flags & PMSEARCH_OPT_SUGGEST) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextSuggest", pmErrStr(sts)); + else + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextIndom", pmErrStr(sts)); + } } static void @@ -302,40 +285,14 @@ on_search_setup(void *arg) on_search_done(sts, arg); } -static void -pmsearch_request(uv_timer_t *arg) -{ - uv_handle_t *handle = (uv_handle_t *)arg; - search_data *dp = (search_data *)handle->data; - - pmSearchSetup(&dp->settings.module, dp); -} - -static int -pmsearch_execute(search_data *dp) -{ - uv_loop_t *loop = dp->loop; - uv_timer_t request; - uv_handle_t *handle = (uv_handle_t *)&request; - - handle->data = (void *)dp; - uv_timer_init(loop, &request); - uv_timer_start(&request, pmsearch_request, 0, 0); - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); - return dp->status; -} - int main(int argc, char *argv[]) { - sds option, query = NULL; - int c, sts, colour = 1; + sds query = NULL; + int c, colour = 1; unsigned int search_count = 0; unsigned int search_offset = 0; const char *inifile = NULL; - const char *keys_host = NULL; - unsigned int keys_port = 6379; /* default key server port */ search_flags flags = 0; search_data *dp; struct dict *config; @@ -355,10 +312,6 @@ main(int argc, char *argv[]) flags |= PMSEARCH_DOCIDS; break; - case 'h': /* key server host to connect to */ - keys_host = opts.optarg; - break; - case 'i': /* report search engine info (metrics) */ flags |= PMSEARCH_OPT_INFO; break; @@ -375,10 +328,6 @@ main(int argc, char *argv[]) search_offset = strtoul(opts.optarg, NULL, 10); break; - case 'p': /* keys server port to connect to */ - keys_port = (unsigned int)strtol(opts.optarg, NULL, 10); - break; - case 'q': /* command line contains query string */ flags |= PMSEARCH_OPT_QUERY; break; @@ -406,35 +355,15 @@ main(int argc, char *argv[]) } /* - * Parse the configuration file, extracting a dictionary of key/value - * pairs. Each key is "section.name" and values are always strings. - * If no config given, default is /etc/pcp/pmproxy.conf (in addition, - * local user path settings in $HOME/.pcp/pmproxy.conf are merged) - - * pmsearch(1) uses keys from the [pmsearch] and [pmseries] sections, - * to share the main pmproxy.conf file (via symlink) for convenience. + * Parse the configuration file for search index path and result + * count settings. Uses [pmsearch] section from pmproxy.conf or + * a dedicated pmsearch.conf file. */ if ((config = pmIniFileSetup(inifile)) == NULL) { - pmprintf("%s: cannot setup from configuration file %s\n", - pmGetProgname(), inifile? inifile : "pmsearch.conf"); - opts.errors++; - } else { - /* - * Push command line options into the configuration, and ensure - * we have some default for attemping Redis server connections. - */ - if ((pmIniFileLookup(config, "pmsearch", "count")) == NULL || - (search_count > 0)) { - option = sdscatfmt(sdsempty(), "%u", search_count); - pmIniFileUpdate(config, "pmsearch", "count", option); - } - - if ((option = pmIniFileLookup(config, "keys", "servers")) == NULL) - if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) - option = pmIniFileLookup(config, "pmseries", "servers"); - if (option == NULL || keys_host != NULL || keys_port != 6379) { - option = sdscatfmt(sdsempty(), "%s:%u", - keys_host? keys_host : "localhost", keys_port); - pmIniFileUpdate(config, "keys", "servers", option); + if (inifile) { + pmprintf("%s: cannot setup from configuration file %s\n", + pmGetProgname(), inifile); + opts.errors++; } } @@ -442,7 +371,7 @@ main(int argc, char *argv[]) opts.errors++; if (opts.errors || (opts.flags & PM_OPTFLAG_EXIT)) { - sts = !(opts.flags & PM_OPTFLAG_EXIT); + int sts = !(opts.flags & PM_OPTFLAG_EXIT); pmUsageMessage(&opts); exit(sts); } @@ -457,7 +386,6 @@ main(int argc, char *argv[]) query = sdsjoin(&argv[opts.optind], argc - opts.optind, " "); dp = search_data_init(flags, query, search_count, search_offset); - dp->loop = uv_default_loop(); dp->settings.callbacks.on_text_result = on_search_result; dp->settings.callbacks.on_metrics = on_search_metrics; @@ -466,8 +394,23 @@ main(int argc, char *argv[]) dp->settings.module.on_info = on_search_info; dp->settings.module.on_setup = on_search_setup; - pmSearchSetEventLoop(&dp->settings.module, dp->loop); - pmSearchSetConfiguration(&dp->settings.module, config); + if (config) + pmSearchSetConfiguration(&dp->settings.module, config); + + if ((c = pmSearchSetup(&dp->settings.module, dp)) < 0) { + if (c == -ENOTSUP) + fprintf(stderr, "%s: search is disabled in configuration\n", + pmGetProgname()); + else + fprintf(stderr, "%s: search setup failed: %s\n", + pmGetProgname(), pmErrStr(c)); + search_data_free(dp); + return 1; + } - return pmsearch_execute(dp); + pmSearchClose(&dp->settings.module); + + c = dp->status; + search_data_free(dp); + return c; } diff --git a/src/pmsearch/pmsearch_daily.service.in b/src/pmsearch/pmsearch_daily.service.in new file mode 100644 index 00000000000..0437b5bb20a --- /dev/null +++ b/src/pmsearch/pmsearch_daily.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=Daily rebuild of the PCP pmsearch full-text index +Documentation=man:pmsearch_daily(1) +ConditionPathExists=!@CRONTAB_PATH@ + +[Service] +Type=@SD_SERVICE_TYPE@ +Restart=no +TimeoutStartSec=1h +ExecStart=@PCP_BINADM_DIR@/pmsearch_daily +WorkingDirectory=@PCP_VAR_DIR@ +Group=@PCP_GROUP@ +User=@PCP_USER@ diff --git a/src/pmsearch/pmsearch_daily.sh b/src/pmsearch/pmsearch_daily.sh new file mode 100755 index 00000000000..e6dd1cc5756 --- /dev/null +++ b/src/pmsearch/pmsearch_daily.sh @@ -0,0 +1,245 @@ +#!/bin/sh +# +# Copyright (c) 2026 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# Rebuild the pmsearch full-text search index from running PMCD +# or a PCP archive. Typically invoked nightly via systemd timer +# or cron. +# + +. $PCP_DIR/etc/pcp.env + +status=1 +tmp=`mktemp -d "$PCP_TMPFILE_DIR/pmsearch_daily.XXXXXXXXX"` || exit 1 +trap "rm -rf \"$tmp\" \"\$NEW\"; exit \$status" 0 1 2 3 15 + +prog=`basename $0` +INDEX="$PCP_VAR_DIR/lib/pmsearch.index" + +cat > $tmp/usage << EOF +Options: + -a=ARCHIVE,--archive=ARCHIVE use archive instead of running PMCD + -N,--showme dry-run, show what would be done + -o=INDEX,--output=INDEX output index file path [default: $INDEX] + -V,--verbose verbose diagnostics + --help +EOF + +_usage() +{ + pmgetopt --progname=$prog --config=$tmp/usage --usage + exit 1 +} + +ARCHIVE="" +VERBOSE=false +SHOWME=false + +ARGS=`pmgetopt --progname=$prog --config=$tmp/usage -- "$@"` || exit 1 +eval set -- "$ARGS" +while [ $# -gt 0 ] +do + case "$1" + in + -a) ARCHIVE="$2"; shift ;; + -N) SHOWME=true ;; + -o) INDEX="$2"; shift ;; + -V) VERBOSE=true ;; + -\?) _usage ;; + --) shift; break ;; + esac + shift +done + +# Build the pminfo/pmprobe flags for archive or live mode +if [ -n "$ARCHIVE" ] +then + SRCFLAGS="-a $ARCHIVE" +else + SRCFLAGS="" +fi + +if $VERBOSE +then + echo "$prog: index target: $INDEX" +fi + +# Build metric→indom mapping first (needed for both metrics and instances). +pminfo $SRCFLAGS -d 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z]/ { metric = $1 } +/InDom:/ && !/PM_INDOM_NULL/ { + for (i = 1; i <= NF; i++) { + if ($i == "InDom:") { + print metric "\t" $(i+1) + break + } + } +}' > $tmp/metric_indom + +# Extract all metric help text in newhelp format. +# Uses "@ M nameindomoneline" for metrics with indoms, +# so newhelp -S can resolve indoms without needing PMNS/PMCD. +pminfo $SRCFLAGS -tT 2>/dev/null | $PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} +/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { + if (index($0, "One-line Help: Error:") > 0) { + skip = 1 + next + } + skip = 0 + name = $1 + oneline = "" + start = index($0, "[") + end = index($0, "]") + if (start > 0 && end > start) { + oneline = substr($0, start + 1, end - start - 1) + } + if (name in metric_indom) + printf "@ M %s\t%s\t%s\n", name, metric_indom[name], oneline + else + printf "@ M %s\t\t%s\n", name, oneline + next +} +/^Help:$/ { next } +/^Full Help: Error:/ { skip = 1; next } +skip { next } +{ + if (substr($0, 1, 1) == "@") + printf " %s\n", $0 + else + print +} +' "$tmp/metric_indom" - > $tmp/helptext + +nhelplines=`wc -l < $tmp/helptext` +if [ "$nhelplines" -eq 0 ] +then + echo >&2 "$prog: warning: no metrics found" + status=0 + exit +fi + +if $VERBOSE +then + nmetrics=`grep -c '^@ ' $tmp/helptext` + echo "$prog: extracted $nmetrics metrics ($nhelplines lines)" +fi + +# Extract instance names. +# Deduplicate by (indom, instance_name) since many metrics share indoms. +pmprobe $SRCFLAGS -I 2>/dev/null > $tmp/pmprobe_output + +$PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} +{ + metric = $1 + count = $2 + 0 + if (count <= 0 || $2 == "PM_IN_NULL") next + + indom = metric_indom[metric] + if (indom == "") next + + rest = $0 + while (match(rest, /"[^"]*"/)) { + inst = substr(rest, RSTART + 1, RLENGTH - 2) + key = indom SUBSEP inst + if (!(key in seen)) { + seen[key] = 1 + inst_data[++ninst] = indom "\t" inst + } + rest = substr(rest, RSTART + RLENGTH) + } +} +END { + for (i = 1; i <= ninst; i++) + printf "@ I %s\n\n", inst_data[i] +} +' "$tmp/metric_indom" "$tmp/pmprobe_output" >> $tmp/helptext + +if $VERBOSE +then + ninst=`grep -c '^@ I ' $tmp/helptext` + echo "$prog: extracted $ninst instances" +fi + +if $SHOWME +then + echo "$prog: would copy $PCP_SHARE_DIR/lib/pmsearch.index to $INDEX and add runtime data" + status=0 + exit +fi + +# Ensure output directory exists +outdir=`dirname "$INDEX"` +if [ ! -d "$outdir" ] +then + mkdir -p "$outdir" 2>/dev/null + if [ ! -d "$outdir" ] + then + echo >&2 "$prog: cannot create directory $outdir" + exit + fi +fi + +# Start from the build-time base index (contains all metric help text). +# The nightly update copies it, then adds/updates runtime data. +NEW="$INDEX.new.$$" +BASE="$PCP_SHARE_DIR/lib/pmsearch.index" +if [ -f "$BASE" ] +then + cp "$BASE" "$NEW" || exit + if $VERBOSE + then + echo "$prog: copied base index from $BASE" + fi +fi + +nentries=`grep -c '^@ ' $tmp/helptext` +if [ "$nentries" -eq 0 ] +then + if $VERBOSE + then + echo "$prog: no runtime data to add" + fi + status=0 + exit +fi + +# Add/update all runtime data (metrics and instances) into the index. +# Metrics already in the base index are updated; new metrics from +# PMDAs without help files (e.g. Python PMDAs) are inserted. +if $PCP_BINADM_DIR/newhelp -S -o "$NEW" $tmp/helptext +then + if mv -f "$NEW" "$INDEX" + then + if $VERBOSE + then + nmetrics=`grep -c '^@ [^I]' $tmp/helptext` + ninst=`grep -c '^@ I ' $tmp/helptext` + echo "$prog: added $nmetrics metrics and $ninst instances to $INDEX" + fi + status=0 + else + echo >&2 "$prog: cannot move $NEW to $INDEX" + fi +else + echo >&2 "$prog: $PCP_BINADM_DIR/newhelp -S failed" +fi diff --git a/src/pmsearch/pmsearch_daily.timer b/src/pmsearch/pmsearch_daily.timer new file mode 100644 index 00000000000..4ad374ba92d --- /dev/null +++ b/src/pmsearch/pmsearch_daily.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Daily rebuild of the PCP pmsearch full-text index +PartOf=pmcd.service + +[Timer] +Persistent=true +# if enabled, runs 10m after boot and just after midnight (and after +# the daily local PCP archive processing at 00:10:00 and proxy +# processing at 00:15:00) +OnBootSec=10min +OnCalendar=*-*-* 00:20:00 +Unit=pmsearch_daily.service + +[Install] +WantedBy=timers.target +WantedBy=pmproxy.service