From 64ebbc33859766514ed48b6a0865c76dddcd6fcb Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 24 Jul 2026 12:27:49 -0400 Subject: [PATCH 01/22] pmsearch: reimplement with SQLite FTS5 replacing RediSearch/Valkey Replace the RediSearch/Valkey dependency with a local SQLite FTS5 full-text search index. This makes pmsearch work out of the box without requiring any external services. Build system: add SQLite3 detection via pkg-config with fallback to AC_CHECK_LIB (configure.ac, builddefs.in). Index builder: add newhelp -S mode that parses help text files and builds a SQLite FTS5 database with porter/unicode61 tokenization, BM25 scoring weights, and prefix indexes (search_sqlite.c/.h, newhelp.c). Search engine: rewrite search.c to query local SQLite indexes instead of Redis. Dual-index strategy: nightly index at $PCP_VAR_DIR/lib/pcp.search (instance names + third-party PMDAs) and build-time index at $PCP_SHARE_DIR/lib/pcp.search (all shipped PMDA help text). Both are queried together via ATTACH DATABASE. CLI: remove libuv dependency from pmsearch, make it synchronous. Remove -h/-p host/port options (no remote server needed). Build-time index: generate from all PMDA help files during make install in src/pmdas/GNUmakefile. Nightly index: add pmsearch_index.sh script with systemd timer and cron fallback for daily index rebuilds capturing runtime instance names. Man pages: update pmsearch.1, pmsearchsetup.3, pmsearchinfo.3, pmsearchtextsuggest.3 to describe local SQLite index. Add new pmsearch_index.1 man page. --- configure.ac | 16 + man/man1/pmsearch.1 | 78 +- man/man1/pmsearch_index.1 | 104 ++ man/man3/pmsearchinfo.3 | 11 +- man/man3/pmsearchsetup.3 | 57 +- man/man3/pmsearchtextsuggest.3 | 13 +- src/include/builddefs.in | 2 + src/libpcp_web/src/GNUmakefile | 1 + src/libpcp_web/src/search.c | 1575 ++++++++++-------------- src/libpcp_web/src/search.h | 92 +- src/newhelp/GNUmakefile | 10 +- src/newhelp/newhelp.c | 166 ++- src/newhelp/search_sqlite.c | 131 ++ src/newhelp/search_sqlite.h | 26 + src/pmdas/GNUmakefile | 10 + src/pmsearch/GNUmakefile | 46 +- src/pmsearch/crontab.in | 5 + src/pmsearch/pmsearch.c | 126 +- src/pmsearch/pmsearch_index.service.in | 10 + src/pmsearch/pmsearch_index.sh | 216 ++++ src/pmsearch/pmsearch_index.timer | 11 + 21 files changed, 1482 insertions(+), 1224 deletions(-) create mode 100644 man/man1/pmsearch_index.1 create mode 100644 src/newhelp/search_sqlite.c create mode 100644 src/newhelp/search_sqlite.h create mode 100644 src/pmsearch/crontab.in create mode 100644 src/pmsearch/pmsearch_index.service.in create mode 100755 src/pmsearch/pmsearch_index.sh create mode 100644 src/pmsearch/pmsearch_index.timer diff --git a/configure.ac b/configure.ac index 307754212d2..f477ab1fad4 100644 --- a/configure.ac +++ b/configure.ac @@ -1996,6 +1996,22 @@ AC_CHECK_LIB(inih, ini_parse, AC_SUBST(HAVE_LIBINIH, [$have_libinih]) AC_SUBST(lib_for_inih) +PKG_CHECK_MODULES([sqlite3], [sqlite3 >= 3.9.0], [have_sqlite3=true], [have_sqlite3=false]) +if test $have_sqlite3 = false; then + AC_CHECK_LIB(sqlite3, sqlite3_open, + [ have_sqlite3=true + lib_for_sqlite3="-lsqlite3" + ], + [ have_sqlite3=false + lib_for_sqlite3="" + ] + ) +else + lib_for_sqlite3="$sqlite3_LIBS" +fi +AC_SUBST(HAVE_SQLITE3, [$have_sqlite3]) +AC_SUBST(lib_for_sqlite3) + PKG_CHECK_MODULES([libnl3], [libnl-3.0], [have_delayacct=true], [have_delayacct=false]) AC_SUBST(HAVE_DELAYACCT, [$have_delayacct]) diff --git a/man/man1/pmsearch.1 b/man/man1/pmsearch.1 index d20f05c48b1..e9f1e18899a 100644 --- a/man/man1/pmsearch.1 +++ b/man/man1/pmsearch.1 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020,2024 Red Hat. +.\" Copyright (c) 2020,2024,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -21,8 +21,6 @@ [\fB\-CdinqsStTV?\fR] [\fB\-c\fR \fIconfig\fR] [\f3\-D\f1 \f2debug\f1] -[\fB\-h\fR \fIhost\fR] -[\fB\-p\fR \fIport\fR] [\fB\-N\fR \fInumber\fR] [\fB\-O\fR \fIoffset\fR] [\fIquery\fR | \fIindom\fR] @@ -41,38 +39,16 @@ performs full text search queries to find metrics using names and help text from metrics, instance domains and instances. .PP -It makes use of capabilities of the Performance Co-Pilot (PCP) -.BR pmproxy (1) -service, the Valkey distributed key-value store and associated -ValkeySearch module. +It uses a local SQLite FTS5 full-text search index that is +built at package install time and updated nightly by +.BR pmsearch_index (1). +The build-time index contains help text for all PCP metrics +from the source tree. +The nightly index adds instance names (disk devices, network +interfaces, etc.) and metrics from any third-party PMDAs +installed on the system. .PP -Note that in order to use these services, it is -.B mandatory -that -.B pmproxy -is communicating with a Valkey key-value server -that has the -.I valkey-search.so -module loaded. -When configured to do so, -.B pmproxy -will then automatically index PCP metric names, instance names, -metric and instance domain help text into the ValkeySearch store, -from PCP archives that it discovers locally. -Refer to -.BR pmlogger (1) -and -.BR pmlogger_daily (1) -for further details. -.PP -By default -.B pmsearch -communicates with a local key-value server -however the \fB\-h\fR and \fB\-p\fR options can be used to -specify an alternate Valket instance. -If this instance is a node of a Valkey cluster, all -other instances in the cluster will be discovered -and used automatically. +No external services (such as Redis or Valkey) are required. .SH OPTIONS .TP 5 \fB\-c\fR \fIconfig\fR, \fB\-\-config\fR=\fIconfig\fR @@ -85,22 +61,15 @@ No highlighting in results text. .TP \fB\-d\fR, \fB\-\-docid\fR Report document ID of each search result. -Documents are identified using a unique SHA-1 hash which -is always displayed in a 40-hexdigit human readable form. -.TP -\fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR -Connect to the key-value server at -.IR host , -rather than the one the localhost. .TP \fB\-i\fR, \fB\-\-info\fR -Report search engine internal metrics. +Report search index metrics (document count). .TP \fB\-n\fR, \fB\-\-indom\fR Perform an .IR instance .IR domain -related entities search +related entities search. .TP \fB\-N\fR \fIN\fR, \fB\-\-number\fR=\fIN\fR Return @@ -110,23 +79,17 @@ search results at most. \fB\-O\fR \fIN\fR, \fB\-\-offset\fR=\fIN\fR Paginated results from given offset. .TP -\fB\-p\fR \fIport\fR, \fB\-\-port\fR=\fIport\fR -Connect to the key-value server at -.IR port , -rather than the default -.BR 6379 . -.TP \fB\-q\fR, \fB\-\-query\fR Perform a general text search (default). .TP \fB\-s\fR, \fB\-\-suggest\fR -Perform a name suggestion search +Perform a name suggestion search. .TP \fB\-S\fR, \fB\-\-score\fR Report score (rank) of each result. .TP \fB\-T\fR, \fB\-\-total\fR -Report total number of of search results. +Report total number of search results. .TP \fB\-t\fR, \fB\-\-times\fR Report elapsed search execution time. @@ -152,6 +115,16 @@ Name: \fBkernel\f1.percpu.cpu.\fBidle\fR InDom: 60.0 One line: percpu \fBidle\fR CPU time metric from /proc/stat .ESAMPLE +.SH FILES +.TP +.I $PCP_VAR_DIR/lib/pcp.search +Nightly search index (SQLite FTS5 database), rebuilt by +.BR pmsearch_index (1). +.TP +.I $PCP_SHARE_DIR/lib/pcp.search +Build-time search index containing all metrics from the PCP source tree. +Used as a fallback when the nightly index is not yet available, and to +fill in help text for PMDAs not installed on the running system. .SH PCP ENVIRONMENT Environment variables with the prefix \fBPCP_\fP are used to parameterize the file and directory names used by PCP. @@ -179,11 +152,12 @@ option to obtain a list of the available debugging options and their meaning. .SH SEE ALSO .BR PCPIntro (1), +.BR pmsearch_index (1), .BR pmproxy (1), .BR pmlogger (1) and .BR pmlogger_daily (1). .\" control lines for scripts/man-spell -.\" +ok+ SHA Valkey ValkeySearch hexdigit idletime +.\" +ok+ idletime .\" +ok+ stat {from /proc/stat} diff --git a/man/man1/pmsearch_index.1 b/man/man1/pmsearch_index.1 new file mode 100644 index 00000000000..dfd97e1dee4 --- /dev/null +++ b/man/man1/pmsearch_index.1 @@ -0,0 +1,104 @@ +'\"macro stdmacro +.\" +.\" Copyright (c) 2026 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PMSEARCH_INDEX 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pmsearch_index\f1 \- rebuild the PCP full-text search index +.SH SYNOPSIS +\fBpmsearch_index\fR +[\fB\-NV?\fR] +[\fB\-o\fR \fIindex\fR] +.SH DESCRIPTION +.B pmsearch_index +rebuilds the runtime full-text search index used by +.BR pmsearch (1) +and the +.BR pmproxy (1) +.I /search +REST API endpoint. +.PP +Its primary purpose is to capture +.B "instance names" +(disk devices, network interfaces, filesystems, etc.) +that are only known on a running system. +It queries the running +.BR pmcd (1) +for all metric names, help text, and instance domain members, +then builds a SQLite FTS5 search index using +.BR newhelp (1). +The resulting index file is written to +.I $PCP_VAR_DIR/lib/pcp.search +by default. +.PP +The search engine uses +.B both +the runtime index and the base index shipped with the PCP package. +The runtime index takes precedence for any entries it contains; +the base index fills in metrics from PMDAs that are not installed +on the current system. +.PP +This command is typically invoked nightly via a systemd timer +or cron job. +.PP +If +.B pmcd +is not running or no metrics are found, the command exits +without modifying any existing index file. +The base index at +.I $PCP_SHARE_DIR/lib/pcp.search +remains available as a fallback. +.SH OPTIONS +.TP 5 +\fB\-N\fR, \fB\-\-showme\fR +Dry-run mode. +Show what would be done without writing any files. +.TP +\fB\-o\fR \fIindex\fR, \fB\-\-output\fR=\fIindex\fR +Write the index to +.IR index +instead of the default +.IR $PCP_VAR_DIR/lib/pcp.search . +.TP +\fB\-V\fR, \fB\-\-verbose\fR +Verbose diagnostics. +Report the number of metrics extracted and the output path. +.TP +\fB\-?\fR, \fB\-\-help\fR +Display usage message and exit. +.SH FILES +.TP 5 +.I $PCP_VAR_DIR/lib/pcp.search +Default output path for the runtime search index. +When present, this index is preferred over the base index. +.TP +.I $PCP_SHARE_DIR/lib/pcp.search +Base search index shipped with the PCP package, +built at package build time from help text source files. +Used as a fallback when the runtime index does not exist. +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.SH SEE ALSO +.BR PCPIntro (1), +.BR newhelp (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmsearch (1) +and +.BR pmproxy (1). diff --git a/man/man3/pmsearchinfo.3 b/man/man3/pmsearchinfo.3 index 4c27b1f1c15..c7f5b7ce360 100644 --- a/man/man3/pmsearchinfo.3 +++ b/man/man3/pmsearchinfo.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -33,13 +33,14 @@ cc ... \-lpcp_web .ad .ft 1 .SH DESCRIPTION -Diagnostic metrics related to the Performance Co-Pilot (PCP) fast, scalable and full-text capable search services. +Provides diagnostic information about the Performance Co-Pilot (PCP) +full-text search index. .PP -Currently, as all full-text related services are using single ValkeySearch index, diagnostic metrics are requestable by passing in +Diagnostic metrics are requestable by passing in .IR key with a value of .BR `text' . -The results will passed to +The results (such as document count) will be passed to the .B on_metrics callback provided by .B settings. @@ -55,5 +56,5 @@ On failure a negative PMAPI error code is returned. and .BR PMWEBAPI (3). .\" control lines for scripts/man-spell -.\" +ok+ ValkeySearch on_metrics {from callback} +.\" +ok+ on_metrics {from callback} .\" +ok+ sds {from typedef char *sds in pcp/sds.h} diff --git a/man/man3/pmsearchsetup.3 b/man/man3/pmsearchsetup.3 index 7d24aafb501..04b733e9bee 100644 --- a/man/man3/pmsearchsetup.3 +++ b/man/man3/pmsearchsetup.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -52,28 +52,45 @@ cc ... \-lpcp_web .SH DESCRIPTION The .B pmSearchSetup -and related API functions prepare an application for accessing the fast and scalable metric, instance and instance domain, full-text capable, searching functionality of the Performance Co-Pilot (PCP). +and related API functions prepare an application for accessing the +full-text search functionality of the Performance Co-Pilot (PCP). .PP -This functionality is provided trough asynchronous APIs, which function in event-driven fashion. The interface described here prepare a given search +The search engine uses a local SQLite FTS5 full-text index. +A build-time index at +.I $PCP_SHARE_DIR/lib/pcp.search +provides help text for all metrics shipped with PCP. +A nightly index at +.I $PCP_VAR_DIR/lib/pcp.search +(rebuilt by +.BR pmsearch_index (1)) +adds instance names and metrics from third-party PMDAs. +When both indexes exist, they are queried together; +the nightly index takes precedence for entries it contains. +.PP +The interfaces described here prepare a given search .IR module -which associates callback routines with certain asynchronous events that occur as part of servicing search requests. +which associates callback routines with events that occur +as part of servicing search requests. .PP As a general pattern, all interfaces in these APIs that need to invoke .I callbacks provided by the calling program will take an opaque (void * pointer) .IR arg parameter. -This pointer will be passed through unchanged and is typically used to access a data structure maintaining state within the calling program. +This pointer will be passed through unchanged and is typically used to +access a data structure maintaining state within the calling program. .PP -Where asynchronous setup is required by +On completion of .B pmSearchSetup -its completion will be indicated through use of the +the .I on_setup -callback, which is part of the passed in +callback will be invoked. +This callback is part of the passed in .I pmSearchModule -structure. This structure also provides for custom diagnostics handling, through the +structure, which also provides for custom diagnostics handling through the .I on_info -callback. These are self-explanatory, see +callback. +See .I for exact calling conventions. .PP @@ -103,15 +120,16 @@ failure (negative PMAPI code) of the operation. .PP The helper functions .B pmSearchSetSlots -(key-value server), +and .B pmSearchSetEventLoop -(libuv), +are retained for API compatibility but have no effect. .B pmSearchSetConfiguration (configuration file) and .B pmSearchSetMetricRegistry (MMV instrumentation) -interfaces provide a mechanism for passing in state for each of the associated subsystems. +provide a mechanism for passing in state for each of the +associated subsystems. .PP Finally, a call to .B pmSearchClose @@ -128,10 +146,19 @@ utilities are the primary users of the interfaces, providing REST API and comman .SH DIAGNOSTICS Where these functions return a status code, this is always zero on success. On failure a negative PMAPI error code is returned. +.SH FILES +.TP +.I $PCP_VAR_DIR/lib/pcp.search +Nightly search index (SQLite FTS5 database), rebuilt by +.BR pmsearch_index (1). +.TP +.I $PCP_SHARE_DIR/lib/pcp.search +Build-time search index shipped with the PCP package. .SH SEE ALSO +.BR pmsearch (1), +.BR pmsearch_index (1), .BR pmproxy (1), .BR pmlogger (1), -.BR pmsearch (1), .BR mmv_stats_registry (3), .BR pmSearchInfo (3), .BR pmSearchTextInDom (3), @@ -144,4 +171,4 @@ and .\" control lines for scripts/man-spell .\" +ok+ on_text_result on_metrics on_setup on_done on_info {from callbacks} .\" +ok+ pmwebapi {from include } -.\" +ok+ libuv dict {from struct dict} +.\" +ok+ dict {from struct dict} diff --git a/man/man3/pmsearchtextsuggest.3 b/man/man3/pmsearchtextsuggest.3 index 1a4c8427e8c..1974de0879b 100644 --- a/man/man3/pmsearchtextsuggest.3 +++ b/man/man3/pmsearchtextsuggest.3 @@ -1,6 +1,6 @@ '\"macro stdmacro .\" -.\" Copyright (c) 2020 Red Hat. +.\" Copyright (c) 2020,2026 Red Hat. .\" .\" This program is free software; you can redistribute it and/or modify it .\" under the terms of the GNU General Public License as published by the @@ -42,11 +42,12 @@ Query string that will be used to search. \fIcount\fR Limits number of results. Defaults to 10. .PP -Words with length less than 2 are omitted. -.PP -This search matches terms inside entity name that either have +This search performs prefix matching on entity names using +FTS5 prefix indexes. +Terms in .B query -prefix or are within Levenshtein distance of 1, with former having slightly higher priority than the latter. +are appended with a wildcard so that partial names +match (e.g.\& \f(CWker\fP matches \f(CWkernel\fP). .PP Each entity matched by the .B request @@ -68,4 +69,4 @@ and .\" control lines for scripts/man-spell .\" +ok+ on_text_result {from callback} -.\" +ok+ Levenshtein {from Levenshtein distance} +.\" +ok+ ker {from example prefix} diff --git a/src/include/builddefs.in b/src/include/builddefs.in index d88c24ec1db..7db05295f87 100644 --- a/src/include/builddefs.in +++ b/src/include/builddefs.in @@ -316,6 +316,8 @@ HAVE_LIBUV = @HAVE_LIBUV@ LIB_FOR_LIBUV = @libuv_LIBS@ HAVE_LIBINIH = @HAVE_LIBINIH@ LIB_FOR_INIH = @lib_for_inih@ +HAVE_SQLITE3 = @HAVE_SQLITE3@ +LIB_FOR_SQLITE3 = @lib_for_sqlite3@ HAVE_LIBELF = @HAVE_LIBELF@ LIB_FOR_LIBELF = @libelf_LIBS@ HAVE_NCURSES = @HAVE_NCURSES@ diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index d032e521b75..3424518090e 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -45,6 +45,7 @@ XFILES = jsmn.c jsmn.h http_parser.c http_parser.h \ sha1.c sha1.h siphash.c LLDLIBS = $(PCPWEBLIB_EXTRAS) $(LIB_FOR_MATH) $(LIB_FOR_REGEX) \ + $(LIB_FOR_SQLITE3) \ $(TOPDIR)/src/libvalkey/src/libvalkey.a ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 6dfd78db984..056e5238183 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -1,5 +1,6 @@ /* * Copyright (c) 2020-2022,2024 Red Hat. + * Copyright (c) 2026 Red Hat. * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -10,39 +11,29 @@ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public * License for more details. + * + * SQLite FTS5 search backend for PCP metric/instance full-text search. */ #include +#include #include "pmapi.h" -#include "pmda.h" -#include "schema.h" +#include "libpcp.h" #include "search.h" -#include "util.h" -#include "sha1.h" -static sds resultcount_str; -static sds DEFAULT_RESULTCOUNT; -static unsigned int resultcount; /* converted resultcount_str */ +#define SEARCH_DOC_METRIC 1 +#define SEARCH_DOC_INDOM 2 +#define SEARCH_DOC_INST 3 -static void -initKeysSearchBaton(keysSearchBaton *baton, keySlots *slots, - pmSearchSettings *settings, void *userdata) -{ - initSeriesBatonMagic(baton, MAGIC_SEARCH); - baton->callbacks = &settings->callbacks; - baton->info = settings->module.on_info; - baton->slots = slots; - baton->module = &settings->module; - baton->userdata = userdata; - pmtimespecNow(&baton->started); -} +static int search_enabled; +static unsigned int default_resultcount = 10; -static void -doneKeysSearchBaton(keysSearchBaton *baton) -{ - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "doneKeysSearchBaton"); - memset(baton, 0, sizeof(keysSearchBaton)); - free(baton); -} +typedef struct searchModuleData { + sqlite3 *db; + int has_base; + struct dict *config; + unsigned int loaded; + unsigned int resultcount; +} searchModuleData; const char * pmSearchTextTypeStr(pmSearchTextType type) @@ -60,1066 +51,770 @@ pmSearchTextTypeStr(pmSearchTextType type) return "unknown"; } +static searchModuleData * +getSearchModuleData(pmSearchModule *module) +{ + if (module->privdata == NULL) + module->privdata = calloc(1, sizeof(searchModuleData)); + return (searchModuleData *)module->privdata; +} + +/* + * Build an FTS5 MATCH expression from the request. + * If infields are restricted, wrap the query in column filters. + * On FTS5 syntax error, fall back to quoting the query as a phrase. + */ static sds -keys_search_docid(const char *key, const char *type, const char *name) +search_build_match(pmSearchTextRequest *request) { - unsigned char hash[20]; - sds docid = sdsempty(); + sds match; + int restricted = 0; - docid = sdscatfmt(docid, "\"key\":\"%s\",\"type\":\"%s\",name:\"%s\"", - key, type, name); - pmwebapi_search_hash(hash, docid, sdslen(docid)); - return pmwebapi_hash_sds(docid, hash); -} + if (request->infields_name || request->infields_oneline || + request->infields_helptext) { + int count = request->infields_name + request->infields_oneline + + request->infields_helptext; + if (count < 3) + restricted = 1; + } + if (restricted) { + sds cols = sdsempty(); + int first = 1; + + if (request->infields_name) { + cols = sdscat(cols, "name"); + first = 0; + } + if (request->infields_oneline) { + if (!first) + cols = sdscat(cols, " "); + cols = sdscat(cols, "oneline"); + first = 0; + } + if (request->infields_helptext) { + if (!first) + cols = sdscat(cols, " "); + cols = sdscat(cols, "helptext"); + } + match = sdscatfmt(sdsempty(), "{%S} : %S", cols, request->query); + sdsfree(cols); + } else { + match = sdsnew(request->query); + } + + return match; +} /* - * This issue isn't fixed in version of search module we are using, - * https://github.com/RediSearch/RediSearch/issues/748 + * Build the type filter clause for SQL queries. + * Returns an sds string like " AND type IN (1,3)" or empty string. */ -static int -keys_search_is_stopword(sds s) +static sds +search_type_filter(pmSearchTextRequest *request) { - size_t i; - static const char* const stopwords[] = { - "a", "is", "the", "an", "and", "are", "as", "at", "be", "but", "by", "for", - "if", "in", "into", "it", "no", "not", "of", "on", "or", "such", "that", "their", - "then", "there", "these", "they", "this", "to", "was", "will", "with", - }; - - for (i = 0; i < sizeof(stopwords) / sizeof(stopwords[0]); i++) { - if (strcmp(s, stopwords[i]) == 0) - return 1; + sds filter = sdsempty(); + int any = request->type_metric + request->type_indom + + request->type_inst; + + if (any == 0 || any == 3) + return filter; + + filter = sdscat(filter, " AND type IN ("); + any = 0; + if (request->type_metric) { + filter = sdscatfmt(filter, "%i", SEARCH_DOC_METRIC); + any = 1; } - return 0; + if (request->type_indom) { + if (any) + filter = sdscat(filter, ","); + filter = sdscatfmt(filter, "%i", SEARCH_DOC_INDOM); + any = 1; + } + if (request->type_inst) { + if (any) + filter = sdscat(filter, ","); + filter = sdscatfmt(filter, "%i", SEARCH_DOC_INST); + } + filter = sdscat(filter, ")"); + return filter; } /* - * Tokenizes text by delimiters described in search module Escaping.html doc + - * ('/' - this one giving troubles to prefix search), - * omits tokens of length greated then *min_length* (0 = don't omit any), optionally prepends *prefix* - * and appends *suffix* (NULL doesnt get prepended / appended), joins tokens by space back into new sds. - * Ideally we would just escape delimtiers with double backslash, - * unfortunately, this doesnt seem to work when searching for indoms and buch of other inconsitencies + * Build a docid string from rowid and name. */ static sds -keys_search_text_prep(sds s, int min_length, char *prefix, char *suffix) +search_make_docid(sqlite3_int64 rowid, const char *name) { - static const char *delimiters = ",.<>{}[]\"\':;!@#$%^&*()-+=~/"; - size_t len = sdslen(s); - size_t i, j; - sds result = sdsempty(); - sds formatted_result; - sds* tokens; - int token_count, non_digit_found; - - for (i = 0; i < len; i++) { - const char *is_found = strchr(delimiters, s[i]); - if (is_found != NULL) { - result = sdscat(result, " "); - } else { - result = sdscatlen(result, &s[i], 1); + char buf[32]; + + pmsprintf(buf, sizeof(buf), "%lld", (long long)rowid); + return sdscatfmt(sdsempty(), "%s:%s", buf, name ? name : ""); +} + +/* + * Execute a text query against a single docs table (qualified name). + * Appends results to the hits array; caller owns the array. + */ +static int +search_query_table(sqlite3 *db, const char *table, + pmSearchTextRequest *request, sds match_expr, sds type_filter, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdscatfmt(sdsempty(), + "SELECT rowid, name, oneline, helptext, type, indom," + " bm25(%s, 9.0, 4.0, 2.0)," + " highlight(%s, 0, '', '')," + " highlight(%s, 1, '', '')," + " highlight(%s, 2, '', '')" + " FROM %s WHERE %s MATCH ?%S" + " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", + table, table, table, table, + table, table, type_filter, + table); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_query_table: prepare: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } + + sqlite3_bind_text(stmt, 1, match_expr, sdslen(match_expr), SQLITE_STATIC); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits) { + *maxhits = *maxhits ? *maxhits * 2 : 64; + *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); + if (*hits == NULL) { + sqlite3_finalize(stmt); + return -ENOMEM; + } } - } - sdstrim(result, " "); - tokens = sdssplitlen(result, sdslen(result), " ", 1, &token_count); - if (tokens == NULL) { - /* Coverity: CID370640 */ - sdsfree(result); - return NULL; - } - formatted_result = sdsempty(); - for (i = 0; i < token_count; i++) { - size_t token_len = sdslen(tokens[i]); - if (min_length != 0 && token_len < min_length) - continue; - if (keys_search_is_stopword(tokens[i])) - continue; - if (prefix == NULL && suffix == NULL) - formatted_result = sdscatsds(formatted_result, tokens[i]); - else if (prefix == NULL) - formatted_result = sdscatfmt(formatted_result, "%S%s", tokens[i], suffix); - else if (suffix == NULL) - formatted_result = sdscatfmt(formatted_result, "%s%S", prefix, tokens[i]); - else { - // Monkey patch for numbers in fuzzy search, not fixed in OSS version of RediSearch - // https://github.com/RediSearch/RediSearch/issues/346 - // Lets just assume that if both prefix and suffix start with %, - // fuzzy search formatting is intended (OSS RediSearch doesn't support LD > 1 anyway) - if (prefix[0] == '%' && suffix[0] == '%') { - non_digit_found = 0; - for (j = 0; j < token_len; j++) { - if (tokens[i][j] < '0' || tokens[i][j] > '9') { - non_digit_found = 1; - } - } - if (!non_digit_found) { - goto outer_loop; - } + + memset(&result, 0, sizeof(result)); + + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + result.type = sqlite3_column_int(stmt, 4); + result.score = -sqlite3_column_double(stmt, 6); + + if (request->return_name) { + if (request->highlight_name) + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 7)); + else + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + } + if (request->return_indom) { + const char *indom = (const char *)sqlite3_column_text(stmt, 5); + if (indom && *indom) + result.indom = sdsnew(indom); + } + if (request->return_oneline) { + if (request->highlight_oneline) + result.oneline = sdsnew((const char *)sqlite3_column_text(stmt, 8)); + else { + const char *ol = (const char *)sqlite3_column_text(stmt, 2); + if (ol && *ol) + result.oneline = sdsnew(ol); } - formatted_result = sdscatfmt(formatted_result, "%s%S%s", prefix, tokens[i], suffix); } - if (i != token_count - 1) { - formatted_result = sdscat(formatted_result, " "); + if (request->return_helptext) { + if (request->highlight_helptext) + result.helptext = sdsnew((const char *)sqlite3_column_text(stmt, 9)); + else { + const char *ht = (const char *)sqlite3_column_text(stmt, 3); + if (ht && *ht) + result.helptext = sdsnew(ht); + } } - outer_loop: - continue; + + (*hits)[*nhits] = result; + (*nhits)++; } - sdsfree(result); - sdsfreesplitres(tokens, token_count); - sdstrim(formatted_result, " "); - return formatted_result; + + sqlite3_finalize(stmt); + return 0; } -static void -keys_search_text_add_callback( - keyClusterAsyncContext *c, void *r, void *arg) +static int +score_cmp(const void *a, const void *b) { - seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesGetContext *context = &baton->pmapi; - respReply *reply = r; + const pmSearchTextResult *ra = a, *rb = b; - checkStatusReplyOK(baton->info, baton->userdata, c, reply, - "%s: %s", FT_ADD, "search text add"); - doneSeriesGetContext(context, "keys_search_text_add_callback"); + if (rb->score > ra->score) return 1; + if (rb->score < ra->score) return -1; + return 0; } -void -keys_search_text_add(keySlots *slots, pmSearchTextType type, - const char *name, const char *indom, - const char *oneline, const char *helptext, void *arg) -{ - seriesLoadBaton *baton = (seriesLoadBaton *)arg; - seriesGetContext *context = &baton->pmapi; - unsigned int length; - const char *typestr = pmSearchTextTypeStr(type); - char buffer[8]; - sds cmd, key, docid; - - seriesBatonCheckMagic(baton, MAGIC_LOAD, "keys_search_text_add"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s %s\n", "keys_search_text_add", typestr, name); - - seriesBatonReference(context, "keys_search_text_add"); - - /* - * FT.ADD pcp:text 1.0 - * REPLACE PARTIAL - * PAYLOAD - * FIELDS NAME TYPE - * [INDOM ] [ONELINE ] [HELPTEXT ] - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - length = 4 + 2 + 2 + 5; - if (indom && *indom != '\0') - length += 2; - if (oneline && *oneline != '\0') - length += 2; - if (helptext && *helptext != '\0') - length += 2; - cmd = resp_command(length); - - cmd = resp_param_str(cmd, FT_ADD, FT_ADD_LEN); - cmd = resp_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - docid = keys_search_docid(FT_TEXT_KEY, typestr, name); - cmd = resp_param_sds(cmd, docid); - sdsfree(docid); - cmd = resp_param_str(cmd, "1", 1); - - cmd = resp_param_str(cmd, FT_REPLACE, FT_REPLACE_LEN); - cmd = resp_param_str(cmd, FT_PARTIAL, FT_PARTIAL_LEN); - - length = pmsprintf(buffer, sizeof(buffer), "%u", type); - cmd = resp_param_str(cmd, FT_PAYLOAD, FT_PAYLOAD_LEN); - cmd = resp_param_str(cmd, buffer, length); - - cmd = resp_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, name, strlen(name)); - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, typestr, strlen(typestr)); - if (indom && *indom != '\0') { - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = resp_param_str(cmd, indom, strlen(indom)); - } - if (oneline && *oneline != '\0') { - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = resp_param_str(cmd, oneline, strlen(oneline)); - } - if (helptext && *helptext != '\0') { - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = resp_param_str(cmd, helptext, strlen(helptext)); - } +static void +search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) +{ + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + sds match, type_filter; + int nhits = 0, maxhits = 0; + unsigned int count, offset, i; + double timer; - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_add_callback, arg); - sdsfree(cmd); -} + pmtimespecNow(&started); -void -pmSearchDiscoverMetric(pmDiscoverEvent *event, - pmDesc *desc, int numnames, char **names, void *arg) -{ - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = (seriesLoadBaton *)p->baton; - context_t *context = &baton->pmapi.context; - char *oneline = NULL, *helptext = NULL; - char buffer[64] = {0}; - pmID id = desc->pmid; - int i; - - if (pmDebugOptions.discovery || pmDebugOptions.search) { - for (i = 0; i < numnames; i++) - fprintf(stderr, "%s: [%d/%d] %s - %s\n", "pmSearchDiscoverMetric", - i + 1, numnames, pmIDStr_r(id, buffer, sizeof(buffer)), - names[i]); + if (!request->return_name && !request->return_indom && + !request->return_oneline && !request->return_helptext && + !request->return_type) { + request->return_name = 1; + request->return_indom = 1; + request->return_oneline = 1; + request->return_helptext = 1; + request->return_type = 1; } - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; + match = search_build_match(request); + type_filter = search_type_filter(request); - /* we have the metric name(s) and desc, has text been discovered yet? */ - pmUseContext(context->context); - pmLookupText(id, PM_TEXT_PMID | PM_TEXT_ONELINE, &oneline); - pmLookupText(id, PM_TEXT_PMID | PM_TEXT_HELP | PM_TEXT_DIRECT, &helptext); + search_query_table(smd->db, "docs", request, match, type_filter, + &hits, &nhits, &maxhits); - if (desc->indom != PM_INDOM_NULL) - pmInDomStr_r(desc->indom, buffer, sizeof(buffer)); + if (smd->has_base) { + search_query_table(smd->db, "base.docs", request, match, type_filter, + &hits, &nhits, &maxhits); + } - for (i = 0; i < numnames; i++) - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, - names[i], buffer, oneline, helptext, baton); + sdsfree(match); + sdsfree(type_filter); - if (oneline) - free(oneline); - if (helptext) - free(helptext); -} + qsort(hits, nhits, sizeof(pmSearchTextResult), score_cmp); -void -pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) -{ - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = p->baton; - pmInDom id = in->indom; - char *oneline = NULL, *helptext = NULL; - char buffer[64]; - int i; - - pmInDomStr_r(id, buffer, sizeof(buffer)); - - if (pmDebugOptions.discovery || pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "pmSearchDiscoverInDom", buffer); - - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; - - /* - * We have the indom and instances, has text been discovered yet? - * Not a problem if not as we use PARTIAL FT.ADD and subsequently - * we will find the text via pmSearchDiscoverText. - */ - pmUseContext(p->ctx); - pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_ONELINE, &oneline); - pmLookupText(id, PM_TEXT_INDOM | PM_TEXT_HELP | PM_TEXT_DIRECT, &helptext); - - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, - buffer, buffer, oneline, helptext, baton); - for (i = 0; i < in->numinst; i++) { - if (in->namelist[i] == NULL) - continue; - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INST, - in->namelist[i], buffer, NULL, NULL, baton); + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); + + offset = request->offset; + count = request->count ? request->count : smd->resultcount; + + for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { + hits[i].total = nhits; + hits[i].count = (i - offset) + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); } - if (oneline) - free(oneline); - if (helptext) - free(helptext); -} -void -pmSearchDiscoverText(pmDiscoverEvent *event, - int ident, int type, char *text, void *arg) -{ - pmDiscover *p = (pmDiscover *)event->data; - seriesLoadBaton *baton = p->baton; - char indom[64] = {0}, **metrics, *oneline, *helptext; - int i, count; - - if (pmDebugOptions.discovery || pmDebugOptions.search) - fprintf(stderr, "%s: ident=%u type=%u arg=" PRINTF_P_PFX "%p\n", - "pmSearchDiscoverText", ident, type, arg); - - if (baton == NULL || baton->slots == NULL || baton->slots->search <= 0) - return; - - oneline = (type & PM_TEXT_ONELINE) ? text : NULL; - helptext = (type & PM_TEXT_HELP) ? text : NULL; - - if (type & PM_TEXT_PMID) { - pmUseContext(p->ctx); - if ((count = pmNameAll(ident, &metrics)) <= 0) - return; - for (i = 0; i < count; i++) - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_METRIC, - metrics[i], NULL, oneline, helptext, baton); - free(metrics); - } else { /* PM_TEXT_INDOM */ - pmInDomStr_r(ident, indom, sizeof(indom)); - keys_search_text_add(baton->slots, PM_SEARCH_TYPE_INDOM, - indom, indom, oneline, helptext, baton); + for (i = 0; i < (unsigned int)nhits; i++) { + sdsfree(hits[i].docid); + sdsfree(hits[i].name); + sdsfree(hits[i].indom); + sdsfree(hits[i].oneline); + sdsfree(hits[i].helptext); } + free(hits); + + callbacks->on_done(0, userdata); } -static void -keys_search_info_callback( - keyClusterAsyncContext *c, void *r, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - respReply *reply = r; - respReply *child, *value; - pmSearchMetrics metrics = {0}; - int i; - sds msg; - - if (reply && reply->type == RESP_REPLY_ARRAY && reply->elements >= 30) { - for (i = 0; i < reply->elements-1; i++) { - value = reply->element[i+1]; - child = reply->element[i]; - if (child->type != RESP_REPLY_STRING && - value->type != RESP_REPLY_STRING) - continue; - else if (strcmp("num_docs", child->str) == 0) - metrics.docs = strtoull(value->str, NULL, 0); - else if (strcmp("num_terms", child->str) == 0) - metrics.terms = strtoull(value->str, NULL, 0); - else if (strcmp("num_records", child->str) == 0) - metrics.records = strtoull(value->str, NULL, 0); - else if (strcmp("inverted_sz_mb", child->str) == 0) - metrics.inverted_sz_mb = strtod(value->str, NULL); - else if (strcmp("inverted_cap_mb", child->str) == 0) - metrics.inverted_cap_mb = strtod(value->str, NULL); - else if (strcmp("inverted_cap_ovh", child->str) == 0) - metrics.inverted_cap_ovh = strtod(value->str, NULL); - else if (strcmp("offset_vectors_sz_mb", child->str) == 0) - metrics.offset_vectors_sz_mb = strtod(value->str, NULL); - else if (strcmp("skip_index_size_mb", child->str) == 0) - metrics.skip_index_size_mb = strtod(value->str, NULL); - else if (strcmp("score_index_size_mb", child->str) == 0) - metrics.score_index_size_mb = strtod(value->str, NULL); - else if (strcmp("records_per_doc_avg", child->str) == 0) - metrics.records_per_doc_avg = strtod(value->str, NULL); - else if (strcmp("bytes_per_record_avg", child->str) == 0) - metrics.bytes_per_record_avg = strtod(value->str, NULL); - else if (strcmp("offsets_per_term_avg", child->str) == 0) - metrics.offsets_per_term_avg = strtod(value->str, NULL); - else if (strcmp("offset_bits_per_record_avg", child->str) == 0) - metrics.offset_bits_per_record_avg = strtod(value->str, NULL); +static int +search_suggest_table(sqlite3 *db, const char *table, sds match, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdscatfmt(sdsempty(), + "SELECT rowid, name, type, bm25(%s, 9.0, 4.0, 2.0)" + " FROM %s WHERE %s MATCH ?" + " AND type IN (1, 3)" + " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", + table, table, table, table); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) + return -EIO; + + sqlite3_bind_text(stmt, 1, match, sdslen(match), SQLITE_STATIC); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits) { + *maxhits = *maxhits ? *maxhits * 2 : 64; + *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); + if (*hits == NULL) { + sqlite3_finalize(stmt); + return -ENOMEM; + } } - baton->callbacks->on_metrics(&metrics, baton->userdata); - } else { - msg = NULL; - infofmt(msg, "expected array from %s (reply=%s)", - FT_INFO, resp_reply_type(reply)); - batoninfo(baton, PMLOG_RESPONSE, msg); - baton->error = -EPROTO; + + memset(&result, 0, sizeof(result)); + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + result.type = sqlite3_column_int(stmt, 2); + result.score = -sqlite3_column_double(stmt, 3); + + (*hits)[(*nhits)++] = result; } - baton->callbacks->on_done(baton->error, baton->userdata); - doneKeysSearchBaton(baton); + sqlite3_finalize(stmt); + return 0; } -void -keys_search_info(keySlots *slots, sds pcpkey, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - sds cmd, key; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_info"); - seriesBatonCheckCount(baton, "keys_search_info"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: search key metrics\n", "keys_search_info"); - - seriesBatonReference(baton, "keys_search_info"); - - /* - * FT.INFO pcp: - */ - key = sdscatfmt(sdsempty(), "pcp:%S", pcpkey); - cmd = resp_command(2); - cmd = resp_param_str(cmd, FT_INFO, FT_INFO_LEN); - cmd = resp_param_sds(cmd, key); - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_info_callback, arg); - sdsfree(cmd); -} +static void +search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) +{ + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + sds match; + int nhits = 0, maxhits = 0; + unsigned int count, i; + double timer; -int -pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) -{ - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + pmtimespecNow(&started); - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_info(data->slots, key, baton); - return 0; + match = sdscatfmt(sdsempty(), "name : %S*", request->query); + + search_suggest_table(smd->db, "docs", match, + &hits, &nhits, &maxhits); + + if (smd->has_base) + search_suggest_table(smd->db, "base.docs", match, + &hits, &nhits, &maxhits); + + sdsfree(match); + + qsort(hits, nhits, sizeof(pmSearchTextResult), score_cmp); + + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); + + count = request->count ? request->count : smd->resultcount; + + for (i = 0; i < (unsigned int)nhits && i < count; i++) { + hits[i].total = nhits; + hits[i].count = i + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); + } + + for (i = 0; i < (unsigned int)nhits; i++) { + sdsfree(hits[i].docid); + sdsfree(hits[i].name); + } + free(hits); + + callbacks->on_done(0, userdata); } -static void -extract_search_results(keysSearchBaton *baton, - unsigned int total, double timer, respReply *reply) -{ - pmSearchTextResult result; - respReply *docid, *score, *payload, *array; - int i, j; - - for (i = 1; i < reply->elements - 3; i += 4) { - docid = reply->element[i]; - score = reply->element[i+1]; - payload = reply->element[i+2]; - array = reply->element[i+3]; - if (payload->type != RESP_REPLY_STRING || - score->type != RESP_REPLY_STRING || - docid->type != RESP_REPLY_STRING || - array->type != RESP_REPLY_ARRAY) { - baton->error = -EPROTO; - break; +static int +search_indom_table(sqlite3 *db, const char *table, const char *query, + int querylen, + pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc; + + sql = sdscatfmt(sdsempty(), + "SELECT rowid, name, oneline, helptext, type, indom" + " FROM %s WHERE indom = ?" + " ORDER BY type", + table); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) + return -EIO; + + sqlite3_bind_text(stmt, 1, query, querylen, SQLITE_STATIC); + + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + pmSearchTextResult result; + + if (*nhits >= *maxhits) { + *maxhits = *maxhits ? *maxhits * 2 : 64; + *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); + if (*hits == NULL) { + sqlite3_finalize(stmt); + return -ENOMEM; + } } memset(&result, 0, sizeof(result)); - result.total = total; - result.timer = timer; - result.count = (i / 2) + 1; - result.docid = sdsnewlen(docid->str, docid->len); - result.score = strtod(score->str, NULL); - - for (j = 0; j < array->elements; j += 2) { - respReply *field = array->element[j]; - respReply *value = array->element[j+1]; - - if (field->type != RESP_REPLY_STRING || - (value->type != RESP_REPLY_STRING && - value->type != RESP_REPLY_NIL)) { - baton->error = -EPROTO; - break; - } - - if (strcmp(field->str, FT_NAME) == 0) - result.name = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_INDOM) == 0) - result.indom = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_ONELINE) == 0) - result.oneline = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_HELPTEXT) == 0) - result.helptext = sdsnewlen(value->str, value->len); - else if (strcmp(field->str, FT_TYPE) == 0) - result.type = atoi(payload->str); + result.docid = search_make_docid( + sqlite3_column_int64(stmt, 0), + (const char *)sqlite3_column_text(stmt, 1)); + result.name = sdsnew((const char *)sqlite3_column_text(stmt, 1)); + result.type = sqlite3_column_int(stmt, 4); + result.score = (result.type == SEARCH_DOC_INDOM) ? 2.0 : + (result.type == SEARCH_DOC_METRIC) ? 1.0 : 0.5; + + { + const char *ol = (const char *)sqlite3_column_text(stmt, 2); + if (ol && *ol) + result.oneline = sdsnew(ol); } - if (baton->error == 0) - baton->callbacks->on_text_result(&result, baton->userdata); - - sdsfree(result.docid); - sdsfree(result.name); - sdsfree(result.indom); - sdsfree(result.oneline); - sdsfree(result.helptext); + { + const char *ht = (const char *)sqlite3_column_text(stmt, 3); + if (ht && *ht) + result.helptext = sdsnew(ht); + } + { + const char *indom = (const char *)sqlite3_column_text(stmt, 5); + if (indom && *indom) + result.indom = sdsnew(indom); + } + + (*hits)[(*nhits)++] = result; } + + sqlite3_finalize(stmt); + return 0; } static void -keys_search_text_query_callback( - keyClusterAsyncContext *c, void *r, void *arg) +search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, + pmSearchCallBacks *callbacks, void *userdata) { - keysSearchBaton *baton = (keysSearchBaton *)arg; - respReply *reply = r; - respReply *value; - struct timespec finished; - unsigned int total; + pmSearchTextResult *hits = NULL; + struct timespec started, finished; + int nhits = 0, maxhits = 0; + unsigned int count, offset, i; double timer; - sds msg; - - if (LIKELY(reply && reply->type == RESP_REPLY_ARRAY)) { - value = reply->element[0]; - if (reply->elements == 1) /* no search results - done! */ - /* do nothing */; - else if (reply->elements < 2) /* expect total then results */ - baton->error = -EPROTO; - else if (value->type != RESP_REPLY_INTEGER) - baton->error = -EPROTO; - else { - pmtimespecNow(&finished); - timer = pmtimespecSub(&finished, &baton->started); - total = (unsigned int)value->integer; - extract_search_results(baton, total, timer, reply); - } - } else { - msg = NULL; - infofmt(msg, "expected array from %s (reply=%s)", - FT_SEARCH, resp_reply_type(reply)); - batoninfo(baton, PMLOG_RESPONSE, msg); - baton->error = -EPROTO; - } - baton->callbacks->on_done(baton->error, baton->userdata); - doneKeysSearchBaton(baton); -} + pmtimespecNow(&started); -static void -keys_search_text_query(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - const char *typestr; - size_t length; - char buffer[64]; - sds cmd, key, query, base_query; - unsigned int types = 0, infields = 0, returns = 0, highlights = 0; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_query"); - seriesBatonCheckCount(baton, "keys_search_text_query"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_query", request->query); - - seriesBatonReference(baton, "keys_search_text_query"); - - types += request->type_metric; - types += request->type_indom; - types += request->type_inst; - - highlights += request->highlight_name; - highlights += request->highlight_oneline; - highlights += request->highlight_helptext; - - infields += request->infields_name; - infields += request->infields_oneline; - infields += request->infields_helptext; - if (infields == 0) { - infields = 3; /* defaults */ - request->infields_name = 1; - request->infields_oneline = 1; - request->infields_helptext = 1; - } + search_indom_table(smd->db, "docs", request->query, + sdslen(request->query), &hits, &nhits, &maxhits); - returns += request->return_name; - returns += request->return_indom; - returns += request->return_oneline; - returns += request->return_helptext; - returns += request->return_type; - if (returns == 0) { - returns = 5; /* defaults */ - request->return_name = 1; - request->return_indom = 1; - request->return_oneline = 1; - request->return_helptext = 1; - request->return_type = 1; - } + if (smd->has_base) + search_indom_table(smd->db, "base.docs", request->query, + sdslen(request->query), &hits, &nhits, &maxhits); - /* - * FT.SEARCH pcp:text "query [@TYPE={ {?type separated by pipe} }]" WITHSCORES WITHPAYLOADS - * [INFIELDS {?field item count} {?field separated by space}] - * [RETURN {?return item count} {?return separated by space}] - * [HIGHLIGHT FIELDS {num} {field} ... ] - * SCORER BM25 - * LIMIT {?pagination offset} {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - length = 5; - if (infields) - length += 2 + infields; - if (returns) - length += 2 + returns; - if (highlights) - length += 3 + highlights; - length += 2 + 3; - - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - - query = sdscatlen(sdsempty(), "\'", 1); - if (types) { - query = sdscatlen(query, "@TYPE:{", 7); - if (request->type_metric) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_METRIC); - query = sdscat(query, typestr); - } - if (request->type_indom && (request->type_metric)) - query = sdscatlen(query, "|", 1); - if (request->type_indom) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_INDOM); - query = sdscat(query, typestr); - } - if (request->type_inst && (request->type_indom || request->type_metric)) - query = sdscatlen(query, "|", 1); - if (request->type_inst) { - typestr = pmSearchTextTypeStr(PM_SEARCH_TYPE_INST); - query = sdscat(query, typestr); - } - query = sdscatlen(query, "} ", 2); - } - base_query = keys_search_text_prep(request->query, 0, NULL, NULL); - query = sdscatfmt(query, "(%S)=>{$inorder:true}\'", base_query); - sdsfree(base_query); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - - if (infields) { - cmd = resp_param_str(cmd, FT_INFIELDS, FT_INFIELDS_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", infields); - cmd = resp_param_str(cmd, buffer, length); - if (request->infields_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->infields_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->infields_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - } + pmtimespecNow(&finished); + timer = pmtimespecSub(&finished, &started); - if (returns) { - cmd = resp_param_str(cmd, FT_RETURN, FT_RETURN_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", returns); - cmd = resp_param_str(cmd, buffer, length); - if (request->return_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->return_indom) - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - if (request->return_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->return_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - if (request->return_type) - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - } + offset = request->offset; + count = request->count ? request->count : smd->resultcount; - if (highlights) { - cmd = resp_param_str(cmd, FT_HIGHLIGHT, FT_HIGHLIGHT_LEN); - cmd = resp_param_str(cmd, FT_FIELDS, FT_FIELDS_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", highlights); - cmd = resp_param_str(cmd, buffer, length); - if (request->highlight_name) - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - if (request->highlight_oneline) - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - if (request->highlight_helptext) - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); + for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { + hits[i].total = nhits; + hits[i].count = (i - offset) + 1; + hits[i].timer = timer; + callbacks->on_text_result(&hits[i], userdata); } - cmd = resp_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = resp_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); - - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = resp_param_str(cmd, buffer, length); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + for (i = 0; i < (unsigned int)nhits; i++) { + sdsfree(hits[i].docid); + sdsfree(hits[i].name); + sdsfree(hits[i].indom); + sdsfree(hits[i].oneline); + sdsfree(hits[i].helptext); } + free(hits); - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + callbacks->on_done(0, userdata); } +/* --- public API --- */ + int -pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + searchModuleData *smd = (searchModuleData *)settings->module.privdata; + pmSearchMetrics metrics; + sqlite3_stmt *stmt; + int rc; - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_query(data->slots, request, baton); - return 0; -} + (void)key; -static void -keys_search_text_suggest(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - size_t length, prefix_length, fuzzy_length; - const char *prefix; - char buffer[64]; - sds cmd, key, query, - prefix_query, fuzzy_query; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_suggest"); - seriesBatonCheckCount(baton, "keys_search_text_suggest"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_suggest", request->query); - - seriesBatonReference(baton, "keys_search_text_suggest"); - - /* by default we cannot use prefix search with words of length less than 2 */ - prefix_query = keys_search_text_prep(request->query, 2, NULL, "*"); - fuzzy_query = keys_search_text_prep(request->query, 2, "%", "%"); - prefix_length = sdslen(prefix_query); - fuzzy_length = sdslen(fuzzy_query); - - query = sdsnewlen("\'", 1); - prefix = ""; - if (prefix_length || fuzzy_length) { - prefix = "("; - } - if (prefix_length) { - query = sdscatfmt(query, "%s@NAME:(%S)", prefix, prefix_query); - prefix = "|"; - } - if (fuzzy_length) { - query = sdscatfmt(query, "%s@NAME:(%S)=>{$weight:0.25;}", prefix, fuzzy_query); - prefix = "|"; + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; } - if (prefix_length || fuzzy_length) { - prefix = ")"; + + memset(&metrics, 0, sizeof(metrics)); + + rc = sqlite3_prepare_v2(smd->db, + "SELECT COUNT(*) FROM docs", -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) + metrics.docs = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); } - query = sdscatfmt( - query, - "%s @TYPE:{%s|%s}", - prefix, - pmSearchTextTypeStr(PM_SEARCH_TYPE_METRIC), - pmSearchTextTypeStr(PM_SEARCH_TYPE_INST) - ); - query = sdscat(query, "\'"); - - sdsfree(prefix_query); - sdsfree(fuzzy_query); - - /* - * FT.SEARCH pcp:text - * "(@NAME:({query}*)|@NAME:(%{query}%)=>{$weight:0.25;}) @TYPE={metric|instance}" - * WITHSCORES WITHPAYLOADS - * RETURN 1 NAME - * SCORER BM25 - * LIMIT 0 {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - - length = 13; // Resp array size - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - cmd = resp_param_str(cmd, FT_RETURN, FT_RETURN_LEN); - cmd = resp_param_str(cmd, "1", 1); - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, FT_SCORER, FT_SCORER_LEN); - cmd = resp_param_str(cmd, FT_SCORER_BM25, FT_SCORER_BM25_LEN); - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - cmd = resp_param_str(cmd, "0", 1); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + + if (smd->has_base) { + rc = sqlite3_prepare_v2(smd->db, + "SELECT COUNT(*) FROM base.docs", -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) + metrics.docs += sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + } } - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + settings->callbacks.on_metrics(&metrics, arg); + settings->callbacks.on_done(0, arg); + return 0; } int -pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; - - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_suggest(data->slots, request, baton); - return 0; -} + searchModuleData *smd = (searchModuleData *)settings->module.privdata; -static void -keys_search_text_indom(keySlots *slots, pmSearchTextRequest *request, void *arg) -{ - keysSearchBaton *baton = (keysSearchBaton *)arg; - size_t length; - char buffer[64]; - sds cmd, key, query; - - seriesBatonCheckMagic(baton, MAGIC_SEARCH, "keys_search_text_indom"); - seriesBatonCheckCount(baton, "keys_search_text_indom"); - - if (pmDebugOptions.search) - fprintf(stderr, "%s: %s\n", "keys_search_text_indom", request->query); - - seriesBatonReference(baton, "keys_search_text_indom"); - - query = sdscatfmt( - sdsnewlen("", 0), - "\'@INDOM:{%s}\'", - request->query - ); - - /* - * FT.SEARCH pcp:text - * "@INDOM:{{query}}" WITHSCORES WITHPAYLOADS - * SORTBY 2 TYPE ASC - *. LIMIT {?pagination offset} {?return result count} - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - - length = 12; // Resp array size - cmd = resp_command(length); - cmd = resp_param_str(cmd, FT_SEARCH, FT_SEARCH_LEN); - cmd = resp_param_sds(cmd, key); - cmd = resp_param_sds(cmd, query); - sdsfree(query); - - cmd = resp_param_str(cmd, FT_WITHSCORES, FT_WITHSCORES_LEN); - cmd = resp_param_str(cmd, FT_WITHPAYLOADS, FT_WITHPAYLOADS_LEN); - - cmd = resp_param_str(cmd, FT_SORTBY, FT_SORTBY_LEN); - cmd = resp_param_str(cmd, "2", 1); - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, FT_ASC, FT_ASC_LEN); - - cmd = resp_param_str(cmd, FT_LIMIT, FT_LIMIT_LEN); - length = pmsprintf(buffer, sizeof(buffer), "%u", request->offset); - cmd = resp_param_str(cmd, buffer, length); - if (request->count == 0) { - cmd = resp_param_sds(cmd, resultcount_str); - request->count = resultcount; - } else { - length = pmsprintf(buffer, sizeof(buffer), "%u", request->count); - cmd = resp_param_str(cmd, buffer, length); + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; } - sdsfree(key); - keySlotsRequestFirstNode(slots, cmd, keys_search_text_query_callback, arg); - sdsfree(cmd); + search_do_text_query(smd, request, &settings->callbacks, arg); + return 0; } int -pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) +pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - seriesModuleData *data = getSeriesModuleData(&settings->module); - keysSearchBaton *baton; + searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (data == NULL) - return -ENOMEM; - if ((baton = calloc(1, sizeof(keysSearchBaton))) == NULL) - return -ENOMEM; - initKeysSearchBaton(baton, data->slots, settings, arg); - keys_search_text_indom(data->slots, request, baton); + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + + search_do_text_suggest(smd, request, &settings->callbacks, arg); return 0; } -static void -keys_search_schema_callback( - keyClusterAsyncContext *c, void *r, void *arg) +int +pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void *arg) { - keySlotsBaton *baton = (keySlotsBaton *)arg; - respReply *reply = r; - - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_search_schema_callback"); + searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (testReplyError(reply, RESP_EDROPINDEX)) { - // index already exists - baton->slots->search = 1; - } - else if (reply && reply->type == RESP_REPLY_STATUS && - (strcmp("OK", reply->str) == 0 || strcmp("QUEUED", reply->str) == 0)) { - // index created - baton->slots->search = 1; - } else { - // probably no search module installed, ignore silently - baton->slots->search = 0; + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; } - keys_slots_end_phase(baton); + search_do_text_indom(smd, request, &settings->callbacks, arg); + return 0; } -void -keys_load_search_schema(void *arg) -{ - keySlotsBaton *baton = (keySlotsBaton *)arg; - sds cmd, key; - - seriesBatonCheckMagic(baton, MAGIC_SLOTS, "keys_load_search_schema"); - - if (pmDebugOptions.search && pmDebugOptions.desperate) - fprintf(stderr, "%s: loading schema\n", "keys_search_schema"); - - seriesBatonReference(baton, "keys_load_search_schema"); - - /* - * FT.CREATE pcp:text SCHEMA - * type TAG SORTABLE - * name TEXT WEIGHT 9 SORTABLE - * indom TAG - * oneline TEXT WEIGHT 4 - * helptext TEXT WEIGHT 2 - */ - key = sdsnewlen(FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = resp_command(3 + 3 + 5 + 2 + 4 + 4); - - cmd = resp_param_str(cmd, FT_CREATE, FT_CREATE_LEN); - cmd = resp_param_str(cmd, FT_TEXT_KEY, FT_TEXT_KEY_LEN); - cmd = resp_param_str(cmd, FT_SCHEMA, FT_SCHEMA_LEN); - - cmd = resp_param_str(cmd, FT_TYPE, FT_TYPE_LEN); - cmd = resp_param_str(cmd, FT_TAG, FT_TAG_LEN); - cmd = resp_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - - cmd = resp_param_str(cmd, FT_NAME, FT_NAME_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "9", sizeof("9")-1); - cmd = resp_param_str(cmd, FT_SORTABLE, FT_SORTABLE_LEN); - - cmd = resp_param_str(cmd, FT_INDOM, FT_INDOM_LEN); - cmd = resp_param_str(cmd, FT_TAG, FT_TAG_LEN); - - cmd = resp_param_str(cmd, FT_ONELINE, FT_ONELINE_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "4", sizeof("4")-1); - - cmd = resp_param_str(cmd, FT_HELPTEXT, FT_HELPTEXT_LEN); - cmd = resp_param_str(cmd, FT_TEXT, FT_TEXT_LEN); - cmd = resp_param_str(cmd, FT_WEIGHT, FT_WEIGHT_LEN); - cmd = resp_param_str(cmd, "2", sizeof("2")-1); - - sdsfree(key); - keySlotsRequestFirstNode(baton->slots, cmd, keys_search_schema_callback, arg); - sdsfree(cmd); -} +/* --- module setup / teardown --- */ int pmSearchSetSlots(pmSearchModule *module, void *slots) { - return pmSeriesSetSlots(module, slots); + (void)module; + (void)slots; + return 0; } int -pmSearchSetConfiguration(pmSearchModule *module, dict *config) +pmSearchSetConfiguration(pmSearchModule *module, struct dict *config) { - return pmSeriesSetConfiguration(module, config); + searchModuleData *smd = getSearchModuleData(module); + + if (smd == NULL) + return -ENOMEM; + smd->config = config; + return 0; } int pmSearchSetEventLoop(pmSearchModule *module, void *events) { - return pmSeriesSetEventLoop(module, events); + (void)module; + (void)events; + return 0; } int -pmSearchSetMetricRegistry(pmSearchModule *module, mmv_registry_t *registry) -{ - return pmSeriesSetMetricRegistry(module, registry); -} - -void -keysSearchInit(struct dict *config) -{ - sds option; - - if (!resultcount) { - if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) - resultcount_str = option; - else - resultcount_str = DEFAULT_RESULTCOUNT = sdsnew("10"); - resultcount = atoi(resultcount_str); - } -} - -void -keysSearchClose(void) +pmSearchSetMetricRegistry(pmSearchModule *module, struct mmv_registry *registry) { - if (DEFAULT_RESULTCOUNT) { - sdsfree(DEFAULT_RESULTCOUNT); - DEFAULT_RESULTCOUNT = NULL; - } + (void)module; + (void)registry; + return 0; } int pmSearchSetup(pmSearchModule *module, void *arg) { - seriesModuleData *data = getSeriesModuleData(module); + searchModuleData *smd = getSearchModuleData(module); + char nightly[MAXPATHLEN]; + char base[MAXPATHLEN]; sds option; - keySlotsFlags flags; + int rc; - if (data == NULL) + if (smd == NULL) return -ENOMEM; - /* create string map caches */ - keysGlobalsInit(data->config); + smd->resultcount = default_resultcount; - /* fast path for when key server has been setup already */ - if (data->slots) { - module->on_setup(arg); - data->shareslots = 1; - } else { - if (!(option = pmIniFileLookup(data->config, "resp", "enabled"))) - option = pmIniFileLookup(data->config, "redis", "enabled"); // compat + if (smd->config) { + option = pmIniFileLookup(smd->config, "pmsearch", "enabled"); if (option && strcmp(option, "false") == 0) return -ENOTSUP; - /* establish an initial connection to key server instance(s) */ - flags = SLOTS_VERSION; + option = pmIniFileLookup(smd->config, "pmsearch", "result.count"); + if (option) + smd->resultcount = atoi(option); + } - option = pmIniFileLookup(data->config, "pmsearch", "enabled"); - if (option && strcmp(option, "true") == 0) - flags |= SLOTS_SEARCH; - else - return -ENOTSUP; + option = smd->config ? + pmIniFileLookup(smd->config, "pmsearch", "index.path") : NULL; + + if (option) { + rc = sqlite3_open_v2(option, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: open %s: %s\n", + option, sqlite3_errmsg(smd->db)); + sqlite3_close(smd->db); + smd->db = NULL; + } + } else { + pmsprintf(nightly, sizeof(nightly), "%s/lib/pcp.search", + pmGetConfig("PCP_VAR_DIR")); + pmsprintf(base, sizeof(base), "%s/lib/pcp.search", + pmGetConfig("PCP_SHARE_DIR")); + + rc = sqlite3_open_v2(nightly, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc == SQLITE_OK) { + char attach[MAXPATHLEN + 64]; + + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: loaded nightly index %s\n", + nightly); + pmsprintf(attach, sizeof(attach), + "ATTACH DATABASE '%s' AS base", base); + rc = sqlite3_exec(smd->db, attach, NULL, NULL, NULL); + if (rc == SQLITE_OK) { + smd->has_base = 1; + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: attached base index %s\n", + base); + } + } else { + sqlite3_close(smd->db); + smd->db = NULL; + + rc = sqlite3_open_v2(base, &smd->db, + SQLITE_OPEN_READONLY, NULL); + if (rc == SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: loaded base index %s\n", + base); + } else { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchSetup: no index found\n"); + sqlite3_close(smd->db); + smd->db = NULL; + } + } + } - /* establish an initial connection to key server instance(s) */ - data->slots = &(keySlotsConnect( - data->config, flags, module->on_info, - module->on_setup, arg, data->events, arg))->slots; - data->shareslots = 0; + if (smd->db != NULL) { + smd->loaded = 1; + search_enabled = 1; + } else { + smd->loaded = 0; } + + if (module->on_setup) + module->on_setup(arg); return 0; } int pmSearchEnabled(void *arg) { - keySlots *slots = (keySlots *)arg; - - if (slots) - return slots->search > 0 ? 1 : 0; - return 0; + (void)arg; + return search_enabled; } void pmSearchClose(pmSearchModule *module) { - seriesModuleData *search = (seriesModuleData *)module->privdata; + searchModuleData *smd = (searchModuleData *)module->privdata; - if (search) { - if (search->slots && !search->shareslots) - keySlotsFree(search->slots); - memset(search, 0, sizeof(*search)); - free(search); + if (smd) { + if (smd->db) + sqlite3_close(smd->db); + memset(smd, 0, sizeof(*smd)); + free(smd); module->privdata = NULL; } + search_enabled = 0; +} + +/* --- stubs for schema.c / keys.c compatibility --- */ + +extern void keys_slots_end_phase(void *); + +void +keysSearchInit(struct dict *config) +{ + sds option; + + if (config) { + if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) + default_resultcount = atoi(option); + } +} + +void +keysSearchClose(void) +{ + default_resultcount = 10; +} + +void +keys_load_search_schema(void *arg) +{ + keys_slots_end_phase(arg); +} + +void +keys_search_text_add(struct keySlots *slots, pmSearchTextType type, + const char *name, const char *indom, + const char *oneline, const char *helptext, void *arg) +{ + (void)slots; (void)type; (void)name; (void)indom; + (void)oneline; (void)helptext; (void)arg; +} + +/* --- discover no-ops (declared in discover.h) --- */ - keysGlobalsClose(); +void +pmSearchDiscoverMetric(pmDiscoverEvent *event, + pmDesc *desc, int numnames, char **names, void *arg) +{ + (void)event; (void)desc; (void)numnames; (void)names; (void)arg; +} + +void +pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) +{ + (void)event; (void)in; (void)arg; +} + +void +pmSearchDiscoverText(pmDiscoverEvent *event, + int ident, int type, char *text, void *arg) +{ + (void)event; (void)ident; (void)type; (void)text; (void)arg; } diff --git a/src/libpcp_web/src/search.h b/src/libpcp_web/src/search.h index b753f7c00b2..8c56a63def2 100644 --- a/src/libpcp_web/src/search.h +++ b/src/libpcp_web/src/search.h @@ -1,6 +1,7 @@ /* * Copyright (c) 2020,2022 Red Hat. - * + * Copyright (c) 2026 Red Hat. + * * This library is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation; either version 2.1 of the License, or @@ -15,95 +16,16 @@ #define SEARCH_SCHEMA_H #include -#include -#include "keys.h" -#include "private.h" -#include "schema.h" -#include "slots.h" - -#define FT_TEXT_KEY "pcp:text" -#define FT_TEXT_KEY_LEN (sizeof(FT_TEXT_KEY)-1) +#include -#define FT_ADD "FT.ADD" -#define FT_ADD_LEN (sizeof(FT_ADD)-1) -#define FT_CREATE "FT.CREATE" -#define FT_CREATE_LEN (sizeof(FT_CREATE)-1) -#define FT_SEARCH "FT.SEARCH" -#define FT_SEARCH_LEN (sizeof(FT_SEARCH)-1) -#define FT_INFIELDS "INFIELDS" -#define FT_INFIELDS_LEN (sizeof(FT_INFIELDS)-1) -#define FT_INFO "FT.INFO" -#define FT_INFO_LEN (sizeof(FT_INFO)-1) -#define FT_ASC "ASC" -#define FT_ASC_LEN (sizeof(FT_ASC)-1) -#define FT_FIELDS "FIELDS" -#define FT_FIELDS_LEN (sizeof(FT_FIELDS)-1) -#define FT_HELPTEXT "HELPTEXT" -#define FT_HELPTEXT_LEN (sizeof(FT_HELPTEXT)-1) -#define FT_HIGHLIGHT "HIGHLIGHT" -#define FT_HIGHLIGHT_LEN (sizeof(FT_HIGHLIGHT)-1) -#define FT_INDOM "INDOM" -#define FT_INDOM_LEN (sizeof(FT_INDOM)-1) -#define FT_LIMIT "LIMIT" -#define FT_LIMIT_LEN (sizeof(FT_LIMIT)-1) -#define FT_NAME "NAME" -#define FT_NAME_LEN (sizeof(FT_NAME)-1) -#define FT_ONELINE "ONELINE" -#define FT_ONELINE_LEN (sizeof(FT_ONELINE)-1) -#define FT_PARTIAL "PARTIAL" -#define FT_PARTIAL_LEN (sizeof(FT_PARTIAL)-1) -#define FT_PAYLOAD "PAYLOAD" -#define FT_PAYLOAD_LEN (sizeof(FT_PAYLOAD)-1) -#define FT_RETURN "RETURN" -#define FT_RETURN_LEN (sizeof(FT_RETURN)-1) -#define FT_REPLACE "REPLACE" -#define FT_REPLACE_LEN (sizeof(FT_REPLACE)-1) -#define FT_SCHEMA "SCHEMA" -#define FT_SCHEMA_LEN (sizeof(FT_SCHEMA)-1) -#define FT_SCORE "SCORE" -#define FT_SCORE_LEN (sizeof(FT_SCORE)-1) -#define FT_SCORER "SCORER" -#define FT_SCORER_LEN (sizeof(FT_SCORER)-1) -#define FT_SCORER_BM25 "BM25" -#define FT_SCORER_BM25_LEN (sizeof(FT_SCORER_BM25)-1) -#define FT_SORTABLE "SORTABLE" -#define FT_SORTABLE_LEN (sizeof(FT_SORTABLE)-1) -#define FT_SORTBY "SORTBY" -#define FT_SORTBY_LEN (sizeof(FT_SORTBY)-1) -#define FT_TAG "TAG" -#define FT_TAG_LEN (sizeof(FT_TAG)-1) -#define FT_TEXT "TEXT" -#define FT_TEXT_LEN (sizeof(FT_TEXT)-1) -#define FT_TYPE "TYPE" -#define FT_TYPE_LEN (sizeof(FT_TYPE)-1) -#define FT_WEIGHT "WEIGHT" -#define FT_WEIGHT_LEN (sizeof(FT_WEIGHT)-1) -#define FT_WITHPAYLOADS "WITHPAYLOADS" -#define FT_WITHPAYLOADS_LEN (sizeof(FT_WITHPAYLOADS)-1) -#define FT_WITHSCORES "WITHSCORES" -#define FT_WITHSCORES_LEN (sizeof(FT_WITHSCORES)-1) +struct dict; +struct keySlots; extern void keysSearchInit(struct dict *); extern void keysSearchClose(void); + extern void keys_load_search_schema(void *); -extern void keys_search_text_add(keySlots *, pmSearchTextType, +extern void keys_search_text_add(struct keySlots *, pmSearchTextType, const char *, const char *, const char *, const char *, void *); -/* - * Asynchronous search baton structures - */ -typedef struct keysSearchBaton { - seriesBatonMagic header; /* MAGIC_SEARCH */ - - keySlots *slots; /* key server slots */ - pmSearchFlags flags; - int error; - void *module; - pmSearchCallBacks *callbacks; - pmLogInfoCallBack info; - struct timespec started; - void *userdata; - void *arg; -} keysSearchBaton; - #endif /* SEARCH_SCHEMA_H */ diff --git a/src/newhelp/GNUmakefile b/src/newhelp/GNUmakefile index c23a8034f5f..0a87771330e 100644 --- a/src/newhelp/GNUmakefile +++ b/src/newhelp/GNUmakefile @@ -18,7 +18,7 @@ include $(TOPDIR)/src/include/builddefs include $(TOPDIR)/src/libpcp/src/GNUlibrarydefs TARGETS = newhelp$(EXECSUFFIX) chkhelp$(EXECSUFFIX) -CFILES = newhelp.c chkhelp.c +CFILES = newhelp.c chkhelp.c search_sqlite.c LLDFLAGS = $(WARN_OFF) LLDLIBS = $(PCP_PMDALIB) @@ -40,12 +40,12 @@ install: default include $(BUILDRULES) -newhelp$(EXECSUFFIX): newhelp.c - $(CCF) -o $@ $(LDFLAGS) newhelp.c $(LDLIBS) +newhelp$(EXECSUFFIX): newhelp.c search_sqlite.c + $(CCF) -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(LDLIBS) $(LIB_FOR_SQLITE3) $(LINKER_MAKERULE) -newhelp.static$(EXECSUFFIX): newhelp.c $(STATIC_LIBPCP) - $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c $(STATIC_LIBPCP) $(STATIC_LDLIBS) +newhelp.static$(EXECSUFFIX): newhelp.c search_sqlite.c $(STATIC_LIBPCP) + $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(STATIC_LIBPCP) $(STATIC_LDLIBS) $(LIB_FOR_SQLITE3) chkhelp$(EXECSUFFIX): chkhelp.o $(CCF) -o $@ $(LDFLAGS) chkhelp.o $(LDLIBS) diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index 997815bc093..922124995a9 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -26,6 +26,7 @@ #include #include "pmapi.h" #include "libpcp.h" +#include "search_sqlite.h" #define DEFAULT_HELP_VERSION 2 @@ -38,6 +39,7 @@ static int ln; static char *filename; static int status; static int version = DEFAULT_HELP_VERSION; +static int search_mode; /* -S: build search index */ static FILE *f; typedef struct { @@ -217,6 +219,130 @@ newentry(char *buf) } } +/* + * Search index mode (-S): collect entries for the FTS5 search index. + * Unlike newentry() which writes .pag files, this just parses the + * name/oneline/helptext and passes them to the SQLite index builder. + */ +static void +newentry_search(char *buf) +{ + char *p; + char *name; + char *oneline; + char *helptext; + int domain, serial; + int type; + char indom_buf[64]; + + /* skip leading white space */ + for (p = buf; isspace((int)*p); p++) + ; + + /* + * Instance entry: @I indom_idinstance name + * Followed by optional oneline and helptext on subsequent lines. + * Tab separates indom_id from instance name (names may contain spaces). + */ + if (*p == 'I' && isspace((int)*(p + 1))) { + char *indom_str, *inst_name; + + p += 2; + while (*p == ' ') + p++; + indom_str = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + return; + } + inst_name = p; + while (*p && *p != '\n') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + /* remainder is oneline[\nhelptext] */ + oneline = p; + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + helptext = p; + if (*helptext) { + int len = (int)strlen(helptext) - 1; + while (len >= 0 && helptext[len] == '\n') + len--; + helptext[len + 1] = '\0'; + } + + if (verbose) + fprintf(stderr, "instance %s [%s]\n", inst_name, indom_str); + + search_sqlite_add(inst_name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + indom_str, SEARCH_DOC_INST); + return; + } + + name = p; + + /* skip over metric name or indom spec */ + for ( ; *p != '\n' && !isspace((int)*p); p++) + ; + *p = '\0'; + p++; + + /* determine type: NNN.NNN = indom, otherwise metric name */ + if (sscanf(name, "%d.%d", &domain, &serial) == 2 && + strchr(name, '.') == strrchr(name, '.')) { + type = SEARCH_DOC_INDOM; + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } else { + type = SEARCH_DOC_METRIC; + indom_buf[0] = '\0'; + } + + /* skip whitespace to start of oneline */ + while (*p != '\n' && isspace((int)*p)) + p++; + oneline = p; + + /* find end of oneline */ + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + /* remainder is helptext - trim trailing newlines */ + helptext = p; + if (*helptext) { + int len = (int)strlen(helptext) - 1; + while (len >= 0 && helptext[len] == '\n') + len--; + helptext[len + 1] = '\0'; + } + + if (verbose) + fprintf(stderr, "%s\n", name); + + search_sqlite_add(name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + (indom_buf[0] != '\0') ? indom_buf : NULL, + type); +} + static int idcomp(const void *a, const void *b) { @@ -248,13 +374,14 @@ static pmLongOptions longopts[] = { PMOPT_HELP, PMAPI_OPTIONS_HEADER("Output options"), { "output", 1, 'o', "FILE", "base name for output files" }, + { "search", 0, 'S', 0, "build search index (output file is a SQLite FTS5 database)" }, { "verbose", 0, 'V', 0, "verbose/diagnostic output" }, { "version", 1, 'v', "N", "deprecated (only version 2 format supported)" }, PMAPI_OPTIONS_END }; static pmOptions opts = { - .short_options = "D:n:o:Vv:?", + .short_options = "D:n:o:SVv:?", .long_options = longopts, .short_usage = "[options] [file ...]", }; @@ -295,6 +422,10 @@ main(int argc, char **argv) fname = opts.optarg; break; + case 'S': /* build search index */ + search_mode = 1; + break; + case 'V': /* more chit-chat */ verbose++; break; @@ -324,9 +455,22 @@ main(int argc, char **argv) exit(2); } - if ((n = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) { - fprintf(stderr, "%s: pmLoadASCIINameSpace(%s, 1): %s\n", pmGetProgname(), pmnsfile, pmErrStr(n)); - exit(2); + if (!search_mode) { + if ((n = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) { + fprintf(stderr, "%s: pmLoadASCIINameSpace(%s, 1): %s\n", + pmGetProgname(), pmnsfile, pmErrStr(n)); + exit(2); + } + } + + if (search_mode) { + if (fname == NULL) { + fprintf(stderr, "%s: -S mode requires -o to name the output file\n", + pmGetProgname()); + exit(2); + } + if (search_sqlite_open(fname) < 0) + exit(2); } do { @@ -341,7 +485,7 @@ main(int argc, char **argv) } else { if (fname == NULL) { - fprintf(stderr, + fprintf(stderr, "%s: need either a -o option or a filename " "argument to name the output file\n", pmGetProgname()); exit(2); @@ -350,7 +494,7 @@ main(int argc, char **argv) inf = stdin; } - if (version == 2 && f == NULL) { + if (!search_mode && version == 2 && f == NULL) { pmsprintf(pathname, sizeof(pathname), "%s.pag", fname); if ((f = fopen(pathname, "w")) == NULL) { fprintf(stderr, "%s: fopen(\"%s\", ...) failed: %s\n", @@ -380,7 +524,10 @@ main(int argc, char **argv) p--; *++p = '\n'; *++p = '\0'; - newentry(buf); + if (search_mode) + newentry_search(buf); + else + newentry(buf); } if (skip == -1) break; @@ -432,7 +579,10 @@ main(int argc, char **argv) opts.optind++; } while (opts.optind < argc); - if (f != NULL) { + if (search_mode) { + if (search_sqlite_close() < 0) + status = 2; + } else if (f != NULL) { fclose(f); /* do the directory index ... */ diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c new file mode 100644 index 00000000000..53319e74a3d --- /dev/null +++ b/src/newhelp/search_sqlite.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * Build a SQLite FTS5 search index from help text entries. + */ +#include +#include +#include "search_sqlite.h" + +static sqlite3 *search_db; +static sqlite3_stmt *search_insert; + +int +search_sqlite_open(const char *path) +{ + int rc; + + rc = sqlite3_open_v2(path, &search_db, + SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: %s: %s\n", + path, sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE VIRTUAL TABLE docs USING fts5(" + " name," + " oneline," + " helptext," + " type UNINDEXED," + " indom UNINDEXED," + " tokenize='porter unicode61'," + " prefix='2,3'" + ");", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create table: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, "BEGIN", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: begin: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "INSERT INTO docs(name, oneline, helptext, type, indom)" + " VALUES(?, ?, ?, ?, ?)", + -1, &search_insert, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + return 0; +} + +void +search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type) +{ + if (search_db == NULL || search_insert == NULL) + return; + + sqlite3_bind_text(search_insert, 1, name, -1, SQLITE_STATIC); + sqlite3_bind_text(search_insert, 2, oneline ? oneline : "", -1, SQLITE_STATIC); + sqlite3_bind_text(search_insert, 3, helptext ? helptext : "", -1, SQLITE_STATIC); + sqlite3_bind_int(search_insert, 4, type); + sqlite3_bind_text(search_insert, 5, indom ? indom : "", -1, SQLITE_STATIC); + + if (sqlite3_step(search_insert) != SQLITE_DONE) { + fprintf(stderr, "search_sqlite_add: %s: %s\n", + name, sqlite3_errmsg(search_db)); + } + + sqlite3_reset(search_insert); +} + +int +search_sqlite_close(void) +{ + int rc; + + if (search_db == NULL) + return -1; + + if (search_insert) { + sqlite3_finalize(search_insert); + search_insert = NULL; + } + + /* optimize the FTS index before closing */ + sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", + NULL, NULL, NULL); + + rc = sqlite3_exec(search_db, "COMMIT", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: commit: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + sqlite3_close(search_db); + search_db = NULL; + return 0; +} diff --git a/src/newhelp/search_sqlite.h b/src/newhelp/search_sqlite.h new file mode 100644 index 00000000000..36404a3c63f --- /dev/null +++ b/src/newhelp/search_sqlite.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#ifndef SEARCH_SQLITE_H +#define SEARCH_SQLITE_H + +#define SEARCH_DOC_METRIC 1 +#define SEARCH_DOC_INDOM 2 +#define SEARCH_DOC_INST 3 + +extern int search_sqlite_open(const char *); +extern void search_sqlite_add(const char *, const char *, + const char *, const char *, int); +extern int search_sqlite_close(void); + +#endif /* SEARCH_SQLITE_H */ diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index f23dc50dd0a..0992731ff36 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -94,5 +94,15 @@ install_pcp :: default_pcp $(INSTALL) -m 644 pmcd.conf $(PCP_PMCDCONF_PATH) $(INSTALL) -m 644 local.conf indom.conf $(PCP_ETC_DIR)/pcp $(INSTALL) -m 644 -t $(PCP_PMNS_DIR)/local.root local.root $(PCP_PMNSADM_DIR)/local.root +ifeq "$(HAVE_SQLITE3)" "true" + help_files=""; \ + for f in `find . -maxdepth 2 -name help -type f`; do \ + help_files="$$help_files $$f"; \ + done; \ + if [ -n "$$help_files" ]; then \ + $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib; \ + $(TOPDIR)/src/newhelp/newhelp -S -o $(PCP_SHARE_DIR)/lib/pcp.search $$help_files; \ + fi +endif include $(BUILDRULES) diff --git a/src/pmsearch/GNUmakefile b/src/pmsearch/GNUmakefile index 21dad170274..c97a18104e7 100644 --- a/src/pmsearch/GNUmakefile +++ b/src/pmsearch/GNUmakefile @@ -1,5 +1,5 @@ # -# Copyright (c) 2017-2020 Red Hat. +# Copyright (c) 2017-2020,2026 Red Hat. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -18,27 +18,57 @@ include $(TOPDIR)/src/include/builddefs CFILES = pmsearch.c CMDTARGET = pmsearch$(EXECSUFFIX) LLDLIBS = $(PCP_WEBLIB) -LCFLAGS = $(LIBUVCFLAGS) +GENSCRIPT = crontab pmsearch_index.service +LDIRT = $(GENSCRIPT) ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 endif -default: build-me +ifeq ($(TARGET_OS),linux) +CRONTAB_USER = $(PCP_USER) +CRONTAB_PATH = $(PCP_ETC_DIR)/cron.d/pcp-pmsearch +ifeq ($(ENABLE_SYSTEMD),true) +CRONTAB_INSTALL = $(PCP_SHARE_DIR)/cron.d/pcp-pmsearch +else +CRONTAB_INSTALL = $(CRONTAB_PATH) +endif +else +CRONTAB_USER = +CRONTAB_PATH = $(PCP_SYSCONF_DIR)/pmsearch/crontab +CRONTAB_INSTALL = $(CRONTAB_PATH) +endif -ifeq "$(HAVE_LIBUV)" "true" -build-me: $(CMDTARGET) +default: $(CMDTARGET) $(GENSCRIPT) install: default $(INSTALL) -m 755 $(CMDTARGET) $(PCP_BIN_DIR)/$(CMDTARGET) + $(INSTALL) -m 755 pmsearch_index.sh $(PCP_BINADM_DIR)/pmsearch_index $(INSTALL) -m 755 -d $(PCP_SYSCONF_DIR)/pmsearch $(INSTALL) -S $(PCP_SYSCONF_DIR)/pmproxy/pmproxy.conf $(PCP_SYSCONF_DIR)/pmsearch/pmsearch.conf -else -build-me: -install: +ifeq ($(ENABLE_SYSTEMD),true) + $(INSTALL) -m 644 pmsearch_index.service $(PCP_SYSTEMDUNIT_DIR)/pmsearch_index.service + $(INSTALL) -m 644 pmsearch_index.timer $(PCP_SYSTEMDUNIT_DIR)/pmsearch_index.timer endif + $(INSTALL) -m 755 -d `dirname $(CRONTAB_INSTALL)` + $(INSTALL) -m 644 crontab $(CRONTAB_INSTALL) include $(BUILDRULES) default_pcp : default install_pcp : install + +pmsearch_index.service : pmsearch_index.service.in + $(SED) <$< >$@ \ + -e 's;@CRONTAB_PATH@;'$(CRONTAB_PATH)';' \ + -e 's;@PCP_BINADM_DIR@;'$(PCP_BINADM_DIR)';' \ + -e 's;@PCP_VAR_DIR@;'$(PCP_VAR_DIR)';' \ + -e 's;@PCP_GROUP@;'$(PCP_GROUP)';' \ + -e 's;@PCP_USER@;'$(PCP_USER)';' \ + # END + +crontab : crontab.in + $(SED) <$< >$@ \ + -e 's;@user@;'$(CRONTAB_USER)';' \ + -e 's;@path@;'$(PCP_BINADM_DIR)';' \ + # END diff --git a/src/pmsearch/crontab.in b/src/pmsearch/crontab.in new file mode 100644 index 00000000000..c1ae95a149a --- /dev/null +++ b/src/pmsearch/crontab.in @@ -0,0 +1,5 @@ +# +# Performance Co-Pilot crontab entry for pmsearch index rebuild +# +# daily rebuild of the full-text search index +20 0 * * * @user@ @path@/pmsearch_index diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index 2b553cd6278..b0deee2ffb7 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Red Hat. + * Copyright (c) 2020-2021,2026 Red Hat. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the @@ -13,7 +13,6 @@ */ #include "pmwebapi.h" -#include typedef enum search_flags { PMSEARCH_COLOUR = (1<<0), /* report in colour if possible */ @@ -32,7 +31,6 @@ typedef enum search_flags { #define ANSI_FG_CYAN "\x1b[36m" /* highlights and statistics */ typedef struct search_data { - uv_loop_t *loop; pmSearchSettings settings; pmSearchTextRequest request; search_flags flags; @@ -66,6 +64,7 @@ static void search_data_free(search_data *dp) { sdsfree(dp->request.query); + free(dp); } static void @@ -94,28 +93,10 @@ on_search_metrics(pmSearchMetrics *metrics, void *arg) on = off = ""; } - printf("RediSearch statistics:\n"); + printf("Search index statistics:\n"); printf(" Documents: %s%llu%s\n", on, metrics->docs, off); printf(" Terms: %s%llu%s\n", on, metrics->terms, off); printf(" Records: %s%llu%s\n", on, metrics->records, off); - printf("- Average records per doc: %s%.2f%s\n", - on, metrics->records_per_doc_avg, off); - printf("- Average bytes per record: %s%.2f%s\n", - on, metrics->bytes_per_record_avg, off); - printf("- Inverted Index\n"); - printf(" Size: %s%.2f MB%s\n", on, metrics->inverted_sz_mb, off); - printf(" Capacity: %s%.2f MB%s\n", on, metrics->inverted_cap_mb, off); - printf(" Overhead: %s%.2f%s\n", on, metrics->inverted_cap_ovh, off); - printf("- Skip Index\n"); - printf(" Size: %s%.2f MB%s\n", - on, metrics->skip_index_size_mb, off); - printf("- Score Index\n"); - printf(" Size: %s%.2f MB%s\n", - on, metrics->score_index_size_mb, off); - printf("- Average offsets per term: %s%.2f%s\n", - on, metrics->offsets_per_term_avg, off); - printf("- Average offset bits per record: %s%.2f%s\n", - on, metrics->offset_bits_per_record_avg, off); } /* @@ -218,21 +199,18 @@ static int pmsearch_overrides(int opt, pmOptions *opts) { switch (opt) { - case 'h': case 'n': case 'N': case 'O': case 'p': case 's': case 'S': case 't': case 'T': + case 'n': case 'N': case 'O': case 's': case 'S': case 't': case 'T': return 1; } return 0; } static pmLongOptions longopts[] = { - PMAPI_OPTIONS_HEADER("Connection Options"), - { "config", 1, 'c', "FILE", "configuration file path"}, - { "host", 1, 'h', "HOST", "connect to Redis using given host name" }, - { "port", 1, 'p', "PORT", "connect to Redis using given TCP/IP port" }, PMAPI_OPTIONS_HEADER("General Options"), + { "config", 1, 'c', "FILE", "configuration file path"}, { "no-colour", 0, 'C', 0, "no highlighting in results text" }, { "docid", 0, 'd', 0, "report document ID of each result" }, - { "info", 0, 'i', 0, "report search engine interal metrics" }, + { "info", 0, 'i', 0, "report search index metrics" }, { "indom", 0, 'n', 0, "perform an instance domain related entities search"}, { "number", 1, 'N', "N", "return N search results at most" }, { "offset", 1, 'O', "N", "paginated results from given offset" }, @@ -248,7 +226,7 @@ static pmLongOptions longopts[] = { }; static pmOptions opts = { - .short_options = "c:CdD:h:inN:O:qp:sStTvV?", + .short_options = "c:CdD:inN:O:qsStTvV?", .long_options = longopts, .short_usage = "[options] [query | indom]", .override = pmsearch_overrides, @@ -275,9 +253,6 @@ on_search_done(int sts, void *arg) else fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), "pmSearchTextIndom", pmErrStr(sts)); - - pmSearchClose(&dp->settings.module); - search_data_free(dp); } static void @@ -302,40 +277,14 @@ on_search_setup(void *arg) on_search_done(sts, arg); } -static void -pmsearch_request(uv_timer_t *arg) -{ - uv_handle_t *handle = (uv_handle_t *)arg; - search_data *dp = (search_data *)handle->data; - - pmSearchSetup(&dp->settings.module, dp); -} - -static int -pmsearch_execute(search_data *dp) -{ - uv_loop_t *loop = dp->loop; - uv_timer_t request; - uv_handle_t *handle = (uv_handle_t *)&request; - - handle->data = (void *)dp; - uv_timer_init(loop, &request); - uv_timer_start(&request, pmsearch_request, 0, 0); - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); - return dp->status; -} - int main(int argc, char *argv[]) { - sds option, query = NULL; - int c, sts, colour = 1; + sds query = NULL; + int c, colour = 1; unsigned int search_count = 0; unsigned int search_offset = 0; const char *inifile = NULL; - const char *keys_host = NULL; - unsigned int keys_port = 6379; /* default key server port */ search_flags flags = 0; search_data *dp; struct dict *config; @@ -355,10 +304,6 @@ main(int argc, char *argv[]) flags |= PMSEARCH_DOCIDS; break; - case 'h': /* key server host to connect to */ - keys_host = opts.optarg; - break; - case 'i': /* report search engine info (metrics) */ flags |= PMSEARCH_OPT_INFO; break; @@ -375,10 +320,6 @@ main(int argc, char *argv[]) search_offset = strtoul(opts.optarg, NULL, 10); break; - case 'p': /* keys server port to connect to */ - keys_port = (unsigned int)strtol(opts.optarg, NULL, 10); - break; - case 'q': /* command line contains query string */ flags |= PMSEARCH_OPT_QUERY; break; @@ -406,35 +347,15 @@ main(int argc, char *argv[]) } /* - * Parse the configuration file, extracting a dictionary of key/value - * pairs. Each key is "section.name" and values are always strings. - * If no config given, default is /etc/pcp/pmproxy.conf (in addition, - * local user path settings in $HOME/.pcp/pmproxy.conf are merged) - - * pmsearch(1) uses keys from the [pmsearch] and [pmseries] sections, - * to share the main pmproxy.conf file (via symlink) for convenience. + * Parse the configuration file for search index path and result + * count settings. Uses [pmsearch] section from pmproxy.conf or + * a dedicated pmsearch.conf file. */ if ((config = pmIniFileSetup(inifile)) == NULL) { - pmprintf("%s: cannot setup from configuration file %s\n", - pmGetProgname(), inifile? inifile : "pmsearch.conf"); - opts.errors++; - } else { - /* - * Push command line options into the configuration, and ensure - * we have some default for attemping Redis server connections. - */ - if ((pmIniFileLookup(config, "pmsearch", "count")) == NULL || - (search_count > 0)) { - option = sdscatfmt(sdsempty(), "%u", search_count); - pmIniFileUpdate(config, "pmsearch", "count", option); - } - - if ((option = pmIniFileLookup(config, "keys", "servers")) == NULL) - if ((option = pmIniFileLookup(config, "redis", "servers")) == NULL) - option = pmIniFileLookup(config, "pmseries", "servers"); - if (option == NULL || keys_host != NULL || keys_port != 6379) { - option = sdscatfmt(sdsempty(), "%s:%u", - keys_host? keys_host : "localhost", keys_port); - pmIniFileUpdate(config, "keys", "servers", option); + if (inifile) { + pmprintf("%s: cannot setup from configuration file %s\n", + pmGetProgname(), inifile); + opts.errors++; } } @@ -442,7 +363,7 @@ main(int argc, char *argv[]) opts.errors++; if (opts.errors || (opts.flags & PM_OPTFLAG_EXIT)) { - sts = !(opts.flags & PM_OPTFLAG_EXIT); + int sts = !(opts.flags & PM_OPTFLAG_EXIT); pmUsageMessage(&opts); exit(sts); } @@ -457,7 +378,6 @@ main(int argc, char *argv[]) query = sdsjoin(&argv[opts.optind], argc - opts.optind, " "); dp = search_data_init(flags, query, search_count, search_offset); - dp->loop = uv_default_loop(); dp->settings.callbacks.on_text_result = on_search_result; dp->settings.callbacks.on_metrics = on_search_metrics; @@ -466,8 +386,14 @@ main(int argc, char *argv[]) dp->settings.module.on_info = on_search_info; dp->settings.module.on_setup = on_search_setup; - pmSearchSetEventLoop(&dp->settings.module, dp->loop); - pmSearchSetConfiguration(&dp->settings.module, config); + if (config) + pmSearchSetConfiguration(&dp->settings.module, config); + + pmSearchSetup(&dp->settings.module, dp); - return pmsearch_execute(dp); + pmSearchClose(&dp->settings.module); + + c = dp->status; + search_data_free(dp); + return c; } diff --git a/src/pmsearch/pmsearch_index.service.in b/src/pmsearch/pmsearch_index.service.in new file mode 100644 index 00000000000..9380de6590f --- /dev/null +++ b/src/pmsearch/pmsearch_index.service.in @@ -0,0 +1,10 @@ +[Unit] +Description=Daily rebuild of the PCP pmsearch full-text index +Documentation=man:pmsearch_index(1) +ConditionPathExists=!@CRONTAB_PATH@ + +[Service] +ExecStart=@PCP_BINADM_DIR@/pmsearch_index +WorkingDirectory=@PCP_VAR_DIR@ +Group=@PCP_GROUP@ +User=@PCP_USER@ diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh new file mode 100755 index 00000000000..a8657240c99 --- /dev/null +++ b/src/pmsearch/pmsearch_index.sh @@ -0,0 +1,216 @@ +#!/bin/sh +# +# Copyright (c) 2026 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# Rebuild the pmsearch full-text search index from running PMCD. +# Typically invoked nightly via systemd timer or cron. +# + +. $PCP_DIR/etc/pcp.env + +status=1 +tmp=`mktemp -d "$PCP_TMPFILE_DIR/pmsearch_index.XXXXXXXXX"` || exit 1 +trap "rm -rf $tmp; exit \$status" 0 1 2 3 15 + +prog=`basename $0` +INDEX="$PCP_VAR_DIR/lib/pcp.search" + +cat > $tmp/usage << EOF +Options: + -N,--showme dry-run, show what would be done + -o=INDEX,--output=INDEX output index file path [default: $INDEX] + -V,--verbose verbose diagnostics + --help +EOF + +_usage() +{ + pmgetopt --progname=$prog --config=$tmp/usage --usage + exit 1 +} + +VERBOSE=false +SHOWME=false + +ARGS=`pmgetopt --progname=$prog --config=$tmp/usage -- "$@"` || exit 1 +eval set -- "$ARGS" +while [ $# -gt 0 ] +do + case "$1" + in + -N) SHOWME=true ;; + -o) INDEX="$2"; shift ;; + -V) VERBOSE=true ;; + -\?) _usage; status=0; exit ;; + --) shift; break ;; + esac + shift +done + +if $VERBOSE +then + echo "$prog: index target: $INDEX" +fi + +# Extract all metric help text from PMCD in newhelp format. +# pminfo -tT output format: +# metricname [oneline text] +# Help: +# multi-line helptext +# +# +# Transform to newhelp format: +# @ metricname oneline text +# multi-line helptext +# +pminfo -tT 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { + # metric line: "name.with.dots [oneline]" or error + if (index($0, "One-line Help: Error:") > 0) { + skip = 1 + next + } + skip = 0 + name = $1 + oneline = "" + start = index($0, "[") + end = index($0, "]") + if (start > 0 && end > start) { + oneline = substr($0, start + 1, end - start - 1) + } + printf "@ %s %s\n", name, oneline + next +} +/^Help:$/ { next } +/^Full Help: Error:/ { skip = 1; next } +skip { next } +{ print } +' > $tmp/helptext + +nhelplines=`wc -l < $tmp/helptext` +if [ "$nhelplines" -eq 0 ] +then + echo >&2 "$prog: warning: no metrics found from PMCD" + status=0 + exit +fi + +if $VERBOSE +then + nmetrics=`grep -c '^@ ' $tmp/helptext` + echo "$prog: extracted $nmetrics metrics ($nhelplines lines)" +fi + +# Extract instance names from running PMCD. +# Build metric→indom mapping, then get instances via pmprobe -I. +# Deduplicate by (indom, instance_name) since many metrics share indoms. +# +pminfo -I 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z]/ { metric = $1 } +/InDom:/ && !/PM_INDOM_NULL/ { + for (i = 1; i <= NF; i++) { + if ($i == "InDom:") { + print metric "\t" $(i+1) + break + } + } +}' > $tmp/metric_indom + +pmprobe -I 2>/dev/null > $tmp/pmprobe_output + +$PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} +FILENAME == ARGV[2] { + # metric→oneline from the helptext file we already built + if (/^@ [a-zA-Z]/) { + name = $2 + start = index($0, " ") + start = index(substr($0, start + 1), " ") + oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" + metric_oneline[name] = oneline + } + next +} +{ + # pmprobe -I: metric count "inst1" "inst2" ... + metric = $1 + count = $2 + 0 + if (count <= 0 || $2 == "PM_IN_NULL") next + + indom = metric_indom[metric] + if (indom == "") next + + if (!(indom in indom_oneline) && metric_oneline[metric] != "") + indom_oneline[indom] = metric_oneline[metric] + + rest = $0 + while (match(rest, /"[^"]*"/)) { + inst = substr(rest, RSTART + 1, RLENGTH - 2) + key = indom SUBSEP inst + if (!(key in seen)) { + seen[key] = 1 + inst_data[++ninst] = indom "\t" inst + inst_indom[ninst] = indom + } + rest = substr(rest, RSTART + RLENGTH) + } +} +END { + for (i = 1; i <= ninst; i++) { + indom = inst_indom[i] + oneline = indom_oneline[indom] + printf "@ I %s\n%s\n\n", inst_data[i], oneline + } +} +' "$tmp/metric_indom" "$tmp/helptext" "$tmp/pmprobe_output" >> $tmp/helptext + +if $VERBOSE +then + ninst=`grep -c '^@ I ' $tmp/helptext` + echo "$prog: extracted $ninst instances" +fi + +if $SHOWME +then + echo "$prog: would run: $PCP_BINADM_DIR/newhelp -S -o $INDEX $tmp/helptext" + status=0 + exit +fi + +# Ensure output directory exists +outdir=`dirname "$INDEX"` +if [ ! -d "$outdir" ] +then + mkdir -p "$outdir" 2>/dev/null + if [ ! -d "$outdir" ] + then + echo >&2 "$prog: cannot create directory $outdir" + exit + fi +fi + +# Build the search index +if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/helptext +then + if $VERBOSE + then + echo "$prog: index written to $INDEX" + fi + status=0 +else + echo >&2 "$prog: $PCP_BINADM_DIR/newhelp -S failed" +fi diff --git a/src/pmsearch/pmsearch_index.timer b/src/pmsearch/pmsearch_index.timer new file mode 100644 index 00000000000..c15dd697f20 --- /dev/null +++ b/src/pmsearch/pmsearch_index.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Daily rebuild of the PCP pmsearch full-text index +PartOf=pmcd.service + +[Timer] +# if enabled, runs 10m after boot and just after midnight (and after +# the daily local PCP archive processing at 00:10:00 and proxy +# processing at 00:15:00) +OnBootSec=10min +OnCalendar=*-*-* 00:20:00 +Unit=pmsearch_index.service From e61fe57f370d0593da6d00ca68cc7c2e27352ec3 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 24 Jul 2026 13:43:43 -0400 Subject: [PATCH 02/22] pmsearch: fix SQLite3 build - add CFLAGS and BuildRequires Propagate sqlite3_CFLAGS from pkg-config through builddefs so that sqlite3.h is found during compilation of search.c and search_sqlite.c. Add sqlite-devel to the RPM spec BuildRequires. --- build/rpm/pcp.spec.in | 1 + src/include/builddefs.in | 1 + src/libpcp_web/src/GNUmakefile | 3 ++- src/newhelp/GNUmakefile | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build/rpm/pcp.spec.in b/build/rpm/pcp.spec.in index 5fa1061e024..afbeca458b9 100755 --- a/build/rpm/pcp.spec.in +++ b/build/rpm/pcp.spec.in @@ -133,6 +133,7 @@ BuildRequires: libpfm-devel >= 4.4 BuildRequires: ncurses-devel BuildRequires: readline-devel +BuildRequires: sqlite-devel >= 3.9.0 # Utilities used indirectly e.g. by scripts we install Requires: bash xz zstd gawk sed grep coreutils diffutils findutils diff --git a/src/include/builddefs.in b/src/include/builddefs.in index 7db05295f87..0e229964059 100644 --- a/src/include/builddefs.in +++ b/src/include/builddefs.in @@ -136,6 +136,7 @@ INVISIBILITY = @INVISIBILITY@ # hide shared library symbols LZMACFLAGS = @lzma_CFLAGS@ LIBUVCFLAGS = @libuv_CFLAGS@ +SQLITE3CFLAGS = @sqlite3_CFLAGS@ OPENSSLCFLAGS = @openssl_CFLAGS@ SASLCFLAGS = @libsasl2_CFLAGS@ CMOCKACFLAGS = @cmocka_CFLAGS@ diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index 3424518090e..750156c2545 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -58,7 +58,8 @@ ifneq "$(HAVE_LIBINIH)" "true" LCFLAGS += -Iinih endif -LCFLAGS += $(C99_CFLAGS) -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ +LCFLAGS += $(C99_CFLAGS) $(SQLITE3CFLAGS) \ + -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/include \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/src diff --git a/src/newhelp/GNUmakefile b/src/newhelp/GNUmakefile index 0a87771330e..d53f8e319e9 100644 --- a/src/newhelp/GNUmakefile +++ b/src/newhelp/GNUmakefile @@ -20,6 +20,7 @@ include $(TOPDIR)/src/libpcp/src/GNUlibrarydefs TARGETS = newhelp$(EXECSUFFIX) chkhelp$(EXECSUFFIX) CFILES = newhelp.c chkhelp.c search_sqlite.c +LCFLAGS += $(SQLITE3CFLAGS) LLDFLAGS = $(WARN_OFF) LLDLIBS = $(PCP_PMDALIB) From c2d4b74869534185c7301ca6c4bcdff3f52a20ca Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 24 Jul 2026 17:08:57 -0400 Subject: [PATCH 03/22] pmsearch: fix dual-index queries and DIST_ROOT packaging Fix FTS5 queries against the attached base index. SQLite FTS5 functions (bm25, highlight) and MATCH require the bare table name ("docs"), while FROM uses the schema-qualified name ("base.docs"). Split the table parameter into separate "from" and "fts" arguments in search_query_table, search_suggest_table, and search_indom_table. Fix build-time index install to write pcp.search to a local temp file then use $(INSTALL) to copy it to $(PCP_SHARE_DIR)/lib/ so that DIST_ROOT and DIST_MANIFEST are handled correctly for RPM packaging. Add pcp.search to LDIRT for make clean. Regenerate configure from configure.ac to include the SQLite3 pkg-config detection block. --- configure | 145 ++++++++++++++++++++++++++++++++++++ src/libpcp_web/src/search.c | 27 +++---- src/pmdas/GNUmakefile | 6 +- 3 files changed, 163 insertions(+), 15 deletions(-) diff --git a/configure b/configure index 64dd8841ba3..547a1892937 100755 --- a/configure +++ b/configure @@ -761,6 +761,10 @@ ncurses_CFLAGS HAVE_DELAYACCT libnl3_LIBS libnl3_CFLAGS +lib_for_sqlite3 +HAVE_SQLITE3 +sqlite3_LIBS +sqlite3_CFLAGS lib_for_inih HAVE_LIBINIH HAVE_LIBUV @@ -1096,6 +1100,8 @@ libdrm_amdgpu_CFLAGS libdrm_amdgpu_LIBS libuv_CFLAGS libuv_LIBS +sqlite3_CFLAGS +sqlite3_LIBS libnl3_CFLAGS libnl3_LIBS ncurses_CFLAGS @@ -1865,6 +1871,10 @@ Some influential environment variables: libuv_CFLAGS C compiler flags for libuv, overriding pkg-config libuv_LIBS linker flags for libuv, overriding pkg-config + sqlite3_CFLAGS + C compiler flags for sqlite3, overriding pkg-config + sqlite3_LIBS + linker flags for sqlite3, overriding pkg-config libnl3_CFLAGS C compiler flags for libnl3, overriding pkg-config libnl3_LIBS linker flags for libnl3, overriding pkg-config @@ -12283,6 +12293,141 @@ HAVE_LIBINIH=$have_libinih +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3 >= 3.9.0" >&5 +printf %s "checking for sqlite3 >= 3.9.0... " >&6; } + +if test -n "$sqlite3_CFLAGS"; then + pkg_cv_sqlite3_CFLAGS="$sqlite3_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"sqlite3 >= 3.9.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "sqlite3 >= 3.9.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_sqlite3_CFLAGS=`$PKG_CONFIG --cflags "sqlite3 >= 3.9.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$sqlite3_LIBS"; then + pkg_cv_sqlite3_LIBS="$sqlite3_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"sqlite3 >= 3.9.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "sqlite3 >= 3.9.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_sqlite3_LIBS=`$PKG_CONFIG --libs "sqlite3 >= 3.9.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + sqlite3_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "sqlite3 >= 3.9.0" 2>&1` + else + sqlite3_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "sqlite3 >= 3.9.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$sqlite3_PKG_ERRORS" >&5 + + have_sqlite3=false +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_sqlite3=false +else + sqlite3_CFLAGS=$pkg_cv_sqlite3_CFLAGS + sqlite3_LIBS=$pkg_cv_sqlite3_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_sqlite3=true +fi +if test $have_sqlite3 = false; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for sqlite3_open in -lsqlite3" >&5 +printf %s "checking for sqlite3_open in -lsqlite3... " >&6; } +if test ${ac_cv_lib_sqlite3_sqlite3_open+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lsqlite3 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char sqlite3_open (void); +int +main (void) +{ +return sqlite3_open (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_sqlite3_sqlite3_open=yes +else case e in #( + e) ac_cv_lib_sqlite3_sqlite3_open=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_sqlite3_sqlite3_open" >&5 +printf "%s\n" "$ac_cv_lib_sqlite3_sqlite3_open" >&6; } +if test "x$ac_cv_lib_sqlite3_sqlite3_open" = xyes +then : + have_sqlite3=true + lib_for_sqlite3="-lsqlite3" + +else case e in #( + e) have_sqlite3=false + lib_for_sqlite3="" + + ;; +esac +fi + +else + lib_for_sqlite3="$sqlite3_LIBS" +fi +HAVE_SQLITE3=$have_sqlite3 + + + + pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libnl-3.0" >&5 printf %s "checking for libnl-3.0... " >&6; } diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 056e5238183..64a230e85d5 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -158,7 +158,7 @@ search_make_docid(sqlite3_int64 rowid, const char *name) * Appends results to the hits array; caller owns the array. */ static int -search_query_table(sqlite3 *db, const char *table, +search_query_table(sqlite3 *db, const char *from, const char *fts, pmSearchTextRequest *request, sds match_expr, sds type_filter, pmSearchTextResult **hits, int *nhits, int *maxhits) { @@ -174,9 +174,9 @@ search_query_table(sqlite3 *db, const char *table, " highlight(%s, 2, '', '')" " FROM %s WHERE %s MATCH ?%S" " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", - table, table, table, table, - table, table, type_filter, - table); + fts, fts, fts, fts, + from, fts, type_filter, + fts); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -283,12 +283,12 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, match = search_build_match(request); type_filter = search_type_filter(request); - search_query_table(smd->db, "docs", request, match, type_filter, + search_query_table(smd->db, "docs", "docs", request, match, type_filter, &hits, &nhits, &maxhits); if (smd->has_base) { - search_query_table(smd->db, "base.docs", request, match, type_filter, - &hits, &nhits, &maxhits); + search_query_table(smd->db, "base.docs", "docs", request, match, + type_filter, &hits, &nhits, &maxhits); } sdsfree(match); @@ -322,7 +322,8 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, } static int -search_suggest_table(sqlite3 *db, const char *table, sds match, +search_suggest_table(sqlite3 *db, const char *from, const char *fts, + sds match, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -334,7 +335,7 @@ search_suggest_table(sqlite3 *db, const char *table, sds match, " FROM %s WHERE %s MATCH ?" " AND type IN (1, 3)" " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", - table, table, table, table); + fts, from, fts, fts); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -385,11 +386,11 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, match = sdscatfmt(sdsempty(), "name : %S*", request->query); - search_suggest_table(smd->db, "docs", match, + search_suggest_table(smd->db, "docs", "docs", match, &hits, &nhits, &maxhits); if (smd->has_base) - search_suggest_table(smd->db, "base.docs", match, + search_suggest_table(smd->db, "base.docs", "docs", match, &hits, &nhits, &maxhits); sdsfree(match); @@ -418,7 +419,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, } static int -search_indom_table(sqlite3 *db, const char *table, const char *query, +search_indom_table(sqlite3 *db, const char *from, const char *query, int querylen, pmSearchTextResult **hits, int *nhits, int *maxhits) { @@ -430,7 +431,7 @@ search_indom_table(sqlite3 *db, const char *table, const char *query, "SELECT rowid, name, oneline, helptext, type, indom" " FROM %s WHERE indom = ?" " ORDER BY type", - table); + from); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index 0992731ff36..25d2c3219db 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -44,7 +44,7 @@ PYPMDAS = gluster zswap unbound haproxy \ uwsgi rocestat hdb rds chrony db2 SUBDIRS = $(CPMDAS) $(PLPMDAS) $(PYPMDAS) -LDIRT = local.conf pmcd.conf local.root +LDIRT = local.conf pmcd.conf local.root pcp.search ifneq ("$(wildcard $(TOPDIR)/src/include/pcp/config.h)","") HAVE_UNIX_DOMAIN_SOCKETS = $(shell grep 'define HAVE_STRUCT_SOCKADDR_UN' $(TOPDIR)/src/include/pcp/config.h >/dev/null && echo yes) @@ -100,8 +100,10 @@ ifeq "$(HAVE_SQLITE3)" "true" help_files="$$help_files $$f"; \ done; \ if [ -n "$$help_files" ]; then \ + $(TOPDIR)/src/newhelp/newhelp -S -o pcp.search $$help_files; \ $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib; \ - $(TOPDIR)/src/newhelp/newhelp -S -o $(PCP_SHARE_DIR)/lib/pcp.search $$help_files; \ + $(INSTALL) -m 644 pcp.search $(PCP_SHARE_DIR)/lib/pcp.search; \ + rm -f pcp.search; \ fi endif From 0f193b3e7894678c7a638106a3111567b6f62f42 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Mon, 27 Jul 2026 18:38:02 -0400 Subject: [PATCH 04/22] pmsearch: update qa tests and fix deduplication when combining the build time and nightly indexes --- qa/1871 | 202 ++++++++++++++++++++++++++---------- src/libpcp_web/src/search.c | 33 +++--- 2 files changed, 169 insertions(+), 66 deletions(-) diff --git a/qa/1871 b/qa/1871 index 7916b86c365..da0d3e473c4 100755 --- a/qa/1871 +++ b/qa/1871 @@ -1,8 +1,10 @@ #!/bin/sh # PCP QA Test No. 1871 -# Exercise pmsearch functionality. +# Exercise pmsearch functionality using local SQLite FTS5 index. +# Tests the dual-index strategy: a build-time base index (from help +# files) with a nightly index (from archive/runtime data) as primary. # -# Copyright (c) 2020 Red Hat. All Rights Reserved. +# Copyright (c) 2020,2026 Red Hat. All Rights Reserved. # seq=`basename $0` @@ -13,13 +15,13 @@ echo "QA output created by $seq" . ./common.filter . ./common.check -_check_series -_check_search -_check_key_server_version_offline +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" _cleanup() { - [ -n "$port" ] && $keys_cli $port shutdown cd $here $sudo rm -rf $tmp $tmp.* } @@ -27,76 +29,170 @@ _cleanup() status=1 # failure is the default! trap "_cleanup; exit \$status" 0 1 2 3 15 -# real QA test starts here -echo "=== Start test key server ===" -key_server_port=`_find_free_port` -key_server_name=`_find_key_server_name` -key_server_path=`_find_key_server_modules` -key_server_search=`_find_key_server_search` -keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" -# workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -$sudo chmod +x $keysearch -# module is only accessible by the associated key server user account -$sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & -port="-p $key_server_port" -_check_key_server_ping $key_server_port -echo +_filter_info() +{ + sed \ + -e 's/: [0-9][0-9]*/: INTEGER/g' \ + #end +} -echo "=== Load an empty index ===" -pmsearch $port no results +_filter_extras() +{ + sed \ + -e 's/\(Score:\) [0-9][0-9]*\.[0-9][0-9]*/\1 N.NN/g' \ + -e 's/\([0-9][0-9]* hit in\) [0-9][0-9]*\.[0-9][0-9]* seconds/\1 N.NNN seconds/g' \ + -e 's/\([0-9][0-9]* hits in\) [0-9][0-9]*\.[0-9][0-9]* seconds/\1 N.NNN seconds/g' \ + #end +} -echo "=== Load deterministic search content ===" -_filter_load() +_filter_docid() { sed \ - -e "s,$here,PATH,g" \ + -e 's/^ID: [0-9][0-9]*:/ID: N:/g' \ #end } -pmseries $port --load $here/archives/sample-labels | _filter_load + +# real QA test starts here + +archive="$here/archives/sample-labels" +[ -f "$archive.0" ] || _notrun "sample-labels archive not found" +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" + +# Build two indexes to mirror the real dual-index strategy: +# base index = build-time index from PMDA help files +# nightly index = runtime index from archive (metrics + instances) + +# --- Base index (like $PCP_SHARE_DIR/lib/pcp.search) --- +mkdir -p $tmp.share/lib +newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Base index built" + +# --- Nightly index (like $PCP_VAR_DIR/lib/pcp.search) --- +# Extract metric help text from archive in newhelp format +pminfo -a $archive -tT 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { + if (index($0, "One-line Help: Error:") > 0) { + skip = 1 + next + } + skip = 0 + name = $1 + oneline = "" + start = index($0, "[") + end = index($0, "]") + if (start > 0 && end > start) { + oneline = substr($0, start + 1, end - start - 1) + } + printf "@ %s %s\n", name, oneline + next +} +/^Help:$/ { next } +/^Full Help: Error:/ { skip = 1; next } +skip { next } +{ print } +' > $tmp.helptext + +# Extract instance names from archive +pminfo -a $archive -I 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z]/ { metric = $1 } +/InDom:/ && !/PM_INDOM_NULL/ { + for (i = 1; i <= NF; i++) { + if ($i == "InDom:") { + print metric "\t" $(i+1) + break + } + } +}' > $tmp.metric_indom + +pmprobe -a $archive -I 2>/dev/null > $tmp.pmprobe_output + +$PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} +FILENAME == ARGV[2] { + if (/^@ [a-zA-Z]/) { + name = $2 + start = index($0, " ") + start = index(substr($0, start + 1), " ") + oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" + metric_oneline[name] = oneline + } + next +} +{ + metric = $1 + count = $2 + 0 + if (count <= 0 || $2 == "PM_IN_NULL") next + + indom = metric_indom[metric] + if (indom == "") next + + if (!(indom in indom_oneline) && metric_oneline[metric] != "") + indom_oneline[indom] = metric_oneline[metric] + + rest = $0 + while (match(rest, /"[^"]*"/)) { + inst = substr(rest, RSTART + 1, RLENGTH - 2) + key = indom SUBSEP inst + if (!(key in seen)) { + seen[key] = 1 + inst_data[++ninst] = indom "\t" inst + inst_indom[ninst] = indom + } + rest = substr(rest, RSTART + RLENGTH) + } +} +END { + for (i = 1; i <= ninst; i++) { + indom = inst_indom[i] + oneline = indom_oneline[indom] + printf "@ I %s\n%s\n\n", inst_data[i], oneline + } +} +' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" >> $tmp.helptext + +mkdir -p $tmp.var/lib +newhelp -S -o $tmp.var/lib/pcp.search $tmp.helptext 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Nightly index built" +echo + +# Override PCP paths so pmsearch finds our test indexes +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share + +echo "=== Empty search ===" +pmsearch -C xyznonexistent search_terms="99 random interesting result" for term in $search_terms do echo "=== Search on $term ===" - pmsearch $port -C $term + pmsearch -C $term done echo "=== Search engine info ===" -_filter_info() -{ - sed \ - -e 's/: [0-9][0-9]*\.[0-9][0-9]*/: DOUBLE/g' \ - -e 's/: [0-9][0-9]*/: INTEGER/g' \ - #end -} -pmsearch $port -i -C | _filter_info +pmsearch -C -i | _filter_info +echo echo "=== Extra reporting - timing, score, hits, doc IDs ===" -_filter_extras() -{ - # need to filter out score, since it seems the search module indexes - # records asynchronously and therefore there may be a delay before - # "score" "stabilizes" until all records are processed - this may - # take longer then the adding of the records themselves - sed \ - -e 's/\(Score:\) 0\.[0-9][0-9]*/\1 0.XX/g' \ - -e 's/\([0-9][0-9]* hit in\) 0\.[0-9][0-9]* seconds/\1 0.X seconds/g' \ - -e 's/\([0-9][0-9]* hits in\) 0\.[0-9][0-9]* seconds/\1 0.Y seconds/g' \ - #end -} -pmsearch $port -dStT 99 | _filter_extras +pmsearch -dStT 99 | _filter_extras | _filter_docid +echo echo "=== Suggestion search 1 ===" -pmsearch $port -s sample | _filter_extras +pmsearch -s -N 3 sample echo "=== Suggestion search 2 ===" -pmsearch $port -s blue | _filter_extras +pmsearch -s blue echo "=== Indom search ===" -pmsearch $port -n 29.3 | _filter_extras - +pmsearch -n 29.3 -cat $tmp.keys >> $seq_full # success, all done status=0 diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 64a230e85d5..46e1851d7f2 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -160,6 +160,7 @@ search_make_docid(sqlite3_int64 rowid, const char *name) static int search_query_table(sqlite3 *db, const char *from, const char *fts, pmSearchTextRequest *request, sds match_expr, sds type_filter, + const char *exclude, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -172,10 +173,10 @@ search_query_table(sqlite3 *db, const char *from, const char *fts, " highlight(%s, 0, '', '')," " highlight(%s, 1, '', '')," " highlight(%s, 2, '', '')" - " FROM %s WHERE %s MATCH ?%S" + " FROM %s WHERE %s MATCH ?%S%s" " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", fts, fts, fts, fts, - from, fts, type_filter, + from, fts, type_filter, exclude, fts); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); @@ -284,11 +285,13 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, type_filter = search_type_filter(request); search_query_table(smd->db, "docs", "docs", request, match, type_filter, - &hits, &nhits, &maxhits); + "", &hits, &nhits, &maxhits); if (smd->has_base) { search_query_table(smd->db, "base.docs", "docs", request, match, - type_filter, &hits, &nhits, &maxhits); + type_filter, + " AND name NOT IN (SELECT name FROM docs)", + &hits, &nhits, &maxhits); } sdsfree(match); @@ -323,7 +326,7 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, static int search_suggest_table(sqlite3 *db, const char *from, const char *fts, - sds match, + sds match, const char *exclude, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -333,9 +336,9 @@ search_suggest_table(sqlite3 *db, const char *from, const char *fts, sql = sdscatfmt(sdsempty(), "SELECT rowid, name, type, bm25(%s, 9.0, 4.0, 2.0)" " FROM %s WHERE %s MATCH ?" - " AND type IN (1, 3)" + " AND type IN (1, 3)%s" " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", - fts, from, fts, fts); + fts, from, fts, exclude, fts); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -386,11 +389,12 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, match = sdscatfmt(sdsempty(), "name : %S*", request->query); - search_suggest_table(smd->db, "docs", "docs", match, + search_suggest_table(smd->db, "docs", "docs", match, "", &hits, &nhits, &maxhits); if (smd->has_base) search_suggest_table(smd->db, "base.docs", "docs", match, + " AND name NOT IN (SELECT name FROM docs)", &hits, &nhits, &maxhits); sdsfree(match); @@ -420,7 +424,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, static int search_indom_table(sqlite3 *db, const char *from, const char *query, - int querylen, + int querylen, const char *exclude, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -429,9 +433,9 @@ search_indom_table(sqlite3 *db, const char *from, const char *query, sql = sdscatfmt(sdsempty(), "SELECT rowid, name, oneline, helptext, type, indom" - " FROM %s WHERE indom = ?" + " FROM %s WHERE indom = ?%s" " ORDER BY type", - from); + from, exclude); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -497,11 +501,14 @@ search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, pmtimespecNow(&started); search_indom_table(smd->db, "docs", request->query, - sdslen(request->query), &hits, &nhits, &maxhits); + sdslen(request->query), "", + &hits, &nhits, &maxhits); if (smd->has_base) search_indom_table(smd->db, "base.docs", request->query, - sdslen(request->query), &hits, &nhits, &maxhits); + sdslen(request->query), + " AND name NOT IN (SELECT name FROM docs)", + &hits, &nhits, &maxhits); pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); From f6875c6c7d7a07921498063aa6fa44bcbf314109 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Tue, 28 Jul 2026 14:08:27 -0400 Subject: [PATCH 05/22] pmsearch: fix qa tests and change nightly index strategy --- qa/1871 | 21 +- qa/1871.out | 240 +++++++----- qa/1872 | 227 ++++++----- qa/1872.out | 673 +++++++++++++++++++++++---------- qa/group | 2 +- src/libpcp_web/src/search.c | 92 ++--- src/newhelp/search_sqlite.c | 2 +- src/pmsearch/pmsearch_index.sh | 38 +- 8 files changed, 822 insertions(+), 473 deletions(-) diff --git a/qa/1871 b/qa/1871 index da0d3e473c4..28f50d2a6be 100755 --- a/qa/1871 +++ b/qa/1871 @@ -59,9 +59,10 @@ archive="$here/archives/sample-labels" help=$PCP_PMDAS_DIR/sample/help [ -f "$help" ] || _notrun "sample PMDA help file not found" -# Build two indexes to mirror the real dual-index strategy: -# base index = build-time index from PMDA help files -# nightly index = runtime index from archive (metrics + instances) +# Mirror the real dual-index strategy: +# 1. Build base index from PMDA help files (like package install) +# 2. Copy base to nightly location (like pmsearch_index.sh does) +# 3. Append instance data from archive to the nightly copy # --- Base index (like $PCP_SHARE_DIR/lib/pcp.search) --- mkdir -p $tmp.share/lib @@ -70,7 +71,11 @@ newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ echo "Base index built" # --- Nightly index (like $PCP_VAR_DIR/lib/pcp.search) --- -# Extract metric help text from archive in newhelp format +# Copy base as starting point +mkdir -p $tmp.var/lib +cp $tmp.share/lib/pcp.search $tmp.var/lib/pcp.search + +# Extract metric help text from archive (used for instance oneline mapping) pminfo -a $archive -tT 2>/dev/null | $PCP_AWK_PROG ' /^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { if (index($0, "One-line Help: Error:") > 0) { @@ -154,12 +159,12 @@ END { printf "@ I %s\n%s\n\n", inst_data[i], oneline } } -' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" >> $tmp.helptext +' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" > $tmp.instances -mkdir -p $tmp.var/lib -newhelp -S -o $tmp.var/lib/pcp.search $tmp.helptext 2>&1 \ +# Append instances to the copied base index +newhelp -S -o $tmp.var/lib/pcp.search $tmp.instances 2>&1 \ | sed -e "s,$tmp,TMP,g" -echo "Nightly index built" +echo "Nightly index built (base + instances)" echo # Override PCP paths so pmsearch finds our test indexes diff --git a/qa/1871.out b/qa/1871.out index dd5514bbc59..313437ec585 100644 --- a/qa/1871.out +++ b/qa/1871.out @@ -1,56 +1,141 @@ QA output created by 1871 -=== Start test key server === -PING -PONG +Base index built +Nightly index built (base + instances) -=== Load an empty index === +=== Empty search === 0 search results -=== Load deterministic search content === -pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === Search on 99 === -Type: indom -Name: 29.3 -InDom: 29.3 +Type: metric +Name: SAMPLE.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" === Search on random === -Type: indom -Name: 29.3 -InDom: 29.3 +Type: metric +Name: sample.drift +One line: A random trended metric +Help: This metric returns a random value (expected mean is approximately 200), +subject to a trending pattern such that the sequence is mainly monotonic, +with a change in direction after on average 4 consecutive samples. + +Use pmStore() to modify the instantaneous value, which becomes the new +expected mean. + +Type: metric +Name: sample.byte_rate +One line: instantaneous bytes/second +Help: random value in the range (0,1023), so avg value is 512 bytes/second + +Type: metric +Name: sample.kbyte_rate +One line: instantaneous Kbytes/second +Help: random value in the range (0,1023), so avg value is 512 Kbytes/second + +Type: metric +Name: sample.byte_rate_perhour +One line: instantaneous bytes/hour +Help: random value in the range (0,1023), so avg value is 512 bytes/hour + +Type: metric +Name: sample.dodgey.value +One line: 5 unreliable instances +Help: The metric is a set of 5 instantaneous values, drawn at random from the +range 0 to 100. The number of instances "visible" is controlled by +sample.dodgey.control. + +Type: metric +Name: SAMPLE.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" Type: metric -Name: sample.mirage -InDom: 29.3 +Name: sample.scramble.bin +One line: Several constant instances, instances scrambled +Help: Like sample.bin, except +1. instances are missing with probability 0.33 +2. order of the instances from pmFetch is random + +Designed to help testing instance matching between pmFetch calls +for PCP clients. + +Type: metric +Name: SAMPLE.6 +One line: Instance domain "dodgey" for sample PMDA. +Help: Instances for an unreliable instance domain, where the instances can +under certain conditions (random or controlled) return holes, errors, +and other interesting scenarios for testing. A sample.dodgey.control +metric store controls this instance domain. + +Type: metric +Name: sample.dodgey.control +One line: control values retured for sample.dodgey.value +Help: If sample.dodgey.control is <= 0, then this is returned as the "numval" +component in the pmResult (0 => no values available, less than 0 => +various errors). + +If sample.dodgey.control is between 1 and 5 (inclusive), then this many +of the values will be "visible". The values will be selected in order +from the underlying 5 instances. + +If sample.dodgey.control is > 5, then at random times (between 1 and +sample.dodgey.control fetches of the metric), the number of instances +available is changed according to the following probabilities ... + 0.9 some number of instances in the range 0 to 5, selected at random + from the underlying 5 instances. + 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION) + +Type: metric +Name: SAMPLE.12 +One line: Instance domain "procs" for sample PMDA +Help: Simulated and small instance domain that mimics the behaviour of the +process instance domain from the proc PMDA. + +The first 5 instances are fixed. Then next 15 instances are variable +such that each time an associated metric is fetched: +- if an instance is currently defined, it may disappear with + probability 0.075 +- for each instance that is not defined, a new instance may + appear with probability 0.075 + +The internal instance identifiers mimic PIDs and are monotonically +increasing up to 1000, when they wrap (duplicates are obviously +avoided). + +The external instance names are the instance number, space, then +a random "executable" name. +=== Search on interesting === +Type: metric +Name: SAMPLE.6 +One line: Instance domain "dodgey" for sample PMDA. +Help: Instances for an unreliable instance domain, where the instances can +under certain conditions (random or controlled) return holes, errors, +and other interesting scenarios for testing. A sample.dodgey.control +metric store controls this instance domain. + +Type: metric +Name: sample.mirage_longlong One line: Simple saw-tooth rate, but instances come and go -Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution +Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution over time. Different instances of the metric have different baselines -for the saw-tooth, but all have an max-to-min range of 100. +for the saw-tooth, but all have an max-to-min range of 100,000,000. -What makes this metric interesting is that instances come and go although -not more often than once every 10 seconds by default. Use pmstore to -change sample.controller.mirage and the frequency of instance domain -changes can be varied. +What makes this metric interesting is that instances come and go (not more +often than once every 10 seconds however). Instance 0 is always present, +but the other instances 1 thru 49 come and go in a cyclic pattern with +a large random component influencing when each instance appears and +disappears. -Instance 0 is always present, but the other instances 1 thru 49 come -and go in a cyclic pattern with a large random component influencing -when each instance appears and disappears. - -The underlying counter starts at 0 and is incremented once -for each pmFetch() to this metric and/or sample.colour and/or -sample.mirage_longlong. +The underlying counter starts at 0 and is incremented once for each +pmFetch() to this metric and/or sample.mirage and/or sample.colour. Use pmStore() to modify the underlying counter (independent of which instance or instances are used). -=== Search on interesting === + Type: metric Name: sample.mirage -InDom: 29.3 One line: Simple saw-tooth rate, but instances come and go Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution over time. Different instances of the metric have different baselines @@ -74,87 +159,72 @@ instance or instances are used). === Search on result === Type: metric Name: sample.rapid -InDom: none One line: count very quickly Help: Base counter increments by 8*10^7 per fetch. Result is 10 x base counter. + +Type: metric +Name: sample.not_ready +One line: interval (in seconds) during which PMDA does not respond to PDUs +Help: Store a positive number of seconds as the value of this metric. The +following PDU received will result in the following sequence of events: + 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd + 2. sleep for the given number of seconds + 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd +If everything went as planned, sample.not_ready returns to 0, otherwise it +has a negative error code as the value. + +Type: metric +Name: sample.not_ready_msec +One line: interval (in milliseconds) during which PMDA does not respond to PDUs +Help: Store a positive number of milliseconds as the value of this metric. The +following PDU received will result in the following sequence of events: + 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd + 2. sleep for the given number of milliseconds + 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd +If everything went as planned, sample.not_ready returns to 0, otherwise it +has a negative error code as the value. === Search engine info === -RediSearch statistics: +Search index statistics: Documents: INTEGER Terms: INTEGER Records: INTEGER -- Average records per doc: DOUBLE -- Average bytes per record: DOUBLE -- Inverted Index - Size: DOUBLE MB - Capacity: DOUBLE MB - Overhead: DOUBLE -- Skip Index - Size: DOUBLE MB -- Score Index - Size: DOUBLE MB -- Average offsets per term: DOUBLE -- Average offset bits per record: DOUBLE + === Extra reporting - timing, score, hits, doc IDs === -1 hit in 0.X seconds +1 hit in N.NNN seconds -ID: 6697a5ee42032f3e914b3475c07a16605d3c8b7a -Score: 0.XX -Type: indom -Name: 29.3 -InDom: 29.3 +ID: N:SAMPLE.3 +Score: N.NN +Type: metric +Name: SAMPLE.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" + === Suggestion search 1 === -Name: sample.mirage +Type: metric +Name: sample.string.null -Name: sample.rapid +Type: metric +Name: sample.aggregate.null -Name: sample.colour +Type: metric +Name: sample.bad.novalues === Suggestion search 2 === +Type: instance Name: blue === Indom search === -Type: indom -Name: 29.3 -InDom: 29.3 -One line: Instance domain "mirage" for sample PMDA -Help: Random number of instances, that change with time. Instance "m-00" (0) -is always present, while the others are numbered 1 .. 49 and named "m-01" -.. "m-99" - Type: instance -Name: m-04 +Name: m-00 InDom: 29.3 +One line: Simple saw-tooth rate, but instances come and go Type: instance Name: m-01 InDom: 29.3 +One line: Simple saw-tooth rate, but instances come and go Type: instance -Name: m-00 -InDom: 29.3 - -Type: metric -Name: sample.mirage +Name: m-04 InDom: 29.3 One line: Simple saw-tooth rate, but instances come and go -Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution -over time. Different instances of the metric have different baselines -for the saw-tooth, but all have an max-to-min range of 100. - -What makes this metric interesting is that instances come and go although -not more often than once every 10 seconds by default. Use pmstore to -change sample.controller.mirage and the frequency of instance domain -changes can be varied. - -Instance 0 is always present, but the other instances 1 thru 49 come -and go in a cyclic pattern with a large random component influencing -when each instance appears and disappears. - -The underlying counter starts at 0 and is incremented once -for each pmFetch() to this metric and/or sample.colour and/or -sample.mirage_longlong. - -Use pmStore() to modify the underlying counter (independent of which -instance or instances are used). diff --git a/qa/1872 b/qa/1872 index adac854962e..be847a2fc56 100755 --- a/qa/1872 +++ b/qa/1872 @@ -1,8 +1,9 @@ #!/bin/sh # PCP QA Test No. 1872 # Exercise pmsearch REST API endpoints using curl(1). +# Uses a local SQLite FTS5 search index (no Redis/Valkey). # -# Copyright (c) 2020 Red Hat. All Rights Reserved. +# Copyright (c) 2020,2026 Red Hat. All Rights Reserved. # seq=`basename $0` @@ -12,142 +13,172 @@ echo "QA output created by $seq" . ./common.product . ./common.filter . ./common.check -. ./common.keys -_check_series -_check_search -_check_key_server_version_offline +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" +which pmproxy >/dev/null 2>&1 || \ + _notrun "pmproxy not installed" +which curl >/dev/null 2>&1 || \ + _notrun "curl not installed" +which pmjson >/dev/null 2>&1 || \ + _notrun "pmjson not installed" _cleanup() { cd $here [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid - [ -n "$port" ] && $keys_cli $port shutdown - if $need_restore - then - need_restore=false - _restore_config $PCP_SYSCONF_DIR/pmproxy - _restore_config $PCP_SYSCONF_DIR/pmseries - fi $sudo rm -rf $tmp $tmp.* } status=1 # failure is the default! signal=$PCP_BINADM_DIR/pmsignal - -userid=`id -u` username=`id -u -n` -hostname=`hostname` -machineid=`_machine_id` -domainname=`_domain_name` -need_restore=false trap "_cleanup; exit \$status" 0 1 2 3 15 -_filter_source() +_filter_elapsed() { sed \ - -e "s,$here,PATH,g" \ - -e "s,$hostname,QAHOST,g" \ + -e 's/\(\"elapsed\":\) [0-9][0-9]*\.[0-9][0-9]*/\1 XXX/g' \ #end } -_filter_archive() +_filter_info() { - pmjson | \ sed \ - -e "s,bozo-laptop,QALOGHOST,g" \ + -e 's/\(\"docs\":\) [0-9][0-9]*/\1 NNN/g' \ + -e 's/\(\"terms\":\) [0-9][0-9]*/\1 NNN/g' \ + -e 's/\(\"records\":\) [0-9][0-9]*/\1 NNN/g' \ #end } # real QA test starts here -_save_config $PCP_SYSCONF_DIR/pmproxy -_save_config $PCP_SYSCONF_DIR/pmseries -$sudo rm -f $PCP_SYSCONF_DIR/pmseries/* -$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/* -need_restore=true - -echo "=== Start test key server ===" -key_server_port=`_find_free_port` -key_server_name=`_find_key_server_name` -key_server_path=`_find_key_server_modules` -key_server_search=`_find_key_server_search` -keysearch="$key_server_path/$key_server_search.$DSO_SUFFIX" -# workaround until https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989385 is fixed -$sudo chmod +x $keysearch -# search module is only accessible by the key server user account -$sudo -u $key_server_name $key_server --port $key_server_port --save "" --loadmodule $keysearch > $tmp.keys 2>&1 & -_check_key_server_ping $key_server_port -port="-p $key_server_port" + +archive="$here/archives/sample-labels" +[ -f "$archive.0" ] || _notrun "sample-labels archive not found" +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" + +# Build a search index using the same dual-index strategy as test 1871: +# base index from help file, copy to nightly, append instance data. + +echo "=== Build search index ===" +newhelp -S -o $tmp.base $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +cp $tmp.base $tmp.index + +# Extract metric help text from archive (for instance oneline mapping) +pminfo -a $archive -tT 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { + if (index($0, "One-line Help: Error:") > 0) { + skip = 1 + next + } + skip = 0 + name = $1 + oneline = "" + start = index($0, "[") + end = index($0, "]") + if (start > 0 && end > start) { + oneline = substr($0, start + 1, end - start - 1) + } + printf "@ %s %s\n", name, oneline + next +} +/^Help:$/ { next } +/^Full Help: Error:/ { skip = 1; next } +skip { next } +{ print } +' > $tmp.helptext + +# Extract instance names from archive +pminfo -a $archive -I 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z]/ { metric = $1 } +/InDom:/ && !/PM_INDOM_NULL/ { + for (i = 1; i <= NF; i++) { + if ($i == "InDom:") { + print metric "\t" $(i+1) + break + } + } +}' > $tmp.metric_indom + +pmprobe -a $archive -I 2>/dev/null > $tmp.pmprobe_output + +$PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} +FILENAME == ARGV[2] { + if (/^@ [a-zA-Z]/) { + name = $2 + start = index($0, " ") + start = index(substr($0, start + 1), " ") + oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" + metric_oneline[name] = oneline + } + next +} +{ + metric = $1 + count = $2 + 0 + if (count <= 0 || $2 == "PM_IN_NULL") next + + indom = metric_indom[metric] + if (indom == "") next + + if (!(indom in indom_oneline) && metric_oneline[metric] != "") + indom_oneline[indom] = metric_oneline[metric] + + rest = $0 + while (match(rest, /"[^"]*"/)) { + inst = substr(rest, RSTART + 1, RLENGTH - 2) + key = indom SUBSEP inst + if (!(key in seen)) { + seen[key] = 1 + inst_data[++ninst] = indom "\t" inst + inst_indom[ninst] = indom + } + rest = substr(rest, RSTART + RLENGTH) + } +} +END { + for (i = 1; i <= ninst; i++) { + indom = inst_indom[i] + oneline = indom_oneline[indom] + printf "@ I %s\n%s\n\n", inst_data[i], oneline + } +} +' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" > $tmp.instances + +newhelp -S -o $tmp.index $tmp.instances 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Index built" echo +# Configure and start pmproxy with search enabled cat > $tmp.conf <>$seq_full -proxyopts="-p $proxyport -r $key_server_port -c $tmp.conf" # -Dseries,http,af -pmproxy -f -U $username -x $seq_full -l $tmp.pmproxy.log $proxyopts & +pmproxy -f -U $username -x $seq_full -l $tmp.pmproxy.log \ + -p $proxyport -c $tmp.conf & pmproxy_pid=$! # check pmproxy has started and is available for requests pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec -# have to wait until everything is properly indexed in the key server -pmsleep 10s - -$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq_full -$keys_cli $port keys pcp:text >> $seq_full -cat $tmp.pmproxy.log >> $seq_full - echo "=== /search/text - query for '99' 'random' 'interesting' 'result' ===" search_terms="99 random interesting result" for term in $search_terms @@ -170,7 +201,7 @@ echo "=== /search/text - query param is required ===" url="http://localhost:$proxyport/search/text" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/text - highlighting is supported on 'name', 'oneline' and 'helptext' by using 'highlight' param ===" @@ -189,7 +220,6 @@ echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full curl --get --silent "$url" | tee -a $seq_full | pmjson | _filter_elapsed -# url query returns 2 results echo "=== /search/text - pagination is supported using cursor params 'limit' and 'offset'" url="http://localhost:$proxyport/search/text?query=random&limit=1" @@ -297,7 +327,7 @@ echo "=== /search/suggest - query param is required ===" url="http://localhost:$proxyport/search/suggest" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/suggest - no match response ===" @@ -330,7 +360,7 @@ echo "=== /search/indom - query param is required ===" url="http://localhost:$proxyport/search/indom" echo "== verify request - $url" | sed -e "s/$proxyport/PORT/" echo "$url" >> $seq_full -curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full +curl --get --silent -s -o /dev/null -w "%{http_code}\n" "$url" | tee -a $seq_full echo "=== /search/indom - pagination is supported using cursor params 'limit' and 'offset'" @@ -352,7 +382,6 @@ echo "$url" >> $seq_full curl --get --silent "$url" | tee -a $seq_full | pmjson | _filter_info -cat $tmp.keys >> $seq_full cat $tmp.pmproxy.log >> $seq_full # success, all done diff --git a/qa/1872.out b/qa/1872.out index c2bd4ca51f9..0079b0d39e2 100644 --- a/qa/1872.out +++ b/qa/1872.out @@ -1,22 +1,18 @@ QA output created by 1872 -=== Start test key server === -PING -PONG +=== Build search index === +Index built -=== Load deterministic search content === -pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/text - query for '99' 'random' 'interesting' 'result' === == verify request - http://localhost:PORT/search/text?query=99 { "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", + "name": "SAMPLE.3", + "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } @@ -24,38 +20,95 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=random { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", + "name": "sample.drift", + "type": "metric", + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "SAMPLE.3", + "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { - "name": "sample.mirage", + "name": "sample.scramble.bin", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "SAMPLE.6", + "type": "metric", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" + }, + { + "name": "SAMPLE.12", + "type": "metric", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } == verify request - http://localhost:PORT/search/text?query=interesting { - "total": 1, + "total": 3, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ + { + "name": "SAMPLE.6", + "type": "metric", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.mirage_longlong", + "type": "metric", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, { "name": "sample.mirage", "type": "metric", - "indom": "29.3", "oneline": "Simple saw-tooth rate, but instances come and go", "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } @@ -63,17 +116,28 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels } == verify request - http://localhost:PORT/search/text?query=result { - "total": 1, + "total": 3, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { "name": "sample.rapid", "type": "metric", - "indom": "none", "oneline": "count very quickly", "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + }, + { + "name": "sample.not_ready", + "type": "metric", + "oneline": "interval (in seconds) during which PMDA does not respond to PDUs", + "helptext": "Store a positive number of seconds as the value of this metric. The\nfollowing PDU received will result in the following sequence of events:\n 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd\n 2. sleep for the given number of seconds\n 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd\nIf everything went as planned, sample.not_ready returns to 0, otherwise it\nhas a negative error code as the value." + }, + { + "name": "sample.not_ready_msec", + "type": "metric", + "oneline": "interval (in milliseconds) during which PMDA does not respond to PDUs", + "helptext": "Store a positive number of milliseconds as the value of this metric. The\nfollowing PDU received will result in the following sequence of events:\n 1. return an error PDU with PM_ERR_PMDANOTREADY to pmcd\n 2. sleep for the given number of milliseconds\n 3. sends an error PDU with PM_ERR_PMDAREADY to pmcd\nIf everything went as planned, sample.not_ready returns to 0, otherwise it\nhas a negative error code as the value." } ] } @@ -83,7 +147,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 0, "elapsed": 0, "offset": 0, - "limit": 10, + "limit": 0, "results": [ ] } === /search/text - query param is required === @@ -92,131 +156,230 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/text - highlighting is supported on 'name', 'oneline' and 'helptext' by using 'highlight' param === == verify request - http://localhost:PORT/search/text?query=domain&highlight=name,oneline,helptext { - "total": 2, + "total": 24, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.1", - "type": "indom", - "indom": "29.1", + "name": "SAMPLE.5", + "type": "metric", + "oneline": "Instance domain \"hordes\" for sample PMDA.", + "helptext": "A relatively large instance domain, with hundreds of numeric instances." + }, + { + "name": "sample.many.int", + "type": "metric", + "oneline": "variable sized instance domain", + "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" + }, + { + "name": "SAMPLE.11", + "type": "metric", + "oneline": "Instance domain \"ghosts\" for sample PMDA.", + "helptext": "Instances are the names of some famous ghosts." + }, + { + "name": "SAMPLE.6", + "type": "metric", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "SAMPLE.10", + "type": "metric", + "oneline": "Instance domain \"events\" for sample PMDA.", + "helptext": "Instances \"fungus\" and \"bogus\" for testing event metrics indoms." + }, + { + "name": "SAMPLE.1", + "type": "metric", "oneline": "Instance domain \"colour\" for sample PMDA", "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + "name": "SAMPLE.8", + "type": "metric", + "oneline": "Instance domain \"many\" for sample PMDA.", + "helptext": "A varable size set of instances controlled by sample.many.count" + }, + { + "name": "sample.many.count", + "type": "metric", + "oneline": "number of instances in sample.many.int's domain", + "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" + }, + { + "name": "SAMPLE.7", + "type": "metric", + "oneline": "Instance domain \"dynamic\" for sample PMDA.", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists." + }, + { + "name": "SAMPLE.4", + "type": "metric", + "oneline": "Instance domain \"family\" for sample PMDA.", + "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" } ] } == verify request - http://localhost:PORT/search/text?query=sample&highlight=name { - "total": 5, + "total": 211, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "sample.mirage", + "name": "sample.long.bin", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "like sample.bin but type 32" }, { - "name": "sample.colour", + "name": "sample.ulong.bin", "type": "metric", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "like sample.bin but type U32" }, { - "name": "sample.rapid", + "name": "sample.longlong.bin", "type": "metric", - "indom": "none", - "oneline": "count very quickly", - "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + "oneline": "like sample.bin but type 64" }, { - "name": "29.1", - "type": "indom", - "indom": "29.1", - "oneline": "Instance domain \"colour\" for sample PMDA", - "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + "name": "sample.float.bin", + "type": "metric", + "oneline": "like sample.bin but type FLOAT" }, { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + "name": "sample.double.bin", + "type": "metric", + "oneline": "like sample.bin but type DOUBLE" + }, + { + "name": "sample.string.bin", + "type": "metric", + "oneline": "a string-valued version of sample.bin" + }, + { + "name": "SAMPLE.0.1000", + "type": "metric", + "oneline": "dynamic sample.secret.bar metric", + "helptext": "Value \"foo\"." + }, + { + "name": "sample.const_rate.gradient", + "type": "metric", + "oneline": "rate per second to set sample.const_rate.value, writable" + }, + { + "name": "sample.string.null", + "type": "metric", + "oneline": "a zero length string" + }, + { + "name": "sample.aggregate.null", + "type": "metric", + "oneline": "a zero length aggregate" } ] } == verify request - http://localhost:PORT/search/text?query=counter&highlight=oneline,helptext { - "total": 3, + "total": 44, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "sample.colour", + "name": "sample.step_counter", "type": "metric", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "A step function (counter)", + "helptext": "This metric changes magnitude every 30 seconds, between a base value and\n3 times the base value.\n\nThe metric has \"counter\" semantics. See also sample.step.\n\nUse pmStore() to modify the base value." }, { - "name": "sample.mirage", + "name": "sample.byte_ctr", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "counter byte counter", + "helptext": "value increments randomly in the range (0,1023) bytes per fetch" }, { - "name": "sample.rapid", + "name": "sample.dynamic.counter", "type": "metric", - "indom": "none", - "oneline": "count very quickly", - "helptext": "Base counter increments by 8*10^7 per fetch. Result is 10 x base counter." + "oneline": "counter metric with dynamic indom", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists.\nEach line in this file is\n\tinternal_id external_id\n\nThis metric increments each time this instance has been seen when scanning\nthe dynamic.indom file, and resets to zero each time the instance appears." + }, + { + "name": "sample.long.bin_ctr", + "type": "metric", + "oneline": "like sample.bin but type 32, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.ulong.bin_ctr", + "type": "metric", + "oneline": "like sample.bin but type U32, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.longlong.bin_ctr", + "type": "metric", + "oneline": "like sample.bin but type 64, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.float.bin_ctr", + "type": "metric", + "oneline": "like sample.bin but type FLOAT, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.double.bin_ctr", + "type": "metric", + "oneline": "like sample.bin but type DOUBLE, SEM_COUNTER and SPACE_KBYTE", + "helptext": "" + }, + { + "name": "sample.kbyte_ctr", + "type": "metric", + "oneline": "counter Kbytes/second", + "helptext": "value increments randomly in the range (0,1023) Kbytes per fetch" + }, + { + "name": "sample.const_rate.value", + "type": "metric", + "oneline": "constant rate counter", + "helptext": "A counter that changes with constant rate between fetches.\n\nThe rate is set by storing the desired rate (counts per second)\ninto sample.const_rate.gradient" } ] } === /search/text - pagination is supported using cursor params 'limit' and 'offset' == verify request - http://localhost:PORT/search/text?query=random&limit=1 { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, "limit": 1, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + "name": "sample.drift", + "type": "metric", + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." } ] } == verify request - http://localhost:PORT/search/text?query=random&limit=1&offset=1 { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 1, "limit": 1, "results": [ { - "name": "sample.mirage", + "name": "sample.byte_rate", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" } ] } @@ -226,7 +389,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 0, "elapsed": 0, "offset": 0, - "limit": 10, + "limit": 0, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=99&field=oneline @@ -234,7 +397,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 0, "elapsed": 0, "offset": 0, - "limit": 10, + "limit": 0, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=99&field=helptext @@ -242,12 +405,11 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", + "name": "SAMPLE.3", + "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } @@ -259,10 +421,11 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3" + "name": "SAMPLE.3", + "type": "metric" } ] } @@ -271,10 +434,10 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "type": "indom" + "type": "metric" } ] } @@ -283,9 +446,10 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { + "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA" } ] @@ -295,9 +459,10 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { + "type": "metric", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } ] @@ -307,114 +472,238 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 1, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "indom": "29.3" + "type": "metric" } ] } == verify request - http://localhost:PORT/search/text?query=random&return=name,type,indom { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3" + "name": "sample.drift", + "type": "metric" }, { - "name": "sample.mirage", - "type": "metric", - "indom": "29.3" + "name": "sample.byte_rate", + "type": "metric" + }, + { + "name": "sample.kbyte_rate", + "type": "metric" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric" + }, + { + "name": "sample.dodgey.value", + "type": "metric" + }, + { + "name": "SAMPLE.3", + "type": "metric" + }, + { + "name": "sample.scramble.bin", + "type": "metric" + }, + { + "name": "SAMPLE.6", + "type": "metric" + }, + { + "name": "sample.dodgey.control", + "type": "metric" + }, + { + "name": "SAMPLE.12", + "type": "metric" } ] } === /search/text - filtering based on entity type (metric, instance, indom) using 'type' param === == verify request - http://localhost:PORT/search/text?query=random&type=metric { - "total": 1, + "total": 12, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "sample.mirage", + "name": "sample.drift", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "SAMPLE.3", + "type": "metric", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, + { + "name": "sample.scramble.bin", + "type": "metric", + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "SAMPLE.6", + "type": "metric", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" + }, + { + "name": "SAMPLE.12", + "type": "metric", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } == verify request - http://localhost:PORT/search/text?query=random&type=indom { - "total": 1, - "elapsed": XXX, + "total": 0, + "elapsed": 0, "offset": 0, - "limit": 10, - "results": [ - { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" - } - ] + "limit": 0, + "results": [ ] } == verify request - http://localhost:PORT/search/text?query=random&type=instance { "total": 0, "elapsed": 0, "offset": 0, - "limit": 10, + "limit": 0, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=random&type=metric,indom { - "total": 2, + "total": 12, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", + "name": "sample.drift", + "type": "metric", + "oneline": "A random trended metric", + "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + }, + { + "name": "sample.byte_rate", + "type": "metric", + "oneline": "instantaneous bytes/second ", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + }, + { + "name": "sample.kbyte_rate", + "type": "metric", + "oneline": "instantaneous Kbytes/second", + "helptext": "random value in the range (0,1023), so avg value is 512 Kbytes/second" + }, + { + "name": "sample.byte_rate_perhour", + "type": "metric", + "oneline": "instantaneous bytes/hour", + "helptext": "random value in the range (0,1023), so avg value is 512 bytes/hour" + }, + { + "name": "sample.dodgey.value", + "type": "metric", + "oneline": "5 unreliable instances", + "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." + }, + { + "name": "SAMPLE.3", + "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { - "name": "sample.mirage", + "name": "sample.scramble.bin", "type": "metric", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "Several constant instances, instances scrambled", + "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." + }, + { + "name": "SAMPLE.6", + "type": "metric", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "sample.dodgey.control", + "type": "metric", + "oneline": "control values retured for sample.dodgey.value", + "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" + }, + { + "name": "SAMPLE.12", + "type": "metric", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } === /search/suggest - generic request === == verify request - http://localhost:PORT/search/suggest?query=sam [ - "sample.mirage", - "sample.colour", - "sample.rapid" + "sample.string.null", + "sample.aggregate.null", + "sample.bad.novalues", + "sample.long.ten", + "sample.long.hundred", + "sample.long.million", + "sample.long.bin", + "sample.ulong.bin", + "sample.longlong.one", + "sample.longlong.ten" ] == verify request - http://localhost:PORT/search/suggest?query=sample.r -[ - "sample.mirage", - "sample.rapid", - "sample.colour" -] +[] === /search/suggest - limit is supported == verify request - http://localhost:PORT/search/suggest?query=sam&limit=2 [ - "sample.mirage", - "sample.colour" + "sample.string.null", + "sample.aggregate.null" ] === /search/suggest - query param is required === == verify request - http://localhost:PORT/search/suggest @@ -425,77 +714,55 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/indom - generic request === == verify request - http://localhost:PORT/search/indom?query=29.1 { - "total": 5, + "total": 3, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "blue", + "name": "red", "type": "instance", - "indom": "29.1" + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time" }, { "name": "green", "type": "instance", - "indom": "29.1" - }, - { - "name": "red", - "type": "instance", - "indom": "29.1" - }, - { - "name": "29.1", - "type": "indom", "indom": "29.1", - "oneline": "Instance domain \"colour\" for sample PMDA", - "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + "oneline": "Metrics with a \"saw-tooth\" trend over time" }, { - "name": "sample.colour", - "type": "metric", + "name": "blue", + "type": "instance", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "Metrics with a \"saw-tooth\" trend over time" } ] } == verify request - http://localhost:PORT/search/indom?query=29.3 { - "total": 5, + "total": 3, "elapsed": XXX, "offset": 0, - "limit": 10, + "limit": 0, "results": [ { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" - }, - { - "name": "m-04", + "name": "m-00", "type": "instance", - "indom": "29.3" + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go" }, { "name": "m-01", "type": "instance", - "indom": "29.3" + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go" }, { - "name": "m-00", + "name": "m-04", "type": "instance", - "indom": "29.3" - }, - { - "name": "sample.mirage", - "type": "metric", "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "Simple saw-tooth rate, but instances come and go" } ] } @@ -505,7 +772,7 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels "total": 0, "elapsed": 0, "offset": 0, - "limit": 10, + "limit": 0, "results": [ ] } === /search/indom - query param is required === @@ -514,57 +781,53 @@ pmseries: [Info] processed 4 archive records from PATH/archives/sample-labels === /search/indom - pagination is supported using cursor params 'limit' and 'offset' == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=0 { - "total": 5, + "total": 3, "elapsed": XXX, "offset": 0, "limit": 2, "results": [ { - "name": "blue", + "name": "red", "type": "instance", - "indom": "29.1" + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time" }, { "name": "green", "type": "instance", - "indom": "29.1" + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time" } ] } == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=2 { - "total": 5, + "total": 3, "elapsed": XXX, "offset": 2, "limit": 2, "results": [ { - "name": "red", + "name": "blue", "type": "instance", - "indom": "29.1" - }, - { - "name": "29.1", - "type": "indom", "indom": "29.1", - "oneline": "Instance domain \"colour\" for sample PMDA", - "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + "oneline": "Metrics with a \"saw-tooth\" trend over time" } ] } === /search/info === == verify request - http://localhost:PORT/search/info { - "docs": 18, - "terms": 174, - "records": XXX, - "records_per_doc_avg": XXX, - "bytes_per_record_avg": XXX, + "docs": NNN, + "terms": NNN, + "records": NNN, + "records_per_doc_avg": 0.00, + "bytes_per_record_avg": 0.00, "inverted_sz_mb": 0.00, "inverted_cap_mb": 0.00, "inverted_cap_ovh": 0.00, "skip_index_size_mb": 0.00, "score_index_size_mb": 0.00, - "offsets_per_term_avg": XXX, - "offset_bits_per_record_avg": 8.00 + "offsets_per_term_avg": 0.00, + "offset_bits_per_record_avg": 0.00 } diff --git a/qa/group b/qa/group index ecdfcb8245c..bda2746f3d4 100644 --- a/qa/group +++ b/qa/group @@ -2351,7 +2351,7 @@ suse 1869 logutil pmlogger_daily local 1870 guidellm2pcp python pmimport libpcp_import local 1871 pmsearch local -1872 pmproxy local pmjson +1872 pmsearch pmproxy local pmjson 1874 pmseries pmproxy local pmjson 1875 pmseries pmproxy libpcp_web local 1876:retired pmcd secure local diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 46e1851d7f2..0148e757386 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -29,7 +29,6 @@ static unsigned int default_resultcount = 10; typedef struct searchModuleData { sqlite3 *db; - int has_base; struct dict *config; unsigned int loaded; unsigned int resultcount; @@ -158,9 +157,8 @@ search_make_docid(sqlite3_int64 rowid, const char *name) * Appends results to the hits array; caller owns the array. */ static int -search_query_table(sqlite3 *db, const char *from, const char *fts, - pmSearchTextRequest *request, sds match_expr, sds type_filter, - const char *exclude, +search_query_table(sqlite3 *db, pmSearchTextRequest *request, + sds match_expr, sds type_filter, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -169,15 +167,13 @@ search_query_table(sqlite3 *db, const char *from, const char *fts, sql = sdscatfmt(sdsempty(), "SELECT rowid, name, oneline, helptext, type, indom," - " bm25(%s, 9.0, 4.0, 2.0)," - " highlight(%s, 0, '', '')," - " highlight(%s, 1, '', '')," - " highlight(%s, 2, '', '')" - " FROM %s WHERE %s MATCH ?%S%s" - " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", - fts, fts, fts, fts, - from, fts, type_filter, exclude, - fts); + " bm25(docs, 9.0, 4.0, 2.0)," + " highlight(docs, 0, '', '')," + " highlight(docs, 1, '', '')," + " highlight(docs, 2, '', '')" + " FROM docs WHERE docs MATCH ?%S" + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)", + type_filter); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -284,15 +280,8 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, match = search_build_match(request); type_filter = search_type_filter(request); - search_query_table(smd->db, "docs", "docs", request, match, type_filter, - "", &hits, &nhits, &maxhits); - - if (smd->has_base) { - search_query_table(smd->db, "base.docs", "docs", request, match, - type_filter, - " AND name NOT IN (SELECT name FROM docs)", - &hits, &nhits, &maxhits); - } + search_query_table(smd->db, request, match, type_filter, + &hits, &nhits, &maxhits); sdsfree(match); sdsfree(type_filter); @@ -325,8 +314,7 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, } static int -search_suggest_table(sqlite3 *db, const char *from, const char *fts, - sds match, const char *exclude, +search_suggest_table(sqlite3 *db, sds match, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -334,11 +322,10 @@ search_suggest_table(sqlite3 *db, const char *from, const char *fts, int rc; sql = sdscatfmt(sdsempty(), - "SELECT rowid, name, type, bm25(%s, 9.0, 4.0, 2.0)" - " FROM %s WHERE %s MATCH ?" - " AND type IN (1, 3)%s" - " ORDER BY bm25(%s, 9.0, 4.0, 2.0)", - fts, from, fts, exclude, fts); + "SELECT rowid, name, type, bm25(docs, 9.0, 4.0, 2.0)" + " FROM docs WHERE docs MATCH ?" + " AND type IN (1, 3)" + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)"); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -389,13 +376,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, match = sdscatfmt(sdsempty(), "name : %S*", request->query); - search_suggest_table(smd->db, "docs", "docs", match, "", - &hits, &nhits, &maxhits); - - if (smd->has_base) - search_suggest_table(smd->db, "base.docs", "docs", match, - " AND name NOT IN (SELECT name FROM docs)", - &hits, &nhits, &maxhits); + search_suggest_table(smd->db, match, &hits, &nhits, &maxhits); sdsfree(match); @@ -423,8 +404,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, } static int -search_indom_table(sqlite3 *db, const char *from, const char *query, - int querylen, const char *exclude, +search_indom_table(sqlite3 *db, const char *query, int querylen, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -433,9 +413,8 @@ search_indom_table(sqlite3 *db, const char *from, const char *query, sql = sdscatfmt(sdsempty(), "SELECT rowid, name, oneline, helptext, type, indom" - " FROM %s WHERE indom = ?%s" - " ORDER BY type", - from, exclude); + " FROM docs WHERE indom = ?" + " ORDER BY type"); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -500,16 +479,10 @@ search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, pmtimespecNow(&started); - search_indom_table(smd->db, "docs", request->query, - sdslen(request->query), "", + search_indom_table(smd->db, request->query, + sdslen(request->query), &hits, &nhits, &maxhits); - if (smd->has_base) - search_indom_table(smd->db, "base.docs", request->query, - sdslen(request->query), - " AND name NOT IN (SELECT name FROM docs)", - &hits, &nhits, &maxhits); - pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); @@ -562,16 +535,6 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) sqlite3_finalize(stmt); } - if (smd->has_base) { - rc = sqlite3_prepare_v2(smd->db, - "SELECT COUNT(*) FROM base.docs", -1, &stmt, NULL); - if (rc == SQLITE_OK) { - if (sqlite3_step(stmt) == SQLITE_ROW) - metrics.docs += sqlite3_column_int64(stmt, 0); - sqlite3_finalize(stmt); - } - } - settings->callbacks.on_metrics(&metrics, arg); settings->callbacks.on_done(0, arg); return 0; @@ -702,20 +665,9 @@ pmSearchSetup(pmSearchModule *module, void *arg) rc = sqlite3_open_v2(nightly, &smd->db, SQLITE_OPEN_READONLY, NULL); if (rc == SQLITE_OK) { - char attach[MAXPATHLEN + 64]; - if (pmDebugOptions.search) fprintf(stderr, "pmSearchSetup: loaded nightly index %s\n", nightly); - pmsprintf(attach, sizeof(attach), - "ATTACH DATABASE '%s' AS base", base); - rc = sqlite3_exec(smd->db, attach, NULL, NULL, NULL); - if (rc == SQLITE_OK) { - smd->has_base = 1; - if (pmDebugOptions.search) - fprintf(stderr, "pmSearchSetup: attached base index %s\n", - base); - } } else { sqlite3_close(smd->db); smd->db = NULL; diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c index 53319e74a3d..63a1a3b8f33 100644 --- a/src/newhelp/search_sqlite.c +++ b/src/newhelp/search_sqlite.c @@ -36,7 +36,7 @@ search_sqlite_open(const char *path) } rc = sqlite3_exec(search_db, - "CREATE VIRTUAL TABLE docs USING fts5(" + "CREATE VIRTUAL TABLE IF NOT EXISTS docs USING fts5(" " name," " oneline," " helptext," diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index a8657240c99..63618fbf291 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -186,7 +186,7 @@ fi if $SHOWME then - echo "$prog: would run: $PCP_BINADM_DIR/newhelp -S -o $INDEX $tmp/helptext" + echo "$prog: would copy $BASE to $INDEX and add instances" status=0 exit fi @@ -203,12 +203,42 @@ then fi fi -# Build the search index -if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/helptext +# Start from the build-time base index (contains all metric help text). +# The nightly update copies it, then appends runtime instance data. +BASE="$PCP_SHARE_DIR/lib/pcp.search" +if [ -f "$BASE" ] then + cp "$BASE" "$INDEX" if $VERBOSE then - echo "$prog: index written to $INDEX" + echo "$prog: copied base index from $BASE" + fi +fi + +# Extract only instance entries for appending to the index +$PCP_AWK_PROG ' +/^@ I / { printing = 1; print; next } +printing && /^$/ { print; printing = 0; next } +printing { print } +' $tmp/helptext > $tmp/instances + +ninst=`grep -c '^@ I ' $tmp/instances` +if [ "$ninst" -eq 0 ] +then + if $VERBOSE + then + echo "$prog: no instance data to add" + fi + status=0 + exit +fi + +# Add instance data to the index +if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/instances +then + if $VERBOSE + then + echo "$prog: added $ninst instances to $INDEX" fi status=0 else From fed5f7fab145478a7ad550539b2a2358acd9655c Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Tue, 28 Jul 2026 14:17:36 -0400 Subject: [PATCH 06/22] pmsearch: man pages for nightly search index creation --- man/man1/pmsearch.1 | 21 ++++++++++----------- man/man1/pmsearch_index.1 | 17 +++++++++-------- man/man3/pmsearchsetup.3 | 19 ++++++++++--------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/man/man1/pmsearch.1 b/man/man1/pmsearch.1 index e9f1e18899a..829473b51eb 100644 --- a/man/man1/pmsearch.1 +++ b/man/man1/pmsearch.1 @@ -39,14 +39,13 @@ performs full text search queries to find metrics using names and help text from metrics, instance domains and instances. .PP -It uses a local SQLite FTS5 full-text search index that is -built at package install time and updated nightly by -.BR pmsearch_index (1). -The build-time index contains help text for all PCP metrics -from the source tree. -The nightly index adds instance names (disk devices, network -interfaces, etc.) and metrics from any third-party PMDAs -installed on the system. +It uses a local SQLite FTS5 full-text search index. +A base index is built at package install time containing +help text for all PCP metrics from the source tree. +Each night, +.BR pmsearch_index (1) +copies the base index and extends it with instance names +(disk devices, network interfaces, etc.) from the running system. .PP No external services (such as Redis or Valkey) are required. .SH OPTIONS @@ -120,11 +119,11 @@ One line: percpu \fBidle\fR CPU time metric from /proc/stat .I $PCP_VAR_DIR/lib/pcp.search Nightly search index (SQLite FTS5 database), rebuilt by .BR pmsearch_index (1). +This is a copy of the base index extended with runtime instance data. .TP .I $PCP_SHARE_DIR/lib/pcp.search -Build-time search index containing all metrics from the PCP source tree. -Used as a fallback when the nightly index is not yet available, and to -fill in help text for PMDAs not installed on the running system. +Build-time base index containing all metrics from the PCP source tree. +Used as a fallback when the nightly index is not yet available. .SH PCP ENVIRONMENT Environment variables with the prefix \fBPCP_\fP are used to parameterize the file and directory names used by PCP. diff --git a/man/man1/pmsearch_index.1 b/man/man1/pmsearch_index.1 index dfd97e1dee4..af560ba1008 100644 --- a/man/man1/pmsearch_index.1 +++ b/man/man1/pmsearch_index.1 @@ -42,12 +42,11 @@ The resulting index file is written to .I $PCP_VAR_DIR/lib/pcp.search by default. .PP -The search engine uses -.B both -the runtime index and the base index shipped with the PCP package. -The runtime index takes precedence for any entries it contains; -the base index fills in metrics from PMDAs that are not installed -on the current system. +Each night, the script copies the base index shipped with the +PCP package and extends the copy with instance data from the +running system. +This ensures the nightly index always starts fresh from the +complete set of metric help text. .PP This command is typically invoked nightly via a systemd timer or cron job. @@ -81,12 +80,14 @@ Display usage message and exit. .TP 5 .I $PCP_VAR_DIR/lib/pcp.search Default output path for the runtime search index. -When present, this index is preferred over the base index. +This is a copy of the base index extended with instance data. +When present, it is preferred over the base index. .TP .I $PCP_SHARE_DIR/lib/pcp.search Base search index shipped with the PCP package, built at package build time from help text source files. -Used as a fallback when the runtime index does not exist. +Copied as the starting point for each nightly rebuild; +used directly as a fallback when the runtime index does not exist. .SH PCP ENVIRONMENT Environment variables with the prefix \fBPCP_\fP are used to parameterize the file and directory names used by PCP. diff --git a/man/man3/pmsearchsetup.3 b/man/man3/pmsearchsetup.3 index 04b733e9bee..1c88bc11cee 100644 --- a/man/man3/pmsearchsetup.3 +++ b/man/man3/pmsearchsetup.3 @@ -56,16 +56,16 @@ and related API functions prepare an application for accessing the full-text search functionality of the Performance Co-Pilot (PCP). .PP The search engine uses a local SQLite FTS5 full-text index. -A build-time index at +A build-time base index at .I $PCP_SHARE_DIR/lib/pcp.search provides help text for all metrics shipped with PCP. -A nightly index at +Each night, +.BR pmsearch_index (1) +copies the base index to .I $PCP_VAR_DIR/lib/pcp.search -(rebuilt by -.BR pmsearch_index (1)) -adds instance names and metrics from third-party PMDAs. -When both indexes exist, they are queried together; -the nightly index takes precedence for entries it contains. +and extends it with instance names from the running system. +When the nightly index exists it is used; +otherwise the base index is used as a fallback. .PP The interfaces described here prepare a given search .IR module @@ -149,11 +149,12 @@ On failure a negative PMAPI error code is returned. .SH FILES .TP .I $PCP_VAR_DIR/lib/pcp.search -Nightly search index (SQLite FTS5 database), rebuilt by +Nightly search index (SQLite FTS5 database), a copy of the base +index extended with runtime instance data by .BR pmsearch_index (1). .TP .I $PCP_SHARE_DIR/lib/pcp.search -Build-time search index shipped with the PCP package. +Build-time base index shipped with the PCP package. .SH SEE ALSO .BR pmsearch (1), .BR pmsearch_index (1), From 38d91b676fd948c9f6ccd29bea7a0bfd84fe5d2d Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Tue, 28 Jul 2026 16:58:16 -0400 Subject: [PATCH 07/22] pmsearch: improve qa tests and add archive support for pmsearch_index --- man/man1/pmsearch_index.1 | 15 +++++ qa/1871 | 114 ++++----------------------------- qa/1872 | 104 +++--------------------------- src/pmsearch/pmsearch_index.sh | 24 +++++-- 4 files changed, 56 insertions(+), 201 deletions(-) diff --git a/man/man1/pmsearch_index.1 b/man/man1/pmsearch_index.1 index af560ba1008..703555064e8 100644 --- a/man/man1/pmsearch_index.1 +++ b/man/man1/pmsearch_index.1 @@ -19,6 +19,7 @@ .SH SYNOPSIS \fBpmsearch_index\fR [\fB\-NV?\fR] +[\fB\-a\fR \fIarchive\fR] [\fB\-o\fR \fIindex\fR] .SH DESCRIPTION .B pmsearch_index @@ -35,6 +36,9 @@ Its primary purpose is to capture that are only known on a running system. It queries the running .BR pmcd (1) +(or a PCP archive when +.B \-a +is specified) for all metric names, help text, and instance domain members, then builds a SQLite FTS5 search index using .BR newhelp (1). @@ -60,6 +64,17 @@ The base index at remains available as a fallback. .SH OPTIONS .TP 5 +\fB\-a\fR \fIarchive\fR, \fB\-\-archive\fR=\fIarchive\fR +Use +.I archive +as the source of metric metadata and instance data instead of +querying the running +.BR pmcd (1). +This is useful for testing or for building an index on a system +where +.B pmcd +is not running. +.TP \fB\-N\fR, \fB\-\-showme\fR Dry-run mode. Show what would be done without writing any files. diff --git a/qa/1871 b/qa/1871 index 28f50d2a6be..abf4cec9857 100755 --- a/qa/1871 +++ b/qa/1871 @@ -1,8 +1,8 @@ #!/bin/sh # PCP QA Test No. 1871 # Exercise pmsearch functionality using local SQLite FTS5 index. -# Tests the dual-index strategy: a build-time base index (from help -# files) with a nightly index (from archive/runtime data) as primary. +# Tests the copy-and-extend strategy: build a base index from help +# files, copy it, then append instance data from an archive. # # Copyright (c) 2020,2026 Red Hat. All Rights Reserved. # @@ -19,6 +19,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" +[ -x $PCP_BINADM_DIR/pmsearch_index ] || \ + _notrun "pmsearch_index not installed" _cleanup() { @@ -59,7 +61,7 @@ archive="$here/archives/sample-labels" help=$PCP_PMDAS_DIR/sample/help [ -f "$help" ] || _notrun "sample PMDA help file not found" -# Mirror the real dual-index strategy: +# Mirror the real copy-and-extend strategy: # 1. Build base index from PMDA help files (like package install) # 2. Copy base to nightly location (like pmsearch_index.sh does) # 3. Append instance data from archive to the nightly copy @@ -71,105 +73,15 @@ newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ echo "Base index built" # --- Nightly index (like $PCP_VAR_DIR/lib/pcp.search) --- -# Copy base as starting point +# Use pmsearch_index to copy base and append instance data from archive mkdir -p $tmp.var/lib -cp $tmp.share/lib/pcp.search $tmp.var/lib/pcp.search - -# Extract metric help text from archive (used for instance oneline mapping) -pminfo -a $archive -tT 2>/dev/null | $PCP_AWK_PROG ' -/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { - if (index($0, "One-line Help: Error:") > 0) { - skip = 1 - next - } - skip = 0 - name = $1 - oneline = "" - start = index($0, "[") - end = index($0, "]") - if (start > 0 && end > start) { - oneline = substr($0, start + 1, end - start - 1) - } - printf "@ %s %s\n", name, oneline - next -} -/^Help:$/ { next } -/^Full Help: Error:/ { skip = 1; next } -skip { next } -{ print } -' > $tmp.helptext - -# Extract instance names from archive -pminfo -a $archive -I 2>/dev/null | $PCP_AWK_PROG ' -/^[a-zA-Z]/ { metric = $1 } -/InDom:/ && !/PM_INDOM_NULL/ { - for (i = 1; i <= NF; i++) { - if ($i == "InDom:") { - print metric "\t" $(i+1) - break - } - } -}' > $tmp.metric_indom - -pmprobe -a $archive -I 2>/dev/null > $tmp.pmprobe_output - -$PCP_AWK_PROG ' -FILENAME == ARGV[1] { - split($0, a, "\t") - metric_indom[a[1]] = a[2] - next -} -FILENAME == ARGV[2] { - if (/^@ [a-zA-Z]/) { - name = $2 - start = index($0, " ") - start = index(substr($0, start + 1), " ") - oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" - metric_oneline[name] = oneline - } - next -} -{ - metric = $1 - count = $2 + 0 - if (count <= 0 || $2 == "PM_IN_NULL") next - - indom = metric_indom[metric] - if (indom == "") next - - if (!(indom in indom_oneline) && metric_oneline[metric] != "") - indom_oneline[indom] = metric_oneline[metric] - - rest = $0 - while (match(rest, /"[^"]*"/)) { - inst = substr(rest, RSTART + 1, RLENGTH - 2) - key = indom SUBSEP inst - if (!(key in seen)) { - seen[key] = 1 - inst_data[++ninst] = indom "\t" inst - inst_indom[ninst] = indom - } - rest = substr(rest, RSTART + RLENGTH) - } -} -END { - for (i = 1; i <= ninst; i++) { - indom = inst_indom[i] - oneline = indom_oneline[indom] - printf "@ I %s\n%s\n\n", inst_data[i], oneline - } -} -' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" > $tmp.instances - -# Append instances to the copied base index -newhelp -S -o $tmp.var/lib/pcp.search $tmp.instances 2>&1 \ - | sed -e "s,$tmp,TMP,g" +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share +$PCP_BINADM_DIR/pmsearch_index -a $archive echo "Nightly index built (base + instances)" echo -# Override PCP paths so pmsearch finds our test indexes -export PCP_VAR_DIR=$tmp.var -export PCP_SHARE_DIR=$tmp.share +# PCP_VAR_DIR and PCP_SHARE_DIR already exported above for pmsearch_index echo "=== Empty search ===" pmsearch -C xyznonexistent @@ -190,13 +102,13 @@ pmsearch -dStT 99 | _filter_extras | _filter_docid echo echo "=== Suggestion search 1 ===" -pmsearch -s -N 3 sample +pmsearch -s -N 3 sample | _filter_extras echo "=== Suggestion search 2 ===" -pmsearch -s blue +pmsearch -s blue | _filter_extras echo "=== Indom search ===" -pmsearch -n 29.3 +pmsearch -n 29.3 | _filter_extras # success, all done diff --git a/qa/1872 b/qa/1872 index be847a2fc56..e719d5f6bde 100755 --- a/qa/1872 +++ b/qa/1872 @@ -18,6 +18,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" +[ -x $PCP_BINADM_DIR/pmsearch_index ] || \ + _notrun "pmsearch_index not installed" which pmproxy >/dev/null 2>&1 || \ _notrun "pmproxy not installed" which curl >/dev/null 2>&1 || \ @@ -61,102 +63,16 @@ archive="$here/archives/sample-labels" help=$PCP_PMDAS_DIR/sample/help [ -f "$help" ] || _notrun "sample PMDA help file not found" -# Build a search index using the same dual-index strategy as test 1871: -# base index from help file, copy to nightly, append instance data. +# Build a search index using the copy-and-extend strategy: +# base index from help file, then pmsearch_index copies and appends instances. echo "=== Build search index ===" -newhelp -S -o $tmp.base $help 2>&1 \ - | sed -e "s,$tmp,TMP,g" -cp $tmp.base $tmp.index - -# Extract metric help text from archive (for instance oneline mapping) -pminfo -a $archive -tT 2>/dev/null | $PCP_AWK_PROG ' -/^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { - if (index($0, "One-line Help: Error:") > 0) { - skip = 1 - next - } - skip = 0 - name = $1 - oneline = "" - start = index($0, "[") - end = index($0, "]") - if (start > 0 && end > start) { - oneline = substr($0, start + 1, end - start - 1) - } - printf "@ %s %s\n", name, oneline - next -} -/^Help:$/ { next } -/^Full Help: Error:/ { skip = 1; next } -skip { next } -{ print } -' > $tmp.helptext - -# Extract instance names from archive -pminfo -a $archive -I 2>/dev/null | $PCP_AWK_PROG ' -/^[a-zA-Z]/ { metric = $1 } -/InDom:/ && !/PM_INDOM_NULL/ { - for (i = 1; i <= NF; i++) { - if ($i == "InDom:") { - print metric "\t" $(i+1) - break - } - } -}' > $tmp.metric_indom - -pmprobe -a $archive -I 2>/dev/null > $tmp.pmprobe_output - -$PCP_AWK_PROG ' -FILENAME == ARGV[1] { - split($0, a, "\t") - metric_indom[a[1]] = a[2] - next -} -FILENAME == ARGV[2] { - if (/^@ [a-zA-Z]/) { - name = $2 - start = index($0, " ") - start = index(substr($0, start + 1), " ") - oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" - metric_oneline[name] = oneline - } - next -} -{ - metric = $1 - count = $2 + 0 - if (count <= 0 || $2 == "PM_IN_NULL") next - - indom = metric_indom[metric] - if (indom == "") next - - if (!(indom in indom_oneline) && metric_oneline[metric] != "") - indom_oneline[indom] = metric_oneline[metric] - - rest = $0 - while (match(rest, /"[^"]*"/)) { - inst = substr(rest, RSTART + 1, RLENGTH - 2) - key = indom SUBSEP inst - if (!(key in seen)) { - seen[key] = 1 - inst_data[++ninst] = indom "\t" inst - inst_indom[ninst] = indom - } - rest = substr(rest, RSTART + RLENGTH) - } -} -END { - for (i = 1; i <= ninst; i++) { - indom = inst_indom[i] - oneline = indom_oneline[indom] - printf "@ I %s\n%s\n\n", inst_data[i], oneline - } -} -' "$tmp.metric_indom" "$tmp.helptext" "$tmp.pmprobe_output" > $tmp.instances - -newhelp -S -o $tmp.index $tmp.instances 2>&1 \ +mkdir -p $tmp.share/lib $tmp.var/lib +newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ | sed -e "s,$tmp,TMP,g" +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share +$PCP_BINADM_DIR/pmsearch_index -a $archive echo "Index built" echo @@ -167,7 +83,7 @@ pcp.enabled = true http.enabled = true [pmsearch] enabled = true -index.path = $tmp.index +index.path = $tmp.var/lib/pcp.search EOF proxyport=`_find_free_port` diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index 63618fbf291..27d8b539312 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -12,8 +12,9 @@ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. # -# Rebuild the pmsearch full-text search index from running PMCD. -# Typically invoked nightly via systemd timer or cron. +# Rebuild the pmsearch full-text search index from running PMCD +# or a PCP archive. Typically invoked nightly via systemd timer +# or cron. # . $PCP_DIR/etc/pcp.env @@ -27,6 +28,7 @@ INDEX="$PCP_VAR_DIR/lib/pcp.search" cat > $tmp/usage << EOF Options: + -a=ARCHIVE,--archive=ARCHIVE use archive instead of running PMCD -N,--showme dry-run, show what would be done -o=INDEX,--output=INDEX output index file path [default: $INDEX] -V,--verbose verbose diagnostics @@ -39,6 +41,7 @@ _usage() exit 1 } +ARCHIVE="" VERBOSE=false SHOWME=false @@ -48,6 +51,7 @@ while [ $# -gt 0 ] do case "$1" in + -a) ARCHIVE="$2"; shift ;; -N) SHOWME=true ;; -o) INDEX="$2"; shift ;; -V) VERBOSE=true ;; @@ -57,6 +61,14 @@ do shift done +# Build the pminfo/pmprobe flags for archive or live mode +if [ -n "$ARCHIVE" ] +then + SRCFLAGS="-a $ARCHIVE" +else + SRCFLAGS="" +fi + if $VERBOSE then echo "$prog: index target: $INDEX" @@ -73,7 +85,7 @@ fi # @ metricname oneline text # multi-line helptext # -pminfo -tT 2>/dev/null | $PCP_AWK_PROG ' +pminfo $SRCFLAGS -tT 2>/dev/null | $PCP_AWK_PROG ' /^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { # metric line: "name.with.dots [oneline]" or error if (index($0, "One-line Help: Error:") > 0) { @@ -100,7 +112,7 @@ skip { next } nhelplines=`wc -l < $tmp/helptext` if [ "$nhelplines" -eq 0 ] then - echo >&2 "$prog: warning: no metrics found from PMCD" + echo >&2 "$prog: warning: no metrics found" status=0 exit fi @@ -115,7 +127,7 @@ fi # Build metric→indom mapping, then get instances via pmprobe -I. # Deduplicate by (indom, instance_name) since many metrics share indoms. # -pminfo -I 2>/dev/null | $PCP_AWK_PROG ' +pminfo $SRCFLAGS -I 2>/dev/null | $PCP_AWK_PROG ' /^[a-zA-Z]/ { metric = $1 } /InDom:/ && !/PM_INDOM_NULL/ { for (i = 1; i <= NF; i++) { @@ -126,7 +138,7 @@ pminfo -I 2>/dev/null | $PCP_AWK_PROG ' } }' > $tmp/metric_indom -pmprobe -I 2>/dev/null > $tmp/pmprobe_output +pmprobe $SRCFLAGS -I 2>/dev/null > $tmp/pmprobe_output $PCP_AWK_PROG ' FILENAME == ARGV[1] { From 0436e7ac1b4aea4df2c17ded1600522e6446f853 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Wed, 29 Jul 2026 17:26:01 -0400 Subject: [PATCH 08/22] pmsearch: fix issues and update tests --- qa/1871 | 2 +- qa/1871.out | 117 +++++++++-- qa/1872 | 1 + qa/1872.out | 358 +++++++++++++++++++++------------ src/libpcp_web/src/search.c | 51 ++++- src/newhelp/newhelp.c | 137 ++++++++++++- src/pmproxy/src/search.c | 18 +- src/pmsearch/pmsearch.c | 11 +- src/pmsearch/pmsearch_index.sh | 24 +-- 9 files changed, 515 insertions(+), 204 deletions(-) diff --git a/qa/1871 b/qa/1871 index abf4cec9857..ebc5467451a 100755 --- a/qa/1871 +++ b/qa/1871 @@ -102,7 +102,7 @@ pmsearch -dStT 99 | _filter_extras | _filter_docid echo echo "=== Suggestion search 1 ===" -pmsearch -s -N 3 sample | _filter_extras +pmsearch -s sample | _filter_extras echo "=== Suggestion search 2 ===" pmsearch -s blue | _filter_extras diff --git a/qa/1871.out b/qa/1871.out index 313437ec585..b57a627724b 100644 --- a/qa/1871.out +++ b/qa/1871.out @@ -5,8 +5,9 @@ Nightly index built (base + instances) === Empty search === 0 search results === Search on 99 === -Type: metric -Name: SAMPLE.3 +Type: indom +Name: 29.3 +InDom: 29.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" @@ -14,6 +15,7 @@ is always present, while the others are numbered 1 .. 49 and named "m-01" === Search on random === Type: metric Name: sample.drift +InDom: none One line: A random trended metric Help: This metric returns a random value (expected mean is approximately 200), subject to a trending pattern such that the sequence is mainly monotonic, @@ -24,28 +26,33 @@ expected mean. Type: metric Name: sample.byte_rate +InDom: none One line: instantaneous bytes/second Help: random value in the range (0,1023), so avg value is 512 bytes/second Type: metric Name: sample.kbyte_rate +InDom: none One line: instantaneous Kbytes/second Help: random value in the range (0,1023), so avg value is 512 Kbytes/second Type: metric Name: sample.byte_rate_perhour +InDom: none One line: instantaneous bytes/hour Help: random value in the range (0,1023), so avg value is 512 bytes/hour Type: metric Name: sample.dodgey.value +InDom: 29.6 One line: 5 unreliable instances Help: The metric is a set of 5 instantaneous values, drawn at random from the range 0 to 100. The number of instances "visible" is controlled by sample.dodgey.control. -Type: metric -Name: SAMPLE.3 +Type: indom +Name: 29.3 +InDom: 29.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" @@ -53,6 +60,7 @@ is always present, while the others are numbered 1 .. 49 and named "m-01" Type: metric Name: sample.scramble.bin +InDom: 29.9 One line: Several constant instances, instances scrambled Help: Like sample.bin, except 1. instances are missing with probability 0.33 @@ -61,8 +69,9 @@ Help: Like sample.bin, except Designed to help testing instance matching between pmFetch calls for PCP clients. -Type: metric -Name: SAMPLE.6 +Type: indom +Name: 29.6 +InDom: 29.6 One line: Instance domain "dodgey" for sample PMDA. Help: Instances for an unreliable instance domain, where the instances can under certain conditions (random or controlled) return holes, errors, @@ -71,6 +80,7 @@ metric store controls this instance domain. Type: metric Name: sample.dodgey.control +InDom: none One line: control values retured for sample.dodgey.value Help: If sample.dodgey.control is <= 0, then this is returned as the "numval" component in the pmResult (0 => no values available, less than 0 => @@ -87,8 +97,9 @@ available is changed according to the following probabilities ... from the underlying 5 instances. 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION) -Type: metric -Name: SAMPLE.12 +Type: indom +Name: 29.12 +InDom: 29.12 One line: Instance domain "procs" for sample PMDA Help: Simulated and small instance domain that mimics the behaviour of the process instance domain from the proc PMDA. @@ -107,8 +118,9 @@ avoided). The external instance names are the instance number, space, then a random "executable" name. === Search on interesting === -Type: metric -Name: SAMPLE.6 +Type: indom +Name: 29.6 +InDom: 29.6 One line: Instance domain "dodgey" for sample PMDA. Help: Instances for an unreliable instance domain, where the instances can under certain conditions (random or controlled) return holes, errors, @@ -117,6 +129,7 @@ metric store controls this instance domain. Type: metric Name: sample.mirage_longlong +InDom: 29.3 One line: Simple saw-tooth rate, but instances come and go Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution over time. Different instances of the metric have different baselines @@ -136,6 +149,7 @@ instance or instances are used). Type: metric Name: sample.mirage +InDom: 29.3 One line: Simple saw-tooth rate, but instances come and go Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution over time. Different instances of the metric have different baselines @@ -159,11 +173,13 @@ instance or instances are used). === Search on result === Type: metric Name: sample.rapid +InDom: none One line: count very quickly Help: Base counter increments by 8*10^7 per fetch. Result is 10 x base counter. Type: metric Name: sample.not_ready +InDom: none One line: interval (in seconds) during which PMDA does not respond to PDUs Help: Store a positive number of seconds as the value of this metric. The following PDU received will result in the following sequence of events: @@ -175,6 +191,7 @@ has a negative error code as the value. Type: metric Name: sample.not_ready_msec +InDom: none One line: interval (in milliseconds) during which PMDA does not respond to PDUs Help: Store a positive number of milliseconds as the value of this metric. The following PDU received will result in the following sequence of events: @@ -192,39 +209,99 @@ Search index statistics: === Extra reporting - timing, score, hits, doc IDs === 1 hit in N.NNN seconds -ID: N:SAMPLE.3 +ID: N:29.3 Score: N.NN -Type: metric -Name: SAMPLE.3 +Type: indom +Name: 29.3 +InDom: 29.3 One line: Instance domain "mirage" for sample PMDA Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" === Suggestion search 1 === -Type: metric Name: sample.string.null -Type: metric Name: sample.aggregate.null -Type: metric Name: sample.bad.novalues + +Name: sample.long.ten + +Name: sample.long.hundred + +Name: sample.long.million + +Name: sample.long.bin + +Name: sample.ulong.bin + +Name: sample.longlong.one + +Name: sample.longlong.ten === Suggestion search 2 === -Type: instance Name: blue === Indom search === +Type: metric +Name: sample.mirage +InDom: 29.3 +One line: Simple saw-tooth rate, but instances come and go +Help: The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution +over time. Different instances of the metric have different baselines +for the saw-tooth, but all have an max-to-min range of 100. + +What makes this metric interesting is that instances come and go although +not more often than once every 10 seconds by default. Use pmstore to +change sample.controller.mirage and the frequency of instance domain +changes can be varied. + +Instance 0 is always present, but the other instances 1 thru 49 come +and go in a cyclic pattern with a large random component influencing +when each instance appears and disappears. + +The underlying counter starts at 0 and is incremented once +for each pmFetch() to this metric and/or sample.colour and/or +sample.mirage_longlong. + +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). + +Type: metric +Name: sample.mirage_longlong +InDom: 29.3 +One line: Simple saw-tooth rate, but instances come and go +Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution +over time. Different instances of the metric have different baselines +for the saw-tooth, but all have an max-to-min range of 100,000,000. + +What makes this metric interesting is that instances come and go (not more +often than once every 10 seconds however). Instance 0 is always present, +but the other instances 1 thru 49 come and go in a cyclic pattern with +a large random component influencing when each instance appears and +disappears. + +The underlying counter starts at 0 and is incremented once for each +pmFetch() to this metric and/or sample.mirage and/or sample.colour. + +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). + +Type: indom +Name: 29.3 +InDom: 29.3 +One line: Instance domain "mirage" for sample PMDA +Help: Random number of instances, that change with time. Instance "m-00" (0) +is always present, while the others are numbered 1 .. 49 and named "m-01" +.. "m-99" + Type: instance Name: m-00 InDom: 29.3 -One line: Simple saw-tooth rate, but instances come and go Type: instance Name: m-01 InDom: 29.3 -One line: Simple saw-tooth rate, but instances come and go Type: instance Name: m-04 InDom: 29.3 -One line: Simple saw-tooth rate, but instances come and go diff --git a/qa/1872 b/qa/1872 index e719d5f6bde..2992afa7974 100755 --- a/qa/1872 +++ b/qa/1872 @@ -84,6 +84,7 @@ http.enabled = true [pmsearch] enabled = true index.path = $tmp.var/lib/pcp.search +result.count = 10 EOF proxyport=`_find_free_port` diff --git a/qa/1872.out b/qa/1872.out index 0079b0d39e2..5c2c3ef8734 100644 --- a/qa/1872.out +++ b/qa/1872.out @@ -8,11 +8,12 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "name": "SAMPLE.3", - "type": "metric", + "name": "29.3", + "type": "indom", + "indom": "29.3", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } @@ -23,7 +24,7 @@ Index built "total": 12, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.drift", @@ -52,24 +53,28 @@ Index built { "name": "sample.dodgey.value", "type": "metric", + "indom": "29.6", "oneline": "5 unreliable instances", "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." }, { - "name": "SAMPLE.3", - "type": "metric", + "name": "29.3", + "type": "indom", + "indom": "29.3", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { "name": "sample.scramble.bin", "type": "metric", + "indom": "29.9", "oneline": "Several constant instances, instances scrambled", "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." }, { - "name": "SAMPLE.6", - "type": "metric", + "name": "29.6", + "type": "indom", + "indom": "29.6", "oneline": "Instance domain \"dodgey\" for sample PMDA.", "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." }, @@ -80,8 +85,9 @@ Index built "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" }, { - "name": "SAMPLE.12", - "type": "metric", + "name": "29.12", + "type": "indom", + "indom": "29.12", "oneline": "Instance domain \"procs\" for sample PMDA", "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } @@ -92,23 +98,26 @@ Index built "total": 3, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "name": "SAMPLE.6", - "type": "metric", + "name": "29.6", + "type": "indom", + "indom": "29.6", "oneline": "Instance domain \"dodgey\" for sample PMDA.", "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." }, { "name": "sample.mirage_longlong", "type": "metric", + "indom": "29.3", "oneline": "Simple saw-tooth rate, but instances come and go", "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { "name": "sample.mirage", "type": "metric", + "indom": "29.3", "oneline": "Simple saw-tooth rate, but instances come and go", "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } @@ -119,7 +128,7 @@ Index built "total": 3, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.rapid", @@ -147,7 +156,7 @@ Index built "total": 0, "elapsed": 0, "offset": 0, - "limit": 0, + "limit": 10, "results": [ ] } === /search/text - query param is required === @@ -159,47 +168,54 @@ Index built "total": 24, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "name": "SAMPLE.5", - "type": "metric", + "name": "29.5", + "type": "indom", + "indom": "29.5", "oneline": "Instance domain \"hordes\" for sample PMDA.", "helptext": "A relatively large instance domain, with hundreds of numeric instances." }, { "name": "sample.many.int", "type": "metric", + "indom": "29.8", "oneline": "variable sized instance domain", "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" }, { - "name": "SAMPLE.11", - "type": "metric", + "name": "29.11", + "type": "indom", + "indom": "29.11", "oneline": "Instance domain \"ghosts\" for sample PMDA.", "helptext": "Instances are the names of some famous ghosts." }, { - "name": "SAMPLE.6", - "type": "metric", + "name": "29.6", + "type": "indom", + "indom": "29.6", "oneline": "Instance domain \"dodgey\" for sample PMDA.", "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." }, { - "name": "SAMPLE.10", - "type": "metric", + "name": "29.10", + "type": "indom", + "indom": "29.10", "oneline": "Instance domain \"events\" for sample PMDA.", "helptext": "Instances \"fungus\" and \"bogus\" for testing event metrics indoms." }, { - "name": "SAMPLE.1", - "type": "metric", + "name": "29.1", + "type": "indom", + "indom": "29.1", "oneline": "Instance domain \"colour\" for sample PMDA", "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "SAMPLE.8", - "type": "metric", + "name": "29.8", + "type": "indom", + "indom": "29.8", "oneline": "Instance domain \"many\" for sample PMDA.", "helptext": "A varable size set of instances controlled by sample.many.count" }, @@ -210,14 +226,16 @@ Index built "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" }, { - "name": "SAMPLE.7", - "type": "metric", + "name": "29.7", + "type": "indom", + "indom": "29.7", "oneline": "Instance domain \"dynamic\" for sample PMDA.", "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists." }, { - "name": "SAMPLE.4", - "type": "metric", + "name": "29.4", + "type": "indom", + "indom": "29.4", "oneline": "Instance domain \"family\" for sample PMDA.", "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" } @@ -228,36 +246,42 @@ Index built "total": 211, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.long.bin", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type 32" }, { "name": "sample.ulong.bin", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type U32" }, { "name": "sample.longlong.bin", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type 64" }, { "name": "sample.float.bin", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type FLOAT" }, { "name": "sample.double.bin", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type DOUBLE" }, { "name": "sample.string.bin", "type": "metric", + "indom": "29.2", "oneline": "a string-valued version of sample.bin" }, { @@ -288,7 +312,7 @@ Index built "total": 44, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.step_counter", @@ -305,36 +329,42 @@ Index built { "name": "sample.dynamic.counter", "type": "metric", + "indom": "29.7", "oneline": "counter metric with dynamic indom", "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists.\nEach line in this file is\n\tinternal_id external_id\n\nThis metric increments each time this instance has been seen when scanning\nthe dynamic.indom file, and resets to zero each time the instance appears." }, { "name": "sample.long.bin_ctr", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type 32, SEM_COUNTER and SPACE_KBYTE", "helptext": "" }, { "name": "sample.ulong.bin_ctr", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type U32, SEM_COUNTER and SPACE_KBYTE", "helptext": "" }, { "name": "sample.longlong.bin_ctr", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type 64, SEM_COUNTER and SPACE_KBYTE", "helptext": "" }, { "name": "sample.float.bin_ctr", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type FLOAT, SEM_COUNTER and SPACE_KBYTE", "helptext": "" }, { "name": "sample.double.bin_ctr", "type": "metric", + "indom": "29.2", "oneline": "like sample.bin but type DOUBLE, SEM_COUNTER and SPACE_KBYTE", "helptext": "" }, @@ -389,7 +419,7 @@ Index built "total": 0, "elapsed": 0, "offset": 0, - "limit": 0, + "limit": 10, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=99&field=oneline @@ -397,7 +427,7 @@ Index built "total": 0, "elapsed": 0, "offset": 0, - "limit": 0, + "limit": 10, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=99&field=helptext @@ -405,11 +435,12 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "name": "SAMPLE.3", - "type": "metric", + "name": "29.3", + "type": "indom", + "indom": "29.3", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } @@ -421,11 +452,10 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "name": "SAMPLE.3", - "type": "metric" + "name": "29.3" } ] } @@ -434,10 +464,10 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "type": "metric" + "type": "indom" } ] } @@ -446,10 +476,9 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "type": "metric", "oneline": "Instance domain \"mirage\" for sample PMDA" } ] @@ -459,10 +488,9 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "type": "metric", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } ] @@ -472,10 +500,10 @@ Index built "total": 1, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { - "type": "metric" + "indom": "29.3" } ] } @@ -484,7 +512,7 @@ Index built "total": 12, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.drift", @@ -504,37 +532,42 @@ Index built }, { "name": "sample.dodgey.value", - "type": "metric" + "type": "metric", + "indom": "29.6" }, { - "name": "SAMPLE.3", - "type": "metric" + "name": "29.3", + "type": "indom", + "indom": "29.3" }, { "name": "sample.scramble.bin", - "type": "metric" + "type": "metric", + "indom": "29.9" }, { - "name": "SAMPLE.6", - "type": "metric" + "name": "29.6", + "type": "indom", + "indom": "29.6" }, { "name": "sample.dodgey.control", "type": "metric" }, { - "name": "SAMPLE.12", - "type": "metric" + "name": "29.12", + "type": "indom", + "indom": "29.12" } ] } === /search/text - filtering based on entity type (metric, instance, indom) using 'type' param === == verify request - http://localhost:PORT/search/text?query=random&type=metric { - "total": 12, + "total": 9, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.drift", @@ -563,27 +596,17 @@ Index built { "name": "sample.dodgey.value", "type": "metric", + "indom": "29.6", "oneline": "5 unreliable instances", "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." }, - { - "name": "SAMPLE.3", - "type": "metric", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" - }, { "name": "sample.scramble.bin", "type": "metric", + "indom": "29.9", "oneline": "Several constant instances, instances scrambled", "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." }, - { - "name": "SAMPLE.6", - "type": "metric", - "oneline": "Instance domain \"dodgey\" for sample PMDA.", - "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." - }, { "name": "sample.dodgey.control", "type": "metric", @@ -591,27 +614,57 @@ Index built "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" }, { - "name": "SAMPLE.12", + "name": "sample.mirage_longlong", "type": "metric", - "oneline": "Instance domain \"procs\" for sample PMDA", - "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, + { + "name": "sample.mirage", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } ] } == verify request - http://localhost:PORT/search/text?query=random&type=indom { - "total": 0, - "elapsed": 0, + "total": 3, + "elapsed": XXX, "offset": 0, - "limit": 0, - "results": [ ] + "limit": 10, + "results": [ + { + "name": "29.3", + "type": "indom", + "indom": "29.3", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + } + ] } == verify request - http://localhost:PORT/search/text?query=random&type=instance { "total": 0, "elapsed": 0, "offset": 0, - "limit": 0, + "limit": 10, "results": [ ] } == verify request - http://localhost:PORT/search/text?query=random&type=metric,indom @@ -619,7 +672,7 @@ Index built "total": 12, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ { "name": "sample.drift", @@ -648,24 +701,28 @@ Index built { "name": "sample.dodgey.value", "type": "metric", + "indom": "29.6", "oneline": "5 unreliable instances", "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." }, { - "name": "SAMPLE.3", - "type": "metric", + "name": "29.3", + "type": "indom", + "indom": "29.3", "oneline": "Instance domain \"mirage\" for sample PMDA", "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" }, { "name": "sample.scramble.bin", "type": "metric", + "indom": "29.9", "oneline": "Several constant instances, instances scrambled", "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." }, { - "name": "SAMPLE.6", - "type": "metric", + "name": "29.6", + "type": "indom", + "indom": "29.6", "oneline": "Instance domain \"dodgey\" for sample PMDA.", "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." }, @@ -676,8 +733,9 @@ Index built "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" }, { - "name": "SAMPLE.12", - "type": "metric", + "name": "29.12", + "type": "indom", + "indom": "29.12", "oneline": "Instance domain \"procs\" for sample PMDA", "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } @@ -698,7 +756,18 @@ Index built "sample.longlong.ten" ] == verify request - http://localhost:PORT/search/suggest?query=sample.r -[] +[ + "sample.const_rate.gradient", + "sample.event.no_indom_records", + "sample.rapid", + "sample.byte_rate", + "sample.kbyte_rate", + "sample.byte_rate_perhour", + "sample.proc.reset", + "sample.recv_pdu", + "sample.const_rate.value", + "sample.percontext.recv_pdu" +] === /search/suggest - limit is supported == verify request - http://localhost:PORT/search/suggest?query=sam&limit=2 [ @@ -714,55 +783,91 @@ Index built === /search/indom - generic request === == verify request - http://localhost:PORT/search/indom?query=29.1 { - "total": 3, + "total": 6, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ + { + "name": "sample.colour", + "type": "metric", + "indom": "29.1", + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, + { + "name": "sample.darkness", + "type": "metric", + "indom": "29.1", + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." + }, + { + "name": "29.1", + "type": "indom", + "indom": "29.1", + "oneline": "Instance domain \"colour\" for sample PMDA", + "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + }, { "name": "red", "type": "instance", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "indom": "29.1" }, { "name": "green", "type": "instance", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "indom": "29.1" }, { "name": "blue", "type": "instance", - "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "indom": "29.1" } ] } == verify request - http://localhost:PORT/search/indom?query=29.3 { - "total": 3, + "total": 6, "elapsed": XXX, "offset": 0, - "limit": 0, + "limit": 10, "results": [ + { + "name": "sample.mirage", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, + { + "name": "sample.mirage_longlong", + "type": "metric", + "indom": "29.3", + "oneline": "Simple saw-tooth rate, but instances come and go", + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + }, + { + "name": "29.3", + "type": "indom", + "indom": "29.3", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, { "name": "m-00", "type": "instance", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go" + "indom": "29.3" }, { "name": "m-01", "type": "instance", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go" + "indom": "29.3" }, { "name": "m-04", "type": "instance", - "indom": "29.3", - "oneline": "Simple saw-tooth rate, but instances come and go" + "indom": "29.3" } ] } @@ -772,7 +877,7 @@ Index built "total": 0, "elapsed": 0, "offset": 0, - "limit": 0, + "limit": 10, "results": [ ] } === /search/indom - query param is required === @@ -781,37 +886,45 @@ Index built === /search/indom - pagination is supported using cursor params 'limit' and 'offset' == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=0 { - "total": 3, + "total": 6, "elapsed": XXX, "offset": 0, "limit": 2, "results": [ { - "name": "red", - "type": "instance", + "name": "sample.colour", + "type": "metric", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { - "name": "green", - "type": "instance", + "name": "sample.darkness", + "type": "metric", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." } ] } == verify request - http://localhost:PORT/search/indom?query=29.1&limit=2&offset=2 { - "total": 3, + "total": 6, "elapsed": XXX, "offset": 2, "limit": 2, "results": [ { - "name": "blue", - "type": "instance", + "name": "29.1", + "type": "indom", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time" + "oneline": "Instance domain \"colour\" for sample PMDA", + "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." + }, + { + "name": "red", + "type": "instance", + "indom": "29.1" } ] } @@ -820,14 +933,5 @@ Index built { "docs": NNN, "terms": NNN, - "records": NNN, - "records_per_doc_avg": 0.00, - "bytes_per_record_avg": 0.00, - "inverted_sz_mb": 0.00, - "inverted_cap_mb": 0.00, - "inverted_cap_ovh": 0.00, - "skip_index_size_mb": 0.00, - "score_index_size_mb": 0.00, - "offsets_per_term_avg": 0.00, - "offset_bits_per_record_avg": 0.00 + "records": NNN } diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 0148e757386..2f97df41fa1 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -203,7 +203,8 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, result.docid = search_make_docid( sqlite3_column_int64(stmt, 0), (const char *)sqlite3_column_text(stmt, 1)); - result.type = sqlite3_column_int(stmt, 4); + if (request->return_type) + result.type = sqlite3_column_int(stmt, 4); result.score = -sqlite3_column_double(stmt, 6); if (request->return_name) { @@ -292,7 +293,9 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, timer = pmtimespecSub(&finished, &started); offset = request->offset; - count = request->count ? request->count : smd->resultcount; + if (!request->count) + request->count = smd->resultcount; + count = request->count; for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { hits[i].total = nhits; @@ -374,7 +377,27 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, pmtimespecNow(&started); - match = sdscatfmt(sdsempty(), "name : %S*", request->query); + { + sds query = request->query; + int len = sdslen(query); + int j, start; + + match = sdsnew("name : ("); + for (j = 0, start = 0; j <= len; j++) { + if (j == len || query[j] == '.') { + if (j > start) { + if (start > 0) + match = sdscat(match, " "); + if (j == len) + match = sdscatlen(match, query + start, j - start); + else + match = sdscatlen(match, query + start, j - start); + } + start = j + 1; + } + } + match = sdscat(match, "*)"); + } search_suggest_table(smd->db, match, &hits, &nhits, &maxhits); @@ -385,7 +408,9 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); - count = request->count ? request->count : smd->resultcount; + if (!request->count) + request->count = smd->resultcount; + count = request->count; for (i = 0; i < (unsigned int)nhits && i < count; i++) { hits[i].total = nhits; @@ -487,7 +512,9 @@ search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, timer = pmtimespecSub(&finished, &started); offset = request->offset; - count = request->count ? request->count : smd->resultcount; + if (!request->count) + request->count = smd->resultcount; + count = request->count; for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { hits[i].total = nhits; @@ -535,6 +562,20 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) sqlite3_finalize(stmt); } + sqlite3_exec(smd->db, + "CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab" + " USING fts5vocab(docs, row)", NULL, NULL, NULL); + rc = sqlite3_prepare_v2(smd->db, + "SELECT COUNT(*), COALESCE(SUM(cnt),0)" + " FROM docs_vocab", -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + metrics.terms = sqlite3_column_int64(stmt, 0); + metrics.records = sqlite3_column_int64(stmt, 1); + } + sqlite3_finalize(stmt); + } + settings->callbacks.on_metrics(&metrics, arg); settings->callbacks.on_done(0, arg); return 0; diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index 922124995a9..1f6194a49f9 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -40,6 +40,7 @@ static char *filename; static int status; static int version = DEFAULT_HELP_VERSION; static int search_mode; /* -S: build search index */ +static char *pmnsfile = PM_NS_DEFAULT; static FILE *f; typedef struct { @@ -219,6 +220,56 @@ newentry(char *buf) } } +/* + * Resolve a symbolic PMDA name (e.g. "SAMPLE") to its numeric domain + * by reading $PCP_VAR_DIR/pmns/stdpmid. Returns -1 on failure. + */ +static int +lookup_domain(const char *pmda_name) +{ + char path[MAXPATHLEN]; + char var_dir[MAXPATHLEN]; + char line[256]; + char name[64]; + int domain; + FILE *fp; + + /* + * Read PCP_VAR_DIR directly from pcp.conf, not the environment, + * since QA tests may override PCP_VAR_DIR to a temp directory. + */ + { + const char *conf = getenv("PCP_CONF"); + if (conf == NULL) + conf = "/etc/pcp.conf"; + if ((fp = fopen(conf, "r")) == NULL) + return -1; + var_dir[0] = '\0'; + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "PCP_VAR_DIR=%s", var_dir) == 1) + break; + } + fclose(fp); + if (var_dir[0] == '\0') + return -1; + } + + pmsprintf(path, sizeof(path), "%s/pmns/stdpmid", var_dir); + if ((fp = fopen(path, "r")) == NULL) + return -1; + + while (fgets(line, sizeof(line), fp) != NULL) { + if (sscanf(line, "#define %63s %d", name, &domain) == 2) { + if (strcmp(name, pmda_name) == 0) { + fclose(fp); + return domain; + } + } + } + fclose(fp); + return -1; +} + /* * Search index mode (-S): collect entries for the FTS5 search index. * Unlike newentry() which writes .pag files, this just parses the @@ -231,7 +282,6 @@ newentry_search(char *buf) char *name; char *oneline; char *helptext; - int domain, serial; int type; char indom_buf[64]; @@ -301,14 +351,79 @@ newentry_search(char *buf) *p = '\0'; p++; - /* determine type: NNN.NNN = indom, otherwise metric name */ - if (sscanf(name, "%d.%d", &domain, &serial) == 2 && - strchr(name, '.') == strrchr(name, '.')) { - type = SEARCH_DOC_INDOM; - pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); - } else { - type = SEARCH_DOC_METRIC; - indom_buf[0] = '\0'; + /* + * Determine type: indom entries have exactly one dot with only + * uppercase letters/digits before it and only digits after it. + * This matches both numeric (29.3) and symbolic (SAMPLE.3) forms. + * Names with multiple dots (SAMPLE.0.1000) or lowercase are metrics. + */ + { + char *dot = strchr(name, '.'); + char *cp; + int is_indom = 0; + + if (dot != NULL && strchr(dot + 1, '.') == NULL) { + is_indom = 1; + for (cp = name; cp < dot; cp++) { + if (!isupper((int)*cp) && !isdigit((int)*cp)) { + is_indom = 0; + break; + } + } + if (is_indom) { + for (cp = dot + 1; *cp != '\0'; cp++) { + if (!isdigit((int)*cp)) { + is_indom = 0; + break; + } + } + } + } + if (is_indom) { + type = SEARCH_DOC_INDOM; + if (isdigit((int)*name)) { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } else { + char pmda_name[64]; + int len = dot - name; + int domain; + + if (len < (int)sizeof(pmda_name)) { + memcpy(pmda_name, name, len); + pmda_name[len] = '\0'; + domain = lookup_domain(pmda_name); + if (domain >= 0) + pmsprintf(indom_buf, sizeof(indom_buf), + "%d.%s", domain, dot + 1); + else + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } else { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", name); + } + } + } else { + static int pmns_loaded = 0; + + type = SEARCH_DOC_METRIC; + indom_buf[0] = '\0'; + if (!pmns_loaded) { + pmns_loaded = 1; + pmLoadASCIINameSpace(pmnsfile, 1); + pmNewContext(PM_CONTEXT_HOST, "localhost"); + } + { + pmID pmid; + pmDesc desc; + const char *np = name; + + if (pmLookupName(1, &np, &pmid) >= 0 && + pmLookupDesc(pmid, &desc) >= 0 && + desc.indom != PM_INDOM_NULL) { + pmsprintf(indom_buf, sizeof(indom_buf), "%s", + pmInDomStr(desc.indom)); + } + } + } } /* skip whitespace to start of oneline */ @@ -336,7 +451,8 @@ newentry_search(char *buf) if (verbose) fprintf(stderr, "%s\n", name); - search_sqlite_add(name, + search_sqlite_add((type == SEARCH_DOC_INDOM && indom_buf[0] != '\0') ? + indom_buf : name, (*oneline != '\0') ? oneline : NULL, (*helptext != '\0') ? helptext : NULL, (indom_buf[0] != '\0') ? indom_buf : NULL, @@ -393,7 +509,6 @@ main(int argc, char **argv) int c; int i; int skip; - char *pmnsfile = PM_NS_DEFAULT; char *fname = NULL; char pathname[MAXPATHLEN]; FILE *inf; diff --git a/src/pmproxy/src/search.c b/src/pmproxy/src/search.c index 4ff7de278b3..3826bc06dad 100644 --- a/src/pmproxy/src/search.c +++ b/src/pmproxy/src/search.c @@ -101,22 +101,8 @@ on_pmsearch_metrics(pmSearchMetrics *metrics, void *arg) baton->suffix = json_push_suffix(baton->suffix, JSON_FLAG_OBJECT); result = sdscatprintf(result, - "{\"docs\":%llu,\"terms\":%llu,\"records\":%llu," - "\"records_per_doc_avg\":%.2f," - "\"bytes_per_record_avg\":%.2f," - "\"inverted_sz_mb\":%.2f," - "\"inverted_cap_mb\":%.2f," - "\"inverted_cap_ovh\":%.2f," - "\"skip_index_size_mb\":%.2f," - "\"score_index_size_mb\":%.2f," - "\"offsets_per_term_avg\":%.2f," - "\"offset_bits_per_record_avg\":%.2f", - metrics->docs, metrics->terms, metrics->records, - metrics->records_per_doc_avg, metrics->bytes_per_record_avg, - metrics->inverted_sz_mb, metrics->inverted_cap_mb, - metrics->inverted_cap_ovh, metrics->skip_index_size_mb, - metrics->score_index_size_mb, metrics->offsets_per_term_avg, - metrics->offset_bits_per_record_avg); + "{\"docs\":%llu,\"terms\":%llu,\"records\":%llu", + metrics->docs, metrics->terms, metrics->records); http_set_buffer(client, result, HTTP_FLAG_JSON); http_transfer(client); diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index b0deee2ffb7..3e88b8b096b 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -183,12 +183,17 @@ on_search_result(pmSearchTextResult *result, void *arg) printf("ID: %s\n", result->docid); if (dp->flags & PMSEARCH_SCORES) printf("Score: %.2f\n", result->score); - if (result->type != PM_SEARCH_TYPE_UNKNOWN) + if (result->type != PM_SEARCH_TYPE_UNKNOWN && + !(dp->flags & PMSEARCH_OPT_SUGGEST)) printf("Type: %s\n", pmSearchTextTypeStr(result->type)); if (result->name != NULL) printv(dp, "Name", result->name); - if (result->indom != NULL) - printv(dp, "InDom", result->indom); + if (!(dp->flags & PMSEARCH_OPT_SUGGEST)) { + if (result->indom != NULL && result->indom[0] != '\0') + printv(dp, "InDom", result->indom); + else if (result->type != PM_SEARCH_TYPE_UNKNOWN) + printv(dp, "InDom", "none"); + } if (result->oneline != NULL) printv(dp, "One line", result->oneline); if (result->helptext != NULL) diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index 27d8b539312..62d27cb0256 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -146,17 +146,6 @@ FILENAME == ARGV[1] { metric_indom[a[1]] = a[2] next } -FILENAME == ARGV[2] { - # metric→oneline from the helptext file we already built - if (/^@ [a-zA-Z]/) { - name = $2 - start = index($0, " ") - start = index(substr($0, start + 1), " ") - oneline = (start > 0) ? substr($0, index($0, $2) + length($2) + 1) : "" - metric_oneline[name] = oneline - } - next -} { # pmprobe -I: metric count "inst1" "inst2" ... metric = $1 @@ -166,9 +155,6 @@ FILENAME == ARGV[2] { indom = metric_indom[metric] if (indom == "") next - if (!(indom in indom_oneline) && metric_oneline[metric] != "") - indom_oneline[indom] = metric_oneline[metric] - rest = $0 while (match(rest, /"[^"]*"/)) { inst = substr(rest, RSTART + 1, RLENGTH - 2) @@ -176,19 +162,15 @@ FILENAME == ARGV[2] { if (!(key in seen)) { seen[key] = 1 inst_data[++ninst] = indom "\t" inst - inst_indom[ninst] = indom } rest = substr(rest, RSTART + RLENGTH) } } END { - for (i = 1; i <= ninst; i++) { - indom = inst_indom[i] - oneline = indom_oneline[indom] - printf "@ I %s\n%s\n\n", inst_data[i], oneline - } + for (i = 1; i <= ninst; i++) + printf "@ I %s\n\n", inst_data[i] } -' "$tmp/metric_indom" "$tmp/helptext" "$tmp/pmprobe_output" >> $tmp/helptext +' "$tmp/metric_indom" "$tmp/pmprobe_output" >> $tmp/helptext if $VERBOSE then From 00f9e3a379e584042ac7f2188667edfc3c0876a5 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Thu, 30 Jul 2026 15:02:40 -0400 Subject: [PATCH 09/22] pmsearch: add metrics without help files and add upsert logic for the index --- qa/1871.out | 40 ++++++++--------- qa/1872.out | 34 +++++++-------- src/newhelp/newhelp.c | 54 +++++++++++++++++++++++ src/newhelp/search_sqlite.c | 45 ++++++++++++++++++- src/pmsearch/pmsearch_index.sh | 80 ++++++++++++++++------------------ 5 files changed, 172 insertions(+), 81 deletions(-) diff --git a/qa/1871.out b/qa/1871.out index b57a627724b..75da8d5a8fd 100644 --- a/qa/1871.out +++ b/qa/1871.out @@ -242,6 +242,26 @@ Name: sample.longlong.ten === Suggestion search 2 === Name: blue === Indom search === +Type: metric +Name: sample.mirage_longlong +InDom: 29.3 +One line: Simple saw-tooth rate, but instances come and go +Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution +over time. Different instances of the metric have different baselines +for the saw-tooth, but all have an max-to-min range of 100,000,000. + +What makes this metric interesting is that instances come and go (not more +often than once every 10 seconds however). Instance 0 is always present, +but the other instances 1 thru 49 come and go in a cyclic pattern with +a large random component influencing when each instance appears and +disappears. + +The underlying counter starts at 0 and is incremented once for each +pmFetch() to this metric and/or sample.mirage and/or sample.colour. + +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). + Type: metric Name: sample.mirage InDom: 29.3 @@ -266,26 +286,6 @@ sample.mirage_longlong. Use pmStore() to modify the underlying counter (independent of which instance or instances are used). -Type: metric -Name: sample.mirage_longlong -InDom: 29.3 -One line: Simple saw-tooth rate, but instances come and go -Help: The metric is a rate (bytes/msec) that varies in a saw-tooth distribution -over time. Different instances of the metric have different baselines -for the saw-tooth, but all have an max-to-min range of 100,000,000. - -What makes this metric interesting is that instances come and go (not more -often than once every 10 seconds however). Instance 0 is always present, -but the other instances 1 thru 49 come and go in a cyclic pattern with -a large random component influencing when each instance appears and -disappears. - -The underlying counter starts at 0 and is incremented once for each -pmFetch() to this metric and/or sample.mirage and/or sample.colour. - -Use pmStore() to modify the underlying counter (independent of which -instance or instances are used). - Type: indom Name: 29.3 InDom: 29.3 diff --git a/qa/1872.out b/qa/1872.out index 5c2c3ef8734..1603d8ab89b 100644 --- a/qa/1872.out +++ b/qa/1872.out @@ -759,9 +759,9 @@ Index built [ "sample.const_rate.gradient", "sample.event.no_indom_records", - "sample.rapid", "sample.byte_rate", "sample.kbyte_rate", + "sample.rapid", "sample.byte_rate_perhour", "sample.proc.reset", "sample.recv_pdu", @@ -789,18 +789,18 @@ Index built "limit": 10, "results": [ { - "name": "sample.colour", + "name": "sample.darkness", "type": "metric", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." }, { - "name": "sample.darkness", + "name": "sample.colour", "type": "metric", "indom": "29.1", - "oneline": "No values available", - "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { "name": "29.1", @@ -834,18 +834,18 @@ Index built "limit": 10, "results": [ { - "name": "sample.mirage", + "name": "sample.mirage_longlong", "type": "metric", "indom": "29.3", "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { - "name": "sample.mirage_longlong", + "name": "sample.mirage", "type": "metric", "indom": "29.3", "oneline": "Simple saw-tooth rate, but instances come and go", - "helptext": "The metric is a rate (bytes/msec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100,000,000.\n\nWhat makes this metric interesting is that instances come and go (not more\noften than once every 10 seconds however). Instance 0 is always present,\nbut the other instances 1 thru 49 come and go in a cyclic pattern with\na large random component influencing when each instance appears and\ndisappears.\n\nThe underlying counter starts at 0 and is incremented once for each\npmFetch() to this metric and/or sample.mirage and/or sample.colour.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "helptext": "The metric is a rate (Kbytes/sec) that varies in a saw-tooth distribution\nover time. Different instances of the metric have different baselines\nfor the saw-tooth, but all have an max-to-min range of 100.\n\nWhat makes this metric interesting is that instances come and go although\nnot more often than once every 10 seconds by default. Use pmstore to\nchange sample.controller.mirage and the frequency of instance domain\nchanges can be varied.\n\nInstance 0 is always present, but the other instances 1 thru 49 come\nand go in a cyclic pattern with a large random component influencing\nwhen each instance appears and disappears.\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.colour and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." }, { "name": "29.3", @@ -892,18 +892,18 @@ Index built "limit": 2, "results": [ { - "name": "sample.colour", + "name": "sample.darkness", "type": "metric", "indom": "29.1", - "oneline": "Metrics with a \"saw-tooth\" trend over time", - "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." + "oneline": "No values available", + "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." }, { - "name": "sample.darkness", + "name": "sample.colour", "type": "metric", "indom": "29.1", - "oneline": "No values available", - "helptext": "Defined over the same instance domain as sample.colour, but this\nmetric returns the \"No values available\" error for every fetch." + "oneline": "Metrics with a \"saw-tooth\" trend over time", + "helptext": "This metric has 3 instances, designated \"red\", \"green\" and \"blue\".\n\nThe value of the metric is monotonic increasing in the range N to\nN+100, then back to N. The different instances have different N\nvalues, namely 100 (red), 200 (green) and 300 (blue).\n\nThe underlying counter starts at 0 and is incremented once\nfor each pmFetch() to this metric and/or sample.mirage and/or\nsample.mirage_longlong.\n\nUse pmStore() to modify the underlying counter (independent of which\ninstance or instances are used)." } ] } diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index 1f6194a49f9..e2cbf271097 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -343,6 +343,60 @@ newentry_search(char *buf) return; } + /* + * Metric with pre-resolved indom: @ M nameindomoneline + * Generated by pmsearch_index to avoid needing PMNS/PMCD lookup. + */ + if (*p == 'M' && isspace((int)*(p + 1))) { + char *indom_str; + + p += 2; + while (*p == ' ') + p++; + name = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + return; + } + indom_str = p; + while (*p && *p != '\t' && *p != '\n') + p++; + if (*p == '\t') { + *p++ = '\0'; + } else { + return; + } + + oneline = p; + while (*p != '\n' && *p != '\0') + p++; + if (*p == '\n') { + *p = '\0'; + p++; + } + + helptext = p; + if (*helptext) { + int len = (int)strlen(helptext) - 1; + while (len >= 0 && helptext[len] == '\n') + len--; + helptext[len + 1] = '\0'; + } + + if (verbose) + fprintf(stderr, "%s [%s]\n", name, indom_str); + + search_sqlite_add(name, + (*oneline != '\0') ? oneline : NULL, + (*helptext != '\0') ? helptext : NULL, + (*indom_str != '\0') ? indom_str : NULL, + SEARCH_DOC_METRIC); + return; + } + name = p; /* skip over metric name or indom spec */ diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c index 63a1a3b8f33..18e1fbc4fa4 100644 --- a/src/newhelp/search_sqlite.c +++ b/src/newhelp/search_sqlite.c @@ -19,6 +19,8 @@ static sqlite3 *search_db; static sqlite3_stmt *search_insert; +static sqlite3_stmt *search_lookup; +static sqlite3_stmt *search_delete; int search_sqlite_open(const char *path) @@ -68,7 +70,29 @@ search_sqlite_open(const char *path) " VALUES(?, ?, ?, ?, ?)", -1, &search_insert, NULL); if (rc != SQLITE_OK) { - fprintf(stderr, "search_sqlite_open: prepare: %s\n", + fprintf(stderr, "search_sqlite_open: prepare insert: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "SELECT rowid FROM docs WHERE name = ? AND type = ?", + -1, &search_lookup, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare lookup: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "DELETE FROM docs WHERE rowid = ?", + -1, &search_delete, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare delete: %s\n", sqlite3_errmsg(search_db)); sqlite3_close(search_db); search_db = NULL; @@ -85,6 +109,17 @@ search_sqlite_add(const char *name, const char *oneline, if (search_db == NULL || search_insert == NULL) return; + /* upsert: remove existing entry with same (name, type) if any */ + sqlite3_bind_text(search_lookup, 1, name, -1, SQLITE_STATIC); + sqlite3_bind_int(search_lookup, 2, type); + if (sqlite3_step(search_lookup) == SQLITE_ROW) { + sqlite3_int64 rowid = sqlite3_column_int64(search_lookup, 0); + sqlite3_bind_int64(search_delete, 1, rowid); + sqlite3_step(search_delete); + sqlite3_reset(search_delete); + } + sqlite3_reset(search_lookup); + sqlite3_bind_text(search_insert, 1, name, -1, SQLITE_STATIC); sqlite3_bind_text(search_insert, 2, oneline ? oneline : "", -1, SQLITE_STATIC); sqlite3_bind_text(search_insert, 3, helptext ? helptext : "", -1, SQLITE_STATIC); @@ -111,6 +146,14 @@ search_sqlite_close(void) sqlite3_finalize(search_insert); search_insert = NULL; } + if (search_lookup) { + sqlite3_finalize(search_lookup); + search_lookup = NULL; + } + if (search_delete) { + sqlite3_finalize(search_delete); + search_delete = NULL; + } /* optimize the FTS index before closing */ sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index 62d27cb0256..cfdc8398735 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -74,20 +74,28 @@ then echo "$prog: index target: $INDEX" fi -# Extract all metric help text from PMCD in newhelp format. -# pminfo -tT output format: -# metricname [oneline text] -# Help: -# multi-line helptext -# -# -# Transform to newhelp format: -# @ metricname oneline text -# multi-line helptext -# +# Build metric→indom mapping first (needed for both metrics and instances). +pminfo $SRCFLAGS -I 2>/dev/null | $PCP_AWK_PROG ' +/^[a-zA-Z]/ { metric = $1 } +/InDom:/ && !/PM_INDOM_NULL/ { + for (i = 1; i <= NF; i++) { + if ($i == "InDom:") { + print metric "\t" $(i+1) + break + } + } +}' > $tmp/metric_indom + +# Extract all metric help text in newhelp format. +# Uses "@ M nameindomoneline" for metrics with indoms, +# so newhelp -S can resolve indoms without needing PMNS/PMCD. pminfo $SRCFLAGS -tT 2>/dev/null | $PCP_AWK_PROG ' +FILENAME == ARGV[1] { + split($0, a, "\t") + metric_indom[a[1]] = a[2] + next +} /^[a-zA-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_.]+ / { - # metric line: "name.with.dots [oneline]" or error if (index($0, "One-line Help: Error:") > 0) { skip = 1 next @@ -100,14 +108,17 @@ pminfo $SRCFLAGS -tT 2>/dev/null | $PCP_AWK_PROG ' if (start > 0 && end > start) { oneline = substr($0, start + 1, end - start - 1) } - printf "@ %s %s\n", name, oneline + if (name in metric_indom) + printf "@ M %s\t%s\t%s\n", name, metric_indom[name], oneline + else + printf "@ M %s\t\t%s\n", name, oneline next } /^Help:$/ { next } /^Full Help: Error:/ { skip = 1; next } skip { next } { print } -' > $tmp/helptext +' "$tmp/metric_indom" - > $tmp/helptext nhelplines=`wc -l < $tmp/helptext` if [ "$nhelplines" -eq 0 ] @@ -123,21 +134,8 @@ then echo "$prog: extracted $nmetrics metrics ($nhelplines lines)" fi -# Extract instance names from running PMCD. -# Build metric→indom mapping, then get instances via pmprobe -I. +# Extract instance names. # Deduplicate by (indom, instance_name) since many metrics share indoms. -# -pminfo $SRCFLAGS -I 2>/dev/null | $PCP_AWK_PROG ' -/^[a-zA-Z]/ { metric = $1 } -/InDom:/ && !/PM_INDOM_NULL/ { - for (i = 1; i <= NF; i++) { - if ($i == "InDom:") { - print metric "\t" $(i+1) - break - } - } -}' > $tmp/metric_indom - pmprobe $SRCFLAGS -I 2>/dev/null > $tmp/pmprobe_output $PCP_AWK_PROG ' @@ -147,7 +145,6 @@ FILENAME == ARGV[1] { next } { - # pmprobe -I: metric count "inst1" "inst2" ... metric = $1 count = $2 + 0 if (count <= 0 || $2 == "PM_IN_NULL") next @@ -198,7 +195,7 @@ then fi # Start from the build-time base index (contains all metric help text). -# The nightly update copies it, then appends runtime instance data. +# The nightly update copies it, then adds/updates runtime data. BASE="$PCP_SHARE_DIR/lib/pcp.search" if [ -f "$BASE" ] then @@ -209,30 +206,27 @@ then fi fi -# Extract only instance entries for appending to the index -$PCP_AWK_PROG ' -/^@ I / { printing = 1; print; next } -printing && /^$/ { print; printing = 0; next } -printing { print } -' $tmp/helptext > $tmp/instances - -ninst=`grep -c '^@ I ' $tmp/instances` -if [ "$ninst" -eq 0 ] +nentries=`grep -c '^@ ' $tmp/helptext` +if [ "$nentries" -eq 0 ] then if $VERBOSE then - echo "$prog: no instance data to add" + echo "$prog: no runtime data to add" fi status=0 exit fi -# Add instance data to the index -if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/instances +# Add/update all runtime data (metrics and instances) into the index. +# Metrics already in the base index are updated; new metrics from +# PMDAs without help files (e.g. Python PMDAs) are inserted. +if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/helptext then if $VERBOSE then - echo "$prog: added $ninst instances to $INDEX" + nmetrics=`grep -c '^@ [^I]' $tmp/helptext` + ninst=`grep -c '^@ I ' $tmp/helptext` + echo "$prog: added $nmetrics metrics and $ninst instances to $INDEX" fi status=0 else From 8c02c1d343c2efa6bb99aac1ef514427dce098e2 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Thu, 30 Jul 2026 21:39:40 -0400 Subject: [PATCH 10/22] pmsearch: improve code quality of recent changes --- src/libpcp_web/src/search.c | 80 ++++++++++++++++++++++++++----------- src/newhelp/newhelp.c | 49 +++++++++++++---------- src/newhelp/search_sqlite.c | 21 ++++++++-- src/newhelp/search_sqlite.h | 6 +-- 4 files changed, 105 insertions(+), 51 deletions(-) diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 2f97df41fa1..7b62dfe2fe0 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -190,12 +190,20 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, pmSearchTextResult result; if (*nhits >= *maxhits) { - *maxhits = *maxhits ? *maxhits * 2 : 64; - *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); - if (*hits == NULL) { + int newmax = *maxhits ? *maxhits * 2 : 64; + pmSearchTextResult *tmp; + + if (newmax < *maxhits) { + sqlite3_finalize(stmt); + return -ENOMEM; + } + tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); + if (tmp == NULL) { sqlite3_finalize(stmt); return -ENOMEM; } + *hits = tmp; + *maxhits = newmax; } memset(&result, 0, sizeof(result)); @@ -332,8 +340,12 @@ search_suggest_table(sqlite3 *db, sds match, rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); - if (rc != SQLITE_OK) + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_suggest_table: prepare: %s\n", + sqlite3_errmsg(db)); return -EIO; + } sqlite3_bind_text(stmt, 1, match, sdslen(match), SQLITE_STATIC); @@ -341,12 +353,20 @@ search_suggest_table(sqlite3 *db, sds match, pmSearchTextResult result; if (*nhits >= *maxhits) { - *maxhits = *maxhits ? *maxhits * 2 : 64; - *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); - if (*hits == NULL) { + int newmax = *maxhits ? *maxhits * 2 : 64; + pmSearchTextResult *tmp; + + if (newmax < *maxhits) { + sqlite3_finalize(stmt); + return -ENOMEM; + } + tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); + if (tmp == NULL) { sqlite3_finalize(stmt); return -ENOMEM; } + *hits = tmp; + *maxhits = newmax; } memset(&result, 0, sizeof(result)); @@ -388,10 +408,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, if (j > start) { if (start > 0) match = sdscat(match, " "); - if (j == len) - match = sdscatlen(match, query + start, j - start); - else - match = sdscatlen(match, query + start, j - start); + match = sdscatlen(match, query + start, j - start); } start = j + 1; } @@ -443,8 +460,12 @@ search_indom_table(sqlite3 *db, const char *query, int querylen, rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); - if (rc != SQLITE_OK) + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_indom_table: prepare: %s\n", + sqlite3_errmsg(db)); return -EIO; + } sqlite3_bind_text(stmt, 1, query, querylen, SQLITE_STATIC); @@ -452,12 +473,20 @@ search_indom_table(sqlite3 *db, const char *query, int querylen, pmSearchTextResult result; if (*nhits >= *maxhits) { - *maxhits = *maxhits ? *maxhits * 2 : 64; - *hits = realloc(*hits, *maxhits * sizeof(pmSearchTextResult)); - if (*hits == NULL) { + int newmax = *maxhits ? *maxhits * 2 : 64; + pmSearchTextResult *tmp; + + if (newmax < *maxhits) { + sqlite3_finalize(stmt); + return -ENOMEM; + } + tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); + if (tmp == NULL) { sqlite3_finalize(stmt); return -ENOMEM; } + *hits = tmp; + *maxhits = newmax; } memset(&result, 0, sizeof(result)); @@ -542,7 +571,7 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) { searchModuleData *smd = (searchModuleData *)settings->module.privdata; pmSearchMetrics metrics; - sqlite3_stmt *stmt; + sqlite3_stmt *stmt = NULL; int rc; (void)key; @@ -562,9 +591,14 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) sqlite3_finalize(stmt); } - sqlite3_exec(smd->db, + rc = sqlite3_exec(smd->db, "CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab" " USING fts5vocab(docs, row)", NULL, NULL, NULL); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "pmSearchInfo: create docs_vocab: %s\n", + sqlite3_errmsg(smd->db)); + } rc = sqlite3_prepare_v2(smd->db, "SELECT COUNT(*), COALESCE(SUM(cnt),0)" " FROM docs_vocab", -1, &stmt, NULL); @@ -586,8 +620,8 @@ pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded) { - settings->callbacks.on_done(-ENOENT, arg); + if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); return 0; } @@ -600,8 +634,8 @@ pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, vo { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded) { - settings->callbacks.on_done(-ENOENT, arg); + if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); return 0; } @@ -614,8 +648,8 @@ pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded) { - settings->callbacks.on_done(-ENOENT, arg); + if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + settings->callbacks.on_done(-EINVAL, arg); return 0; } diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index e2cbf271097..c129d404692 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -270,6 +270,19 @@ lookup_domain(const char *pmda_name) return -1; } +static void +trim_trailing_newlines(char *str) +{ + int len; + + if (*str) { + len = (int)strlen(str) - 1; + while (len >= 0 && str[len] == '\n') + len--; + str[len + 1] = '\0'; + } +} + /* * Search index mode (-S): collect entries for the FTS5 search index. * Unlike newentry() which writes .pag files, this just parses the @@ -284,6 +297,8 @@ newentry_search(char *buf) char *helptext; int type; char indom_buf[64]; + pmID pmid; + pmDesc desc; /* skip leading white space */ for (p = buf; isspace((int)*p); p++) @@ -306,6 +321,9 @@ newentry_search(char *buf) if (*p == '\t') { *p++ = '\0'; } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] instance entry missing tab separator\n", + pmGetProgname(), filename, ln); return; } inst_name = p; @@ -326,12 +344,7 @@ newentry_search(char *buf) } helptext = p; - if (*helptext) { - int len = (int)strlen(helptext) - 1; - while (len >= 0 && helptext[len] == '\n') - len--; - helptext[len + 1] = '\0'; - } + trim_trailing_newlines(helptext); if (verbose) fprintf(stderr, "instance %s [%s]\n", inst_name, indom_str); @@ -359,6 +372,9 @@ newentry_search(char *buf) if (*p == '\t') { *p++ = '\0'; } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] metric entry missing first tab separator\n", + pmGetProgname(), filename, ln); return; } indom_str = p; @@ -367,6 +383,9 @@ newentry_search(char *buf) if (*p == '\t') { *p++ = '\0'; } else { + if (verbose) + fprintf(stderr, "%s: [%s:%d] metric entry missing second tab separator\n", + pmGetProgname(), filename, ln); return; } @@ -379,12 +398,7 @@ newentry_search(char *buf) } helptext = p; - if (*helptext) { - int len = (int)strlen(helptext) - 1; - while (len >= 0 && helptext[len] == '\n') - len--; - helptext[len + 1] = '\0'; - } + trim_trailing_newlines(helptext); if (verbose) fprintf(stderr, "%s [%s]\n", name, indom_str); @@ -466,8 +480,6 @@ newentry_search(char *buf) pmNewContext(PM_CONTEXT_HOST, "localhost"); } { - pmID pmid; - pmDesc desc; const char *np = name; if (pmLookupName(1, &np, &pmid) >= 0 && @@ -493,14 +505,9 @@ newentry_search(char *buf) p++; } - /* remainder is helptext - trim trailing newlines */ + /* remainder is helptext */ helptext = p; - if (*helptext) { - int len = (int)strlen(helptext) - 1; - while (len >= 0 && helptext[len] == '\n') - len--; - helptext[len + 1] = '\0'; - } + trim_trailing_newlines(helptext); if (verbose) fprintf(stderr, "%s\n", name); diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c index 18e1fbc4fa4..936c01a834a 100644 --- a/src/newhelp/search_sqlite.c +++ b/src/newhelp/search_sqlite.c @@ -83,6 +83,8 @@ search_sqlite_open(const char *path) if (rc != SQLITE_OK) { fprintf(stderr, "search_sqlite_open: prepare lookup: %s\n", sqlite3_errmsg(search_db)); + sqlite3_finalize(search_insert); + search_insert = NULL; sqlite3_close(search_db); search_db = NULL; return -1; @@ -94,6 +96,10 @@ search_sqlite_open(const char *path) if (rc != SQLITE_OK) { fprintf(stderr, "search_sqlite_open: prepare delete: %s\n", sqlite3_errmsg(search_db)); + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; sqlite3_close(search_db); search_db = NULL; return -1; @@ -106,7 +112,7 @@ void search_sqlite_add(const char *name, const char *oneline, const char *helptext, const char *indom, int type) { - if (search_db == NULL || search_insert == NULL) + if (search_db == NULL || search_insert == NULL || name == NULL) return; /* upsert: remove existing entry with same (name, type) if any */ @@ -115,7 +121,10 @@ search_sqlite_add(const char *name, const char *oneline, if (sqlite3_step(search_lookup) == SQLITE_ROW) { sqlite3_int64 rowid = sqlite3_column_int64(search_lookup, 0); sqlite3_bind_int64(search_delete, 1, rowid); - sqlite3_step(search_delete); + if (sqlite3_step(search_delete) != SQLITE_DONE) { + fprintf(stderr, "search_sqlite_add: delete: %s\n", + sqlite3_errmsg(search_db)); + } sqlite3_reset(search_delete); } sqlite3_reset(search_lookup); @@ -156,8 +165,12 @@ search_sqlite_close(void) } /* optimize the FTS index before closing */ - sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", - NULL, NULL, NULL); + rc = sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: optimize: %s\n", + sqlite3_errmsg(search_db)); + } rc = sqlite3_exec(search_db, "COMMIT", NULL, NULL, NULL); if (rc != SQLITE_OK) { diff --git a/src/newhelp/search_sqlite.h b/src/newhelp/search_sqlite.h index 36404a3c63f..5791eefd710 100644 --- a/src/newhelp/search_sqlite.h +++ b/src/newhelp/search_sqlite.h @@ -18,9 +18,9 @@ #define SEARCH_DOC_INDOM 2 #define SEARCH_DOC_INST 3 -extern int search_sqlite_open(const char *); -extern void search_sqlite_add(const char *, const char *, - const char *, const char *, int); +extern int search_sqlite_open(const char *path); +extern void search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type); extern int search_sqlite_close(void); #endif /* SEARCH_SQLITE_H */ From 5fcb452e1d98c7756d4f101f8c53afe921aad68c Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 31 Jul 2026 12:17:03 -0400 Subject: [PATCH 11/22] pmsearch: changes after code review of recent rework of pmsearch --- src/newhelp/newhelp.c | 4 ++-- src/pmsearch/pmsearch.c | 2 +- src/pmsearch/pmsearch_index.service.in | 3 +++ src/pmsearch/pmsearch_index.sh | 2 +- src/pmsearch/pmsearch_index.timer | 5 +++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index c129d404692..fb7262c61cc 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -31,8 +31,8 @@ #define DEFAULT_HELP_VERSION 2 /* maximum bytes per line and bytes per entry */ -#define MAXLINE 128 -#define MAXENTRY 1024 +#define MAXLINE 1024 +#define MAXENTRY 2048 static int verbose; static int ln; diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index 3e88b8b096b..fc5be25bc43 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -107,7 +107,7 @@ static void printv(search_data *dp, const char *name, sds input) { const char *on, *off; - char *start, *end, *tmp; + char *start, *end, *tmp; sds value; if (input == NULL || input[0] == '\0') diff --git a/src/pmsearch/pmsearch_index.service.in b/src/pmsearch/pmsearch_index.service.in index 9380de6590f..2166d9cc9c3 100644 --- a/src/pmsearch/pmsearch_index.service.in +++ b/src/pmsearch/pmsearch_index.service.in @@ -4,6 +4,9 @@ Documentation=man:pmsearch_index(1) ConditionPathExists=!@CRONTAB_PATH@ [Service] +Type=@SD_SERVICE_TYPE@ +Restart=no +TimeoutStartSec=1h ExecStart=@PCP_BINADM_DIR@/pmsearch_index WorkingDirectory=@PCP_VAR_DIR@ Group=@PCP_GROUP@ diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index cfdc8398735..43e77c9db58 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -177,7 +177,7 @@ fi if $SHOWME then - echo "$prog: would copy $BASE to $INDEX and add instances" + echo "$prog: would copy $PCP_SHARE_DIR/lib/pcp.search to $INDEX and add runtime data" status=0 exit fi diff --git a/src/pmsearch/pmsearch_index.timer b/src/pmsearch/pmsearch_index.timer index c15dd697f20..63b41492ab3 100644 --- a/src/pmsearch/pmsearch_index.timer +++ b/src/pmsearch/pmsearch_index.timer @@ -3,9 +3,14 @@ Description=Daily rebuild of the PCP pmsearch full-text index PartOf=pmcd.service [Timer] +Persistent=true # if enabled, runs 10m after boot and just after midnight (and after # the daily local PCP archive processing at 00:10:00 and proxy # processing at 00:15:00) OnBootSec=10min OnCalendar=*-*-* 00:20:00 Unit=pmsearch_index.service + +[Install] +WantedBy=timers.target +RequiredBy=pmproxy.service From 1eda63522e734e1f8368ca39262ba5eff4ff3bb8 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 31 Jul 2026 12:42:24 -0400 Subject: [PATCH 12/22] add sqlite to list of packages installed in QA containers --- qa/admin/other-packages/manifest | 11 +++++++++++ qa/admin/package-lists/AmazonLinux+2023+aarch64 | 1 + qa/admin/package-lists/AmazonLinux+2023+x86_64 | 1 + qa/admin/package-lists/CentOS+7+x86_64 | 1 + qa/admin/package-lists/CentOS+8+x86_64 | 1 + qa/admin/package-lists/CentOS+Stream10+x86_64 | 1 + qa/admin/package-lists/CentOS+Stream8+x86_64 | 1 + qa/admin/package-lists/CentOS+Stream9+x86_64 | 1 + qa/admin/package-lists/Debian+11+x86_64 | 1 + qa/admin/package-lists/Debian+12+aarch64 | 1 + qa/admin/package-lists/Debian+12+i686 | 1 + qa/admin/package-lists/Debian+12+x86_64 | 1 + qa/admin/package-lists/Debian+13+i686 | 1 + qa/admin/package-lists/Debian+13+x86_64 | 1 + qa/admin/package-lists/Debian+14+x86_64 | 1 + qa/admin/package-lists/Fedora+42+aarch64 | 1 + qa/admin/package-lists/Fedora+42+x86_64 | 1 + qa/admin/package-lists/Fedora+43+aarch64 | 1 + qa/admin/package-lists/Fedora+43+x86_64 | 1 + qa/admin/package-lists/Fedora+44+aarch64 | 1 + qa/admin/package-lists/Fedora+44+x86_64 | 1 + qa/admin/package-lists/Fedora+45+aarch64 | 1 + qa/admin/package-lists/Fedora+45+x86_64 | 1 + qa/admin/package-lists/MX+23.6+x86_64 | 1 + qa/admin/package-lists/RHEL+10+x86_64 | 1 + qa/admin/package-lists/RHEL+8+x86_64 | 1 + qa/admin/package-lists/RHEL+9+x86_64 | 1 + qa/admin/package-lists/Ubuntu+18.04+i686 | 1 + qa/admin/package-lists/Ubuntu+18.04+x86_64 | 1 + qa/admin/package-lists/Ubuntu+20.04+x86_64 | 1 + qa/admin/package-lists/Ubuntu+22.04+x86_64 | 1 + qa/admin/package-lists/Ubuntu+24.04+any | 1 + qa/admin/package-lists/Ubuntu+24.04+x86_64 | 1 + qa/admin/package-lists/Ubuntu+26.04+any | 1 + qa/admin/package-lists/Ubuntu+26.04+x86_64 | 1 + qa/admin/package-lists/openSUSE+15.6+x86_64 | 1 + qa/admin/package-lists/openSUSE+16.0+x86_64 | 1 + 37 files changed, 47 insertions(+) diff --git a/qa/admin/other-packages/manifest b/qa/admin/other-packages/manifest index 88262e3f66e..88e97a0012e 100644 --- a/qa/admin/other-packages/manifest +++ b/qa/admin/other-packages/manifest @@ -1018,6 +1018,17 @@ S_pkg? /usr/lib/*/libsqlite3.so [database/sqlite-3 (QA optional)] slackpkg? /usr/lib*/libsqlite3.so [sqlite (QA optional)] pacman? /usr/lib/libsqlite3.so [core/sqlite (QA optional)] brew? /usr/lib/libsqlite3.dylib [sqlite (QA optional)] +# -- libsqlite3 headers for pmsearch FTS5 index +dpkg? /usr/include/sqlite3.h [libsqlite3-dev (build optional)] +rpm? /usr/include/sqlite3.h [sqlite-devel (build optional)] +emerge? /usr/include/sqlite3.h [dev-db/sqlite (build optional)] +pkgin? /usr/pkg/include/sqlite3.h [sqlite3 (build optional)] +pkg_add? /usr/local/include/sqlite3.h [sqlite3 (build optional)] +F_pkg? /usr/local/include/sqlite3.h [sqlite3 (build optional)] +S_pkg? /usr/include/sqlite3.h [database/sqlite-3 (build optional)] +slackpkg? /usr/include/sqlite3.h [sqlite (build optional)] +pacman? /usr/include/sqlite3.h [core/sqlite (build optional)] +brew? /usr/local/include/sqlite3.h [sqlite (build optional)] # -- libicui18n for certutil dpkg? /usr/lib/*/libicui18n.so.* [libicu[0-9][0-9]* (QA optional)] rpm? /usr/lib*/libicui18n.so [libicu-devel (QA optional)] diff --git a/qa/admin/package-lists/AmazonLinux+2023+aarch64 b/qa/admin/package-lists/AmazonLinux+2023+aarch64 index 3bfb97d4768..290f544fc03 100644 --- a/qa/admin/package-lists/AmazonLinux+2023+aarch64 +++ b/qa/admin/package-lists/AmazonLinux+2023+aarch64 @@ -105,6 +105,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/AmazonLinux+2023+x86_64 b/qa/admin/package-lists/AmazonLinux+2023+x86_64 index 94649929248..e876c81e9c9 100644 --- a/qa/admin/package-lists/AmazonLinux+2023+x86_64 +++ b/qa/admin/package-lists/AmazonLinux+2023+x86_64 @@ -108,6 +108,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+7+x86_64 b/qa/admin/package-lists/CentOS+7+x86_64 index fa3b92ebc93..7c6ed7966f7 100644 --- a/qa/admin/package-lists/CentOS+7+x86_64 +++ b/qa/admin/package-lists/CentOS+7+x86_64 @@ -117,6 +117,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sudo sysstat systemd-devel diff --git a/qa/admin/package-lists/CentOS+8+x86_64 b/qa/admin/package-lists/CentOS+8+x86_64 index 9765fbfb3a4..f0c7a106c4e 100644 --- a/qa/admin/package-lists/CentOS+8+x86_64 +++ b/qa/admin/package-lists/CentOS+8+x86_64 @@ -114,6 +114,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat systemd-devel diff --git a/qa/admin/package-lists/CentOS+Stream10+x86_64 b/qa/admin/package-lists/CentOS+Stream10+x86_64 index c5cbba5aa38..18cd1118879 100644 --- a/qa/admin/package-lists/CentOS+Stream10+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream10+x86_64 @@ -126,6 +126,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+Stream8+x86_64 b/qa/admin/package-lists/CentOS+Stream8+x86_64 index e662358a1cc..6c6226613ae 100644 --- a/qa/admin/package-lists/CentOS+Stream8+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream8+x86_64 @@ -121,6 +121,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/CentOS+Stream9+x86_64 b/qa/admin/package-lists/CentOS+Stream9+x86_64 index 652169939b9..638d7b4fce4 100644 --- a/qa/admin/package-lists/CentOS+Stream9+x86_64 +++ b/qa/admin/package-lists/CentOS+Stream9+x86_64 @@ -128,6 +128,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Debian+11+x86_64 b/qa/admin/package-lists/Debian+11+x86_64 index c0589faa99a..d8b20f5de96 100644 --- a/qa/admin/package-lists/Debian+11+x86_64 +++ b/qa/admin/package-lists/Debian+11+x86_64 @@ -72,6 +72,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+aarch64 b/qa/admin/package-lists/Debian+12+aarch64 index ee56a676b80..9f16aac4233 100644 --- a/qa/admin/package-lists/Debian+12+aarch64 +++ b/qa/admin/package-lists/Debian+12+aarch64 @@ -71,6 +71,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+i686 b/qa/admin/package-lists/Debian+12+i686 index 30daf7bdae3..cd3058914b2 100644 --- a/qa/admin/package-lists/Debian+12+i686 +++ b/qa/admin/package-lists/Debian+12+i686 @@ -74,6 +74,7 @@ libspreadsheet-read-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+12+x86_64 b/qa/admin/package-lists/Debian+12+x86_64 index 8ff6f199545..79ff8ea7453 100644 --- a/qa/admin/package-lists/Debian+12+x86_64 +++ b/qa/admin/package-lists/Debian+12+x86_64 @@ -77,6 +77,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+13+i686 b/qa/admin/package-lists/Debian+13+i686 index c976f4c821d..198206b0f90 100644 --- a/qa/admin/package-lists/Debian+13+i686 +++ b/qa/admin/package-lists/Debian+13+i686 @@ -73,6 +73,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+13+x86_64 b/qa/admin/package-lists/Debian+13+x86_64 index fb8c5a73429..9435e6be9b1 100644 --- a/qa/admin/package-lists/Debian+13+x86_64 +++ b/qa/admin/package-lists/Debian+13+x86_64 @@ -74,6 +74,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Debian+14+x86_64 b/qa/admin/package-lists/Debian+14+x86_64 index 9464ac98b51..7e88e8eb3ef 100644 --- a/qa/admin/package-lists/Debian+14+x86_64 +++ b/qa/admin/package-lists/Debian+14+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Fedora+42+aarch64 b/qa/admin/package-lists/Fedora+42+aarch64 index c4a2a1b389a..1438d3654ac 100644 --- a/qa/admin/package-lists/Fedora+42+aarch64 +++ b/qa/admin/package-lists/Fedora+42+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+42+x86_64 b/qa/admin/package-lists/Fedora+42+x86_64 index 2c76c948c02..37a64060e1c 100644 --- a/qa/admin/package-lists/Fedora+42+x86_64 +++ b/qa/admin/package-lists/Fedora+42+x86_64 @@ -132,6 +132,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+43+aarch64 b/qa/admin/package-lists/Fedora+43+aarch64 index e96d484ed2c..5c6d144e2e2 100644 --- a/qa/admin/package-lists/Fedora+43+aarch64 +++ b/qa/admin/package-lists/Fedora+43+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+43+x86_64 b/qa/admin/package-lists/Fedora+43+x86_64 index e4b5219ee59..160e1d878e7 100644 --- a/qa/admin/package-lists/Fedora+43+x86_64 +++ b/qa/admin/package-lists/Fedora+43+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+44+aarch64 b/qa/admin/package-lists/Fedora+44+aarch64 index 36c69166c18..351e89a30be 100644 --- a/qa/admin/package-lists/Fedora+44+aarch64 +++ b/qa/admin/package-lists/Fedora+44+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+44+x86_64 b/qa/admin/package-lists/Fedora+44+x86_64 index f1648cd0cfd..f749ca745ed 100644 --- a/qa/admin/package-lists/Fedora+44+x86_64 +++ b/qa/admin/package-lists/Fedora+44+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+45+aarch64 b/qa/admin/package-lists/Fedora+45+aarch64 index 0f8936901d3..0bc98dcae68 100644 --- a/qa/admin/package-lists/Fedora+45+aarch64 +++ b/qa/admin/package-lists/Fedora+45+aarch64 @@ -127,6 +127,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/Fedora+45+x86_64 b/qa/admin/package-lists/Fedora+45+x86_64 index 1578325f3cc..a115773aa91 100644 --- a/qa/admin/package-lists/Fedora+45+x86_64 +++ b/qa/admin/package-lists/Fedora+45+x86_64 @@ -131,6 +131,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/MX+23.6+x86_64 b/qa/admin/package-lists/MX+23.6+x86_64 index 797804a9cb2..12b34c621e1 100644 --- a/qa/admin/package-lists/MX+23.6+x86_64 +++ b/qa/admin/package-lists/MX+23.6+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/RHEL+10+x86_64 b/qa/admin/package-lists/RHEL+10+x86_64 index fe5f4f47a22..0a3a65717a2 100644 --- a/qa/admin/package-lists/RHEL+10+x86_64 +++ b/qa/admin/package-lists/RHEL+10+x86_64 @@ -122,6 +122,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat sudo diff --git a/qa/admin/package-lists/RHEL+8+x86_64 b/qa/admin/package-lists/RHEL+8+x86_64 index fa2e3db4d75..390d30e219d 100644 --- a/qa/admin/package-lists/RHEL+8+x86_64 +++ b/qa/admin/package-lists/RHEL+8+x86_64 @@ -119,6 +119,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sudo sysstat diff --git a/qa/admin/package-lists/RHEL+9+x86_64 b/qa/admin/package-lists/RHEL+9+x86_64 index 3d40352cd6f..dc78842e947 100644 --- a/qa/admin/package-lists/RHEL+9+x86_64 +++ b/qa/admin/package-lists/RHEL+9+x86_64 @@ -125,6 +125,7 @@ setools-console smartmontools socat sqlite +sqlite-devel sqlite-libs sysstat sudo diff --git a/qa/admin/package-lists/Ubuntu+18.04+i686 b/qa/admin/package-lists/Ubuntu+18.04+i686 index 7eb6111f0c5..04b624c6216 100644 --- a/qa/admin/package-lists/Ubuntu+18.04+i686 +++ b/qa/admin/package-lists/Ubuntu+18.04+i686 @@ -70,6 +70,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+18.04+x86_64 b/qa/admin/package-lists/Ubuntu+18.04+x86_64 index 4516770b7c3..59fa19cf16e 100644 --- a/qa/admin/package-lists/Ubuntu+18.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+18.04+x86_64 @@ -68,6 +68,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+20.04+x86_64 b/qa/admin/package-lists/Ubuntu+20.04+x86_64 index 2bab5e76ae7..b125b081521 100644 --- a/qa/admin/package-lists/Ubuntu+20.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+20.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+22.04+x86_64 b/qa/admin/package-lists/Ubuntu+22.04+x86_64 index b9f4cd3a9b0..68a38fbe520 100644 --- a/qa/admin/package-lists/Ubuntu+22.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+22.04+x86_64 @@ -76,6 +76,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+24.04+any b/qa/admin/package-lists/Ubuntu+24.04+any index 4ef4c87d85a..cbd1edc98aa 100644 --- a/qa/admin/package-lists/Ubuntu+24.04+any +++ b/qa/admin/package-lists/Ubuntu+24.04+any @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+24.04+x86_64 b/qa/admin/package-lists/Ubuntu+24.04+x86_64 index bfa71e37d1b..ee9686d4dbf 100644 --- a/qa/admin/package-lists/Ubuntu+24.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+24.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+26.04+any b/qa/admin/package-lists/Ubuntu+26.04+any index 0fec7a41ccf..82aaaaa1034 100644 --- a/qa/admin/package-lists/Ubuntu+26.04+any +++ b/qa/admin/package-lists/Ubuntu+26.04+any @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/Ubuntu+26.04+x86_64 b/qa/admin/package-lists/Ubuntu+26.04+x86_64 index 87c80d499a9..08afca7c26d 100644 --- a/qa/admin/package-lists/Ubuntu+26.04+x86_64 +++ b/qa/admin/package-lists/Ubuntu+26.04+x86_64 @@ -75,6 +75,7 @@ libspreadsheet-readsxc-perl libspreadsheet-writeexcel-perl libspreadsheet-xlsx-perl libsqlite3-0 +libsqlite3-dev libssl-dev libsystemd-dev libtext-csv-xs-perl diff --git a/qa/admin/package-lists/openSUSE+15.6+x86_64 b/qa/admin/package-lists/openSUSE+15.6+x86_64 index 22b4016ab0b..25873f5c9a6 100644 --- a/qa/admin/package-lists/openSUSE+15.6+x86_64 +++ b/qa/admin/package-lists/openSUSE+15.6+x86_64 @@ -119,6 +119,7 @@ sensors setools-console smartmontools socat +sqlite3-devel sudo sysstat systemd diff --git a/qa/admin/package-lists/openSUSE+16.0+x86_64 b/qa/admin/package-lists/openSUSE+16.0+x86_64 index b11b78028d9..3e7d0ab4e3b 100644 --- a/qa/admin/package-lists/openSUSE+16.0+x86_64 +++ b/qa/admin/package-lists/openSUSE+16.0+x86_64 @@ -118,6 +118,7 @@ sensors setools-console smartmontools socat +sqlite3-devel sudo sysstat systemd From 9e7a74bc0d06dae9799ecaad6a629808ff72aa61 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 31 Jul 2026 13:45:28 -0400 Subject: [PATCH 13/22] pmsearch: fix man3 build error on fedora --- man/man3/pmsearchtextsuggest.3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/man3/pmsearchtextsuggest.3 b/man/man3/pmsearchtextsuggest.3 index 1974de0879b..7c10a5e0722 100644 --- a/man/man3/pmsearchtextsuggest.3 +++ b/man/man3/pmsearchtextsuggest.3 @@ -47,7 +47,7 @@ FTS5 prefix indexes. Terms in .B query are appended with a wildcard so that partial names -match (e.g.\& \f(CWker\fP matches \f(CWkernel\fP). +match (e.g.\& \fBker\fP matches \fBkernel\fP). .PP Each entity matched by the .B request From 9e32bb8f0ee3596ab398b70d05ffeac1660c3970 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 31 Jul 2026 17:57:09 -0400 Subject: [PATCH 14/22] pmsearch: incorporate code review feedback --- configure | 145 +++++++---- configure.ac | 31 +++ qa/admin/other-packages/manifest | 20 +- qa/admin/package-lists/CentOS+7+x86_64 | 1 - qa/group | 2 +- src/libpcp_web/src/search.c | 335 +++++++++++++++---------- src/libpcp_web/src/search.h | 6 + src/newhelp/GNUmakefile | 4 +- src/newhelp/newhelp.c | 18 +- src/newhelp/search_sqlite.c | 95 ++++++- src/pmdas/GNUmakefile | 8 +- src/pmsearch/pmsearch.c | 38 ++- src/pmsearch/pmsearch_index.sh | 9 +- src/pmsearch/pmsearch_index.timer | 2 +- 14 files changed, 495 insertions(+), 219 deletions(-) diff --git a/configure b/configure index 547a1892937..19fd9a77745 100755 --- a/configure +++ b/configure @@ -2186,6 +2186,50 @@ fi } # ac_fn_c_try_cpp +# ac_fn_c_try_run LINENO +# ---------------------- +# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that +# executables *can* be run. +ac_fn_c_try_run () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf "%s\n" "$ac_try_echo"; } >&5 + (eval "$ac_link") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' + { { case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf "%s\n" "$ac_try_echo"; } >&5 + (eval "$ac_try") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; } +then : + ac_retval=0 +else case e in #( + e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 + printf "%s\n" "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=$ac_status ;; +esac +fi + rm -rf conftest.dSYM conftest_ipa8_conftest.oo + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_run + # ac_fn_c_check_func LINENO FUNC VAR # ---------------------------------- # Tests whether FUNC exists, setting the cache variable VAR accordingly @@ -2308,50 +2352,6 @@ printf "%s\n" "$ac_res" >&6; } eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_type - -# ac_fn_c_try_run LINENO -# ---------------------- -# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that -# executables *can* be run. -ac_fn_c_try_run () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' - { { case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 - (eval "$ac_try") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; } -then : - ac_retval=0 -else case e in #( - e) printf "%s\n" "$as_me: program exited with status $ac_status" >&5 - printf "%s\n" "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=$ac_status ;; -esac -fi - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_run ac_configure_args_raw= for ac_arg do @@ -12423,6 +12423,63 @@ fi else lib_for_sqlite3="$sqlite3_LIBS" fi +if test $have_sqlite3 = true; then + save_LIBS="$LIBS" + LIBS="$lib_for_sqlite3 $LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for SQLite3 >= 3.9.0 with FTS5 support" >&5 +printf %s "checking for SQLite3 >= 3.9.0 with FTS5 support... " >&6; } + if test "$cross_compiling" = yes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: assuming yes (cross-compiling)" >&5 +printf "%s\n" "assuming yes (cross-compiling)" >&6; } +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#if SQLITE_VERSION_NUMBER < 3009000 +#error "SQLite >= 3.9.0 required for FTS5" +#endif + +int +main (void) +{ + +sqlite3 *db; +int sts = 1; + +if (sqlite3_open(":memory:", &db) == SQLITE_OK) + sts = (sqlite3_exec(db, "CREATE VIRTUAL TABLE t USING fts5(c)", + 0, 0, 0) == SQLITE_OK) ? 0 : 1; +sqlite3_close(db); +return sts; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_sqlite3=false + lib_for_sqlite3="" + ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + + LIBS="$save_LIBS" +fi +if test $have_sqlite3 = false; then + as_fn_error $? "SQLite >= 3.9.0 with FTS5 support is required" "$LINENO" 5 +fi HAVE_SQLITE3=$have_sqlite3 diff --git a/configure.ac b/configure.ac index f477ab1fad4..23cfaeb4ac6 100644 --- a/configure.ac +++ b/configure.ac @@ -2009,6 +2009,37 @@ if test $have_sqlite3 = false; then else lib_for_sqlite3="$sqlite3_LIBS" fi +dnl Verify SQLite >= 3.9.0 and that the linked library provides FTS5 +if test $have_sqlite3 = true; then + save_LIBS="$LIBS" + LIBS="$lib_for_sqlite3 $LIBS" + AC_MSG_CHECKING([for SQLite3 >= 3.9.0 with FTS5 support]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([[ +#include +#if SQLITE_VERSION_NUMBER < 3009000 +#error "SQLite >= 3.9.0 required for FTS5" +#endif + ]], [[ +sqlite3 *db; +int sts = 1; + +if (sqlite3_open(":memory:", &db) == SQLITE_OK) + sts = (sqlite3_exec(db, "CREATE VIRTUAL TABLE t USING fts5(c)", + 0, 0, 0) == SQLITE_OK) ? 0 : 1; +sqlite3_close(db); +return sts; + ]])], + [AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) + have_sqlite3=false + lib_for_sqlite3="" + ], + [AC_MSG_RESULT([assuming yes (cross-compiling)])]) + LIBS="$save_LIBS" +fi +if test $have_sqlite3 = false; then + AC_MSG_ERROR([SQLite >= 3.9.0 with FTS5 support is required]) +fi AC_SUBST(HAVE_SQLITE3, [$have_sqlite3]) AC_SUBST(lib_for_sqlite3) diff --git a/qa/admin/other-packages/manifest b/qa/admin/other-packages/manifest index 88e97a0012e..ebfbe28d290 100644 --- a/qa/admin/other-packages/manifest +++ b/qa/admin/other-packages/manifest @@ -1019,16 +1019,16 @@ slackpkg? /usr/lib*/libsqlite3.so [sqlite (QA optional)] pacman? /usr/lib/libsqlite3.so [core/sqlite (QA optional)] brew? /usr/lib/libsqlite3.dylib [sqlite (QA optional)] # -- libsqlite3 headers for pmsearch FTS5 index -dpkg? /usr/include/sqlite3.h [libsqlite3-dev (build optional)] -rpm? /usr/include/sqlite3.h [sqlite-devel (build optional)] -emerge? /usr/include/sqlite3.h [dev-db/sqlite (build optional)] -pkgin? /usr/pkg/include/sqlite3.h [sqlite3 (build optional)] -pkg_add? /usr/local/include/sqlite3.h [sqlite3 (build optional)] -F_pkg? /usr/local/include/sqlite3.h [sqlite3 (build optional)] -S_pkg? /usr/include/sqlite3.h [database/sqlite-3 (build optional)] -slackpkg? /usr/include/sqlite3.h [sqlite (build optional)] -pacman? /usr/include/sqlite3.h [core/sqlite (build optional)] -brew? /usr/local/include/sqlite3.h [sqlite (build optional)] +dpkg? /usr/include/sqlite3.h [libsqlite3-dev] +rpm? /usr/include/sqlite3.h [sqlite-devel or sqlite3-devel] +emerge? /usr/include/sqlite3.h [dev-db/sqlite] +pkgin? /usr/pkg/include/sqlite3.h [sqlite3] +pkg_add? /usr/local/include/sqlite3.h [sqlite3] +F_pkg? /usr/local/include/sqlite3.h [sqlite3] +S_pkg? /usr/include/sqlite3.h [database/sqlite-3] +slackpkg? /usr/include/sqlite3.h [sqlite] +pacman? /usr/include/sqlite3.h [core/sqlite] +brew? /usr/local/include/sqlite3.h [sqlite] # -- libicui18n for certutil dpkg? /usr/lib/*/libicui18n.so.* [libicu[0-9][0-9]* (QA optional)] rpm? /usr/lib*/libicui18n.so [libicu-devel (QA optional)] diff --git a/qa/admin/package-lists/CentOS+7+x86_64 b/qa/admin/package-lists/CentOS+7+x86_64 index 7c6ed7966f7..fa3b92ebc93 100644 --- a/qa/admin/package-lists/CentOS+7+x86_64 +++ b/qa/admin/package-lists/CentOS+7+x86_64 @@ -117,7 +117,6 @@ setools-console smartmontools socat sqlite -sqlite-devel sudo sysstat systemd-devel diff --git a/qa/group b/qa/group index bda2746f3d4..fb5bb172740 100644 --- a/qa/group +++ b/qa/group @@ -2351,7 +2351,7 @@ suse 1869 logutil pmlogger_daily local 1870 guidellm2pcp python pmimport libpcp_import local 1871 pmsearch local -1872 pmsearch pmproxy local pmjson +1872 pmsearch pmproxy help local pmjson 1874 pmseries pmproxy local pmjson 1875 pmseries pmproxy libpcp_web local 1876:retired pmcd secure local diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 7b62dfe2fe0..00d1198a643 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -14,15 +14,15 @@ * * SQLite FTS5 search backend for PCP metric/instance full-text search. */ -#include #include #include "pmapi.h" #include "libpcp.h" #include "search.h" -#define SEARCH_DOC_METRIC 1 -#define SEARCH_DOC_INDOM 2 -#define SEARCH_DOC_INST 3 +/* Type values stored in the SQLite index — must match pmSearchTextType */ +#define SEARCH_DOC_METRIC PM_SEARCH_TYPE_METRIC +#define SEARCH_DOC_INDOM PM_SEARCH_TYPE_INDOM +#define SEARCH_DOC_INST PM_SEARCH_TYPE_INST static int search_enabled; static unsigned int default_resultcount = 10; @@ -61,7 +61,6 @@ getSearchModuleData(pmSearchModule *module) /* * Build an FTS5 MATCH expression from the request. * If infields are restricted, wrap the query in column filters. - * On FTS5 syntax error, fall back to quoting the query as a phrase. */ static sds search_build_match(pmSearchTextRequest *request) @@ -152,13 +151,73 @@ search_make_docid(sqlite3_int64 rowid, const char *name) return sdscatfmt(sdsempty(), "%s:%s", buf, name ? name : ""); } -/* - * Execute a text query against a single docs table (qualified name). - * Appends results to the hits array; caller owns the array. - */ +/* Grow the hits array, doubling capacity (starting at 64). Returns 0 or -ENOMEM. */ +static int +search_hits_grow(pmSearchTextResult **hits, int *nhits, int *maxhits) +{ + int newmax = *maxhits ? *maxhits * 2 : 64; + pmSearchTextResult *tmp; + + if (newmax < *maxhits) + return -ENOMEM; + tmp = realloc(*hits, (size_t)newmax * sizeof(pmSearchTextResult)); + if (tmp == NULL) + return -ENOMEM; + *hits = tmp; + *maxhits = newmax; + return 0; +} + +static void +search_hits_free(pmSearchTextResult *hits, int nhits) +{ + int i; + + for (i = 0; i < nhits; i++) { + sdsfree(hits[i].docid); + sdsfree(hits[i].name); + sdsfree(hits[i].indom); + sdsfree(hits[i].oneline); + sdsfree(hits[i].helptext); + } + free(hits); +} + +/* Count matching docs for a text query; returns 0 on failure. */ +static int +search_count_table(sqlite3 *db, sds match_expr, sds type_filter) +{ + sqlite3_stmt *stmt = NULL; + sds sql; + int rc, total = 0; + + sql = sdscatfmt(sdsempty(), + "SELECT count(*) FROM docs WHERE docs MATCH ?%S", + type_filter); + + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_count_table: prepare: %s\n", + sqlite3_errmsg(db)); + return 0; + } + + sqlite3_bind_text(stmt, 1, match_expr, sdslen(match_expr), SQLITE_STATIC); + + if (sqlite3_step(stmt) == SQLITE_ROW) + total = sqlite3_column_int(stmt, 0); + + sqlite3_finalize(stmt); + return total; +} + +/* Execute a text query against docs; appends results to the hits array. */ static int search_query_table(sqlite3 *db, pmSearchTextRequest *request, sds match_expr, sds type_filter, + unsigned int limit, unsigned int offset, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -172,7 +231,8 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, " highlight(docs, 1, '', '')," " highlight(docs, 2, '', '')" " FROM docs WHERE docs MATCH ?%S" - " ORDER BY bm25(docs, 9.0, 4.0, 2.0)", + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)" + " LIMIT ? OFFSET ?", type_filter); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); @@ -185,25 +245,16 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, } sqlite3_bind_text(stmt, 1, match_expr, sdslen(match_expr), SQLITE_STATIC); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64)limit); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64)offset); while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { pmSearchTextResult result; - if (*nhits >= *maxhits) { - int newmax = *maxhits ? *maxhits * 2 : 64; - pmSearchTextResult *tmp; - - if (newmax < *maxhits) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); - if (tmp == NULL) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - *hits = tmp; - *maxhits = newmax; + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; } memset(&result, 0, sizeof(result)); @@ -250,16 +301,12 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, } sqlite3_finalize(stmt); - return 0; -} - -static int -score_cmp(const void *a, const void *b) -{ - const pmSearchTextResult *ra = a, *rb = b; - - if (rb->score > ra->score) return 1; - if (rb->score < ra->score) return -1; + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_query_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } return 0; } @@ -270,7 +317,7 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, pmSearchTextResult *hits = NULL; struct timespec started, finished; sds match, type_filter; - int nhits = 0, maxhits = 0; + int nhits = 0, maxhits = 0, total, sts; unsigned int count, offset, i; double timer; @@ -286,42 +333,41 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, request->return_type = 1; } + offset = request->offset; + if (!request->count) + request->count = smd->resultcount; + count = request->count; + match = search_build_match(request); type_filter = search_type_filter(request); - search_query_table(smd->db, request, match, type_filter, - &hits, &nhits, &maxhits); + total = search_count_table(smd->db, match, type_filter); + + sts = search_query_table(smd->db, request, match, type_filter, + count, offset, &hits, &nhits, &maxhits); sdsfree(match); sdsfree(type_filter); - qsort(hits, nhits, sizeof(pmSearchTextResult), score_cmp); + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; + } pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); - offset = request->offset; - if (!request->count) - request->count = smd->resultcount; - count = request->count; - - for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { - hits[i].total = nhits; - hits[i].count = (i - offset) + 1; + for (i = 0; i < (unsigned int)nhits; i++) { + hits[i].total = total; + hits[i].count = i + 1; hits[i].timer = timer; callbacks->on_text_result(&hits[i], userdata); } - for (i = 0; i < (unsigned int)nhits; i++) { - sdsfree(hits[i].docid); - sdsfree(hits[i].name); - sdsfree(hits[i].indom); - sdsfree(hits[i].oneline); - sdsfree(hits[i].helptext); - } - free(hits); + search_hits_free(hits, nhits); - callbacks->on_done(0, userdata); + callbacks->on_done(sts, userdata); } static int @@ -335,8 +381,9 @@ search_suggest_table(sqlite3 *db, sds match, sql = sdscatfmt(sdsempty(), "SELECT rowid, name, type, bm25(docs, 9.0, 4.0, 2.0)" " FROM docs WHERE docs MATCH ?" - " AND type IN (1, 3)" - " ORDER BY bm25(docs, 9.0, 4.0, 2.0)"); + " AND type IN (%i, %i)" + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)", + SEARCH_DOC_METRIC, SEARCH_DOC_INST); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); @@ -352,21 +399,10 @@ search_suggest_table(sqlite3 *db, sds match, while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { pmSearchTextResult result; - if (*nhits >= *maxhits) { - int newmax = *maxhits ? *maxhits * 2 : 64; - pmSearchTextResult *tmp; - - if (newmax < *maxhits) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); - if (tmp == NULL) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - *hits = tmp; - *maxhits = newmax; + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; } memset(&result, 0, sizeof(result)); @@ -381,6 +417,12 @@ search_suggest_table(sqlite3 *db, sds match, } sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_suggest_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } return 0; } @@ -391,7 +433,7 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, pmSearchTextResult *hits = NULL; struct timespec started, finished; sds match; - int nhits = 0, maxhits = 0; + int nhits = 0, maxhits = 0, sts; unsigned int count, i; double timer; @@ -416,11 +458,15 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, match = sdscat(match, "*)"); } - search_suggest_table(smd->db, match, &hits, &nhits, &maxhits); + sts = search_suggest_table(smd->db, match, &hits, &nhits, &maxhits); sdsfree(match); - qsort(hits, nhits, sizeof(pmSearchTextResult), score_cmp); + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; + } pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); @@ -436,13 +482,9 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, callbacks->on_text_result(&hits[i], userdata); } - for (i = 0; i < (unsigned int)nhits; i++) { - sdsfree(hits[i].docid); - sdsfree(hits[i].name); - } - free(hits); + search_hits_free(hits, nhits); - callbacks->on_done(0, userdata); + callbacks->on_done(sts, userdata); } static int @@ -453,18 +495,28 @@ search_indom_table(sqlite3 *db, const char *query, int querylen, sds sql; int rc; - sql = sdscatfmt(sdsempty(), - "SELECT rowid, name, oneline, helptext, type, indom" - " FROM docs WHERE indom = ?" - " ORDER BY type"); + sql = sdsnew( + "SELECT d.rowid, d.name, d.oneline, d.helptext, d.type, d.indom" + " FROM indom_map m JOIN docs d ON d.rowid = m.docid" + " WHERE m.indom = ?" + " ORDER BY d.type"); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); if (rc != SQLITE_OK) { - if (pmDebugOptions.search) - fprintf(stderr, "search_indom_table: prepare: %s\n", - sqlite3_errmsg(db)); - return -EIO; + /* fall back for indexes built without indom_map */ + sql = sdsnew( + "SELECT rowid, name, oneline, helptext, type, indom" + " FROM docs WHERE indom = ?" + " ORDER BY type"); + rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); + sdsfree(sql); + if (rc != SQLITE_OK) { + if (pmDebugOptions.search) + fprintf(stderr, "search_indom_table: prepare: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } } sqlite3_bind_text(stmt, 1, query, querylen, SQLITE_STATIC); @@ -472,21 +524,10 @@ search_indom_table(sqlite3 *db, const char *query, int querylen, while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { pmSearchTextResult result; - if (*nhits >= *maxhits) { - int newmax = *maxhits ? *maxhits * 2 : 64; - pmSearchTextResult *tmp; - - if (newmax < *maxhits) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - tmp = realloc(*hits, newmax * sizeof(pmSearchTextResult)); - if (tmp == NULL) { - sqlite3_finalize(stmt); - return -ENOMEM; - } - *hits = tmp; - *maxhits = newmax; + if (*nhits >= *maxhits && + search_hits_grow(hits, nhits, maxhits) < 0) { + sqlite3_finalize(stmt); + return -ENOMEM; } memset(&result, 0, sizeof(result)); @@ -518,6 +559,12 @@ search_indom_table(sqlite3 *db, const char *query, int querylen, } sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + if (pmDebugOptions.search) + fprintf(stderr, "search_indom_table: step: %s\n", + sqlite3_errmsg(db)); + return -EIO; + } return 0; } @@ -527,16 +574,22 @@ search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, { pmSearchTextResult *hits = NULL; struct timespec started, finished; - int nhits = 0, maxhits = 0; + int nhits = 0, maxhits = 0, sts; unsigned int count, offset, i; double timer; pmtimespecNow(&started); - search_indom_table(smd->db, request->query, + sts = search_indom_table(smd->db, request->query, sdslen(request->query), &hits, &nhits, &maxhits); + if (sts < 0) { + search_hits_free(hits, nhits); + callbacks->on_done(sts, userdata); + return; + } + pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); @@ -545,23 +598,16 @@ search_do_text_indom(searchModuleData *smd, pmSearchTextRequest *request, request->count = smd->resultcount; count = request->count; - for (i = offset; i < (unsigned int)nhits && i < offset + count; i++) { + for (i = offset; i < (unsigned int)nhits && (i - offset) < count; i++) { hits[i].total = nhits; hits[i].count = (i - offset) + 1; hits[i].timer = timer; callbacks->on_text_result(&hits[i], userdata); } - for (i = 0; i < (unsigned int)nhits; i++) { - sdsfree(hits[i].docid); - sdsfree(hits[i].name); - sdsfree(hits[i].indom); - sdsfree(hits[i].oneline); - sdsfree(hits[i].helptext); - } - free(hits); + search_hits_free(hits, nhits); - callbacks->on_done(0, userdata); + callbacks->on_done(sts, userdata); } /* --- public API --- */ @@ -591,14 +637,6 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) sqlite3_finalize(stmt); } - rc = sqlite3_exec(smd->db, - "CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab" - " USING fts5vocab(docs, row)", NULL, NULL, NULL); - if (rc != SQLITE_OK) { - if (pmDebugOptions.search) - fprintf(stderr, "pmSearchInfo: create docs_vocab: %s\n", - sqlite3_errmsg(smd->db)); - } rc = sqlite3_prepare_v2(smd->db, "SELECT COUNT(*), COALESCE(SUM(cnt),0)" " FROM docs_vocab", -1, &stmt, NULL); @@ -608,6 +646,9 @@ pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) metrics.records = sqlite3_column_int64(stmt, 1); } sqlite3_finalize(stmt); + } else if (pmDebugOptions.search) { + fprintf(stderr, "pmSearchInfo: docs_vocab: %s\n", + sqlite3_errmsg(smd->db)); } settings->callbacks.on_metrics(&metrics, arg); @@ -620,7 +661,11 @@ pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, void { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + if (request == NULL || request->query == NULL) { settings->callbacks.on_done(-EINVAL, arg); return 0; } @@ -634,7 +679,11 @@ pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, vo { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + if (request == NULL || request->query == NULL) { settings->callbacks.on_done(-EINVAL, arg); return 0; } @@ -648,7 +697,11 @@ pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, void { searchModuleData *smd = (searchModuleData *)settings->module.privdata; - if (smd == NULL || !smd->loaded || request == NULL || request->query == NULL) { + if (smd == NULL || !smd->loaded) { + settings->callbacks.on_done(-ENOENT, arg); + return 0; + } + if (request == NULL || request->query == NULL) { settings->callbacks.on_done(-EINVAL, arg); return 0; } @@ -714,8 +767,18 @@ pmSearchSetup(pmSearchModule *module, void *arg) return -ENOTSUP; option = pmIniFileLookup(smd->config, "pmsearch", "result.count"); - if (option) - smd->resultcount = atoi(option); + if (option) { + char *endp; + unsigned long value; + + errno = 0; + value = strtoul(option, &endp, 10); + if (errno == 0 && *endp == '\0' && value > 0 && value <= UINT_MAX) + smd->resultcount = (unsigned int)value; + else + pmNotifyErr(LOG_WARNING, "ignoring invalid " + "pmsearch result.count \"%s\"", option); + } } option = smd->config ? @@ -725,9 +788,9 @@ pmSearchSetup(pmSearchModule *module, void *arg) rc = sqlite3_open_v2(option, &smd->db, SQLITE_OPEN_READONLY, NULL); if (rc != SQLITE_OK) { - if (pmDebugOptions.search) - fprintf(stderr, "pmSearchSetup: open %s: %s\n", - option, sqlite3_errmsg(smd->db)); + pmNotifyErr(LOG_WARNING, + "cannot open pmsearch index %s: %s", + option, sqlite3_errmsg(smd->db)); sqlite3_close(smd->db); smd->db = NULL; } @@ -754,8 +817,10 @@ pmSearchSetup(pmSearchModule *module, void *arg) fprintf(stderr, "pmSearchSetup: loaded base index %s\n", base); } else { - if (pmDebugOptions.search) - fprintf(stderr, "pmSearchSetup: no index found\n"); + pmNotifyErr(LOG_WARNING, + "no pmsearch index found " + "(tried %s and %s); run pmsearch_index(1)", + nightly, base); sqlite3_close(smd->db); smd->db = NULL; } diff --git a/src/libpcp_web/src/search.h b/src/libpcp_web/src/search.h index 8c56a63def2..8790d8959be 100644 --- a/src/libpcp_web/src/search.h +++ b/src/libpcp_web/src/search.h @@ -21,6 +21,12 @@ struct dict; struct keySlots; +/* + * The functions below are retained only for link compatibility with + * schema.c and keys.c. They are implemented as no-ops now that the + * search index is built by newhelp(1) into a local SQLite FTS5 file. + * Do not add new callers. + */ extern void keysSearchInit(struct dict *); extern void keysSearchClose(void); diff --git a/src/newhelp/GNUmakefile b/src/newhelp/GNUmakefile index d53f8e319e9..f228fa48282 100644 --- a/src/newhelp/GNUmakefile +++ b/src/newhelp/GNUmakefile @@ -41,11 +41,11 @@ install: default include $(BUILDRULES) -newhelp$(EXECSUFFIX): newhelp.c search_sqlite.c +newhelp$(EXECSUFFIX): newhelp.c search_sqlite.c search_sqlite.h $(CCF) -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(LDLIBS) $(LIB_FOR_SQLITE3) $(LINKER_MAKERULE) -newhelp.static$(EXECSUFFIX): newhelp.c search_sqlite.c $(STATIC_LIBPCP) +newhelp.static$(EXECSUFFIX): newhelp.c search_sqlite.c search_sqlite.h $(STATIC_LIBPCP) $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(STATIC_LIBPCP) $(STATIC_LDLIBS) $(LIB_FOR_SQLITE3) chkhelp$(EXECSUFFIX): chkhelp.o diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index fb7262c61cc..f2ddb96fe1c 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -246,7 +246,7 @@ lookup_domain(const char *pmda_name) return -1; var_dir[0] = '\0'; while (fgets(line, sizeof(line), fp) != NULL) { - if (sscanf(line, "PCP_VAR_DIR=%s", var_dir) == 1) + if (sscanf(line, "PCP_VAR_DIR=%1023s", var_dir) == 1) break; } fclose(fp); @@ -471,15 +471,25 @@ newentry_search(char *buf) } } else { static int pmns_loaded = 0; + static int pmns_usable = 0; type = SEARCH_DOC_METRIC; indom_buf[0] = '\0'; if (!pmns_loaded) { + int sts; + pmns_loaded = 1; - pmLoadASCIINameSpace(pmnsfile, 1); - pmNewContext(PM_CONTEXT_HOST, "localhost"); + if ((sts = pmLoadASCIINameSpace(pmnsfile, 1)) < 0) + fprintf(stderr, "%s: pmLoadASCIINameSpace(%s): %s\n", + pmGetProgname(), + pmnsfile ? pmnsfile : "default", pmErrStr(sts)); + else if ((sts = pmNewContext(PM_CONTEXT_HOST, "localhost")) < 0) + fprintf(stderr, "%s: pmNewContext(localhost): %s\n", + pmGetProgname(), pmErrStr(sts)); + else + pmns_usable = 1; } - { + if (pmns_usable) { const char *np = name; if (pmLookupName(1, &np, &pmid) >= 0 && diff --git a/src/newhelp/search_sqlite.c b/src/newhelp/search_sqlite.c index 936c01a834a..fa7618d16de 100644 --- a/src/newhelp/search_sqlite.c +++ b/src/newhelp/search_sqlite.c @@ -21,6 +21,8 @@ static sqlite3 *search_db; static sqlite3_stmt *search_insert; static sqlite3_stmt *search_lookup; static sqlite3_stmt *search_delete; +static sqlite3_stmt *search_indom_insert; +static sqlite3_stmt *search_indom_delete; int search_sqlite_open(const char *path) @@ -56,6 +58,32 @@ search_sqlite_open(const char *path) return -1; } + rc = sqlite3_exec(search_db, + "CREATE TABLE IF NOT EXISTS indom_map(" + " docid INTEGER PRIMARY KEY," + " indom TEXT NOT NULL" + ");", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create indom_map: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_exec(search_db, + "CREATE INDEX IF NOT EXISTS idx_indom_map" + " ON indom_map(indom);", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: create index: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + rc = sqlite3_exec(search_db, "BEGIN", NULL, NULL, NULL); if (rc != SQLITE_OK) { fprintf(stderr, "search_sqlite_open: begin: %s\n", @@ -105,6 +133,42 @@ search_sqlite_open(const char *path) return -1; } + rc = sqlite3_prepare_v2(search_db, + "INSERT INTO indom_map(docid, indom) VALUES(?, ?)", + -1, &search_indom_insert, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare indom insert: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_delete); + search_delete = NULL; + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + + rc = sqlite3_prepare_v2(search_db, + "DELETE FROM indom_map WHERE docid = ?", + -1, &search_indom_delete, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_open: prepare indom delete: %s\n", + sqlite3_errmsg(search_db)); + sqlite3_finalize(search_indom_insert); + search_indom_insert = NULL; + sqlite3_finalize(search_delete); + search_delete = NULL; + sqlite3_finalize(search_lookup); + search_lookup = NULL; + sqlite3_finalize(search_insert); + search_insert = NULL; + sqlite3_close(search_db); + search_db = NULL; + return -1; + } + return 0; } @@ -112,7 +176,8 @@ void search_sqlite_add(const char *name, const char *oneline, const char *helptext, const char *indom, int type) { - if (search_db == NULL || search_insert == NULL || name == NULL) + if (search_db == NULL || search_insert == NULL || name == NULL || + search_lookup == NULL || search_delete == NULL) return; /* upsert: remove existing entry with same (name, type) if any */ @@ -120,6 +185,9 @@ search_sqlite_add(const char *name, const char *oneline, sqlite3_bind_int(search_lookup, 2, type); if (sqlite3_step(search_lookup) == SQLITE_ROW) { sqlite3_int64 rowid = sqlite3_column_int64(search_lookup, 0); + sqlite3_bind_int64(search_indom_delete, 1, rowid); + sqlite3_step(search_indom_delete); + sqlite3_reset(search_indom_delete); sqlite3_bind_int64(search_delete, 1, rowid); if (sqlite3_step(search_delete) != SQLITE_DONE) { fprintf(stderr, "search_sqlite_add: delete: %s\n", @@ -140,6 +208,14 @@ search_sqlite_add(const char *name, const char *oneline, name, sqlite3_errmsg(search_db)); } + if (indom && *indom) { + sqlite3_int64 docid = sqlite3_last_insert_rowid(search_db); + sqlite3_bind_int64(search_indom_insert, 1, docid); + sqlite3_bind_text(search_indom_insert, 2, indom, -1, SQLITE_STATIC); + sqlite3_step(search_indom_insert); + sqlite3_reset(search_indom_insert); + } + sqlite3_reset(search_insert); } @@ -163,6 +239,14 @@ search_sqlite_close(void) sqlite3_finalize(search_delete); search_delete = NULL; } + if (search_indom_insert) { + sqlite3_finalize(search_indom_insert); + search_indom_insert = NULL; + } + if (search_indom_delete) { + sqlite3_finalize(search_indom_delete); + search_indom_delete = NULL; + } /* optimize the FTS index before closing */ rc = sqlite3_exec(search_db, "INSERT INTO docs(docs) VALUES('optimize')", @@ -181,6 +265,15 @@ search_sqlite_close(void) return -1; } + rc = sqlite3_exec(search_db, + "CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab" + " USING fts5vocab(docs, row)", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + fprintf(stderr, "search_sqlite_close: create docs_vocab: %s\n", + sqlite3_errmsg(search_db)); + } + sqlite3_close(search_db); search_db = NULL; return 0; diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index 25d2c3219db..93ec5d22e65 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -94,16 +94,18 @@ install_pcp :: default_pcp $(INSTALL) -m 644 pmcd.conf $(PCP_PMCDCONF_PATH) $(INSTALL) -m 644 local.conf indom.conf $(PCP_ETC_DIR)/pcp $(INSTALL) -m 644 -t $(PCP_PMNS_DIR)/local.root local.root $(PCP_PMNSADM_DIR)/local.root -ifeq "$(HAVE_SQLITE3)" "true" +ifneq "$(CROSS_COMPILING)" "yes" help_files=""; \ for f in `find . -maxdepth 2 -name help -type f`; do \ help_files="$$help_files $$f"; \ done; \ if [ -n "$$help_files" ]; then \ - $(TOPDIR)/src/newhelp/newhelp -S -o pcp.search $$help_files; \ - $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib; \ + $(TOPDIR)/src/newhelp/newhelp.static -S -o pcp.search $$help_files && \ + $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib && \ $(INSTALL) -m 644 pcp.search $(PCP_SHARE_DIR)/lib/pcp.search; \ + rc=$$?; \ rm -f pcp.search; \ + test $$rc -eq 0 || exit 1; \ fi endif diff --git a/src/pmsearch/pmsearch.c b/src/pmsearch/pmsearch.c index fc5be25bc43..5480104eb5d 100644 --- a/src/pmsearch/pmsearch.c +++ b/src/pmsearch/pmsearch.c @@ -246,18 +246,21 @@ on_search_done(int sts, void *arg) if ((dp->flags & (PMSEARCH_OPT_QUERY | PMSEARCH_OPT_SUGGEST | PMSEARCH_OPT_INDOM)) && (dp->count == 0)) printf("0 search results\n"); - } else if (dp->flags & PMSEARCH_OPT_INFO) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchInfo", pmErrStr(sts)); - else if (dp->flags & PMSEARCH_OPT_QUERY) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextQuery", pmErrStr(sts)); - else if (dp->flags & PMSEARCH_OPT_SUGGEST) - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextSuggest", pmErrStr(sts)); - else - fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), - "pmSearchTextIndom", pmErrStr(sts)); + } else { + dp->status = 1; + if (dp->flags & PMSEARCH_OPT_INFO) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchInfo", pmErrStr(sts)); + else if (dp->flags & PMSEARCH_OPT_QUERY) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextQuery", pmErrStr(sts)); + else if (dp->flags & PMSEARCH_OPT_SUGGEST) + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextSuggest", pmErrStr(sts)); + else + fprintf(stderr, "%s: %s failed - %s\n", pmGetProgname(), + "pmSearchTextIndom", pmErrStr(sts)); + } } static void @@ -394,7 +397,16 @@ main(int argc, char *argv[]) if (config) pmSearchSetConfiguration(&dp->settings.module, config); - pmSearchSetup(&dp->settings.module, dp); + if ((c = pmSearchSetup(&dp->settings.module, dp)) < 0) { + if (c == -ENOTSUP) + fprintf(stderr, "%s: search is disabled in configuration\n", + pmGetProgname()); + else + fprintf(stderr, "%s: search setup failed: %s\n", + pmGetProgname(), pmErrStr(c)); + search_data_free(dp); + return 1; + } pmSearchClose(&dp->settings.module); diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index 43e77c9db58..07f91adda6e 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -21,7 +21,7 @@ status=1 tmp=`mktemp -d "$PCP_TMPFILE_DIR/pmsearch_index.XXXXXXXXX"` || exit 1 -trap "rm -rf $tmp; exit \$status" 0 1 2 3 15 +trap "rm -rf \"$tmp\" \"\$NEW\"; exit \$status" 0 1 2 3 15 prog=`basename $0` INDEX="$PCP_VAR_DIR/lib/pcp.search" @@ -75,7 +75,7 @@ then fi # Build metric→indom mapping first (needed for both metrics and instances). -pminfo $SRCFLAGS -I 2>/dev/null | $PCP_AWK_PROG ' +pminfo $SRCFLAGS -d 2>/dev/null | $PCP_AWK_PROG ' /^[a-zA-Z]/ { metric = $1 } /InDom:/ && !/PM_INDOM_NULL/ { for (i = 1; i <= NF; i++) { @@ -196,10 +196,11 @@ fi # Start from the build-time base index (contains all metric help text). # The nightly update copies it, then adds/updates runtime data. +NEW="$INDEX.new.$$" BASE="$PCP_SHARE_DIR/lib/pcp.search" if [ -f "$BASE" ] then - cp "$BASE" "$INDEX" + cp "$BASE" "$NEW" || exit if $VERBOSE then echo "$prog: copied base index from $BASE" @@ -220,7 +221,7 @@ fi # Add/update all runtime data (metrics and instances) into the index. # Metrics already in the base index are updated; new metrics from # PMDAs without help files (e.g. Python PMDAs) are inserted. -if $PCP_BINADM_DIR/newhelp -S -o "$INDEX" $tmp/helptext +if $PCP_BINADM_DIR/newhelp -S -o "$NEW" $tmp/helptext && mv -f "$NEW" "$INDEX" then if $VERBOSE then diff --git a/src/pmsearch/pmsearch_index.timer b/src/pmsearch/pmsearch_index.timer index 63b41492ab3..760f319e266 100644 --- a/src/pmsearch/pmsearch_index.timer +++ b/src/pmsearch/pmsearch_index.timer @@ -13,4 +13,4 @@ Unit=pmsearch_index.service [Install] WantedBy=timers.target -RequiredBy=pmproxy.service +WantedBy=pmproxy.service From eaa37683712f20002bc724a7933de80259bd880f Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Mon, 3 Aug 2026 17:45:15 -0400 Subject: [PATCH 15/22] pmsearch: incorporate more feedback from code review --- qa/1861 | 121 ++++++++++++++++++++------------- qa/1861.out | 88 +++++++++++++++++++++++- qa/group | 2 +- src/libpcp_web/src/search.c | 29 +++++--- src/pmsearch/pmsearch_index.sh | 2 +- 5 files changed, 181 insertions(+), 61 deletions(-) diff --git a/qa/1861 b/qa/1861 index 9c4f00264de..ec446c956ff 100755 --- a/qa/1861 +++ b/qa/1861 @@ -1,34 +1,12 @@ -#!/bin/sh -# PCP QA Test No. 1861 -# Test reserved by nathans on Wed 5 Aug 2026 09:23:40 AEST -# [what am I here for?] -# -# Copyright (c) 2026 [who are you?]. All Rights Reserved. -# -if [ $# -eq 0 ] -then - seq=`basename $0` - echo "QA output created by $seq" -else - # use $seq from caller, unless not set - [ -n "$seq" ] || seq=`basename $0` - echo "QA output created by `basename $0` $*" -fi - -# get standard environment, filters and checks . ./common.product . ./common.filter . ./common.check -do_valgrind=false -if [ "$1" = "--valgrind" ] -then - _check_valgrind - do_valgrind=true -fi - -# test for-some-thing || _notrun No support for some-thing +which pmsearch >/dev/null 2>&1 || \ + _notrun "pmsearch not installed" +which newhelp >/dev/null 2>&1 || \ + _notrun "newhelp not installed" _cleanup() { @@ -36,34 +14,81 @@ _cleanup() $sudo rm -rf $tmp $tmp.* } -status=0 # success is the default! +status=1 # failure is the default! trap "_cleanup; exit \$status" 0 1 2 3 15 -_filter() -{ - sed \ - -e "s@$tmp@TMP@g" \ - -e "s@/privateTMP@TMP@g" \ - # end -} - # real QA test starts here -if $do_valgrind -then - _run_valgrind [--save-output] ...your test goes here... -else - ...your test goes here... 2>&1 -fi \ -| _filter +help=$PCP_PMDAS_DIR/sample/help +[ -f "$help" ] || _notrun "sample PMDA help file not found" -# if really bad error -status=1 -exit +# --- Step 1: Build base index from sample PMDA help files --- +mkdir -p $tmp.share/lib +newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Base index built" + +# --- Step 2: Copy base to nightly location (as pmsearch_index does) --- +mkdir -p $tmp.var/lib +cp $tmp.share/lib/pcp.search $tmp.var/lib/pcp.search +echo "Nightly index initialized from base" + +export PCP_VAR_DIR=$tmp.var +export PCP_SHARE_DIR=$tmp.share + +# --- Step 3: Verify Python PMDA metrics are NOT in base index --- +echo +echo "=== Verify Python PMDA metrics absent from base index ===" +pmsearch -C "qa_python" + +# --- Step 4: Create synthetic @ M entries mirroring Python PMDA patterns --- +# Python PMDAs define help text via add_metric() in code, not help files. +# pmsearch_index scrapes running PMDAs with pminfo -tT and generates +# @ M nameindomoneline entries for newhelp -S. +# +# Pattern (a): one-line help only (bpftrace style - most common) +# Pattern (b): both one-line and long help (openmetrics style) +# Pattern (c): metric with an indom (bpftrace per-script style) +# +cat > $tmp.helptext <<'EOF' +@ M qa_python.control.status status of the test python agent +@ M qa_python.control.debug debug flag to enable verbose log messages +Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 +@ M qa_python.data.counter 99.0 count of events processed by python agent +@ M qa_python.data.value 99.0 instantaneous value from python agent +EOF + +# --- Step 5: Insert Python PMDA metrics into nightly index --- +echo +echo "=== Inserting Python PMDA metrics ===" +newhelp -S -o $tmp.var/lib/pcp.search $tmp.helptext 2>&1 \ + | sed -e "s,$tmp,TMP,g" +echo "Python PMDA metrics inserted" + +# --- Step 6: Verify Python PMDA metrics now searchable --- +echo +echo "=== Search for Python PMDA metrics by name ===" +pmsearch -C "qa_python" + +echo +echo "=== Search for Python PMDA metric by one-line help ===" +pmsearch -C "verbose log messages" + +echo +echo "=== Search for Python PMDA metric with indom ===" +pmsearch -C "events processed" + +echo +echo "=== Suggest search for Python PMDA metrics ===" +pmsearch -s qa_python -# optional stuff if your test has verbose output to help resolve problems -#echo -#echo "If failure, check $seq_full" +# --- Step 7: Verify original sample PMDA metrics still present --- +echo +echo "=== Verify sample PMDA metrics still present ===" +pmsearch -C "trended" # success, all done +status=0 exit diff --git a/qa/1861.out b/qa/1861.out index c0537127120..6f57b16e216 100644 --- a/qa/1861.out +++ b/qa/1861.out @@ -1,2 +1,86 @@ -QA output created by 1861 -[skeleton from qa/new, replace me] +QA output created by 1728 +Base index built +Nightly index initialized from base + +=== Verify Python PMDA metrics absent from base index === +0 search results + +=== Inserting Python PMDA metrics === +Python PMDA metrics inserted + +=== Search for Python PMDA metrics by name === +Type: metric +Name: qa_python.data.value +InDom: 99.0 +One line: instantaneous value from python agent + +Type: metric +Name: qa_python.control.status +InDom: none +One line: status of the test python agent + +Type: metric +Name: qa_python.data.counter +InDom: 99.0 +One line: count of events processed by python agent + +Type: metric +Name: qa_python.control.debug +InDom: none +One line: debug flag to enable verbose log messages +Help: Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 + +=== Search for Python PMDA metric by one-line help === +Type: metric +Name: qa_python.control.debug +InDom: none +One line: debug flag to enable verbose log messages +Help: Debugging flag to enable verbose log messages in the test +python PMDA, to enable: +> pmstore qa_python.control.debug 1 + +=== Search for Python PMDA metric with indom === +Type: metric +Name: qa_python.data.counter +InDom: 99.0 +One line: count of events processed by python agent + +=== Suggest search for Python PMDA metrics === +Name: qa_python.data.value + +Name: qa_python.control.status + +Name: qa_python.data.counter + +Name: qa_python.control.debug + +=== Verify sample PMDA metrics still present === +Type: metric +Name: sample.drift +InDom: none +One line: A random trended metric +Help: This metric returns a random value (expected mean is approximately 200), +subject to a trending pattern such that the sequence is mainly monotonic, +with a change in direction after on average 4 consecutive samples. + +Use pmStore() to modify the instantaneous value, which becomes the new +expected mean. + +Type: metric +Name: sample.colour +InDom: 29.1 +One line: Metrics with a "saw-tooth" trend over time +Help: This metric has 3 instances, designated "red", "green" and "blue". + +The value of the metric is monotonic increasing in the range N to +N+100, then back to N. The different instances have different N +values, namely 100 (red), 200 (green) and 300 (blue). + +The underlying counter starts at 0 and is incremented once +for each pmFetch() to this metric and/or sample.mirage and/or +sample.mirage_longlong. + +Use pmStore() to modify the underlying counter (independent of which +instance or instances are used). diff --git a/qa/group b/qa/group index fb5bb172740..74451468130 100644 --- a/qa/group +++ b/qa/group @@ -2344,7 +2344,7 @@ suse 1854 pmlogger pmimport local 1855 pmda.rabbitmq local 1856 dstat python local derive -1861:reserved pmsearch local +1861 pmsearch help local 1862 other local 1863 logutil pmlogger_daily local 1864 logutil pmlogger_daily local diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 00d1198a643..85e49450104 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -371,7 +371,7 @@ search_do_text_query(searchModuleData *smd, pmSearchTextRequest *request, } static int -search_suggest_table(sqlite3 *db, sds match, +search_suggest_table(sqlite3 *db, sds match, unsigned int limit, pmSearchTextResult **hits, int *nhits, int *maxhits) { sqlite3_stmt *stmt = NULL; @@ -382,7 +382,8 @@ search_suggest_table(sqlite3 *db, sds match, "SELECT rowid, name, type, bm25(docs, 9.0, 4.0, 2.0)" " FROM docs WHERE docs MATCH ?" " AND type IN (%i, %i)" - " ORDER BY bm25(docs, 9.0, 4.0, 2.0)", + " ORDER BY bm25(docs, 9.0, 4.0, 2.0)" + " LIMIT ?", SEARCH_DOC_METRIC, SEARCH_DOC_INST); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); @@ -395,6 +396,7 @@ search_suggest_table(sqlite3 *db, sds match, } sqlite3_bind_text(stmt, 1, match, sdslen(match), SQLITE_STATIC); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64)limit); while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { pmSearchTextResult result; @@ -458,7 +460,11 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, match = sdscat(match, "*)"); } - sts = search_suggest_table(smd->db, match, &hits, &nhits, &maxhits); + if (!request->count) + request->count = smd->resultcount; + count = request->count; + + sts = search_suggest_table(smd->db, match, count, &hits, &nhits, &maxhits); sdsfree(match); @@ -471,10 +477,6 @@ search_do_text_suggest(searchModuleData *smd, pmSearchTextRequest *request, pmtimespecNow(&finished); timer = pmtimespecSub(&finished, &started); - if (!request->count) - request->count = smd->resultcount; - count = request->count; - for (i = 0; i < (unsigned int)nhits && i < count; i++) { hits[i].total = nhits; hits[i].count = i + 1; @@ -869,10 +871,19 @@ void keysSearchInit(struct dict *config) { sds option; + char *endp; + unsigned long value; if (config) { - if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) - default_resultcount = atoi(option); + if ((option = pmIniFileLookup(config, "pmsearch", "result.count"))) { + errno = 0; + value = strtoul(option, &endp, 10); + if (errno == 0 && *endp == '\0' && value > 0 && value <= UINT_MAX) + default_resultcount = (unsigned int)value; + else + pmNotifyErr(LOG_WARNING, "ignoring invalid " + "pmsearch result.count \"%s\"", option); + } } } diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index 07f91adda6e..a52ba53838f 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -55,7 +55,7 @@ do -N) SHOWME=true ;; -o) INDEX="$2"; shift ;; -V) VERBOSE=true ;; - -\?) _usage; status=0; exit ;; + -\?) _usage ;; --) shift; break ;; esac shift From 3aeedff3a5d681a1dbcde24bc4137cf2dc4377f6 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Mon, 10 Aug 2026 12:05:17 -0400 Subject: [PATCH 16/22] pmsearch: fix more feedback --- configure | 3 +++ configure.ac | 3 +++ src/pmsearch/pmsearch_index.sh | 17 +++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/configure b/configure index 19fd9a77745..39757055485 100755 --- a/configure +++ b/configure @@ -12424,6 +12424,8 @@ else lib_for_sqlite3="$sqlite3_LIBS" fi if test $have_sqlite3 = true; then + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$sqlite3_CFLAGS $CPPFLAGS" save_LIBS="$LIBS" LIBS="$lib_for_sqlite3 $LIBS" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for SQLite3 >= 3.9.0 with FTS5 support" >&5 @@ -12476,6 +12478,7 @@ esac fi LIBS="$save_LIBS" + CPPFLAGS="$save_CPPFLAGS" fi if test $have_sqlite3 = false; then as_fn_error $? "SQLite >= 3.9.0 with FTS5 support is required" "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 23cfaeb4ac6..a0902ad34e5 100644 --- a/configure.ac +++ b/configure.ac @@ -2011,6 +2011,8 @@ else fi dnl Verify SQLite >= 3.9.0 and that the linked library provides FTS5 if test $have_sqlite3 = true; then + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$sqlite3_CFLAGS $CPPFLAGS" save_LIBS="$LIBS" LIBS="$lib_for_sqlite3 $LIBS" AC_MSG_CHECKING([for SQLite3 >= 3.9.0 with FTS5 support]) @@ -2036,6 +2038,7 @@ return sts; ], [AC_MSG_RESULT([assuming yes (cross-compiling)])]) LIBS="$save_LIBS" + CPPFLAGS="$save_CPPFLAGS" fi if test $have_sqlite3 = false; then AC_MSG_ERROR([SQLite >= 3.9.0 with FTS5 support is required]) diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_index.sh index a52ba53838f..42dd0634b3c 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_index.sh @@ -221,15 +221,20 @@ fi # Add/update all runtime data (metrics and instances) into the index. # Metrics already in the base index are updated; new metrics from # PMDAs without help files (e.g. Python PMDAs) are inserted. -if $PCP_BINADM_DIR/newhelp -S -o "$NEW" $tmp/helptext && mv -f "$NEW" "$INDEX" +if $PCP_BINADM_DIR/newhelp -S -o "$NEW" $tmp/helptext then - if $VERBOSE + if mv -f "$NEW" "$INDEX" then - nmetrics=`grep -c '^@ [^I]' $tmp/helptext` - ninst=`grep -c '^@ I ' $tmp/helptext` - echo "$prog: added $nmetrics metrics and $ninst instances to $INDEX" + if $VERBOSE + then + nmetrics=`grep -c '^@ [^I]' $tmp/helptext` + ninst=`grep -c '^@ I ' $tmp/helptext` + echo "$prog: added $nmetrics metrics and $ninst instances to $INDEX" + fi + status=0 + else + echo >&2 "$prog: cannot move $NEW to $INDEX" fi - status=0 else echo >&2 "$prog: $PCP_BINADM_DIR/newhelp -S failed" fi From e2ee82cf1d49e394a2d3f81c3950c83d201e4fbe Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Mon, 10 Aug 2026 18:42:28 -0400 Subject: [PATCH 17/22] pmsearch: more changes based on feedback --- man/man1/pmsearch.1 | 10 ++++----- .../{pmsearch_index.1 => pmsearch_daily.1} | 18 +++++++-------- man/man3/pmsearchsetup.3 | 14 ++++++------ qa/1861 | 22 ++++++++++++++----- qa/1861.out | 2 +- qa/1871 | 18 +++++++-------- qa/1872 | 12 +++++----- src/libpcp_web/src/search.c | 6 ++--- src/newhelp/newhelp.c | 2 +- src/pmdas/GNUmakefile | 8 +++---- src/pmsearch/GNUmakefile | 10 ++++----- src/pmsearch/crontab.in | 2 +- ...x.service.in => pmsearch_daily.service.in} | 4 ++-- .../{pmsearch_index.sh => pmsearch_daily.sh} | 8 +++---- ...earch_index.timer => pmsearch_daily.timer} | 2 +- 15 files changed, 75 insertions(+), 63 deletions(-) rename man/man1/{pmsearch_index.1 => pmsearch_daily.1} (90%) rename src/pmsearch/{pmsearch_index.service.in => pmsearch_daily.service.in} (75%) rename src/pmsearch/{pmsearch_index.sh => pmsearch_daily.sh} (95%) rename src/pmsearch/{pmsearch_index.timer => pmsearch_daily.timer} (93%) diff --git a/man/man1/pmsearch.1 b/man/man1/pmsearch.1 index 829473b51eb..05a69cea5ff 100644 --- a/man/man1/pmsearch.1 +++ b/man/man1/pmsearch.1 @@ -43,7 +43,7 @@ It uses a local SQLite FTS5 full-text search index. A base index is built at package install time containing help text for all PCP metrics from the source tree. Each night, -.BR pmsearch_index (1) +.BR pmsearch_daily (1) copies the base index and extends it with instance names (disk devices, network interfaces, etc.) from the running system. .PP @@ -116,12 +116,12 @@ One line: percpu \fBidle\fR CPU time metric from /proc/stat .ESAMPLE .SH FILES .TP -.I $PCP_VAR_DIR/lib/pcp.search +.I $PCP_VAR_DIR/lib/pmsearch.index Nightly search index (SQLite FTS5 database), rebuilt by -.BR pmsearch_index (1). +.BR pmsearch_daily (1). This is a copy of the base index extended with runtime instance data. .TP -.I $PCP_SHARE_DIR/lib/pcp.search +.I $PCP_SHARE_DIR/lib/pmsearch.index Build-time base index containing all metrics from the PCP source tree. Used as a fallback when the nightly index is not yet available. .SH PCP ENVIRONMENT @@ -151,7 +151,7 @@ option to obtain a list of the available debugging options and their meaning. .SH SEE ALSO .BR PCPIntro (1), -.BR pmsearch_index (1), +.BR pmsearch_daily (1), .BR pmproxy (1), .BR pmlogger (1) and diff --git a/man/man1/pmsearch_index.1 b/man/man1/pmsearch_daily.1 similarity index 90% rename from man/man1/pmsearch_index.1 rename to man/man1/pmsearch_daily.1 index 703555064e8..5775e4eb73d 100644 --- a/man/man1/pmsearch_index.1 +++ b/man/man1/pmsearch_daily.1 @@ -13,16 +13,16 @@ .\" for more details. .\" .\" -.TH PMSEARCH_INDEX 1 "PCP" "Performance Co-Pilot" +.TH PMSEARCH_DAILY 1 "PCP" "Performance Co-Pilot" .SH NAME -\f3pmsearch_index\f1 \- rebuild the PCP full-text search index +\f3pmsearch_daily\f1 \- rebuild the PCP full-text search index .SH SYNOPSIS -\fBpmsearch_index\fR +\fBpmsearch_daily\fR [\fB\-NV?\fR] [\fB\-a\fR \fIarchive\fR] [\fB\-o\fR \fIindex\fR] .SH DESCRIPTION -.B pmsearch_index +.B pmsearch_daily rebuilds the runtime full-text search index used by .BR pmsearch (1) and the @@ -43,7 +43,7 @@ for all metric names, help text, and instance domain members, then builds a SQLite FTS5 search index using .BR newhelp (1). The resulting index file is written to -.I $PCP_VAR_DIR/lib/pcp.search +.I $PCP_VAR_DIR/lib/pmsearch.index by default. .PP Each night, the script copies the base index shipped with the @@ -60,7 +60,7 @@ If is not running or no metrics are found, the command exits without modifying any existing index file. The base index at -.I $PCP_SHARE_DIR/lib/pcp.search +.I $PCP_SHARE_DIR/lib/pmsearch.index remains available as a fallback. .SH OPTIONS .TP 5 @@ -83,7 +83,7 @@ Show what would be done without writing any files. Write the index to .IR index instead of the default -.IR $PCP_VAR_DIR/lib/pcp.search . +.IR $PCP_VAR_DIR/lib/pmsearch.index . .TP \fB\-V\fR, \fB\-\-verbose\fR Verbose diagnostics. @@ -93,12 +93,12 @@ Report the number of metrics extracted and the output path. Display usage message and exit. .SH FILES .TP 5 -.I $PCP_VAR_DIR/lib/pcp.search +.I $PCP_VAR_DIR/lib/pmsearch.index Default output path for the runtime search index. This is a copy of the base index extended with instance data. When present, it is preferred over the base index. .TP -.I $PCP_SHARE_DIR/lib/pcp.search +.I $PCP_SHARE_DIR/lib/pmsearch.index Base search index shipped with the PCP package, built at package build time from help text source files. Copied as the starting point for each nightly rebuild; diff --git a/man/man3/pmsearchsetup.3 b/man/man3/pmsearchsetup.3 index 1c88bc11cee..859ceec5865 100644 --- a/man/man3/pmsearchsetup.3 +++ b/man/man3/pmsearchsetup.3 @@ -57,12 +57,12 @@ full-text search functionality of the Performance Co-Pilot (PCP). .PP The search engine uses a local SQLite FTS5 full-text index. A build-time base index at -.I $PCP_SHARE_DIR/lib/pcp.search +.I $PCP_SHARE_DIR/lib/pmsearch.index provides help text for all metrics shipped with PCP. Each night, -.BR pmsearch_index (1) +.BR pmsearch_daily (1) copies the base index to -.I $PCP_VAR_DIR/lib/pcp.search +.I $PCP_VAR_DIR/lib/pmsearch.index and extends it with instance names from the running system. When the nightly index exists it is used; otherwise the base index is used as a fallback. @@ -148,16 +148,16 @@ Where these functions return a status code, this is always zero on success. On failure a negative PMAPI error code is returned. .SH FILES .TP -.I $PCP_VAR_DIR/lib/pcp.search +.I $PCP_VAR_DIR/lib/pmsearch.index Nightly search index (SQLite FTS5 database), a copy of the base index extended with runtime instance data by -.BR pmsearch_index (1). +.BR pmsearch_daily (1). .TP -.I $PCP_SHARE_DIR/lib/pcp.search +.I $PCP_SHARE_DIR/lib/pmsearch.index Build-time base index shipped with the PCP package. .SH SEE ALSO .BR pmsearch (1), -.BR pmsearch_index (1), +.BR pmsearch_daily (1), .BR pmproxy (1), .BR pmlogger (1), .BR mmv_stats_registry (3), diff --git a/qa/1861 b/qa/1861 index ec446c956ff..92acfb8a67f 100755 --- a/qa/1861 +++ b/qa/1861 @@ -1,4 +1,16 @@ +#!/bin/sh +# PCP QA Test No. 1861 +# Exercise inserting metrics from a PMDA without build-time help files +# (e.g. Python PMDAs) into the pmsearch SQLite FTS5 index using the +# @ M format that pmsearch_daily generates at runtime. +# +# Copyright (c) 2026 Red Hat. All Rights Reserved. +# + +seq=`basename $0` +echo "QA output created by $seq" +# get standard environment, filters and checks . ./common.product . ./common.filter . ./common.check @@ -24,13 +36,13 @@ help=$PCP_PMDAS_DIR/sample/help # --- Step 1: Build base index from sample PMDA help files --- mkdir -p $tmp.share/lib -newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ | sed -e "s,$tmp,TMP,g" echo "Base index built" -# --- Step 2: Copy base to nightly location (as pmsearch_index does) --- +# --- Step 2: Copy base to nightly location (as pmsearch_daily does) --- mkdir -p $tmp.var/lib -cp $tmp.share/lib/pcp.search $tmp.var/lib/pcp.search +cp $tmp.share/lib/pmsearch.index $tmp.var/lib/pmsearch.index echo "Nightly index initialized from base" export PCP_VAR_DIR=$tmp.var @@ -43,7 +55,7 @@ pmsearch -C "qa_python" # --- Step 4: Create synthetic @ M entries mirroring Python PMDA patterns --- # Python PMDAs define help text via add_metric() in code, not help files. -# pmsearch_index scrapes running PMDAs with pminfo -tT and generates +# pmsearch_daily scrapes running PMDAs with pminfo -tT and generates # @ M nameindomoneline entries for newhelp -S. # # Pattern (a): one-line help only (bpftrace style - most common) @@ -63,7 +75,7 @@ EOF # --- Step 5: Insert Python PMDA metrics into nightly index --- echo echo "=== Inserting Python PMDA metrics ===" -newhelp -S -o $tmp.var/lib/pcp.search $tmp.helptext 2>&1 \ +newhelp -S -o $tmp.var/lib/pmsearch.index $tmp.helptext 2>&1 \ | sed -e "s,$tmp,TMP,g" echo "Python PMDA metrics inserted" diff --git a/qa/1861.out b/qa/1861.out index 6f57b16e216..92729d52bc0 100644 --- a/qa/1861.out +++ b/qa/1861.out @@ -1,4 +1,4 @@ -QA output created by 1728 +QA output created by 1861 Base index built Nightly index initialized from base diff --git a/qa/1871 b/qa/1871 index ebc5467451a..877e7216957 100755 --- a/qa/1871 +++ b/qa/1871 @@ -19,8 +19,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" -[ -x $PCP_BINADM_DIR/pmsearch_index ] || \ - _notrun "pmsearch_index not installed" +[ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ + _notrun "pmsearch_daily not installed" _cleanup() { @@ -63,25 +63,25 @@ help=$PCP_PMDAS_DIR/sample/help # Mirror the real copy-and-extend strategy: # 1. Build base index from PMDA help files (like package install) -# 2. Copy base to nightly location (like pmsearch_index.sh does) +# 2. Copy base to nightly location (like pmsearch_daily.sh does) # 3. Append instance data from archive to the nightly copy -# --- Base index (like $PCP_SHARE_DIR/lib/pcp.search) --- +# --- Base index (like $PCP_SHARE_DIR/lib/pmsearch.index) --- mkdir -p $tmp.share/lib -newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ | sed -e "s,$tmp,TMP,g" echo "Base index built" -# --- Nightly index (like $PCP_VAR_DIR/lib/pcp.search) --- -# Use pmsearch_index to copy base and append instance data from archive +# --- Nightly index (like $PCP_VAR_DIR/lib/pmsearch.index) --- +# Use pmsearch_daily to copy base and append instance data from archive mkdir -p $tmp.var/lib export PCP_VAR_DIR=$tmp.var export PCP_SHARE_DIR=$tmp.share -$PCP_BINADM_DIR/pmsearch_index -a $archive +$PCP_BINADM_DIR/pmsearch_daily -a $archive echo "Nightly index built (base + instances)" echo -# PCP_VAR_DIR and PCP_SHARE_DIR already exported above for pmsearch_index +# PCP_VAR_DIR and PCP_SHARE_DIR already exported above for pmsearch_daily echo "=== Empty search ===" pmsearch -C xyznonexistent diff --git a/qa/1872 b/qa/1872 index 2992afa7974..2d7ba061a5d 100755 --- a/qa/1872 +++ b/qa/1872 @@ -18,8 +18,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" -[ -x $PCP_BINADM_DIR/pmsearch_index ] || \ - _notrun "pmsearch_index not installed" +[ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ + _notrun "pmsearch_daily not installed" which pmproxy >/dev/null 2>&1 || \ _notrun "pmproxy not installed" which curl >/dev/null 2>&1 || \ @@ -64,15 +64,15 @@ help=$PCP_PMDAS_DIR/sample/help [ -f "$help" ] || _notrun "sample PMDA help file not found" # Build a search index using the copy-and-extend strategy: -# base index from help file, then pmsearch_index copies and appends instances. +# base index from help file, then pmsearch_daily copies and appends instances. echo "=== Build search index ===" mkdir -p $tmp.share/lib $tmp.var/lib -newhelp -S -o $tmp.share/lib/pcp.search $help 2>&1 \ +newhelp -S -o $tmp.share/lib/pmsearch.index $help 2>&1 \ | sed -e "s,$tmp,TMP,g" export PCP_VAR_DIR=$tmp.var export PCP_SHARE_DIR=$tmp.share -$PCP_BINADM_DIR/pmsearch_index -a $archive +$PCP_BINADM_DIR/pmsearch_daily -a $archive echo "Index built" echo @@ -83,7 +83,7 @@ pcp.enabled = true http.enabled = true [pmsearch] enabled = true -index.path = $tmp.var/lib/pcp.search +index.path = $tmp.var/lib/pmsearch.index result.count = 10 EOF diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 85e49450104..128a374f0fd 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -797,9 +797,9 @@ pmSearchSetup(pmSearchModule *module, void *arg) smd->db = NULL; } } else { - pmsprintf(nightly, sizeof(nightly), "%s/lib/pcp.search", + pmsprintf(nightly, sizeof(nightly), "%s/lib/pmsearch.index", pmGetConfig("PCP_VAR_DIR")); - pmsprintf(base, sizeof(base), "%s/lib/pcp.search", + pmsprintf(base, sizeof(base), "%s/lib/pmsearch.index", pmGetConfig("PCP_SHARE_DIR")); rc = sqlite3_open_v2(nightly, &smd->db, @@ -821,7 +821,7 @@ pmSearchSetup(pmSearchModule *module, void *arg) } else { pmNotifyErr(LOG_WARNING, "no pmsearch index found " - "(tried %s and %s); run pmsearch_index(1)", + "(tried %s and %s); run pmsearch_daily(1)", nightly, base); sqlite3_close(smd->db); smd->db = NULL; diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index f2ddb96fe1c..b03561585a5 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -358,7 +358,7 @@ newentry_search(char *buf) /* * Metric with pre-resolved indom: @ M nameindomoneline - * Generated by pmsearch_index to avoid needing PMNS/PMCD lookup. + * Generated by pmsearch_daily to avoid needing PMNS/PMCD lookup. */ if (*p == 'M' && isspace((int)*(p + 1))) { char *indom_str; diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index 93ec5d22e65..290eebb8eeb 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -44,7 +44,7 @@ PYPMDAS = gluster zswap unbound haproxy \ uwsgi rocestat hdb rds chrony db2 SUBDIRS = $(CPMDAS) $(PLPMDAS) $(PYPMDAS) -LDIRT = local.conf pmcd.conf local.root pcp.search +LDIRT = local.conf pmcd.conf local.root pmsearch.index ifneq ("$(wildcard $(TOPDIR)/src/include/pcp/config.h)","") HAVE_UNIX_DOMAIN_SOCKETS = $(shell grep 'define HAVE_STRUCT_SOCKADDR_UN' $(TOPDIR)/src/include/pcp/config.h >/dev/null && echo yes) @@ -100,11 +100,11 @@ ifneq "$(CROSS_COMPILING)" "yes" help_files="$$help_files $$f"; \ done; \ if [ -n "$$help_files" ]; then \ - $(TOPDIR)/src/newhelp/newhelp.static -S -o pcp.search $$help_files && \ + $(TOPDIR)/src/newhelp/newhelp.static -S -o pmsearch.index $$help_files && \ $(INSTALL) -m 755 -d $(PCP_SHARE_DIR)/lib && \ - $(INSTALL) -m 644 pcp.search $(PCP_SHARE_DIR)/lib/pcp.search; \ + $(INSTALL) -m 644 pmsearch.index $(PCP_SHARE_DIR)/lib/pmsearch.index; \ rc=$$?; \ - rm -f pcp.search; \ + rm -f pmsearch.index; \ test $$rc -eq 0 || exit 1; \ fi endif diff --git a/src/pmsearch/GNUmakefile b/src/pmsearch/GNUmakefile index c97a18104e7..890dcb13674 100644 --- a/src/pmsearch/GNUmakefile +++ b/src/pmsearch/GNUmakefile @@ -18,7 +18,7 @@ include $(TOPDIR)/src/include/builddefs CFILES = pmsearch.c CMDTARGET = pmsearch$(EXECSUFFIX) LLDLIBS = $(PCP_WEBLIB) -GENSCRIPT = crontab pmsearch_index.service +GENSCRIPT = crontab pmsearch_daily.service LDIRT = $(GENSCRIPT) ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 @@ -42,12 +42,12 @@ default: $(CMDTARGET) $(GENSCRIPT) install: default $(INSTALL) -m 755 $(CMDTARGET) $(PCP_BIN_DIR)/$(CMDTARGET) - $(INSTALL) -m 755 pmsearch_index.sh $(PCP_BINADM_DIR)/pmsearch_index + $(INSTALL) -m 755 pmsearch_daily.sh $(PCP_BINADM_DIR)/pmsearch_daily $(INSTALL) -m 755 -d $(PCP_SYSCONF_DIR)/pmsearch $(INSTALL) -S $(PCP_SYSCONF_DIR)/pmproxy/pmproxy.conf $(PCP_SYSCONF_DIR)/pmsearch/pmsearch.conf ifeq ($(ENABLE_SYSTEMD),true) - $(INSTALL) -m 644 pmsearch_index.service $(PCP_SYSTEMDUNIT_DIR)/pmsearch_index.service - $(INSTALL) -m 644 pmsearch_index.timer $(PCP_SYSTEMDUNIT_DIR)/pmsearch_index.timer + $(INSTALL) -m 644 pmsearch_daily.service $(PCP_SYSTEMDUNIT_DIR)/pmsearch_daily.service + $(INSTALL) -m 644 pmsearch_daily.timer $(PCP_SYSTEMDUNIT_DIR)/pmsearch_daily.timer endif $(INSTALL) -m 755 -d `dirname $(CRONTAB_INSTALL)` $(INSTALL) -m 644 crontab $(CRONTAB_INSTALL) @@ -58,7 +58,7 @@ default_pcp : default install_pcp : install -pmsearch_index.service : pmsearch_index.service.in +pmsearch_daily.service : pmsearch_daily.service.in $(SED) <$< >$@ \ -e 's;@CRONTAB_PATH@;'$(CRONTAB_PATH)';' \ -e 's;@PCP_BINADM_DIR@;'$(PCP_BINADM_DIR)';' \ diff --git a/src/pmsearch/crontab.in b/src/pmsearch/crontab.in index c1ae95a149a..b75130b3ea3 100644 --- a/src/pmsearch/crontab.in +++ b/src/pmsearch/crontab.in @@ -2,4 +2,4 @@ # Performance Co-Pilot crontab entry for pmsearch index rebuild # # daily rebuild of the full-text search index -20 0 * * * @user@ @path@/pmsearch_index +20 0 * * * @user@ @path@/pmsearch_daily diff --git a/src/pmsearch/pmsearch_index.service.in b/src/pmsearch/pmsearch_daily.service.in similarity index 75% rename from src/pmsearch/pmsearch_index.service.in rename to src/pmsearch/pmsearch_daily.service.in index 2166d9cc9c3..0437b5bb20a 100644 --- a/src/pmsearch/pmsearch_index.service.in +++ b/src/pmsearch/pmsearch_daily.service.in @@ -1,13 +1,13 @@ [Unit] Description=Daily rebuild of the PCP pmsearch full-text index -Documentation=man:pmsearch_index(1) +Documentation=man:pmsearch_daily(1) ConditionPathExists=!@CRONTAB_PATH@ [Service] Type=@SD_SERVICE_TYPE@ Restart=no TimeoutStartSec=1h -ExecStart=@PCP_BINADM_DIR@/pmsearch_index +ExecStart=@PCP_BINADM_DIR@/pmsearch_daily WorkingDirectory=@PCP_VAR_DIR@ Group=@PCP_GROUP@ User=@PCP_USER@ diff --git a/src/pmsearch/pmsearch_index.sh b/src/pmsearch/pmsearch_daily.sh similarity index 95% rename from src/pmsearch/pmsearch_index.sh rename to src/pmsearch/pmsearch_daily.sh index 42dd0634b3c..d250ec46546 100755 --- a/src/pmsearch/pmsearch_index.sh +++ b/src/pmsearch/pmsearch_daily.sh @@ -20,11 +20,11 @@ . $PCP_DIR/etc/pcp.env status=1 -tmp=`mktemp -d "$PCP_TMPFILE_DIR/pmsearch_index.XXXXXXXXX"` || exit 1 +tmp=`mktemp -d "$PCP_TMPFILE_DIR/pmsearch_daily.XXXXXXXXX"` || exit 1 trap "rm -rf \"$tmp\" \"\$NEW\"; exit \$status" 0 1 2 3 15 prog=`basename $0` -INDEX="$PCP_VAR_DIR/lib/pcp.search" +INDEX="$PCP_VAR_DIR/lib/pmsearch.index" cat > $tmp/usage << EOF Options: @@ -177,7 +177,7 @@ fi if $SHOWME then - echo "$prog: would copy $PCP_SHARE_DIR/lib/pcp.search to $INDEX and add runtime data" + echo "$prog: would copy $PCP_SHARE_DIR/lib/pmsearch.index to $INDEX and add runtime data" status=0 exit fi @@ -197,7 +197,7 @@ fi # Start from the build-time base index (contains all metric help text). # The nightly update copies it, then adds/updates runtime data. NEW="$INDEX.new.$$" -BASE="$PCP_SHARE_DIR/lib/pcp.search" +BASE="$PCP_SHARE_DIR/lib/pmsearch.index" if [ -f "$BASE" ] then cp "$BASE" "$NEW" || exit diff --git a/src/pmsearch/pmsearch_index.timer b/src/pmsearch/pmsearch_daily.timer similarity index 93% rename from src/pmsearch/pmsearch_index.timer rename to src/pmsearch/pmsearch_daily.timer index 760f319e266..4ad374ba92d 100644 --- a/src/pmsearch/pmsearch_index.timer +++ b/src/pmsearch/pmsearch_daily.timer @@ -9,7 +9,7 @@ Persistent=true # processing at 00:15:00) OnBootSec=10min OnCalendar=*-*-* 00:20:00 -Unit=pmsearch_index.service +Unit=pmsearch_daily.service [Install] WantedBy=timers.target From deceaa560807210d2568efcf4f1d1fb0fe6863d4 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Tue, 11 Aug 2026 17:29:41 -0400 Subject: [PATCH 18/22] pmsearch: changes based on review --- qa/1871 | 2 ++ qa/1872 | 2 ++ src/newhelp/newhelp.c | 3 +++ src/pmsearch/GNUmakefile | 1 + src/pmsearch/pmsearch_daily.sh | 7 ++++++- 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/qa/1871 b/qa/1871 index 877e7216957..9946690f6b4 100755 --- a/qa/1871 +++ b/qa/1871 @@ -19,6 +19,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" [ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ _notrun "pmsearch_daily not installed" diff --git a/qa/1872 b/qa/1872 index 2d7ba061a5d..925894146cb 100755 --- a/qa/1872 +++ b/qa/1872 @@ -18,6 +18,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" [ -x $PCP_BINADM_DIR/pmsearch_daily ] || \ _notrun "pmsearch_daily not installed" which pmproxy >/dev/null 2>&1 || \ diff --git a/src/newhelp/newhelp.c b/src/newhelp/newhelp.c index b03561585a5..8ab874b2112 100644 --- a/src/newhelp/newhelp.c +++ b/src/newhelp/newhelp.c @@ -324,6 +324,7 @@ newentry_search(char *buf) if (verbose) fprintf(stderr, "%s: [%s:%d] instance entry missing tab separator\n", pmGetProgname(), filename, ln); + if (!status) status = 1; return; } inst_name = p; @@ -375,6 +376,7 @@ newentry_search(char *buf) if (verbose) fprintf(stderr, "%s: [%s:%d] metric entry missing first tab separator\n", pmGetProgname(), filename, ln); + if (!status) status = 1; return; } indom_str = p; @@ -386,6 +388,7 @@ newentry_search(char *buf) if (verbose) fprintf(stderr, "%s: [%s:%d] metric entry missing second tab separator\n", pmGetProgname(), filename, ln); + if (!status) status = 1; return; } diff --git a/src/pmsearch/GNUmakefile b/src/pmsearch/GNUmakefile index 890dcb13674..b9f6eabb4f2 100644 --- a/src/pmsearch/GNUmakefile +++ b/src/pmsearch/GNUmakefile @@ -65,6 +65,7 @@ pmsearch_daily.service : pmsearch_daily.service.in -e 's;@PCP_VAR_DIR@;'$(PCP_VAR_DIR)';' \ -e 's;@PCP_GROUP@;'$(PCP_GROUP)';' \ -e 's;@PCP_USER@;'$(PCP_USER)';' \ + -e 's;@SD_SERVICE_TYPE@;'$(SD_SERVICE_TYPE)';' \ # END crontab : crontab.in diff --git a/src/pmsearch/pmsearch_daily.sh b/src/pmsearch/pmsearch_daily.sh index d250ec46546..e6dd1cc5756 100755 --- a/src/pmsearch/pmsearch_daily.sh +++ b/src/pmsearch/pmsearch_daily.sh @@ -117,7 +117,12 @@ FILENAME == ARGV[1] { /^Help:$/ { next } /^Full Help: Error:/ { skip = 1; next } skip { next } -{ print } +{ + if (substr($0, 1, 1) == "@") + printf " %s\n", $0 + else + print +} ' "$tmp/metric_indom" - > $tmp/helptext nhelplines=`wc -l < $tmp/helptext` From 244d2d7468895ff20a0f1d73990176544c9b2eb7 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 14 Aug 2026 13:23:11 -0400 Subject: [PATCH 19/22] pmsearch: make SQLite an optional build dependency --- configure | 3 +- configure.ac | 2 +- src/libpcp_web/src/GNUmakefile | 10 +- src/libpcp_web/src/nosearch.c | 156 +++++++++++++++++++++++++++++++ src/newhelp/GNUmakefile | 16 ++-- src/newhelp/search_sqlite_stub.c | 36 +++++++ src/pmdas/GNUmakefile | 2 + src/pmsearch/GNUmakefile | 22 +++-- 8 files changed, 230 insertions(+), 17 deletions(-) create mode 100644 src/libpcp_web/src/nosearch.c create mode 100644 src/newhelp/search_sqlite_stub.c diff --git a/configure b/configure index 39757055485..6be0ffbad31 100755 --- a/configure +++ b/configure @@ -12481,7 +12481,8 @@ fi CPPFLAGS="$save_CPPFLAGS" fi if test $have_sqlite3 = false; then - as_fn_error $? "SQLite >= 3.9.0 with FTS5 support is required" "$LINENO" 5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built" >&5 +printf "%s\n" "$as_me: WARNING: SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built" >&2;} fi HAVE_SQLITE3=$have_sqlite3 diff --git a/configure.ac b/configure.ac index a0902ad34e5..da95e7f8f78 100644 --- a/configure.ac +++ b/configure.ac @@ -2041,7 +2041,7 @@ return sts; CPPFLAGS="$save_CPPFLAGS" fi if test $have_sqlite3 = false; then - AC_MSG_ERROR([SQLite >= 3.9.0 with FTS5 support is required]) + AC_MSG_WARN([SQLite >= 3.9.0 with FTS5 not found, pmsearch will not be built]) fi AC_SUBST(HAVE_SQLITE3, [$have_sqlite3]) AC_SUBST(lib_for_sqlite3) diff --git a/src/libpcp_web/src/GNUmakefile b/src/libpcp_web/src/GNUmakefile index 750156c2545..d902946eac6 100644 --- a/src/libpcp_web/src/GNUmakefile +++ b/src/libpcp_web/src/GNUmakefile @@ -45,7 +45,6 @@ XFILES = jsmn.c jsmn.h http_parser.c http_parser.h \ sha1.c sha1.h siphash.c LLDLIBS = $(PCPWEBLIB_EXTRAS) $(LIB_FOR_MATH) $(LIB_FOR_REGEX) \ - $(LIB_FOR_SQLITE3) \ $(TOPDIR)/src/libvalkey/src/libvalkey.a ifeq "$(TARGET_OS)" "mingw" LLDLIBS += -lws2_32 @@ -58,7 +57,7 @@ ifneq "$(HAVE_LIBINIH)" "true" LCFLAGS += -Iinih endif -LCFLAGS += $(C99_CFLAGS) $(SQLITE3CFLAGS) \ +LCFLAGS += $(C99_CFLAGS) \ -DJSMN_PARENT_LINKS=1 -DJSMN_STRICT=1 -DHTTP_PARSER_STRICT=0 -Ideps \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/include \ -I$(TOPDIR)/vendor/github.com/valkey-io/libvalkey/src @@ -71,6 +70,13 @@ else CFILES += nodiscover.c nologgroup.c nowebgroup.c notimer.c endif +ifeq "$(HAVE_SQLITE3)" "true" +LCFLAGS += $(SQLITE3CFLAGS) +LLDLIBS += $(LIB_FOR_SQLITE3) +else +CFILES := $(subst search.c,nosearch.c,$(CFILES)) +endif + ifeq "$(HAVE_LIBINIH)" "true" LLDLIBS += $(LIB_FOR_INIH) endif diff --git a/src/libpcp_web/src/nosearch.c b/src/libpcp_web/src/nosearch.c new file mode 100644 index 00000000000..357d19c9441 --- /dev/null +++ b/src/libpcp_web/src/nosearch.c @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * Stub routines for builds without SQLite FTS5 support. + */ +#include "pmwebapi.h" +#include "search.h" +#include "discover.h" + +const char * +pmSearchTextTypeStr(pmSearchTextType type) +{ + (void)type; + return "unknown"; +} + +int +pmSearchInfo(pmSearchSettings *settings, sds key, void *arg) +{ + (void)settings; (void)key; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextQuery(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextSuggest(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchTextInDom(pmSearchSettings *settings, pmSearchTextRequest *request, + void *arg) +{ + (void)settings; (void)request; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchSetSlots(pmSearchModule *module, void *slots) +{ + (void)module; (void)slots; + return 0; +} + +int +pmSearchSetConfiguration(pmSearchModule *module, struct dict *config) +{ + (void)module; (void)config; + return 0; +} + +int +pmSearchSetEventLoop(pmSearchModule *module, void *events) +{ + (void)module; (void)events; + return 0; +} + +int +pmSearchSetMetricRegistry(pmSearchModule *module, struct mmv_registry *registry) +{ + (void)module; (void)registry; + return 0; +} + +int +pmSearchSetup(pmSearchModule *module, void *arg) +{ + (void)module; (void)arg; + return -EOPNOTSUPP; +} + +int +pmSearchEnabled(void *arg) +{ + (void)arg; + return 0; +} + +void +pmSearchClose(pmSearchModule *module) +{ + (void)module; +} + +/* --- stubs for schema.c / keys.c compatibility --- */ + +extern void keys_slots_end_phase(void *); + +void +keysSearchInit(struct dict *config) +{ + (void)config; +} + +void +keysSearchClose(void) +{ +} + +void +keys_load_search_schema(void *arg) +{ + keys_slots_end_phase(arg); +} + +void +keys_search_text_add(struct keySlots *slots, pmSearchTextType type, + const char *name, const char *indom, + const char *oneline, const char *helptext, void *arg) +{ + (void)slots; (void)type; (void)name; (void)indom; + (void)oneline; (void)helptext; (void)arg; +} + +/* --- discover no-ops --- */ + +void +pmSearchDiscoverMetric(pmDiscoverEvent *event, + pmDesc *desc, int numnames, char **names, void *arg) +{ + (void)event; (void)desc; (void)numnames; (void)names; (void)arg; +} + +void +pmSearchDiscoverInDom(pmDiscoverEvent *event, pmInResult *in, void *arg) +{ + (void)event; (void)in; (void)arg; +} + +void +pmSearchDiscoverText(pmDiscoverEvent *event, + int ident, int type, char *text, void *arg) +{ + (void)event; (void)ident; (void)type; (void)text; (void)arg; +} diff --git a/src/newhelp/GNUmakefile b/src/newhelp/GNUmakefile index f228fa48282..34e34c3ea38 100644 --- a/src/newhelp/GNUmakefile +++ b/src/newhelp/GNUmakefile @@ -18,9 +18,13 @@ include $(TOPDIR)/src/include/builddefs include $(TOPDIR)/src/libpcp/src/GNUlibrarydefs TARGETS = newhelp$(EXECSUFFIX) chkhelp$(EXECSUFFIX) -CFILES = newhelp.c chkhelp.c search_sqlite.c - +ifeq "$(HAVE_SQLITE3)" "true" +SEARCH_SQLITE = search_sqlite.c LCFLAGS += $(SQLITE3CFLAGS) +else +SEARCH_SQLITE = search_sqlite_stub.c +endif +CFILES = newhelp.c chkhelp.c $(SEARCH_SQLITE) LLDFLAGS = $(WARN_OFF) LLDLIBS = $(PCP_PMDALIB) @@ -41,12 +45,12 @@ install: default include $(BUILDRULES) -newhelp$(EXECSUFFIX): newhelp.c search_sqlite.c search_sqlite.h - $(CCF) -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(LDLIBS) $(LIB_FOR_SQLITE3) +newhelp$(EXECSUFFIX): newhelp.c $(SEARCH_SQLITE) search_sqlite.h + $(CCF) -o $@ $(LDFLAGS) newhelp.c $(SEARCH_SQLITE) $(LDLIBS) $(LIB_FOR_SQLITE3) $(LINKER_MAKERULE) -newhelp.static$(EXECSUFFIX): newhelp.c search_sqlite.c search_sqlite.h $(STATIC_LIBPCP) - $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c search_sqlite.c $(STATIC_LIBPCP) $(STATIC_LDLIBS) $(LIB_FOR_SQLITE3) +newhelp.static$(EXECSUFFIX): newhelp.c $(SEARCH_SQLITE) search_sqlite.h $(STATIC_LIBPCP) + $(CCF) -DPCP_STATIC -o $@ $(LDFLAGS) newhelp.c $(SEARCH_SQLITE) $(STATIC_LIBPCP) $(STATIC_LDLIBS) $(LIB_FOR_SQLITE3) chkhelp$(EXECSUFFIX): chkhelp.o $(CCF) -o $@ $(LDFLAGS) chkhelp.o $(LDLIBS) diff --git a/src/newhelp/search_sqlite_stub.c b/src/newhelp/search_sqlite_stub.c new file mode 100644 index 00000000000..0566ca1ff76 --- /dev/null +++ b/src/newhelp/search_sqlite_stub.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026 Red Hat. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * Stub routines when SQLite is not available. + */ +#include "search_sqlite.h" + +int +search_sqlite_open(const char *path) +{ + (void)path; + return -1; +} + +void +search_sqlite_add(const char *name, const char *oneline, + const char *helptext, const char *indom, int type) +{ + (void)name; (void)oneline; (void)helptext; (void)indom; (void)type; +} + +int +search_sqlite_close(void) +{ + return -1; +} diff --git a/src/pmdas/GNUmakefile b/src/pmdas/GNUmakefile index 290eebb8eeb..eab900b77ba 100644 --- a/src/pmdas/GNUmakefile +++ b/src/pmdas/GNUmakefile @@ -95,6 +95,7 @@ install_pcp :: default_pcp $(INSTALL) -m 644 local.conf indom.conf $(PCP_ETC_DIR)/pcp $(INSTALL) -m 644 -t $(PCP_PMNS_DIR)/local.root local.root $(PCP_PMNSADM_DIR)/local.root ifneq "$(CROSS_COMPILING)" "yes" +ifeq "$(HAVE_SQLITE3)" "true" help_files=""; \ for f in `find . -maxdepth 2 -name help -type f`; do \ help_files="$$help_files $$f"; \ @@ -108,5 +109,6 @@ ifneq "$(CROSS_COMPILING)" "yes" test $$rc -eq 0 || exit 1; \ fi endif +endif include $(BUILDRULES) diff --git a/src/pmsearch/GNUmakefile b/src/pmsearch/GNUmakefile index b9f6eabb4f2..038e0b30d1d 100644 --- a/src/pmsearch/GNUmakefile +++ b/src/pmsearch/GNUmakefile @@ -15,6 +15,9 @@ TOPDIR = ../.. include $(TOPDIR)/src/include/builddefs +default: build-me + +ifeq "$(HAVE_SQLITE3)" "true" CFILES = pmsearch.c CMDTARGET = pmsearch$(EXECSUFFIX) LLDLIBS = $(PCP_WEBLIB) @@ -38,7 +41,7 @@ CRONTAB_PATH = $(PCP_SYSCONF_DIR)/pmsearch/crontab CRONTAB_INSTALL = $(CRONTAB_PATH) endif -default: $(CMDTARGET) $(GENSCRIPT) +build-me: $(CMDTARGET) $(GENSCRIPT) install: default $(INSTALL) -m 755 $(CMDTARGET) $(PCP_BIN_DIR)/$(CMDTARGET) @@ -52,12 +55,6 @@ endif $(INSTALL) -m 755 -d `dirname $(CRONTAB_INSTALL)` $(INSTALL) -m 644 crontab $(CRONTAB_INSTALL) -include $(BUILDRULES) - -default_pcp : default - -install_pcp : install - pmsearch_daily.service : pmsearch_daily.service.in $(SED) <$< >$@ \ -e 's;@CRONTAB_PATH@;'$(CRONTAB_PATH)';' \ @@ -73,3 +70,14 @@ crontab : crontab.in -e 's;@user@;'$(CRONTAB_USER)';' \ -e 's;@path@;'$(PCP_BINADM_DIR)';' \ # END + +else +build-me: +install: +endif + +include $(BUILDRULES) + +default_pcp : default + +install_pcp : install From b61025d02015ee4ef2be0834ada5cb61791f3456 Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 14 Aug 2026 13:24:13 -0400 Subject: [PATCH 20/22] qa: notrun 1861 when missing sqlite support --- qa/1861 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/1861 b/qa/1861 index 92acfb8a67f..43f756b2296 100755 --- a/qa/1861 +++ b/qa/1861 @@ -19,6 +19,8 @@ which pmsearch >/dev/null 2>&1 || \ _notrun "pmsearch not installed" which newhelp >/dev/null 2>&1 || \ _notrun "newhelp not installed" +newhelp -\? 2>&1 | grep -q '\-S, --search' || \ + _notrun "newhelp not built with SQLite search index support" _cleanup() { From a64a3fe159613d364aa2deffff99257213ce563a Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Fri, 14 Aug 2026 13:25:33 -0400 Subject: [PATCH 21/22] pmsearch: weight indom help text higher --- qa/1871.out | 76 +++++------ qa/1872.out | 246 +++++++++++++++++++----------------- src/libpcp_web/src/search.c | 6 +- 3 files changed, 173 insertions(+), 155 deletions(-) diff --git a/qa/1871.out b/qa/1871.out index 75da8d5a8fd..a1314a65297 100644 --- a/qa/1871.out +++ b/qa/1871.out @@ -13,6 +13,23 @@ Help: Random number of instances, that change with time. Instance "m-00" (0) is always present, while the others are numbered 1 .. 49 and named "m-01" .. "m-99" === Search on random === +Type: indom +Name: 29.3 +InDom: 29.3 +One line: Instance domain "mirage" for sample PMDA +Help: Random number of instances, that change with time. Instance "m-00" (0) +is always present, while the others are numbered 1 .. 49 and named "m-01" +.. "m-99" + +Type: indom +Name: 29.6 +InDom: 29.6 +One line: Instance domain "dodgey" for sample PMDA. +Help: Instances for an unreliable instance domain, where the instances can +under certain conditions (random or controlled) return holes, errors, +and other interesting scenarios for testing. A sample.dodgey.control +metric store controls this instance domain. + Type: metric Name: sample.drift InDom: none @@ -24,6 +41,27 @@ with a change in direction after on average 4 consecutive samples. Use pmStore() to modify the instantaneous value, which becomes the new expected mean. +Type: indom +Name: 29.12 +InDom: 29.12 +One line: Instance domain "procs" for sample PMDA +Help: Simulated and small instance domain that mimics the behaviour of the +process instance domain from the proc PMDA. + +The first 5 instances are fixed. Then next 15 instances are variable +such that each time an associated metric is fetched: +- if an instance is currently defined, it may disappear with + probability 0.075 +- for each instance that is not defined, a new instance may + appear with probability 0.075 + +The internal instance identifiers mimic PIDs and are monotonically +increasing up to 1000, when they wrap (duplicates are obviously +avoided). + +The external instance names are the instance number, space, then +a random "executable" name. + Type: metric Name: sample.byte_rate InDom: none @@ -50,14 +88,6 @@ Help: The metric is a set of 5 instantaneous values, drawn at random from the range 0 to 100. The number of instances "visible" is controlled by sample.dodgey.control. -Type: indom -Name: 29.3 -InDom: 29.3 -One line: Instance domain "mirage" for sample PMDA -Help: Random number of instances, that change with time. Instance "m-00" (0) -is always present, while the others are numbered 1 .. 49 and named "m-01" -.. "m-99" - Type: metric Name: sample.scramble.bin InDom: 29.9 @@ -69,15 +99,6 @@ Help: Like sample.bin, except Designed to help testing instance matching between pmFetch calls for PCP clients. -Type: indom -Name: 29.6 -InDom: 29.6 -One line: Instance domain "dodgey" for sample PMDA. -Help: Instances for an unreliable instance domain, where the instances can -under certain conditions (random or controlled) return holes, errors, -and other interesting scenarios for testing. A sample.dodgey.control -metric store controls this instance domain. - Type: metric Name: sample.dodgey.control InDom: none @@ -96,27 +117,6 @@ available is changed according to the following probabilities ... 0.9 some number of instances in the range 0 to 5, selected at random from the underlying 5 instances. 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION) - -Type: indom -Name: 29.12 -InDom: 29.12 -One line: Instance domain "procs" for sample PMDA -Help: Simulated and small instance domain that mimics the behaviour of the -process instance domain from the proc PMDA. - -The first 5 instances are fixed. Then next 15 instances are variable -such that each time an associated metric is fetched: -- if an instance is currently defined, it may disappear with - probability 0.075 -- for each instance that is not defined, a new instance may - appear with probability 0.075 - -The internal instance identifiers mimic PIDs and are monotonically -increasing up to 1000, when they wrap (duplicates are obviously -avoided). - -The external instance names are the instance number, space, then -a random "executable" name. === Search on interesting === Type: indom Name: 29.6 diff --git a/qa/1872.out b/qa/1872.out index 1603d8ab89b..90c70ff4f55 100644 --- a/qa/1872.out +++ b/qa/1872.out @@ -26,12 +26,33 @@ Index built "offset": 0, "limit": 10, "results": [ + { + "name": "29.3", + "type": "indom", + "indom": "29.3", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, { "name": "sample.drift", "type": "metric", "oneline": "A random trended metric", "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + }, { "name": "sample.byte_rate", "type": "metric", @@ -57,13 +78,6 @@ Index built "oneline": "5 unreliable instances", "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." }, - { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" - }, { "name": "sample.scramble.bin", "type": "metric", @@ -71,25 +85,11 @@ Index built "oneline": "Several constant instances, instances scrambled", "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." }, - { - "name": "29.6", - "type": "indom", - "indom": "29.6", - "oneline": "Instance domain \"dodgey\" for sample PMDA.", - "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." - }, { "name": "sample.dodgey.control", "type": "metric", "oneline": "control values retured for sample.dodgey.value", "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" - }, - { - "name": "29.12", - "type": "indom", - "indom": "29.12", - "oneline": "Instance domain \"procs\" for sample PMDA", - "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } @@ -177,13 +177,6 @@ Index built "oneline": "Instance domain \"hordes\" for sample PMDA.", "helptext": "A relatively large instance domain, with hundreds of numeric instances." }, - { - "name": "sample.many.int", - "type": "metric", - "indom": "29.8", - "oneline": "variable sized instance domain", - "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" - }, { "name": "29.11", "type": "indom", @@ -219,12 +212,6 @@ Index built "oneline": "Instance domain \"many\" for sample PMDA.", "helptext": "A varable size set of instances controlled by sample.many.count" }, - { - "name": "sample.many.count", - "type": "metric", - "oneline": "number of instances in sample.many.int's domain", - "helptext": "store a value in sample.many.count to change the number of instances\nthat appear in sample.many.int's instance domain" - }, { "name": "29.7", "type": "indom", @@ -238,6 +225,20 @@ Index built "indom": "29.4", "oneline": "Instance domain \"family\" for sample PMDA.", "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" + }, + { + "name": "29.2", + "type": "indom", + "indom": "29.2", + "oneline": "Instance domain \"bin\" for sample PMDA", + "helptext": "Universally 9 instances numbered 100 .. 900 in steps of 100, and named\n\"bin-100\" .. \"bin-900\"" + }, + { + "name": "29.9", + "type": "indom", + "indom": "29.9", + "oneline": "Instance domain \"scramble\" for sample PMDA.", + "helptext": "Several constant instances, instances reordered on successive pmFetch\ncalls, and sometimes (probability 0.33) some instances are missing." } ] } @@ -249,61 +250,74 @@ Index built "limit": 10, "results": [ { - "name": "sample.long.bin", - "type": "metric", - "indom": "29.2", - "oneline": "like sample.bin but type 32" + "name": "29.8", + "type": "indom", + "indom": "29.8", + "oneline": "Instance domain \"many\" for sample PMDA.", + "helptext": "A varable size set of instances controlled by sample.many.count" }, { - "name": "sample.ulong.bin", - "type": "metric", - "indom": "29.2", - "oneline": "like sample.bin but type U32" + "name": "29.7", + "type": "indom", + "indom": "29.7", + "oneline": "Instance domain \"dynamic\" for sample PMDA.", + "helptext": "Instances come from $PCP_PMDAS_DIR/sample/dynamic.indom, if it exists." }, { - "name": "sample.longlong.bin", - "type": "metric", - "indom": "29.2", - "oneline": "like sample.bin but type 64" + "name": "29.11", + "type": "indom", + "indom": "29.11", + "oneline": "Instance domain \"ghosts\" for sample PMDA.", + "helptext": "Instances are the names of some famous ghosts." }, { - "name": "sample.float.bin", - "type": "metric", - "indom": "29.2", - "oneline": "like sample.bin but type FLOAT" + "name": "29.10", + "type": "indom", + "indom": "29.10", + "oneline": "Instance domain \"events\" for sample PMDA.", + "helptext": "Instances \"fungus\" and \"bogus\" for testing event metrics indoms." }, { - "name": "sample.double.bin", - "type": "metric", - "indom": "29.2", - "oneline": "like sample.bin but type DOUBLE" + "name": "29.1", + "type": "indom", + "indom": "29.1", + "oneline": "Instance domain \"colour\" for sample PMDA", + "helptext": "Universally 3 instances, \"red\" (0), \"green\" (1) and \"blue\" (3)." }, { - "name": "sample.string.bin", - "type": "metric", - "indom": "29.2", - "oneline": "a string-valued version of sample.bin" + "name": "29.5", + "type": "indom", + "indom": "29.5", + "oneline": "Instance domain \"hordes\" for sample PMDA.", + "helptext": "A relatively large instance domain, with hundreds of numeric instances." }, { - "name": "SAMPLE.0.1000", - "type": "metric", - "oneline": "dynamic sample.secret.bar metric", - "helptext": "Value \"foo\"." + "name": "29.4", + "type": "indom", + "indom": "29.4", + "oneline": "Instance domain \"family\" for sample PMDA.", + "helptext": "A fixed set of instances:\n \"colleen\", \"terry\", \"emma\", \"cathy\" and \"fat bald bastard\"" }, { - "name": "sample.const_rate.gradient", - "type": "metric", - "oneline": "rate per second to set sample.const_rate.value, writable" + "name": "29.2", + "type": "indom", + "indom": "29.2", + "oneline": "Instance domain \"bin\" for sample PMDA", + "helptext": "Universally 9 instances numbered 100 .. 900 in steps of 100, and named\n\"bin-100\" .. \"bin-900\"" }, { - "name": "sample.string.null", - "type": "metric", - "oneline": "a zero length string" + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." }, { - "name": "sample.aggregate.null", - "type": "metric", - "oneline": "a zero length aggregate" + "name": "29.9", + "type": "indom", + "indom": "29.9", + "oneline": "Instance domain \"scramble\" for sample PMDA.", + "helptext": "Several constant instances, instances reordered on successive pmFetch\ncalls, and sometimes (probability 0.33) some instances are missing." } ] } @@ -391,10 +405,11 @@ Index built "limit": 1, "results": [ { - "name": "sample.drift", - "type": "metric", - "oneline": "A random trended metric", - "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." + "name": "29.3", + "type": "indom", + "indom": "29.3", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" } ] } @@ -406,10 +421,11 @@ Index built "limit": 1, "results": [ { - "name": "sample.byte_rate", - "type": "metric", - "oneline": "instantaneous bytes/second ", - "helptext": "random value in the range (0,1023), so avg value is 512 bytes/second" + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." } ] } @@ -514,10 +530,25 @@ Index built "offset": 0, "limit": 10, "results": [ + { + "name": "29.3", + "type": "indom", + "indom": "29.3" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6" + }, { "name": "sample.drift", "type": "metric" }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12" + }, { "name": "sample.byte_rate", "type": "metric" @@ -535,29 +566,14 @@ Index built "type": "metric", "indom": "29.6" }, - { - "name": "29.3", - "type": "indom", - "indom": "29.3" - }, { "name": "sample.scramble.bin", "type": "metric", "indom": "29.9" }, - { - "name": "29.6", - "type": "indom", - "indom": "29.6" - }, { "name": "sample.dodgey.control", "type": "metric" - }, - { - "name": "29.12", - "type": "indom", - "indom": "29.12" } ] } @@ -674,12 +690,33 @@ Index built "offset": 0, "limit": 10, "results": [ + { + "name": "29.3", + "type": "indom", + "indom": "29.3", + "oneline": "Instance domain \"mirage\" for sample PMDA", + "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" + }, + { + "name": "29.6", + "type": "indom", + "indom": "29.6", + "oneline": "Instance domain \"dodgey\" for sample PMDA.", + "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." + }, { "name": "sample.drift", "type": "metric", "oneline": "A random trended metric", "helptext": "This metric returns a random value (expected mean is approximately 200),\nsubject to a trending pattern such that the sequence is mainly monotonic,\nwith a change in direction after on average 4 consecutive samples.\n\nUse pmStore() to modify the instantaneous value, which becomes the new\nexpected mean." }, + { + "name": "29.12", + "type": "indom", + "indom": "29.12", + "oneline": "Instance domain \"procs\" for sample PMDA", + "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." + }, { "name": "sample.byte_rate", "type": "metric", @@ -705,13 +742,6 @@ Index built "oneline": "5 unreliable instances", "helptext": "The metric is a set of 5 instantaneous values, drawn at random from the\nrange 0 to 100. The number of instances \"visible\" is controlled by\nsample.dodgey.control." }, - { - "name": "29.3", - "type": "indom", - "indom": "29.3", - "oneline": "Instance domain \"mirage\" for sample PMDA", - "helptext": "Random number of instances, that change with time. Instance \"m-00\" (0)\nis always present, while the others are numbered 1 .. 49 and named \"m-01\"\n.. \"m-99\"" - }, { "name": "sample.scramble.bin", "type": "metric", @@ -719,25 +749,11 @@ Index built "oneline": "Several constant instances, instances scrambled", "helptext": "Like sample.bin, except\n1. instances are missing with probability 0.33\n2. order of the instances from pmFetch is random\n\nDesigned to help testing instance matching between pmFetch calls\nfor PCP clients." }, - { - "name": "29.6", - "type": "indom", - "indom": "29.6", - "oneline": "Instance domain \"dodgey\" for sample PMDA.", - "helptext": "Instances for an unreliable instance domain, where the instances can\nunder certain conditions (random or controlled) return holes, errors,\nand other interesting scenarios for testing. A sample.dodgey.control\nmetric store controls this instance domain." - }, { "name": "sample.dodgey.control", "type": "metric", "oneline": "control values retured for sample.dodgey.value", "helptext": "If sample.dodgey.control is <= 0, then this is returned as the \"numval\"\ncomponent in the pmResult (0 => no values available, less than 0 =>\nvarious errors).\n\nIf sample.dodgey.control is between 1 and 5 (inclusive), then this many\nof the values will be \"visible\". The values will be selected in order\nfrom the underlying 5 instances.\n\nIf sample.dodgey.control is > 5, then at random times (between 1 and\nsample.dodgey.control fetches of the metric), the number of instances\navailable is changed according to the following probabilities ...\n 0.9 some number of instances in the range 0 to 5, selected at random\n from the underlying 5 instances.\n 0.1 error (PM_ERR_NOAGENT or PM_ERR_AGAIN or PM_ERR_APPVERSION)" - }, - { - "name": "29.12", - "type": "indom", - "indom": "29.12", - "oneline": "Instance domain \"procs\" for sample PMDA", - "helptext": "Simulated and small instance domain that mimics the behaviour of the\nprocess instance domain from the proc PMDA.\n\nThe first 5 instances are fixed. Then next 15 instances are variable\nsuch that each time an associated metric is fetched:\n- if an instance is currently defined, it may disappear with\n probability 0.075\n- for each instance that is not defined, a new instance may\n appear with probability 0.075\n\nThe internal instance identifiers mimic PIDs and are monotonically\nincreasing up to 1000, when they wrap (duplicates are obviously\navoided).\n\nThe external instance names are the instance number, space, then\na random \"executable\" name." } ] } diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 128a374f0fd..38903e85536 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -231,9 +231,11 @@ search_query_table(sqlite3 *db, pmSearchTextRequest *request, " highlight(docs, 1, '', '')," " highlight(docs, 2, '', '')" " FROM docs WHERE docs MATCH ?%S" - " ORDER BY bm25(docs, 9.0, 4.0, 2.0)" + " ORDER BY CASE WHEN type = %i" + " THEN bm25(docs, 9.0, 4.0, 2.0) * 2.0" + " ELSE bm25(docs, 9.0, 4.0, 2.0) END" " LIMIT ? OFFSET ?", - type_filter); + type_filter, SEARCH_DOC_INDOM); rc = sqlite3_prepare_v2(db, sql, sdslen(sql), &stmt, NULL); sdsfree(sql); From 22a19be7c4ccdbac40da4320265cb0fc61a8a20d Mon Sep 17 00:00:00 2001 From: Sam Feifer Date: Wed, 19 Aug 2026 17:37:56 -0400 Subject: [PATCH 22/22] libpcp_web: parenthesize the complete query in search.c --- src/libpcp_web/src/search.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libpcp_web/src/search.c b/src/libpcp_web/src/search.c index 38903e85536..143391f64ab 100644 --- a/src/libpcp_web/src/search.c +++ b/src/libpcp_web/src/search.c @@ -95,7 +95,7 @@ search_build_match(pmSearchTextRequest *request) cols = sdscat(cols, " "); cols = sdscat(cols, "helptext"); } - match = sdscatfmt(sdsempty(), "{%S} : %S", cols, request->query); + match = sdscatfmt(sdsempty(), "{%S} : (%S)", cols, request->query); sdsfree(cols); } else { match = sdsnew(request->query);