Cvs 169692 enable on commit tests part iii

Makefile

@@ -600,7 +600,7 @@ test_throughput_dummy_model: venv
 	@docker rm --force $(OVMS_CPP_CONTAINER_NAME)
 
 test_functional: venv
-	@. $(ACTIVATE); pytest --json=report.json -v -s $(TEST_PATH)
+	@export OVMS_CPP_DOCKER_IMAGE=$(OVMS_CPP_DOCKER_IMAGE) && export OVMS_CPP_IMAGE_TAG=$(OVMS_CPP_IMAGE_TAG);. $(ACTIVATE); pytest --json=report.json -v -s $(TEST_PATH)
 
 test_python_clients:
 	@echo "Prepare docker image"

ci/build_test_OnCommit.groovy

@@ -71,7 +71,7 @@ pipeline {
               if (git_diff =~ /(\n|^)client/) {
                   client_test_needed = "true"
               }
-              if (git_diff =~ /(\n|^)tests\/functional/) {
+              if (git_diff =~ /(\n|^)(tests\/functional|tests\/requirements\.txt)/) {
                   functional_tests_changed = "true"
               }
               if (git_diff =~ /(\n|^)(demos\/common\/export_models\/|prepare_llm_models\.sh$)/) {
@@ -185,6 +185,15 @@ pipeline {
                   sh 'make sdl-check'
               }
             }
+            stage('Pylint functional tests') {
+              agent {
+                label "${agent_name_linux}"
+              }
+              when { expression { functional_tests_changed == "true" } }
+              steps {
+                  sh 'python3 -m venv .venv-pylint && . .venv-pylint/bin/activate && pip install -r tests/requirements.txt && python3 -m pylint --rcfile=tests/functional/pylintrc tests/functional'
+              }
+            }
           }
         }
         stage('Cleanup node') {
@@ -344,7 +353,7 @@ pipeline {
                       sh "make create-venv && rm -f tests/functional && ln -s ${pwd}/../tests/functional tests/functional"
                       def cmd_venv_activate = ". .venv/bin/activate"
                       def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500 && export CORE_BRANCH=${env.CHANGE_BRANCH ?: 'main'}"
-                      def cmd_pytest = "pytest tests/non_functional/documentation -k '${test_doc_files_str}' -n 0 --dist loadgroup"
+                      def cmd_pytest = "pytest tests/non_functional/documentation -k 'test_links_ovms or ${test_doc_files_str}' -n 0 --dist loadgroup"
                       def cmd = ""
                       if ( image_build_needed == "true" ) {
                           unstash 'ovms-release-image'
@@ -407,7 +416,7 @@ pipeline {
                       def current_path = bat(returnStdout: true, script: 'cd').trim().split('\n').last().trim()
                       def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim()
                       def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional"
-                      def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt"
+                      def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r ${ovms_c_repo_path}\\tests\\requirements.txt -r requirements.txt"
                       def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\" && set \"CORE_BRANCH=${env.CHANGE_BRANCH ?: 'main'}\""
                       def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\""
                       def cmd = ""

ci/functional_tests_pytest.groovy

@@ -0,0 +1,104 @@
+pipeline {
+    agent none
+    options {
+        timeout(time: 3, unit: 'HOURS')
+    }
+    parameters {
+        string(
+            name: 'TARGET_HOST',
+            defaultValue: 'ovms_icelake',
+            description: 'Worker label to run functional tests on'
+        )
+        string(
+            name: 'CORE_BRANCH',
+            defaultValue: 'main',
+            description: 'ovms-c branch to use for this test run'
+        )
+        string(
+            name: 'PYTEST_PARAMS',
+            defaultValue: 'tests/functional',
+            description: 'Pytest target(s) and options, e.g. tests/functional or tests/functional/test_something.py -k smoke'
+        )
+        string(
+            name: 'TT_XDIST_WORKERS',
+            defaultValue: '4',
+            description: 'Number of pytest-xdist workers to use for parallel execution'
+        )
+        string(
+            name: 'TT_TARGET_DEVICE',
+            defaultValue: 'CPU',
+            description: 'Target device(s) for OVMS tests, e.g. CPU or CPU,GPU,NPU'
+        )
+
+        string(
+            name: 'TT_OVMS_IMAGE_NAME',
+            defaultValue: 'openvino/model_server:latest',
+            description: 'Full OVMS image name, e.g. openvino/model_server:latest. Empty means config default (None)'
+        )
+        booleanParam(
+            name: 'TT_OVMS_IMAGE_LOCAL',
+            defaultValue: false,
+            description: 'Whether the OVMS image is available only locally. Default matches config.py: False'
+        )
+        string(
+            name: 'TT_LOGGING_LEVEL_OVMS',
+            defaultValue: 'INFO',
+            description: 'OVMS container log level. Default matches config.py: INFO'
+        )
+        booleanParam(
+            name: 'TT_ON_COMMIT_TESTS',
+            defaultValue: true,
+            description: 'Run on-commit tests. Default matches config.py: True'
+        )
+        booleanParam(
+            name: 'TT_RUN_REGRESSION_TESTS',
+            defaultValue: false,
+            description: 'Run regression tests. Default matches config.py: False'
+        )
+        text(
+            name: 'TEST_PARAMETERS',
+            defaultValue: '',
+            description: 'Extra shell environment assignments to apply before pytest, one per line. Example: TT_TEST=21'
+        )
+    }
+    stages {
+        stage('Run functional tests') {
+            agent {
+                label "${params.TARGET_HOST}"
+            }
+            steps {
+                script {
+                    if (!(params.TARGET_HOST ==~ /[a-zA-Z0-9_.-]+/)) {
+                        error "Invalid TARGET_HOST '${params.TARGET_HOST}'. Allowed characters: letters, digits, dot, underscore, hyphen."
+                    }
+                    def envAssignments = params.TEST_PARAMETERS
+                        .readLines()
+                        .findAll { line -> !line.trim().isEmpty() }
+                        .collect { line -> line.trim() }
+                        .join(' ')
+                    def buildDir = "${env.WORKSPACE}/job-${env.BUILD_NUMBER}"
+                    ws(buildDir) {
+                        checkout([$class: 'GitSCM', branches: [[name: "*/${params.CORE_BRANCH}"]], userRemoteConfigs: [[url: scm.userRemoteConfigs[0].url, credentialsId: scm.userRemoteConfigs[0].credentialsId]]])
+                        sh """
+                            set -eux
+                            test -d .venv || python3 -m venv .venv
+                            . .venv/bin/activate
+                            python -m pip install --upgrade pip
+                            python -m pip install -r tests/requirements.txt
+                            export TT_XDIST_WORKERS='${params.TT_XDIST_WORKERS}'
+                            export TT_TARGET_DEVICE='${params.TT_TARGET_DEVICE}'
+                            export TT_OVMS_IMAGE_NAME='${params.TT_OVMS_IMAGE_NAME}'
+                            export TT_OVMS_IMAGE_LOCAL='${params.TT_OVMS_IMAGE_LOCAL}'
+                            export TT_LOGGING_LEVEL_OVMS='${params.TT_LOGGING_LEVEL_OVMS}'
+                            export TT_ON_COMMIT_TESTS='${params.TT_ON_COMMIT_TESTS}'
+                            export TT_RUN_REGRESSION_TESTS='${params.TT_RUN_REGRESSION_TESTS}'
+                            ${envAssignments} pytest ${params.PYTEST_PARAMS} -n ${params.TT_XDIST_WORKERS} --junitxml=pytest-functional.xml
+                        """
+                        junit allowEmptyResults: true, testResults: 'pytest-functional.xml'
+                        archiveArtifacts allowEmptyArchive: true, artifacts: 'pytest-functional.xml,test_log/**,tests/functional/test_log_build/**'
+                    }
+                }
+            }
+        }
+    }
+}

docs/developer_guide.md

@@ -174,16 +174,14 @@ make test_functional
 
 - Configuration options are :
 
-| Variable    | Description |
-| :---        |    :----   |
-| `IMAGE`  | Docker image name for the tests.|
-| `TEST_DIR_CACHE`| Location from which models and test data are downloaded.|
-| `TEST_DIR` | Location to which models and test data are copied during tests.|
-| `TEST_DIR_CLEANUP` | Set to `True` to remove the directory under `TEST_DIR` after the tests.|
-| `LOG_LEVEL` | The log level.|
-| `BUILD_LOGS` | Path to save artifacts.|
-| `START_CONTAINER_COMMAND` | The command to start the OpenVINO Model Storage container.|
-| `CONTAINER_LOG_LINE` | The log line in the container that confirms the container started properly.|
+| Variable                  | Description                                                |
+|:--------------------------|:-----------------------------------------------------------|
+| `OVMS_CPP_DOCKER_IMAGE`   | Docker image name for the tests.                           |
+| `OVMS_CPP_IMAGE_TAG`      | Docker image tag for the tests.                            |
+| `TT_LOGGING_LEVEL`        | The log level for tests.                                   |
+| `TT_LOGGING_LEVEL_OVMS`   | The log level for OVMS.                                    |
+| `BUILD_LOGS`              | Path to save artifacts.                                    |
+| `START_CONTAINER_COMMAND` | The command to start the OpenVINO Model Storage container. |
 
 2. Add any configuration variables to the command line in this format :
 
@@ -464,7 +462,7 @@ Use OpenVINO Model Server build image because it installs the necessary tools.
 
 4. Run a test in this terminal. Change `TEST_PATH` to point to the test you want to debug:
 	```bash
-	make test_functional TEST_PATH=tests/functional/test_batching.py::TestBatchModelInference::test_run_inference_rest IMAGE=openvino/model_server-build:latest
+	make test_functional TEST_PATH=tests/functional/test_embeddings.py::TestEmbeddings::test_on_commit_embeddings_endpoints TT_OVMS_IMAGE_NAME=openvino/model_server-build:latest
 	```
 
 5. Open a second terminal.

tests/functional/config.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# pylint: disable=pointless-string-statement
 
 import os
 import re
+
+from collections import defaultdict
 from pathlib import Path
 
 from tests.functional.constants.os_type import OsType
@@ -33,14 +36,14 @@
 
 try:
     # In user_config.py, user might export custom environment variables
-    import user_config
+    import user_config  # pylint: disable=unused-import
 except ImportError:
     pass
 
 
 def get_uses_mapping():
     _uses_mapping = get_list("TT_USES_MAPPING", fallback=[None])
-    _uses_mapping = list(set([str(x).upper() for x in _uses_mapping]))  # make upper & remove duplicates
+    _uses_mapping = list({str(x).upper() for x in _uses_mapping})  # make upper & remove duplicates
     # Reduce to True/False/None
     _uses_mapping = [x == "TRUE" if x in ["TRUE", "FALSE"] else None for x in _uses_mapping]
     validate_supported_values(_uses_mapping, [True, False, None])
@@ -52,17 +55,14 @@ def get_uses_mapping():
     Possible TT_USES_MAPPING values (case insensitive):
     - (empty)/""/NONE - Default leave mapping.json provided alongside model untouched (if exists).
     - FALSE - forcibly remove mapping.json if provided with model.
-    - TRUE - remove any previous mapping and add generic mapping.json 
+    - TRUE - remove any previous mapping and add generic mapping.json
             (see: ovms/object_model/ovms_mapping_config.py for details)
     - TRUE,FALSE,NONE - Iterate each test case from listed values in single test session.
 """
 uses_mapping = get_uses_mapping()
 
 """TEST_DIR -  location where models and test data should be copied from TEST_DIR_CACHE and deleted after tests"""
-test_dir = os.environ.get("TEST_DIR", "/tmp/{}".format(generate_test_object_name(prefix='ovms_models')))
-
-"""TEST_DIR_CACHE -  location where models and test data should be downloaded to and serve as cache for TEST_DIR"""
-test_dir_cache = os.environ.get("TEST_DIR_CACHE", "/tmp/ovms_models_cache")
+test_dir = os.environ.get("TEST_DIR", f"/tmp/{generate_test_object_name(prefix='ovms_models')}")
 
 """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """
 ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./"))
@@ -104,22 +104,25 @@ def get_uses_mapping():
 """ TT_WIN_PY_VERSION - Python version for virtualenv on Windows OS """
 windows_python_version = os.environ.get("TT_WIN_PY_VERSION", "3.12")
 
-""" TT_DOCKER_REGISTRY - Docker registry"""
+""" TT_DOCKER_REGISTRY - Docker registry """
 docker_registry = os.environ.get("TT_DOCKER_REGISTRY", None)
 
 """ OVMS_CPP_DOCKER_IMAGE """
-ovms_cpp_docker_image = os.environ.get("OVMS_CPP_DOCKER_IMAGE", None)
-
-""" TT_OVMS_IMAGE_NAME """
-ovms_image = os.environ.get("TT_OVMS_IMAGE_NAME", None)
+ovms_cpp_docker_image = os.environ.get("OVMS_CPP_DOCKER_IMAGE", "openvino/model_server")
 
 """ OVMS_CPP_IMAGE_TAG - tag of OVMS image to test (compatible with build parameter) """
-ovms_image_tag = os.environ.get("OVMS_CPP_IMAGE_TAG", None)
+ovms_image_tag = os.environ.get("OVMS_CPP_IMAGE_TAG", "latest")
+
+""" TT_DEFAULT_OVMS_IMAGE_TAG - default value of ovms image tag (based on OS) """
+ovms_image_tag_dict = defaultdict(lambda: ovms_image_tag)
+
+""" TT_OVMS_IMAGE_NAME - full image name (name + tag) """
+ovms_image = os.environ.get("TT_OVMS_IMAGE_NAME", None)
 
 """ TT_OVMS_TEST_IMAGE_NAME - image name for cpu extensions and custom nodes """
 ovms_test_image_name = os.environ.get("TT_OVMS_TEST_IMAGE_NAME", None)
 
-""" TT_FORCE_USE_OVMS_IMAGE - force to use TT_OVMS_IMAGE_NAME """
+""" TT_FORCE_USE_OVMS_IMAGE - force to use given image parameters (skip automatic suffix updates) """
 force_use_ovms_image = get_bool("TT_FORCE_USE_OVMS_IMAGE", False)
 
 """ TT_OVMS_C_RELEASE_ARTIFACTS_PATH - path to current release artifacts """
@@ -130,22 +133,9 @@ def get_uses_mapping():
 """START_CONTAINER_COMMAND - command to start ovms container"""
 start_container_command = os.environ.get("START_CONTAINER_COMMAND", "")
 
-"""CONTAINER_LOG_LINE - log line to check in container"""
-# For multiple log lines, pass them separated with ':'
-container_log_line = os.environ.get("CONTAINER_LOG_LINE", "Started model manager thread")
-container_log_line = container_log_line.split(":")
-
 """OVMS_BINARY_PATH - path to ovms binary file; when specified, tests are executed against provided binary."""
 ovms_binary_path = os.environ.get("OVMS_BINARY_PATH", None)
 
-"""LOG_LEVEL - set log level """
-log_level = os.environ.get("LOG_LEVEL", "INFO")
-
-path_to_mount = os.path.join(test_dir, "saved_models")
-os.makedirs(path_to_mount, exist_ok=True)
-
-path_to_mount_cache = os.path.join(test_dir_cache, "saved_models")
-
 """ TT_MINIO_IMAGE_NAME - Docker image for Minio"""
 minio_image = os.environ.get(
     "TT_MINIO_IMAGE_NAME",
@@ -430,3 +420,11 @@ def get_ovms_types():
 
 """ TT_HUGGINGFACE_TOKEN - huggingface token value. Env var takes priority, then file. """
 huggingface_token = os.environ.get("TT_HUGGINGFACE_TOKEN") or get_token_value(huggingface_token_file_path, "")
+
+"""TT_ON_COMMIT_TESTS - False -> api-on-commit tests are not run,
+                        True -> api-on-commit tests are run, default: True """
+run_on_commit_tests = get_bool("TT_ON_COMMIT_TESTS", True)
+
+"""TT_RUN_REGRESSION_TESTS - False -> api-regression tests are not run,
+                             True -> api-regression tests are run, default: False """
+run_regression_tests = get_bool("TT_RUN_REGRESSION_TESTS", False)

tests/functional/conftest.py

@@ -16,8 +16,10 @@
 
 import random
 import sys
+import pytest
 
 from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter, machine_is_reserved_for_test_session
+from tests.functional.constants.components import OvmsComponents
 from tests.functional.constants.ovms import (
     CURRENT_TARGET_DEVICE_DICT_ARGUMENT,
     TMP_REPOS_DIR_ARGUMENT,
@@ -32,8 +34,6 @@
 
 if enable_pytest_plugins:
 
-    raise NotImplementedError("OVMS tests not enabled")
-
     pytest_plugins = [      # pylint: disable=unreachable
         "tests.functional.fixtures.ovms",
         "tests.functional.fixtures.server",