GSoC Module_A-week7: deduplication and metrics (stacked on top of #1029)

application/tests/harvester_test/artifact_registry_test.py

@@ -0,0 +1,99 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.artifact_registry import ArtifactRegistry
+from application.utils.harvester.models import ArtifactRegistryRecord
+
+
+class ArtifactRegistryTests(unittest.TestCase):
+    def test_insert_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        self.assertTrue(registry.exists(record.artifact_id))
+
+    def test_get_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        stored = registry.get(record.artifact_id)
+        assert stored is not None
+
+        self.assertEqual(stored.content_hash, "abc")
+
+    def test_update_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        record.content_hash = "xyz"
+        record.status = "updated"
+
+        registry.upsert(record)
+
+        stored = registry.get(record.artifact_id)
+        assert stored is not None
+
+        self.assertEqual(stored.content_hash, "xyz")
+        self.assertEqual(stored.status, "updated")
+
+    def test_all_records(self):
+        registry = ArtifactRegistry()
+
+        for i in range(3):
+            registry.upsert(
+                ArtifactRegistryRecord(
+                    artifact_id=f"art:{i}",
+                    repository="repo",
+                    locator_path=f"{i}.md",
+                    content_hash=str(i),
+                    last_commit_sha="sha",
+                    last_pipeline_run="run",
+                    last_processed_at=datetime.now(),
+                    status="new",
+                )
+            )
+
+        self.assertEqual(
+            len(registry.all()),
+            3,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/checkpoint_manager_test.py

@@ -0,0 +1,68 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.checkpoint_manager import CheckpointManager
+from application.utils.harvester.models import CheckpointRecord
+
+
+class CheckpointManagerTests(unittest.TestCase):
+    def test_save_checkpoint(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+
+        self.assertIsNotNone(manager.get("OWASP/ASVS"))
+
+    def test_update_commit(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+
+        manager.update_commit(
+            "OWASP/ASVS",
+            "deadbeef",
+        )
+
+        stored = manager.get("OWASP/ASVS")
+        assert stored is not None
+
+        self.assertEqual(stored.last_processed_commit, "deadbeef")
+
+    def test_mark_completed(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+        manager.mark_completed("OWASP/ASVS")
+
+        stored = manager.get("OWASP/ASVS")
+        assert stored is not None
+
+        self.assertEqual(stored.status, "completed")
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/content_hash_test.py

@@ -0,0 +1,35 @@
+import unittest
+
+from application.utils.harvester.content_hash import (
+    generate_content_hash,
+)
+
+
+class ContentHashTests(unittest.TestCase):
+    def test_same_text_same_hash(self):
+        text = "Hello World"
+
+        self.assertEqual(
+            generate_content_hash(text),
+            generate_content_hash(text),
+        )
+
+    def test_different_text_different_hash(self):
+        self.assertNotEqual(
+            generate_content_hash("Hello"),
+            generate_content_hash("World"),
+        )
+
+    def test_empty_string(self):
+        digest = generate_content_hash("")
+
+        self.assertEqual(len(digest), 64)
+
+    def test_hash_is_hex(self):
+        digest = generate_content_hash("OpenCRE")
+
+        int(digest, 16)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/deduplication_metrics_test.py

@@ -0,0 +1,35 @@
+import unittest
+
+from application.utils.harvester.deduplication_metrics import DeduplicationMetrics
+from application.utils.harvester.models import DeduplicationStatus
+
+
+class DeduplicationMetricsTests(unittest.TestCase):
+    def test_records_new_document(self):
+        metrics = DeduplicationMetrics()
+
+        metrics.record(DeduplicationStatus.NEW)
+
+        self.assertEqual(metrics.total_artifacts_scanned, 1)
+        self.assertEqual(metrics.artifacts_new, 1)
+        self.assertEqual(metrics.artifacts_emitted, 1)
+
+    def test_records_updated_document(self):
+        metrics = DeduplicationMetrics()
+
+        metrics.record(DeduplicationStatus.UPDATED)
+
+        self.assertEqual(metrics.artifacts_updated, 1)
+        self.assertEqual(metrics.artifacts_emitted, 1)
+
+    def test_records_unchanged_document(self):
+        metrics = DeduplicationMetrics()
+
+        metrics.record(DeduplicationStatus.UNCHANGED)
+
+        self.assertEqual(metrics.artifacts_unchanged, 1)
+        self.assertEqual(metrics.artifacts_skipped, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/diff_parser_test.py

@@ -10,6 +10,11 @@
 TEST_COMMITTED_AT = datetime.now(UTC)
 
 
+TEST_REPOSITORY = "OWASP/ASVS"
+TEST_COMMIT_SHA = "abc123"
+TEST_COMMITTED_AT = datetime.now(UTC)
+
+
 class DiffParserTests(unittest.TestCase):
     def test_single_file_diff(self):
         parser = DiffParser()

application/tests/harvester_test/diff_pipeline_test.py

@@ -1,8 +1,8 @@
 from datetime import UTC, datetime
+import os
 import subprocess
 import time
 import unittest
-import os
 
 from application.utils.harvester.diff_normalizer import DiffNormalizer
 from application.utils.harvester.diff_parser import DiffParser
@@ -19,7 +19,6 @@ class DiffPipelineBenchmark(unittest.TestCase):
     """
 
     def test_pipeline_benchmark(self):
-
         if os.getenv("OPENCRE_RUN_NETWORK_TESTS") != "1":
             self.skipTest("Network benchmark disabled")
 
@@ -29,9 +28,7 @@ def test_pipeline_benchmark(self):
             "master",
         )
         client.sync()
-
         head_commit = client.get_current_commit_sha()
-
         previous_commit = subprocess.run(
             [
                 "git",
@@ -51,23 +48,18 @@ def test_pipeline_benchmark(self):
         normalizer = DiffNormalizer()
 
         start = time.perf_counter()
-
         diff = retriever.get_diff(
             previous_commit,
             head_commit,
         )
-
         blocks = parser.parse(
             diff,
             repository="OWASP/ASVS",
             commit_sha=head_commit,
             committed_at=datetime.now(UTC),
         )
-
         normalizer.normalize(blocks)
-
         elapsed = time.perf_counter() - start
 
         print(f"\nPipeline took {elapsed:.3f}s")
-
         self.assertLess(elapsed, 5)

application/tests/harvester_test/diff_retriever_test.py

@@ -1,7 +1,7 @@
 import unittest
 from unittest.mock import MagicMock
-from unittest.mock import patch
 from unittest.mock import call
+from unittest.mock import patch
 
 from application.utils.harvester.diff_retriever import (
     DiffRetriever,
@@ -16,10 +16,8 @@ def test_get_diff(self, mock_run):
             MagicMock(stdout="def456\n"),
             MagicMock(stdout=b"diff --git a/README.md b/README.md\n"),
         ]
-
         client = MagicMock()
         client.get_local_path.return_value = "/tmp/repo"
-
         retriever = DiffRetriever(client)
 
         diff = retriever.get_diff(
@@ -31,7 +29,6 @@ def test_get_diff(self, mock_run):
             diff,
             "diff --git a/README.md b/README.md\n",
         )
-
         mock_run.assert_has_calls(
             [
                 call(
@@ -88,7 +85,6 @@ def test_large_diff_raises(self, mock_run):
 
         client = MagicMock()
         client.get_local_path.return_value = "/tmp/repo"
-
         retriever = DiffRetriever(client)
 
         with self.assertRaises(ValueError):