From 2e4071b010e02b62db5b738a8a43f0788554bfae Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Thu, 9 Jul 2026 12:01:48 +0200 Subject: [PATCH 1/6] Tests: fix common module import in test_get_entity_eids --- tests/test_api/test_get_entity_eids.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_api/test_get_entity_eids.py b/tests/test_api/test_get_entity_eids.py index 43cd2b92..07a5f1c9 100644 --- a/tests/test_api/test_get_entity_eids.py +++ b/tests/test_api/test_get_entity_eids.py @@ -2,9 +2,8 @@ import sys from time import sleep -import common - from dp3.api.internal.entity_response_models import EntityEidList +from tests.test_api import common class GetEntityEids(common.APITest): From da098ea81847e214ca2d3facb0b3681008bcd823 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Mon, 13 Jul 2026 13:54:15 +0200 Subject: [PATCH 2/6] API(entity.py): add EID sorting on ednpoint entity/{etype}/get --- dp3/api/routers/entity.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index 7b3b5f52..bc4ca85a 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -144,6 +144,7 @@ def _validate_sort_params(etype: str, sort: list[str] | None) -> list[tuple[str, sort: list of sort specifications in format 'attribute:direction' where direction is 1 (ascending) or -1 (descending) e.g., ['hostname:-1', 'ip:1'] + Special attribute 'eid' is also supported for sorting by entity ID. Returns: List of (attribute, direction) tuples for sorting, or None if no sorting specified @@ -172,6 +173,11 @@ def _validate_sort_params(etype: str, sort: list[str] | None) -> list[tuple[str, attr, direction_str = match.groups() direction = int(direction_str) if direction_str else 1 # Default to ascending (1) + # Support sorting by entity ID using pseudo-attribute 'eid' + if attr == "eid": + sort_criteria.append((attr, direction)) + continue + # Check if attribute exists if attr not in entity_attribs: raise RequestValidationError( @@ -306,11 +312,13 @@ async def get_entity_type_eids( Generic and fulltext filters are merged - fulltext overrides conflicting keys. - Sorting is supported for plain and observations attributes with primitive data types - (excluding json and multi_value observations). To sort by multiple attributes, provide - multiple sort parameters in the format 'attribute:direction' where direction is 1 (ascending) - or -1 (descending). Direction defaults to 1 (ascending) if not provided. Example: - `?sort=hostname:-1&sort=ip:1` + Sorting is supported by entity ID using the special attribute name `eid`, + as well as for plain and observations attributes with primitive data types + (excluding json and multi_value observations). To sort by multiple attributes, + provide multiple sort parameters in the format 'attribute:direction' where + direction is 1 (ascending) or -1 (descending). Direction defaults to 1 (ascending) + if not provided. Examples: + `?sort=eid:1`, `?sort=hostname:-1&sort=ip:1` """ fulltext_filters, generic_filter = _validate_snapshot_filters(fulltext_filters, generic_filter) sort_criteria = _validate_sort_params(etype, sort) @@ -320,8 +328,12 @@ async def get_entity_type_eids( # Apply sorting if specified if sort_criteria: - # Prepare sort specification with 'last.' prefix for snapshot data - sort_spec = [("last." + attr, direction) for attr, direction in sort_criteria] + # Prepare sort specification with 'last.' prefix for snapshot data. + # The special attribute 'eid' is mapped to the EID stored in the snapshot. + sort_spec = [ + ("last.eid" if attr == "eid" else "last." + attr, direction) + for attr, direction in sort_criteria + ] cursor = cursor.sort(sort_spec) cursor_page = cursor.skip(skip).limit(limit) From 7bb46386cc0a493dea148b97b6a7c4425aec58d6 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Mon, 13 Jul 2026 13:56:09 +0200 Subject: [PATCH 3/6] Tests(test_get_entity_eids): add EID sorting tests --- tests/test_api/test_get_entity_eids.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_api/test_get_entity_eids.py b/tests/test_api/test_get_entity_eids.py index 07a5f1c9..262ff039 100644 --- a/tests/test_api/test_get_entity_eids.py +++ b/tests/test_api/test_get_entity_eids.py @@ -59,3 +59,13 @@ def test_get_entity_eids_generic_filters_eid(self): {5, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59}, {x["eid"] for x in eids.data}, ) + + def test_get_entity_eids_sort_by_eid_asc(self): + eids = self.get_entity_data("entity/A/get", EntityEidList, sort=["eid:1"], limit=0) + self.assertEqual(100, len(eids.data)) + self.assertEqual(list(range(100)), [x["eid"] for x in eids.data]) + + def test_get_entity_eids_sort_by_eid_desc(self): + eids = self.get_entity_data("entity/A/get", EntityEidList, sort=["eid:-1"], limit=0) + self.assertEqual(100, len(eids.data)) + self.assertEqual(list(range(99, -1, -1)), [x["eid"] for x in eids.data]) From b6432fef1abe6b4d5f7ed40804b8c5ed3c070d2d Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Mon, 13 Jul 2026 17:22:59 +0200 Subject: [PATCH 4/6] TESTS API(test_get_entity_eids): fix bad import common module --- tests/test_api/test_get_entity_eids.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_api/test_get_entity_eids.py b/tests/test_api/test_get_entity_eids.py index 262ff039..902e53c5 100644 --- a/tests/test_api/test_get_entity_eids.py +++ b/tests/test_api/test_get_entity_eids.py @@ -2,8 +2,9 @@ import sys from time import sleep +import common + from dp3.api.internal.entity_response_models import EntityEidList -from tests.test_api import common class GetEntityEids(common.APITest): From 931f5e9c9e7c4837c4659f7d8473ec5ded7972f3 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Tue, 14 Jul 2026 12:45:20 +0200 Subject: [PATCH 5/6] API TESTS(get_entity_edids): fix query param when test eid sort, add test for multi attr sort --- tests/test_api/test_get_entity_eids.py | 60 +++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/test_api/test_get_entity_eids.py b/tests/test_api/test_get_entity_eids.py index 902e53c5..a8cc1f72 100644 --- a/tests/test_api/test_get_entity_eids.py +++ b/tests/test_api/test_get_entity_eids.py @@ -62,11 +62,67 @@ def test_get_entity_eids_generic_filters_eid(self): ) def test_get_entity_eids_sort_by_eid_asc(self): - eids = self.get_entity_data("entity/A/get", EntityEidList, sort=["eid:1"], limit=0) + eids = self.get_entity_data("entity/A/get", EntityEidList, sort="eid:1", limit=0) self.assertEqual(100, len(eids.data)) self.assertEqual(list(range(100)), [x["eid"] for x in eids.data]) def test_get_entity_eids_sort_by_eid_desc(self): - eids = self.get_entity_data("entity/A/get", EntityEidList, sort=["eid:-1"], limit=0) + eids = self.get_entity_data("entity/A/get", EntityEidList, sort="eid:-1", limit=0) self.assertEqual(100, len(eids.data)) self.assertEqual(list(range(99, -1, -1)), [x["eid"] for x in eids.data]) + + def test_get_entity_eids_pagination_with_sort(self): + received_eids = [] + for i in range(0, 100, 10): + eids = self.get_entity_data( + "entity/A/get", EntityEidList, sort="eid:1", skip=i, limit=10 + ) + self.assertEqual(10, len(eids.data), f"Failed at {i}") + received_eids.extend(x["eid"] for x in eids.data) + self.assertEqual(list(range(100)), received_eids) + + received_eids = [] + for i in range(0, 100, 10): + eids = self.get_entity_data( + "entity/A/get", EntityEidList, sort="eid:-1", skip=i, limit=10 + ) + self.assertEqual(10, len(eids.data), f"Failed at {i}") + received_eids.extend(x["eid"] for x in eids.data) + self.assertEqual(list(range(99, -1, -1)), received_eids) + + def _get_sorted_eids(self, sort_params: list[str], **kwargs) -> EntityEidList: + """Fetch entity EIDs with multiple sort query parameters. + + The shared `get_request` helper joins kwargs with '&' and cannot emit + multiple values for the same key, which is required for multi-column + sorting. Build the query string explicitly here. + """ + query_parts = [f"sort={param}" for param in sort_params] + for key, value in kwargs.items(): + query_parts.append(f"{key}={value}") + response = self.get_request(f"entity/A/get?{'&'.join(query_parts)}") + self.assertEqual(response.status_code, 200) + return EntityEidList.model_validate_json(response.content) + + def test_get_entity_eids_sort_by_multiple_attrs(self): + res = self.push_datapoints( + [ + {"src": "setup@test", "attr": "data2", "type": "A", "id": i, "v": f"g{i % 5}"} + for i in range(0, 100) + ] + ) + self.assertEqual(res.status_code, 200) + sleep(8) + self.get_request("control/make_snapshots") + sleep(6) + + expected = sorted(range(100), key=lambda i: (f"g{i % 5}", i)) + # omit :1 if sort is ascending + eids = self._get_sorted_eids(["data2", "eid"], limit=0) + self.assertEqual(100, len(eids.data)) + self.assertEqual(expected, [x["eid"] for x in eids.data]) + + expected = sorted(range(100), key=lambda i: (f"g{i % 5}", -i)) + eids = self._get_sorted_eids(["data2:1", "eid:-1"], limit=0) + self.assertEqual(100, len(eids.data)) + self.assertEqual(expected, [x["eid"] for x in eids.data]) From eaa1f68854e393cb7e4c69d271857bceca99ed72 Mon Sep 17 00:00:00 2001 From: Lukas Simonik Date: Tue, 14 Jul 2026 14:01:08 +0200 Subject: [PATCH 6/6] chore(entity.py): delete meaningless if --- dp3/api/routers/entity.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index bc4ca85a..e5c4231b 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -328,12 +328,8 @@ async def get_entity_type_eids( # Apply sorting if specified if sort_criteria: - # Prepare sort specification with 'last.' prefix for snapshot data. - # The special attribute 'eid' is mapped to the EID stored in the snapshot. - sort_spec = [ - ("last.eid" if attr == "eid" else "last." + attr, direction) - for attr, direction in sort_criteria - ] + # Prepare sort specification with 'last.' prefix for snapshot data + sort_spec = [("last." + attr, direction) for attr, direction in sort_criteria] cursor = cursor.sort(sort_spec) cursor_page = cursor.skip(skip).limit(limit)