diff --git a/datawrapper/__init__.py b/datawrapper/__init__.py index c2d0f00..2323d2d 100644 --- a/datawrapper/__init__.py +++ b/datawrapper/__init__.py @@ -23,6 +23,8 @@ ColumnChart, ColumnFormat, ConnectorLine, + DataChange, + DataChangeList, Describe, Line, LineChart, @@ -96,6 +98,8 @@ "BaseChart", "Annotate", "ColumnFormat", + "DataChange", + "DataChangeList", "Transform", "Describe", "BarChart", diff --git a/datawrapper/charts/__init__.py b/datawrapper/charts/__init__.py index 53fde8d..1d839db 100644 --- a/datawrapper/charts/__init__.py +++ b/datawrapper/charts/__init__.py @@ -40,6 +40,8 @@ ColumnFormatList, CustomRangeMixin, CustomTicksMixin, + DataChange, + DataChangeList, Describe, GridDisplayMixin, GridFormatMixin, @@ -83,6 +85,8 @@ "ColumnFormatList", "CustomRangeMixin", "CustomTicksMixin", + "DataChange", + "DataChangeList", "GridFormatMixin", "GridDisplayMixin", "ArrowHead", diff --git a/datawrapper/charts/models/__init__.py b/datawrapper/charts/models/__init__.py index 0240be7..3569eac 100644 --- a/datawrapper/charts/models/__init__.py +++ b/datawrapper/charts/models/__init__.py @@ -24,7 +24,13 @@ YRangeAnnotation, ) from .text_annotations import ConnectorLine, TextAnnotation -from .transforms import ColumnFormat, ColumnFormatList, Transform +from .transforms import ( + ColumnFormat, + ColumnFormatList, + DataChange, + DataChangeList, + Transform, +) __all__ = [ "Annotate", @@ -34,6 +40,8 @@ "ConnectorLine", "CustomRangeMixin", "CustomTicksMixin", + "DataChange", + "DataChangeList", "Describe", "GridDisplayMixin", "GridFormatMixin", diff --git a/datawrapper/charts/models/transforms.py b/datawrapper/charts/models/transforms.py index 2372db6..2907963 100644 --- a/datawrapper/charts/models/transforms.py +++ b/datawrapper/charts/models/transforms.py @@ -236,6 +236,158 @@ def __getitem__(self, index): return self.formats[index] +class DataChange(BaseModel): + """A data-value correction from Datawrapper's Check & Describe step. + + Datawrapper stores individual cell edits in ``metadata.data.changes``. Row + and column are zero-based indexes into the uploaded dataset, value is the + replacement value, and time is the edit timestamp in Unix milliseconds when + Datawrapper includes it. Overwritten cells can include previous, while + added-column/header edits can omit it. + """ + + model_config = ConfigDict( + populate_by_name=True, + strict=True, + json_schema_extra={ + "examples": [ + { + "row": 9, + "column": 3, + "value": "1.7", + "time": 1573134075869, + "previous": "0.7", + } + ] + }, + ) + + #: The zero-based row index for the change + row: int = Field( + ge=0, + description="The zero-based row index for the data change", + ) + + #: The zero-based column index for the change + column: int = Field( + ge=0, + description="The zero-based column index for the data change", + ) + + #: The replacement value stored by Datawrapper + value: str = Field(description="The replacement value for the changed cell") + + #: Unix time in milliseconds when the change was made, when available + time: int | None = Field( + default=None, + ge=0, + description="Unix time in milliseconds when the change was made", + ) + + #: The original value overwritten by the correction, when Datawrapper includes it + previous: str | None = Field( + default=None, + description="The original value overwritten by the correction", + ) + + #: Datawrapper's internal change ID, when returned by the API + id: str | None = Field( + default=None, + description="Datawrapper's internal change ID", + ) + + #: Whether Datawrapper marks the change as ignored + ignored: bool | None = Field( + default=None, + description="Whether Datawrapper marks the change as ignored", + ) + + #: Datawrapper's internal ordering index, when returned by the API + index: int | None = Field( + default=None, + alias="_index", + ge=0, + description="Datawrapper's internal ordering index", + ) + + @model_serializer + def serialize_change(self) -> dict[str, Any]: + """Serialize only fields that Datawrapper supplied or the user set.""" + result: dict[str, Any] = { + "row": self.row, + "column": self.column, + "value": self.value, + } + if self.time is not None: + result["time"] = self.time + if "previous" in self.model_fields_set: + result["previous"] = self.previous + if self.id is not None: + result["id"] = self.id + if self.ignored is not None: + result["ignored"] = self.ignored + if self.index is not None: + result["_index"] = self.index + return result + + +class DataChangeList(BaseModel): + """Collection wrapper for Datawrapper's ``metadata.data.changes`` value. + + Datawrapper's public docs show ``changes`` as a list of change objects. Some + API responses return an object keyed by internal change IDs instead. This + wrapper accepts both forms and serializes back to the same form it received; + direct user-created lists remain lists. + """ + + model_config = ConfigDict(populate_by_name=True) + + #: The list of data changes + changes: list[DataChange] = Field(default_factory=list) + + #: Internal keys from API object-shaped changes, when that shape was supplied + keys: list[str] | None = Field(default=None, exclude=True) + + @model_validator(mode="before") + @classmethod + def convert_from_dict_or_list(cls, data: Any) -> dict[str, Any]: + """Accept docs-style lists, API object maps, or wrapped values.""" + if isinstance(data, cls): + return {"changes": data.changes, "keys": data.keys} + if isinstance(data, dict) and "changes" in data: + return data + if isinstance(data, dict): + keys = list(data.keys()) + return {"changes": list(data.values()), "keys": keys} + if isinstance(data, list): + return {"changes": data} + return data + + @model_serializer + def serialize_changes(self) -> list[dict[str, Any]] | dict[str, dict[str, Any]]: + """Serialize to the same changes container shape that was provided.""" + serialized = [change.model_dump(by_alias=True) for change in self.changes] + if self.keys is not None and len(self.keys) == len(serialized): + return dict(zip(self.keys, serialized, strict=True)) + return serialized + + def __iter__(self): + """Allow iteration over the changes list.""" + return iter(self.changes) + + def __len__(self): + """Return the number of changes.""" + return len(self.changes) + + def __getitem__(self, index): + """Allow indexing into the changes list.""" + return self.changes[index] + + def __bool__(self): + """Return whether any changes are present.""" + return bool(self.changes) + + class Transform(BaseModel): """A model for the Datawrapper API's 'data' metadata attribute.""" @@ -251,6 +403,15 @@ class Transform(BaseModel): "column-format": [ {"column": "sales", "type": "number", "number-prepend": "$"} ], + "changes": [ + { + "row": 9, + "column": 3, + "value": "1.7", + "time": 1573134075869, + "previous": "0.7", + } + ], "external-data": "", "use-datawrapper-cdn": True, "upload-method": "copy", @@ -288,6 +449,13 @@ class Transform(BaseModel): description="The formatting options for the data columns", ) + #: Individual value corrections made in the Check & Describe tab + changes: DataChangeList = Field( + default_factory=DataChangeList, + exclude_if=lambda value: not value, + description="Individual value corrections made in the Check & Describe tab", + ) + #: An external data source URL external_data: str = Field( default="", alias="external-data", description="An external data source URL" diff --git a/docs/user-guide/api/models.rst b/docs/user-guide/api/models.rst index 4ecc5aa..183fb7d 100644 --- a/docs/user-guide/api/models.rst +++ b/docs/user-guide/api/models.rst @@ -53,6 +53,57 @@ Column Format :members: :show-inheritance: +Data Changes +------------ +.. currentmodule:: datawrapper.charts.models + +``DataChange`` models individual cell corrections made in Datawrapper's +**Check & Describe** tab. Datawrapper stores these corrections under +``metadata.data.changes`` as zero-based row/column indexes, the replacement +``value``, an edit ``time`` in Unix milliseconds when available, and +``previous`` for overwritten cells. The ``previous`` field is optional because +Datawrapper omits it for entries such as added-column headers. + +Datawrapper's developer docs show ``changes`` as a list. Existing API responses +can also return an object keyed by internal change IDs, with fields such as +``id``, ``ignored``, and ``_index``. ``Transform`` accepts both shapes and +round-trips object-shaped API responses without converting them to lists. + +Live API smoke verification for this behavior used an authorized unpublished +Datawrapper test chart. The API accepted an object-map ``changes`` payload, +returned that object-map shape on read-back, and preserved the relevant +``id``, ``ignored``, and ``_index`` fields after a ``Transform`` +deserialize/serialize write-back. The documented list-shaped compatibility +input was also accepted and read back as a list. The smoke restored the test +chart's corrections to an empty object and did not publish or verify behavior +for production-facing charts. + +.. code-block:: python + + chart = dw.BarChart( + title="Corrected values", + data=df, + transformations=dw.Transform( + changes=[ + dw.DataChange( + row=9, + column=3, + value="1.7", + time=1573134075869, + previous="0.7", + ) + ] + ), + ) + +.. autoclass:: DataChange + :members: + :show-inheritance: + +.. autoclass:: DataChangeList + :members: + :show-inheritance: + Line Configuration ------------------ .. currentmodule:: datawrapper.charts.line diff --git a/tests/fixtures/metadata_data_changes.json b/tests/fixtures/metadata_data_changes.json new file mode 100644 index 0000000..cbf908d --- /dev/null +++ b/tests/fixtures/metadata_data_changes.json @@ -0,0 +1,38 @@ +{ + "type": "d3-bars", + "title": "Corrections fixture", + "language": "en-US", + "metadata": { + "data": { + "transpose": false, + "vertical-header": true, + "horizontal-header": true, + "changes": [ + { + "row": 9, + "time": 1573134075869, + "value": "1.7", + "column": 3, + "previous": "0.7" + }, + { + "row": 0, + "time": 1573134111144, + "value": "Extra column", + "column": 5 + } + ], + "column-order": [2, 0, 1], + "column-format": { + "Column A": { + "type": "text", + "ignore": false, + "number-append": "", + "number-format": "n1", + "number-divisor": 0, + "number-prepend": "" + } + } + } + } +} diff --git a/tests/integration/test_scatter_chart.py b/tests/integration/test_scatter_chart.py index 943daf7..ee3c08f 100644 --- a/tests/integration/test_scatter_chart.py +++ b/tests/integration/test_scatter_chart.py @@ -475,7 +475,9 @@ def test_automation_fixture_loads_with_explicit_utf8_encoding(self): chart_metadata = load_sample_json("automation.json") sample_csv = load_sample_csv("automation.csv") - encoding_kwargs = [call.kwargs["encoding"] for call in mocked_open.call_args_list] + encoding_kwargs = [ + call.kwargs["encoding"] for call in mocked_open.call_args_list + ] assert encoding_kwargs == ["utf-8", "utf-8"] assert chart_metadata["title"] == ( diff --git a/tests/unit/models/test_transform.py b/tests/unit/models/test_transform.py index 38bc5fc..b05ba3e 100644 --- a/tests/unit/models/test_transform.py +++ b/tests/unit/models/test_transform.py @@ -262,3 +262,170 @@ def test_transform_with_list_column_format(self): assert len(transform.column_format) == 2 assert transform.column_format[0].column == "sales" assert transform.column_format[1].column == "date" + + +def test_data_change_serializes_documented_api_shape(): + """Individual value corrections serialize to metadata.data.changes entries.""" + change = datawrapper.DataChange( + row=9, + column=3, + value="1.7", + time=1573134075869, + previous="0.7", + ) + + assert change.model_dump(by_alias=True) == { + "row": 9, + "column": 3, + "value": "1.7", + "time": 1573134075869, + "previous": "0.7", + } + + +def test_data_change_previous_is_optional_for_added_column_entries(): + """Datawrapper omits previous for added-column entries in changes.""" + change = datawrapper.DataChange( + row=0, + column=5, + value="Extra column", + time=1573134111144, + ) + + assert change.previous is None + assert change.model_dump(by_alias=True) == { + "row": 0, + "column": 5, + "value": "Extra column", + "time": 1573134111144, + } + + +def test_transform_accepts_data_change_objects_and_dicts(): + """Transform exposes changes ergonomically while preserving API output.""" + transform = datawrapper.Transform( + changes=[ + datawrapper.DataChange( + row=9, + column=3, + value="1.7", + time=1573134075869, + previous="0.7", + ), + { + "row": 0, + "column": 5, + "value": "Extra column", + "time": 1573134111144, + }, + ] + ) + + assert all( + isinstance(change, datawrapper.DataChange) for change in transform.changes + ) + assert transform.model_dump(by_alias=True)["changes"] == [ + { + "row": 9, + "column": 3, + "value": "1.7", + "time": 1573134075869, + "previous": "0.7", + }, + { + "row": 0, + "column": 5, + "value": "Extra column", + "time": 1573134111144, + }, + ] + + +def test_transform_deserializes_documented_data_changes_fixture(): + """The documented Check & Describe fixture round-trips through Transform.""" + import json + from pathlib import Path + + fixture_path = Path(__file__).parents[2] / "fixtures" / "metadata_data_changes.json" + fixture = json.loads(fixture_path.read_text()) + + transform = datawrapper.Transform.model_validate(fixture["metadata"]["data"]) + + assert list(transform.changes) == [ + datawrapper.DataChange( + row=9, + column=3, + value="1.7", + time=1573134075869, + previous="0.7", + ), + datawrapper.DataChange( + row=0, + column=5, + value="Extra column", + time=1573134111144, + ), + ] + assert ( + transform.model_dump(by_alias=True)["changes"] + == fixture["metadata"]["data"]["changes"] + ) + + +def test_transform_defaults_do_not_emit_empty_changes(): + """Existing default transformation serialization remains unchanged.""" + transform = datawrapper.Transform() + + assert len(transform.changes) == 0 + assert "changes" not in transform.model_dump(by_alias=True) + + +def test_data_change_rejects_negative_indexes(): + """Row and column indexes are zero-based and cannot be negative.""" + with pytest.raises(ValidationError): + datawrapper.DataChange(row=-1, column=0, value="x", time=1) + + with pytest.raises(ValidationError): + datawrapper.DataChange(row=0, column=-1, value="x", time=1) + + +def test_transform_accepts_empty_changes_object_as_no_changes(): + """Existing API responses can use changes: {} when no corrections exist.""" + transform = datawrapper.Transform.model_validate({"changes": {}}) + + assert len(transform.changes) == 0 + assert "changes" not in transform.model_dump(by_alias=True) + + +def test_transform_preserves_api_object_shaped_changes(): + """Existing API fixtures can key changes by Datawrapper's internal IDs.""" + api_changes = { + "20azca858m": { + "id": "20azca858m", + "row": 0, + "time": 1592404219100, + "value": "New cases", + "column": 1, + "ignored": False, + "previous": "new", + "_index": 0, + }, + "sfr5gsdyqd": { + "id": "sfr5gsdyqd", + "row": 0, + "time": 1592404227900, + "value": "Deaths per day", + "column": 2, + "ignored": False, + "previous": "deaths", + "_index": 1, + }, + } + + transform = datawrapper.Transform.model_validate({"changes": api_changes}) + + assert [change.id for change in transform.changes] == [ + "20azca858m", + "sfr5gsdyqd", + ] + assert transform.model_dump(by_alias=True)["changes"] == api_changes