From 066c8bc8b0fb5f913a3865139f01635c60c55bd8 Mon Sep 17 00:00:00 2001 From: Akim Juillerat Date: Fri, 10 Jul 2026 23:17:03 +0200 Subject: [PATCH 1/6] Copy fs_attachment_s3 into fs_attachment_azure --- fs_attachment_azure/README.rst | 201 +++++++ fs_attachment_azure/__init__.py | 1 + fs_attachment_azure/__manifest__.py | 21 + fs_attachment_azure/i18n/fs_attachment_s3.pot | 48 ++ fs_attachment_azure/i18n/it.po | 56 ++ fs_attachment_azure/models/__init__.py | 2 + fs_attachment_azure/models/fs_storage.py | 54 ++ fs_attachment_azure/models/ir_attachment.py | 86 +++ fs_attachment_azure/pyproject.toml | 3 + fs_attachment_azure/readme/CONFIGURE.md | 64 ++ fs_attachment_azure/readme/CONTRIBUTORS.md | 2 + fs_attachment_azure/readme/CREDITS.md | 4 + fs_attachment_azure/readme/DESCRIPTION.md | 7 + fs_attachment_azure/readme/HISTORY.md | 14 + .../readme/newsfragments/.gitkeep | 0 .../static/description/icon.png | Bin 0 -> 9455 bytes .../static/description/index.html | 552 ++++++++++++++++++ fs_attachment_azure/tests/__init__.py | 1 + fs_attachment_azure/tests/common.py | 63 ++ .../tests/test_fs_attachment_s3.py | 69 +++ fs_attachment_azure/views/fs_storage.xml | 23 + 21 files changed, 1271 insertions(+) create mode 100644 fs_attachment_azure/README.rst create mode 100644 fs_attachment_azure/__init__.py create mode 100644 fs_attachment_azure/__manifest__.py create mode 100644 fs_attachment_azure/i18n/fs_attachment_s3.pot create mode 100644 fs_attachment_azure/i18n/it.po create mode 100644 fs_attachment_azure/models/__init__.py create mode 100644 fs_attachment_azure/models/fs_storage.py create mode 100644 fs_attachment_azure/models/ir_attachment.py create mode 100644 fs_attachment_azure/pyproject.toml create mode 100644 fs_attachment_azure/readme/CONFIGURE.md create mode 100644 fs_attachment_azure/readme/CONTRIBUTORS.md create mode 100644 fs_attachment_azure/readme/CREDITS.md create mode 100644 fs_attachment_azure/readme/DESCRIPTION.md create mode 100644 fs_attachment_azure/readme/HISTORY.md create mode 100644 fs_attachment_azure/readme/newsfragments/.gitkeep create mode 100644 fs_attachment_azure/static/description/icon.png create mode 100644 fs_attachment_azure/static/description/index.html create mode 100644 fs_attachment_azure/tests/__init__.py create mode 100644 fs_attachment_azure/tests/common.py create mode 100644 fs_attachment_azure/tests/test_fs_attachment_s3.py create mode 100644 fs_attachment_azure/views/fs_storage.xml diff --git a/fs_attachment_azure/README.rst b/fs_attachment_azure/README.rst new file mode 100644 index 0000000000..788dc97b94 --- /dev/null +++ b/fs_attachment_azure/README.rst @@ -0,0 +1,201 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +================ +Fs Attachment S3 +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:c01d32f225802fc30d7d79a6130c073016c0d98c69ba20dd6d6c0d1213e9f92d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fstorage-lightgray.png?logo=github + :target: https://github.com/OCA/storage/tree/17.0/fs_attachment_s3 + :alt: OCA/storage +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/storage-17-0/storage-17-0-fs_attachment_s3 + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/storage&target_branch=17.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module extends the functionality of +`fs_attachment `__ +to better support Amazon S3 storage. It includes features such as: + +- Special handling of X-Accel-Redirect headers for S3 storages. +- Options for using signed URLs in X-Accel-Redirect. (This is required + to be able to serve files from a private S3 bucket using + X-Accel-Redirect without exposing the files publicly.) +- Enforcing the mimetype of files stored in S3. + +**Table of contents** + +.. contents:: + :local: + +Configuration +============= + +On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File +Storage*. + +When you create a new storage for s3 or modify an existing one, when you +activate the option "Use X-Sendfile To Serve Internal Url", 2 additional +fields will appear: + +- **S3 Uses Signed URL For X-Accel-Redirect**: If checked, the + X-Accel-Redirect path will be a signed URL, which is useful for S3 + storages that require signed URLs for access. +- **S3 Signed URL Expiration**: The expiration time for the signed URL + in seconds. This field is only relevant if the previous option is + checked. By default, it is set to 30 seconds but it could be less + since the url generated into the X-Accel-Redirect process is directly + used by the web server to serve the file. + +The value of these fields can also be set in the server environment +variables using the keys: + +- *s3_uses_signed_url_for_x_sendfile* +- *s3_signed_url_expiration* + +When the option "Use X-Sendfile To Serve Internal Url" is enabled, the +system will generate an X-Accel-Redirect header in the response to a +request to get a file. In the case of S3 storages, it will follow the +format: + +.. code:: text + + X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any} + +Where: + +- ``{scheme}``: The URL scheme (http or https). +- ``{host}``: The host of the S3 storage. +- ``{path with query if any}``: The path to the file in the S3 storage, + including any query parameters. (Query parameters are set when the + ``s3_uses_signed_url_for_x_sendfile`` option is enabled.) + +In order to serve files using X-Accel-Redirect, you must ensure that +your web server is configured to handle these headers correctly. This +typically involves setting up a location block in your web server +configuration that matches the X-Accel-Redirect path and proxies the +request to the S3 storage. + +For example, if you are using Nginx, you would add a location block like +this: + +.. code:: nginx + + + location ~ ^/fs_x_sendfile/(.*?)/(.*?)/(.*) { + internal; + set $url_scheme $1; + set $url_host $2; + set $url_path $3; + set $url $url_scheme://$url_host/$url_path; + + proxy_pass $url$is_args$args; + proxy_set_header Host $url_host; + proxy_ssl_server_name on; + + } + +Unlike the standard implementation of X-Accel-Redirect on non S3 +storages, the S3 implementation does not require a base URL to be set in +the storage configuration. The X-Accel-Redirect path is constructed +directly from the S3 storage's URL defined for the connection, the +directory name as bucket name, and the file path. + +Changelog +========= + +17.0.1.2.1 (2026-05-27) +----------------------- + +Bugfixes +~~~~~~~~ + +- Allow to use a prefix path and bucket in the directory_path on + fs.storage When the directory_path parameter is configured as / the + presigned url generation failed with a botocore error: "Invalid bucket + name". (`#b17de9 `__) + +17.0.1.2.0 (2025-10-22) +----------------------- + +Features +~~~~~~~~ + +- Adapt to handle {db_name} in directory_path. + (`#db_name `__) + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV + +Contributors +------------ + +- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) +- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) + +Other credits +------------- + +The development of this module has been financially supported by: + +- ACSONE SA/NV (https://www.acsone.eu) +- Alcyon Belux + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-lmignon| image:: https://github.com/lmignon.png?size=40px + :target: https://github.com/lmignon + :alt: lmignon + +Current `maintainer `__: + +|maintainer-lmignon| + +This module is part of the `OCA/storage `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/fs_attachment_azure/__init__.py b/fs_attachment_azure/__init__.py new file mode 100644 index 0000000000..0650744f6b --- /dev/null +++ b/fs_attachment_azure/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/fs_attachment_azure/__manifest__.py b/fs_attachment_azure/__manifest__.py new file mode 100644 index 0000000000..fb52781cdc --- /dev/null +++ b/fs_attachment_azure/__manifest__.py @@ -0,0 +1,21 @@ +# Copyright 2025 ACSONE SA/NV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +{ + "name": "Fs Attachment S3", + "summary": """Store attachments into S3 complient filesystem""", + "version": "17.0.1.2.1", + "license": "AGPL-3", + "author": "ACSONE SA/NV,Odoo Community Association (OCA)", + "website": "https://github.com/OCA/storage", + "depends": ["fs_attachment"], + "external_dependencies": { + "python": [ + "fsspec[s3]", + ], + }, + "data": [ + "views/fs_storage.xml", + ], + "maintainers": ["lmignon"], +} diff --git a/fs_attachment_azure/i18n/fs_attachment_s3.pot b/fs_attachment_azure/i18n/fs_attachment_s3.pot new file mode 100644 index 0000000000..bd3d203e80 --- /dev/null +++ b/fs_attachment_azure/i18n/fs_attachment_s3.pot @@ -0,0 +1,48 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fs_attachment_s3 +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 17.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: fs_attachment_s3 +#: model:ir.model,name:fs_attachment_s3.model_ir_attachment +msgid "Attachment" +msgstr "" + +#. module: fs_attachment_s3 +#: model:ir.model,name:fs_attachment_s3.model_fs_storage +msgid "FS Storage" +msgstr "" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile +msgid "" +"If checked, the storage will use signed URLs for attachments when using " +"X-Accel-Redirect. This is useful for S3 storage where the file path is not " +"directly accessible without authentication." +msgstr "" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration +msgid "Signed URL Expiration (seconds)" +msgstr "" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration +msgid "" +"The expiration time for the signed URL in seconds. Default is 30 seconds." +msgstr "" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile +msgid "Use signed URL for X-Accel-Redirect" +msgstr "" diff --git a/fs_attachment_azure/i18n/it.po b/fs_attachment_azure/i18n/it.po new file mode 100644 index 0000000000..d03ad91248 --- /dev/null +++ b/fs_attachment_azure/i18n/it.po @@ -0,0 +1,56 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * fs_attachment_s3 +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 17.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2025-10-23 08:43+0000\n" +"Last-Translator: mymage \n" +"Language-Team: none\n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 5.10.4\n" + +#. module: fs_attachment_s3 +#: model:ir.model,name:fs_attachment_s3.model_ir_attachment +msgid "Attachment" +msgstr "Allegato" + +#. module: fs_attachment_s3 +#: model:ir.model,name:fs_attachment_s3.model_fs_storage +msgid "FS Storage" +msgstr "Deposito FS" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile +msgid "" +"If checked, the storage will use signed URLs for attachments when using " +"X-Accel-Redirect. This is useful for S3 storage where the file path is not " +"directly accessible without authentication." +msgstr "" +"Se selezionata, l'archiviazione utilizzerà URL firmati per gli allegati " +"quando si utilizza X-Accel-Redirect. Questa opzione è utile per " +"l'archiviazione S3 in cui il percorso del file non è direttamente " +"accessibile senza autenticazione." + +#. module: fs_attachment_s3 +#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration +msgid "Signed URL Expiration (seconds)" +msgstr "Scadenza URL firmato (secondi)" + +#. module: fs_attachment_s3 +#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration +msgid "" +"The expiration time for the signed URL in seconds. Default is 30 seconds." +msgstr "" +"Il tempo di scadenza per l'URL firmato in secondi. Predefinito 30 secondi." + +#. module: fs_attachment_s3 +#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile +msgid "Use signed URL for X-Accel-Redirect" +msgstr "Utilizza URL firmato per X-Accel-Redirect" diff --git a/fs_attachment_azure/models/__init__.py b/fs_attachment_azure/models/__init__.py new file mode 100644 index 0000000000..45a28cbdca --- /dev/null +++ b/fs_attachment_azure/models/__init__.py @@ -0,0 +1,2 @@ +from . import fs_storage +from . import ir_attachment diff --git a/fs_attachment_azure/models/fs_storage.py b/fs_attachment_azure/models/fs_storage.py new file mode 100644 index 0000000000..1b737c722d --- /dev/null +++ b/fs_attachment_azure/models/fs_storage.py @@ -0,0 +1,54 @@ +# Copyright 2025 ACSONE SA/NV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import fsspec.asyn + +from odoo import api, fields, models + + +class FsStorage(models.Model): + _inherit = "fs.storage" + + s3_uses_signed_url_for_x_sendfile = fields.Boolean( + string="Use signed URL for X-Accel-Redirect", + help="If checked, the storage will use signed URLs for attachments " + "when using X-Accel-Redirect. This is useful for S3 storage where the " + "file path is not directly accessible without authentication.", + ) + s3_signed_url_expiration = fields.Integer( + string="Signed URL Expiration (seconds)", + default=30, + help="The expiration time for the signed URL in seconds. " + "Default is 30 seconds.", + ) + + @property + def _server_env_fields(self): + """Override to include S3 specific fields.""" + fields = super()._server_env_fields + fields.update( + { + "s3_uses_signed_url_for_x_sendfile": {}, + "s3_signed_url_expiration": {}, + } + ) + return fields + + @property + def is_s3_storage(self): + """Check if the storage is an S3 storage.""" + self.ensure_one() + return hasattr(self._get_root_filesystem(self.fs), "s3") + + @api.model + def _s3_call_generate_presigned_url(self, s3_client, *args, **kwargs): + """Generate a presigned URL for S3 operations.""" + # s3fs uses aiobotocore as s3 client, which is asynchronous. + # We need to run the async function in a synchronous context. + return fsspec.asyn.sync( + fsspec.asyn.get_loop(), + s3_client.generate_presigned_url, + *args, + timeout=None, + **kwargs, + ) diff --git a/fs_attachment_azure/models/ir_attachment.py b/fs_attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000000..8bdc2c94ac --- /dev/null +++ b/fs_attachment_azure/models/ir_attachment.py @@ -0,0 +1,86 @@ +# Copyright 2025 ACSONE SA/NV +# Copyright 2025 XCG SAS +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +from urllib.parse import urlparse + +from odoo import models + + +class IrAttachment(models.Model): + _inherit = "ir.attachment" + + def _storage_write_option(self, fs): + option = super()._storage_write_option(fs) + mimetype = self.env.context.get("mimetype") + if mimetype: + root_fs = self.env["fs.storage"]._get_root_filesystem(fs) + if hasattr(root_fs, "s3"): + option["ContentType"] = mimetype + return option + + def _get_x_sendfile_path(self): + self.ensure_one() + storage = self.fs_storage_id + if storage.is_s3_storage: + return self._get_s3_x_sendfile_path() + return super()._get_x_sendfile_path() + + def _fs_use_x_sendfile(self): + self.ensure_one() + storage = self.fs_storage_id + if storage.is_s3_storage: + return storage.use_x_sendfile_to_serve_internal_url + return super()._fs_use_x_sendfile() + + def _get_s3_x_sendfile_path(self): + """Generate the X-Accel-Redirect path for S3 storage. + + This method is used to generate the path for S3 storage when using + X-Accel-Redirect. It constructs the path based on the S3 bucket and + file path, ensuring that it is compatible with the S3 storage + configuration and the Odoo file storage system. + + Args: + attachment (IrAttachment): The attachment record for which the + X-Accel-Redirect path is being generated. + Returns: + str: The X-Accel-Redirect path for the S3 storage. + + The path is formatted as: + /fs_x_sendfile/// + + where: + - `` is the scheme of the base URL (e.g., 'https'). + - `` is the netloc of the base URL (e.g., 's3.amazonaws.com'). + - `` is the path to the file in the S3 bucket, including the + bucket name + """ + fs, storage_code, file_path = self._get_fs_parts() + storage = self.env["fs.storage"].sudo().get_by_code(storage_code) + root_fs = storage._get_root_filesystem(fs) + s3_client = root_fs.s3 + # The directory path might contain the bucket and a prefix + # the part before the first "/" is the bucket + # the rest is the prefix + bucket_name, *prefix_parts = storage.get_directory_path().strip("/").split("/") + s3_key = "/".join(prefix_parts + [file_path.lstrip("/")]) + if storage.s3_uses_signed_url_for_x_sendfile: + file_url = storage._s3_call_generate_presigned_url( + s3_client, + "get_object", + Params={"Bucket": bucket_name, "Key": s3_key}, + ExpiresIn=storage.s3_signed_url_expiration, + ) + else: + file_url = ( + f"{s3_client.meta.endpoint_url.rstrip('/')}/" + f"{bucket_name}/{s3_key.lstrip('/')}" + ) + + parsed_url = urlparse(file_url) + path = parsed_url.path.strip("/") + query = parsed_url.query + redirect_path = f"/fs_x_sendfile/{parsed_url.scheme}/{parsed_url.netloc}/{path}" + if query: + redirect_path += f"?{query}" + return redirect_path diff --git a/fs_attachment_azure/pyproject.toml b/fs_attachment_azure/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/fs_attachment_azure/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/fs_attachment_azure/readme/CONFIGURE.md b/fs_attachment_azure/readme/CONFIGURE.md new file mode 100644 index 0000000000..219778ad4d --- /dev/null +++ b/fs_attachment_azure/readme/CONFIGURE.md @@ -0,0 +1,64 @@ +On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File Storage*. + +When you create a new storage for s3 or modify an existing one, when you activate +the option "Use X-Sendfile To Serve Internal Url", 2 additional fields will appear: + +- **S3 Uses Signed URL For X-Accel-Redirect**: If checked, the X-Accel-Redirect + path will be a signed URL, which is useful for S3 storages that require + signed URLs for access. +- **S3 Signed URL Expiration**: The expiration time for the signed URL in seconds. + This field is only relevant if the previous option is checked. By default, + it is set to 30 seconds but it could be less since the url generated into + the X-Accel-Redirect process is directly used by the web server to serve the file. + +The value of these fields can also be set in the server environment variables using +the keys: + +- *s3_uses_signed_url_for_x_sendfile* +- *s3_signed_url_expiration* + +When the option "Use X-Sendfile To Serve Internal Url" is enabled, the system will +generate an X-Accel-Redirect header in the response to a request to get a file. +In the case of S3 storages, it will follow the format: + +```text +X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any} +``` + +Where: + +- `{scheme}`: The URL scheme (http or https). +- `{host}`: The host of the S3 storage. +- `{path with query if any}`: The path to the file in the S3 storage, + including any query parameters. (Query parameters are set when the + `s3_uses_signed_url_for_x_sendfile` option is enabled.) + +In order to serve files using X-Accel-Redirect, you must ensure that your +web server is configured to handle these headers correctly. This typically +involves setting up a location block in your web server configuration that +matches the X-Accel-Redirect path and proxies the request to the S3 storage. + +For example, if you are using Nginx, you would add a location block like this: + +```nginx + + location ~ ^/fs_x_sendfile/(.*?)/(.*?)/(.*) { + internal; + set $url_scheme $1; + set $url_host $2; + set $url_path $3; + set $url $url_scheme://$url_host/$url_path; + + proxy_pass $url$is_args$args; + proxy_set_header Host $url_host; + proxy_ssl_server_name on; + + } +``` + + +Unlike the standard implementation of X-Accel-Redirect on non S3 storages, +the S3 implementation does not require a base URL to be set in the storage +configuration. The X-Accel-Redirect path is constructed directly from the +S3 storage's URL defined for the connection, the directory name as +bucket name, and the file path. diff --git a/fs_attachment_azure/readme/CONTRIBUTORS.md b/fs_attachment_azure/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..06d49341ab --- /dev/null +++ b/fs_attachment_azure/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Laurent Mignon (https://www.acsone.eu) +- Stéphane Bidoul (https://www.acsone.eu) diff --git a/fs_attachment_azure/readme/CREDITS.md b/fs_attachment_azure/readme/CREDITS.md new file mode 100644 index 0000000000..c733078a75 --- /dev/null +++ b/fs_attachment_azure/readme/CREDITS.md @@ -0,0 +1,4 @@ +The development of this module has been financially supported by: + +- ACSONE SA/NV (https://www.acsone.eu) +- Alcyon Belux \ No newline at end of file diff --git a/fs_attachment_azure/readme/DESCRIPTION.md b/fs_attachment_azure/readme/DESCRIPTION.md new file mode 100644 index 0000000000..c37359b415 --- /dev/null +++ b/fs_attachment_azure/readme/DESCRIPTION.md @@ -0,0 +1,7 @@ +This module extends the functionality of [fs_attachment](https://github.com/OCA/storage/tree/16.0/fs_attachment) +to better support Amazon S3 storage. It includes features such as: + +- Special handling of X-Accel-Redirect headers for S3 storages. +- Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private S3 bucket + using X-Accel-Redirect without exposing the files publicly.) +- Enforcing the mimetype of files stored in S3. diff --git a/fs_attachment_azure/readme/HISTORY.md b/fs_attachment_azure/readme/HISTORY.md new file mode 100644 index 0000000000..b4c434c401 --- /dev/null +++ b/fs_attachment_azure/readme/HISTORY.md @@ -0,0 +1,14 @@ +## 17.0.1.2.1 (2026-05-27) + +### Bugfixes + +- Allow to use a prefix path and bucket in the directory_path on fs.storage + When the directory_path parameter is configured as / + the presigned url generation failed with a botocore error: "Invalid bucket name". ([#b17de9](https://github.com/OCA/storage/issues/b17de9)) + + +## 17.0.1.2.0 (2025-10-22) + +### Features + +- Adapt to handle {db_name} in directory_path. ([#db_name](https://github.com/OCA/storage/issues/db_name)) diff --git a/fs_attachment_azure/readme/newsfragments/.gitkeep b/fs_attachment_azure/readme/newsfragments/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fs_attachment_azure/static/description/icon.png b/fs_attachment_azure/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/fs_attachment_azure/static/description/index.html b/fs_attachment_azure/static/description/index.html new file mode 100644 index 0000000000..233c7217f1 --- /dev/null +++ b/fs_attachment_azure/static/description/index.html @@ -0,0 +1,552 @@ + + + + + +README.rst + + + +
+ + + +Odoo Community Association + +
+

Fs Attachment S3

+ +

Beta License: AGPL-3 OCA/storage Translate me on Weblate Try me on Runboat

+

This module extends the functionality of +fs_attachment +to better support Amazon S3 storage. It includes features such as:

+
    +
  • Special handling of X-Accel-Redirect headers for S3 storages.
    • +
  • Options for using signed URLs in X-Accel-Redirect. (This is required +to be able to serve files from a private S3 bucket using +X-Accel-Redirect without exposing the files publicly.)
    • +
  • Enforcing the mimetype of files stored in S3.
    • +
+

Table of contents

+ +
+

Configuration

+

On the Odoo instance, go to Settings > Technical > Storage > File +Storage.

+

When you create a new storage for s3 or modify an existing one, when you +activate the option “Use X-Sendfile To Serve Internal Url”, 2 additional +fields will appear:

+
    +
  • S3 Uses Signed URL For X-Accel-Redirect: If checked, the +X-Accel-Redirect path will be a signed URL, which is useful for S3 +storages that require signed URLs for access.
    • +
  • S3 Signed URL Expiration: The expiration time for the signed URL +in seconds. This field is only relevant if the previous option is +checked. By default, it is set to 30 seconds but it could be less +since the url generated into the X-Accel-Redirect process is directly +used by the web server to serve the file.
    • +
+

The value of these fields can also be set in the server environment +variables using the keys:

+
    +
  • s3_uses_signed_url_for_x_sendfile
    • +
  • s3_signed_url_expiration
    • +
+

When the option “Use X-Sendfile To Serve Internal Url” is enabled, the +system will generate an X-Accel-Redirect header in the response to a +request to get a file. In the case of S3 storages, it will follow the +format:

+
+X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any}
+
+

Where:

+
    +
  • {scheme}: The URL scheme (http or https).
    • +
  • {host}: The host of the S3 storage.
    • +
  • {path with query if any}: The path to the file in the S3 storage, +including any query parameters. (Query parameters are set when the +s3_uses_signed_url_for_x_sendfile option is enabled.)
    • +
+

In order to serve files using X-Accel-Redirect, you must ensure that +your web server is configured to handle these headers correctly. This +typically involves setting up a location block in your web server +configuration that matches the X-Accel-Redirect path and proxies the +request to the S3 storage.

+

For example, if you are using Nginx, you would add a location block like +this:

+
+location ~ ^/fs_x_sendfile/(.*?)/(.*?)/(.*) {
+    internal;
+    set $url_scheme $1;
+    set $url_host $2;
+    set $url_path $3;
+    set $url $url_scheme://$url_host/$url_path;
+
+    proxy_pass $url$is_args$args;
+    proxy_set_header Host $url_host;
+    proxy_ssl_server_name on;
+
+}
+
+

Unlike the standard implementation of X-Accel-Redirect on non S3 +storages, the S3 implementation does not require a base URL to be set in +the storage configuration. The X-Accel-Redirect path is constructed +directly from the S3 storage’s URL defined for the connection, the +directory name as bucket name, and the file path.

+
+
+

Changelog

+
+

17.0.1.2.1 (2026-05-27)

+
+

Bugfixes

+
    +
  • Allow to use a prefix path and bucket in the directory_path on +fs.storage When the directory_path parameter is configured as / the +presigned url generation failed with a botocore error: “Invalid bucket +name”. (#b17de9)
    • +
+
+
+
+

17.0.1.2.0 (2025-10-22)

+
+

Features

+
    +
  • Adapt to handle {db_name} in directory_path. +(#db_name)
    • +
+
+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE SA/NV
    • +
+
+ +
+

Other credits

+

The development of this module has been financially supported by:

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

lmignon

+

This module is part of the OCA/storage project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+
+ + diff --git a/fs_attachment_azure/tests/__init__.py b/fs_attachment_azure/tests/__init__.py new file mode 100644 index 0000000000..a788fc4772 --- /dev/null +++ b/fs_attachment_azure/tests/__init__.py @@ -0,0 +1 @@ +from . import test_fs_attachment_s3 diff --git a/fs_attachment_azure/tests/common.py b/fs_attachment_azure/tests/common.py new file mode 100644 index 0000000000..3f2c61b581 --- /dev/null +++ b/fs_attachment_azure/tests/common.py @@ -0,0 +1,63 @@ +# Copyright 2025 ACSONE SA/NV (http://acsone.eu). +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.tests.common import TransactionCase + +from odoo.addons.base.tests.common import DISABLED_MAIL_CONTEXT + + +class TestFSAttachmentS3Common(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, **DISABLED_MAIL_CONTEXT)) + cls.s3_backend_config = { + "name": "S3 Storage", + "protocol": "s3", + "code": "s3tst", + "directory_path": "test-bucket", + "json_options": { + "key": "aws-key", + "secret": "aws-secret", + "client_kwargs": { + "endpoint_url": "http://minio.minio/", + "region_name": "aws-region", + }, + }, + "base_url": False, # S3 does not use base_url for x-sendfile + } + cls.s3_backend = cls.env["fs.storage"].create(cls.s3_backend_config) + cls.ir_attachment_model = cls.env["ir.attachment"] + + cls.fake_attachment_s3 = cls.env["ir.attachment"].create( + { + "name": "fake_s3_file.txt", + "fs_storage_id": cls.s3_backend.id, + } + ) + cls.fake_attachment_s3.flush_recordset() + # update the attachment into database since we don't have a real S3 bucket + # and we can't use moto to mock S3 since s3fs rely on aiobotocore + cls.env.cr.execute( + """ + UPDATE + ir_attachment + SET + store_fname = 's3tst://dir/sub/fake_s3_file.txt', + fs_filename = 'fake_s3_file.txt', + fs_storage_code = 's3tst', + checksum = 234, + file_size = 1234, + fs_storage_id = %s + WHERE + id = %s + """, + (cls.s3_backend.id, cls.fake_attachment_s3.id), + ) + cls.fake_attachment_s3.invalidate_recordset() + + def setUp(self): + super().setUp() + # enforce backend_config fields since it seems that they are reset on + # savepoint rollback when managed by server_environment -> TO Be investigated + self.s3_backend.write(self.s3_backend_config) diff --git a/fs_attachment_azure/tests/test_fs_attachment_s3.py b/fs_attachment_azure/tests/test_fs_attachment_s3.py new file mode 100644 index 0000000000..67488c7fff --- /dev/null +++ b/fs_attachment_azure/tests/test_fs_attachment_s3.py @@ -0,0 +1,69 @@ +# Copyright 2025 ACSONE SA/NV (http://acsone.eu). +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + + +from .common import TestFSAttachmentS3Common + + +class TestFSAttachementS3(TestFSAttachmentS3Common): + def test_get_x_sendfile_path_s3_signed(self): + """Test the X-Accel-Redirect path generation for S3 storage.""" + self.s3_backend.write( + { + "s3_uses_signed_url_for_x_sendfile": True, + "s3_signed_url_expiration": 60, + } + ) + + url = self.fake_attachment_s3._get_x_sendfile_path() + self.assertTrue( + url.startswith( + "/fs_x_sendfile/http/minio.minio/" + "test-bucket/dir/sub/fake_s3_file.txt?" + ), + "The end of the path should contain the path to the file " + f"name and query parameters. ({url})", + ) + + def test_get_x_sendfile_path(self): + """Test the X-Accel-Redirect path generation.""" + url = self.fake_attachment_s3._get_x_sendfile_path() + self.assertEqual( + url, + "/fs_x_sendfile/http/minio.minio/test-bucket/dir/sub/fake_s3_file.txt", + f"The X-Accel-Redirect path should match the expected format. ({url})", + ) + + def test_get_x_sendfile_path_s3_signed_with_prefix(self): + """Test the path generation when the directory path has a prefix.""" + self.s3_backend.write( + { + "s3_uses_signed_url_for_x_sendfile": True, + "s3_signed_url_expiration": 60, + "directory_path": "test-bucket/test-prefix", + } + ) + + url = self.fake_attachment_s3._get_x_sendfile_path() + self.assertTrue( + url.startswith( + "/fs_x_sendfile/http/minio.minio/" + "test-bucket/test-prefix/dir/sub/fake_s3_file.txt?" + ), + f"The URL should include the prefix. ({url})", + ) + + def test_get_x_sendfile_path_with_prefix(self): + """Test the unsigned path generation with a prefix.""" + self.s3_backend.write( + { + "s3_uses_signed_url_for_x_sendfile": False, + "directory_path": "test-bucket/test-prefix", + } + ) + url = self.fake_attachment_s3._get_x_sendfile_path() + self.assertEqual( + url, + "/fs_x_sendfile/http/minio.minio/test-bucket/test-prefix/dir/sub/fake_s3_file.txt", + f"The URL should include the prefix. ({url})", + ) diff --git a/fs_attachment_azure/views/fs_storage.xml b/fs_attachment_azure/views/fs_storage.xml new file mode 100644 index 0000000000..8bd9160413 --- /dev/null +++ b/fs_attachment_azure/views/fs_storage.xml @@ -0,0 +1,23 @@ + + + + + + fs.storage + + + + + + + + + + From 53150a133385039a8b848b1783e430f47beea4e4 Mon Sep 17 00:00:00 2001 From: Akim Juillerat Date: Mon, 13 Jul 2026 21:14:44 +0200 Subject: [PATCH 2/6] fs_attachment_azure: Adapt to azure --- fs_attachment_azure/README.rst | 120 +++++++---------- fs_attachment_azure/__manifest__.py | 13 +- fs_attachment_azure/i18n/fs_attachment_s3.pot | 48 ------- fs_attachment_azure/i18n/it.po | 56 -------- fs_attachment_azure/models/fs_storage.py | 28 ++-- fs_attachment_azure/models/ir_attachment.py | 93 +++++++------ fs_attachment_azure/readme/CONFIGURE.md | 26 ++-- fs_attachment_azure/readme/CONTRIBUTORS.md | 1 + fs_attachment_azure/readme/CREDITS.md | 4 - fs_attachment_azure/readme/DESCRIPTION.md | 7 +- fs_attachment_azure/readme/HISTORY.md | 15 +-- .../static/description/index.html | 127 +++++++----------- fs_attachment_azure/tests/__init__.py | 2 +- fs_attachment_azure/tests/common.py | 52 +++---- .../tests/test_fs_attachment_azure.py | 62 +++++++++ .../tests/test_fs_attachment_s3.py | 69 ---------- fs_attachment_azure/views/fs_storage.xml | 9 +- requirements.txt | 1 + 18 files changed, 275 insertions(+), 458 deletions(-) delete mode 100644 fs_attachment_azure/i18n/fs_attachment_s3.pot delete mode 100644 fs_attachment_azure/i18n/it.po delete mode 100644 fs_attachment_azure/readme/CREDITS.md create mode 100644 fs_attachment_azure/tests/test_fs_attachment_azure.py delete mode 100644 fs_attachment_azure/tests/test_fs_attachment_s3.py diff --git a/fs_attachment_azure/README.rst b/fs_attachment_azure/README.rst index 788dc97b94..8c3bd487b5 100644 --- a/fs_attachment_azure/README.rst +++ b/fs_attachment_azure/README.rst @@ -1,10 +1,6 @@ -.. image:: https://odoo-community.org/readme-banner-image - :target: https://odoo-community.org/get-involved?utm_source=readme - :alt: Odoo Community Association - -================ -Fs Attachment S3 -================ +=================== +Fs Attachment Azure +=================== .. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -17,14 +13,14 @@ Fs Attachment S3 .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status :alt: Beta -.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html :alt: License: AGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fstorage-lightgray.png?logo=github - :target: https://github.com/OCA/storage/tree/17.0/fs_attachment_s3 + :target: https://github.com/OCA/storage/tree/17.0/fs_attachment_azure :alt: OCA/storage .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/storage-17-0/storage-17-0-fs_attachment_s3 + :target: https://translation.odoo-community.org/projects/storage-17-0/storage-17-0-fs_attachment_azure :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png :target: https://runboat.odoo-community.org/builds?repo=OCA/storage&target_branch=17.0 @@ -34,13 +30,12 @@ Fs Attachment S3 This module extends the functionality of `fs_attachment `__ -to better support Amazon S3 storage. It includes features such as: +to better support Azure storage. It includes features such as: -- Special handling of X-Accel-Redirect headers for S3 storages. -- Options for using signed URLs in X-Accel-Redirect. (This is required - to be able to serve files from a private S3 bucket using - X-Accel-Redirect without exposing the files publicly.) -- Enforcing the mimetype of files stored in S3. +- Special handling of X-Accel-Redirect headers for Azure storages. +- Options for using signed URLs in X-Accel-Redirect. (This is required + to be able to serve files from a private Azure Blob Storage using + X-Accel-Redirect without exposing the files publicly.) **Table of contents** @@ -53,28 +48,28 @@ Configuration On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File Storage*. -When you create a new storage for s3 or modify an existing one, when you -activate the option "Use X-Sendfile To Serve Internal Url", 2 additional -fields will appear: +When you create a new storage for Azure or modify an existing one, when +you activate the option "Use X-Sendfile To Serve Internal Url", 2 +additional fields will appear: -- **S3 Uses Signed URL For X-Accel-Redirect**: If checked, the - X-Accel-Redirect path will be a signed URL, which is useful for S3 - storages that require signed URLs for access. -- **S3 Signed URL Expiration**: The expiration time for the signed URL - in seconds. This field is only relevant if the previous option is - checked. By default, it is set to 30 seconds but it could be less - since the url generated into the X-Accel-Redirect process is directly - used by the web server to serve the file. +- **Azure Uses Signed URL For X-Accel-Redirect**: If checked, the + X-Accel-Redirect path will be a signed URL, which is useful for S3 + storages that require signed URLs for access. +- **Azure Signed URL Expiration**: The expiration time for the signed + URL in seconds. This field is only relevant if the previous option is + checked. By default, it is set to 30 seconds but it could be less + since the url generated into the X-Accel-Redirect process is directly + used by the web server to serve the file. The value of these fields can also be set in the server environment variables using the keys: -- *s3_uses_signed_url_for_x_sendfile* -- *s3_signed_url_expiration* +- *azure_uses_signed_url_for_x_sendfile* +- *azure_signed_url_expiration* When the option "Use X-Sendfile To Serve Internal Url" is enabled, the system will generate an X-Accel-Redirect header in the response to a -request to get a file. In the case of S3 storages, it will follow the +request to get a file. In the case of Azure storages, it will follow the format: .. code:: text @@ -83,17 +78,17 @@ format: Where: -- ``{scheme}``: The URL scheme (http or https). -- ``{host}``: The host of the S3 storage. -- ``{path with query if any}``: The path to the file in the S3 storage, - including any query parameters. (Query parameters are set when the - ``s3_uses_signed_url_for_x_sendfile`` option is enabled.) +- ``{scheme}``: The URL scheme (http or https). +- ``{host}``: The host of the Azure storage. +- ``{path with query if any}``: The path to the file in the Azure + storage, including any query parameters. (Query parameters are set + when the ``azure_uses_signed_url_for_x_sendfile`` option is enabled.) In order to serve files using X-Accel-Redirect, you must ensure that your web server is configured to handle these headers correctly. This typically involves setting up a location block in your web server configuration that matches the X-Accel-Redirect path and proxies the -request to the S3 storage. +request to the Azure storage. For example, if you are using Nginx, you would add a location block like this: @@ -114,34 +109,19 @@ this: } -Unlike the standard implementation of X-Accel-Redirect on non S3 -storages, the S3 implementation does not require a base URL to be set in -the storage configuration. The X-Accel-Redirect path is constructed -directly from the S3 storage's URL defined for the connection, the +Unlike the standard implementation of X-Accel-Redirect on non Azure +storages, the Azure implementation does not require a base URL to be set +in the storage configuration. The X-Accel-Redirect path is constructed +directly from the Azure storage's URL defined for the connection, the directory name as bucket name, and the file path. Changelog ========= -17.0.1.2.1 (2026-05-27) ------------------------ - -Bugfixes -~~~~~~~~ - -- Allow to use a prefix path and bucket in the directory_path on - fs.storage When the directory_path parameter is configured as / the - presigned url generation failed with a botocore error: "Invalid bucket - name". (`#b17de9 `__) - -17.0.1.2.0 (2025-10-22) +17.0.1.0.0 (2026-07-13) ----------------------- -Features -~~~~~~~~ - -- Adapt to handle {db_name} in directory_path. - (`#db_name `__) +- This module was "forked" from fs_attachment_s3 v17.0.1.2.1 Bug Tracker =========== @@ -149,7 +129,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -160,20 +140,14 @@ Authors ------- * ACSONE SA/NV +* Camptocamp Contributors ------------ -- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) -- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) - -Other credits -------------- - -The development of this module has been financially supported by: - -- ACSONE SA/NV (https://www.acsone.eu) -- Alcyon Belux +- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) +- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) +- Akim Juillerat akim.juillerat@camptocamp.com Maintainers ----------- @@ -188,14 +162,14 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -.. |maintainer-lmignon| image:: https://github.com/lmignon.png?size=40px - :target: https://github.com/lmignon - :alt: lmignon +.. |maintainer-grindtildeath| image:: https://github.com/grindtildeath.png?size=40px + :target: https://github.com/grindtildeath + :alt: grindtildeath Current `maintainer `__: -|maintainer-lmignon| +|maintainer-grindtildeath| -This module is part of the `OCA/storage `_ project on GitHub. +This module is part of the `OCA/storage `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/fs_attachment_azure/__manifest__.py b/fs_attachment_azure/__manifest__.py index fb52781cdc..8244fc6974 100644 --- a/fs_attachment_azure/__manifest__.py +++ b/fs_attachment_azure/__manifest__.py @@ -1,21 +1,22 @@ # Copyright 2025 ACSONE SA/NV +# Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { - "name": "Fs Attachment S3", - "summary": """Store attachments into S3 complient filesystem""", - "version": "17.0.1.2.1", + "name": "Fs Attachment Azure", + "summary": """Store attachments into Azure Blob storage""", + "version": "17.0.1.0.0", "license": "AGPL-3", - "author": "ACSONE SA/NV,Odoo Community Association (OCA)", + "author": "ACSONE SA/NV,Camptocamp,Odoo Community Association (OCA)", "website": "https://github.com/OCA/storage", "depends": ["fs_attachment"], "external_dependencies": { "python": [ - "fsspec[s3]", + "adlfs", ], }, "data": [ "views/fs_storage.xml", ], - "maintainers": ["lmignon"], + "maintainers": ["grindtildeath"], } diff --git a/fs_attachment_azure/i18n/fs_attachment_s3.pot b/fs_attachment_azure/i18n/fs_attachment_s3.pot deleted file mode 100644 index bd3d203e80..0000000000 --- a/fs_attachment_azure/i18n/fs_attachment_s3.pot +++ /dev/null @@ -1,48 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * fs_attachment_s3 -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 17.0\n" -"Report-Msgid-Bugs-To: \n" -"Last-Translator: \n" -"Language-Team: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: \n" - -#. module: fs_attachment_s3 -#: model:ir.model,name:fs_attachment_s3.model_ir_attachment -msgid "Attachment" -msgstr "" - -#. module: fs_attachment_s3 -#: model:ir.model,name:fs_attachment_s3.model_fs_storage -msgid "FS Storage" -msgstr "" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile -msgid "" -"If checked, the storage will use signed URLs for attachments when using " -"X-Accel-Redirect. This is useful for S3 storage where the file path is not " -"directly accessible without authentication." -msgstr "" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration -msgid "Signed URL Expiration (seconds)" -msgstr "" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration -msgid "" -"The expiration time for the signed URL in seconds. Default is 30 seconds." -msgstr "" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile -msgid "Use signed URL for X-Accel-Redirect" -msgstr "" diff --git a/fs_attachment_azure/i18n/it.po b/fs_attachment_azure/i18n/it.po deleted file mode 100644 index d03ad91248..0000000000 --- a/fs_attachment_azure/i18n/it.po +++ /dev/null @@ -1,56 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * fs_attachment_s3 -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 17.0\n" -"Report-Msgid-Bugs-To: \n" -"PO-Revision-Date: 2025-10-23 08:43+0000\n" -"Last-Translator: mymage \n" -"Language-Team: none\n" -"Language: it\n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 5.10.4\n" - -#. module: fs_attachment_s3 -#: model:ir.model,name:fs_attachment_s3.model_ir_attachment -msgid "Attachment" -msgstr "Allegato" - -#. module: fs_attachment_s3 -#: model:ir.model,name:fs_attachment_s3.model_fs_storage -msgid "FS Storage" -msgstr "Deposito FS" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile -msgid "" -"If checked, the storage will use signed URLs for attachments when using " -"X-Accel-Redirect. This is useful for S3 storage where the file path is not " -"directly accessible without authentication." -msgstr "" -"Se selezionata, l'archiviazione utilizzerà URL firmati per gli allegati " -"quando si utilizza X-Accel-Redirect. Questa opzione è utile per " -"l'archiviazione S3 in cui il percorso del file non è direttamente " -"accessibile senza autenticazione." - -#. module: fs_attachment_s3 -#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration -msgid "Signed URL Expiration (seconds)" -msgstr "Scadenza URL firmato (secondi)" - -#. module: fs_attachment_s3 -#: model:ir.model.fields,help:fs_attachment_s3.field_fs_storage__s3_signed_url_expiration -msgid "" -"The expiration time for the signed URL in seconds. Default is 30 seconds." -msgstr "" -"Il tempo di scadenza per l'URL firmato in secondi. Predefinito 30 secondi." - -#. module: fs_attachment_s3 -#: model:ir.model.fields,field_description:fs_attachment_s3.field_fs_storage__s3_uses_signed_url_for_x_sendfile -msgid "Use signed URL for X-Accel-Redirect" -msgstr "Utilizza URL firmato per X-Accel-Redirect" diff --git a/fs_attachment_azure/models/fs_storage.py b/fs_attachment_azure/models/fs_storage.py index 1b737c722d..31c0acb2d7 100644 --- a/fs_attachment_azure/models/fs_storage.py +++ b/fs_attachment_azure/models/fs_storage.py @@ -1,4 +1,5 @@ # Copyright 2025 ACSONE SA/NV +# Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). import fsspec.asyn @@ -9,13 +10,13 @@ class FsStorage(models.Model): _inherit = "fs.storage" - s3_uses_signed_url_for_x_sendfile = fields.Boolean( + azure_uses_signed_url_for_x_sendfile = fields.Boolean( string="Use signed URL for X-Accel-Redirect", help="If checked, the storage will use signed URLs for attachments " - "when using X-Accel-Redirect. This is useful for S3 storage where the " + "when using X-Accel-Redirect. This is useful for Azure storage where the " "file path is not directly accessible without authentication.", ) - s3_signed_url_expiration = fields.Integer( + azure_signed_url_expiration = fields.Integer( string="Signed URL Expiration (seconds)", default=30, help="The expiration time for the signed URL in seconds. " @@ -24,30 +25,31 @@ class FsStorage(models.Model): @property def _server_env_fields(self): - """Override to include S3 specific fields.""" + """Override to include Azure specific fields.""" fields = super()._server_env_fields fields.update( { - "s3_uses_signed_url_for_x_sendfile": {}, - "s3_signed_url_expiration": {}, + "azure_uses_signed_url_for_x_sendfile": {}, + "azure_signed_url_expiration": {}, } ) return fields @property - def is_s3_storage(self): - """Check if the storage is an S3 storage.""" + def is_azure_storage(self): + """Check if the storage is an Azure storage.""" self.ensure_one() - return hasattr(self._get_root_filesystem(self.fs), "s3") + fs = self._get_root_filesystem(self.fs) + protocol = getattr(fs, "protocol", []) + return self.protocol in protocol @api.model - def _s3_call_generate_presigned_url(self, s3_client, *args, **kwargs): - """Generate a presigned URL for S3 operations.""" - # s3fs uses aiobotocore as s3 client, which is asynchronous. + def _azure_call_synchronous(self, azure_client_function, *args, **kwargs): + # adlfs uses asynchronous client # We need to run the async function in a synchronous context. return fsspec.asyn.sync( fsspec.asyn.get_loop(), - s3_client.generate_presigned_url, + azure_client_function, *args, timeout=None, **kwargs, diff --git a/fs_attachment_azure/models/ir_attachment.py b/fs_attachment_azure/models/ir_attachment.py index 8bdc2c94ac..c171bae118 100644 --- a/fs_attachment_azure/models/ir_attachment.py +++ b/fs_attachment_azure/models/ir_attachment.py @@ -1,81 +1,98 @@ # Copyright 2025 ACSONE SA/NV # Copyright 2025 XCG SAS +# Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import datetime from urllib.parse import urlparse +from adlfs.spec import BlobSasPermissions, generate_blob_sas + from odoo import models class IrAttachment(models.Model): _inherit = "ir.attachment" - def _storage_write_option(self, fs): - option = super()._storage_write_option(fs) - mimetype = self.env.context.get("mimetype") - if mimetype: - root_fs = self.env["fs.storage"]._get_root_filesystem(fs) - if hasattr(root_fs, "s3"): - option["ContentType"] = mimetype - return option - def _get_x_sendfile_path(self): self.ensure_one() storage = self.fs_storage_id - if storage.is_s3_storage: - return self._get_s3_x_sendfile_path() + if storage.is_azure_storage: + return self._get_azure_x_sendfile_path() return super()._get_x_sendfile_path() def _fs_use_x_sendfile(self): self.ensure_one() storage = self.fs_storage_id - if storage.is_s3_storage: + if storage.is_azure_storage: return storage.use_x_sendfile_to_serve_internal_url return super()._fs_use_x_sendfile() - def _get_s3_x_sendfile_path(self): - """Generate the X-Accel-Redirect path for S3 storage. + def _get_azure_x_sendfile_path(self): + """Generate the X-Accel-Redirect path for Azure storage. - This method is used to generate the path for S3 storage when using - X-Accel-Redirect. It constructs the path based on the S3 bucket and - file path, ensuring that it is compatible with the S3 storage + This method is used to generate the path for Azure storage when using + X-Accel-Redirect. It constructs the path based on the Azure container and + file path, ensuring that it is compatible with the Azure storage configuration and the Odoo file storage system. Args: attachment (IrAttachment): The attachment record for which the X-Accel-Redirect path is being generated. Returns: - str: The X-Accel-Redirect path for the S3 storage. + str: The X-Accel-Redirect path for the Azure storage. The path is formatted as: /fs_x_sendfile/// where: - `` is the scheme of the base URL (e.g., 'https'). - - `` is the netloc of the base URL (e.g., 's3.amazonaws.com'). - - `` is the path to the file in the S3 bucket, including the - bucket name + - `` is the netloc of the base URL + (e.g., 'myaccount.blob.core.windows.net'). + - `` is the path to the file in the Azure container, including the + container name """ fs, storage_code, file_path = self._get_fs_parts() storage = self.env["fs.storage"].sudo().get_by_code(storage_code) root_fs = storage._get_root_filesystem(fs) - s3_client = root_fs.s3 - # The directory path might contain the bucket and a prefix - # the part before the first "/" is the bucket - # the rest is the prefix - bucket_name, *prefix_parts = storage.get_directory_path().strip("/").split("/") - s3_key = "/".join(prefix_parts + [file_path.lstrip("/")]) - if storage.s3_uses_signed_url_for_x_sendfile: - file_url = storage._s3_call_generate_presigned_url( - s3_client, - "get_object", - Params={"Bucket": bucket_name, "Key": s3_key}, - ExpiresIn=storage.s3_signed_url_expiration, - ) + azure_client = root_fs.service_client + container_name = storage.get_directory_path() + blob_client = azure_client.get_blob_client(container_name, file_path) + if storage.azure_uses_signed_url_for_x_sendfile: + if ( + azure_client.connection_string + or azure_client.account_name + and azure_client.account_key + ): + file_url = azure_client.url( + file_path, expires=storage.azure_signed_url_expiration + ) + else: + # Ideally we would be able to call azure_client.url() as it is calling + # generate_blob_sas. However, it expects to use an account shared key + # (i.e either a connection string or account name/key pair). + # For this we need to get a delegation key first + now = datetime.datetime.now() + expiry_time = now + datetime.timedelta( + seconds=storage.azure_signed_url_expiration + ) + delegation_key = storage._azure_call_synchronous( + azure_client.get_user_delegation_key, + key_start_time=now, + key_expiry_time=expiry_time, + ) + # Then we can call generate_blob_sas + sas_token = storage._azure_call_synchronous( + generate_blob_sas, + fs.fs.account_name, + fs.path, + file_path, + user_delegation_key=delegation_key, + permission=BlobSasPermissions(read=True), + expiry=expiry_time, + ) + file_url = f"{blob_client.url}?{sas_token}" else: - file_url = ( - f"{s3_client.meta.endpoint_url.rstrip('/')}/" - f"{bucket_name}/{s3_key.lstrip('/')}" - ) + file_url = blob_client.url parsed_url = urlparse(file_url) path = parsed_url.path.strip("/") diff --git a/fs_attachment_azure/readme/CONFIGURE.md b/fs_attachment_azure/readme/CONFIGURE.md index 219778ad4d..84842eef89 100644 --- a/fs_attachment_azure/readme/CONFIGURE.md +++ b/fs_attachment_azure/readme/CONFIGURE.md @@ -1,12 +1,12 @@ On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File Storage*. -When you create a new storage for s3 or modify an existing one, when you activate +When you create a new storage for Azure or modify an existing one, when you activate the option "Use X-Sendfile To Serve Internal Url", 2 additional fields will appear: -- **S3 Uses Signed URL For X-Accel-Redirect**: If checked, the X-Accel-Redirect +- **Azure Uses Signed URL For X-Accel-Redirect**: If checked, the X-Accel-Redirect path will be a signed URL, which is useful for S3 storages that require signed URLs for access. -- **S3 Signed URL Expiration**: The expiration time for the signed URL in seconds. +- **Azure Signed URL Expiration**: The expiration time for the signed URL in seconds. This field is only relevant if the previous option is checked. By default, it is set to 30 seconds but it could be less since the url generated into the X-Accel-Redirect process is directly used by the web server to serve the file. @@ -14,12 +14,12 @@ the option "Use X-Sendfile To Serve Internal Url", 2 additional fields will appe The value of these fields can also be set in the server environment variables using the keys: -- *s3_uses_signed_url_for_x_sendfile* -- *s3_signed_url_expiration* +- *azure_uses_signed_url_for_x_sendfile* +- *azure_signed_url_expiration* When the option "Use X-Sendfile To Serve Internal Url" is enabled, the system will generate an X-Accel-Redirect header in the response to a request to get a file. -In the case of S3 storages, it will follow the format: +In the case of Azure storages, it will follow the format: ```text X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any} @@ -28,15 +28,15 @@ X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any} Where: - `{scheme}`: The URL scheme (http or https). -- `{host}`: The host of the S3 storage. -- `{path with query if any}`: The path to the file in the S3 storage, +- `{host}`: The host of the Azure storage. +- `{path with query if any}`: The path to the file in the Azure storage, including any query parameters. (Query parameters are set when the - `s3_uses_signed_url_for_x_sendfile` option is enabled.) + `azure_uses_signed_url_for_x_sendfile` option is enabled.) In order to serve files using X-Accel-Redirect, you must ensure that your web server is configured to handle these headers correctly. This typically involves setting up a location block in your web server configuration that -matches the X-Accel-Redirect path and proxies the request to the S3 storage. +matches the X-Accel-Redirect path and proxies the request to the Azure storage. For example, if you are using Nginx, you would add a location block like this: @@ -57,8 +57,8 @@ For example, if you are using Nginx, you would add a location block like this: ``` -Unlike the standard implementation of X-Accel-Redirect on non S3 storages, -the S3 implementation does not require a base URL to be set in the storage +Unlike the standard implementation of X-Accel-Redirect on non Azure storages, +the Azure implementation does not require a base URL to be set in the storage configuration. The X-Accel-Redirect path is constructed directly from the -S3 storage's URL defined for the connection, the directory name as +Azure storage's URL defined for the connection, the directory name as bucket name, and the file path. diff --git a/fs_attachment_azure/readme/CONTRIBUTORS.md b/fs_attachment_azure/readme/CONTRIBUTORS.md index 06d49341ab..cb7b22f28c 100644 --- a/fs_attachment_azure/readme/CONTRIBUTORS.md +++ b/fs_attachment_azure/readme/CONTRIBUTORS.md @@ -1,2 +1,3 @@ - Laurent Mignon (https://www.acsone.eu) - Stéphane Bidoul (https://www.acsone.eu) +- Akim Juillerat \ No newline at end of file diff --git a/fs_attachment_azure/readme/CREDITS.md b/fs_attachment_azure/readme/CREDITS.md deleted file mode 100644 index c733078a75..0000000000 --- a/fs_attachment_azure/readme/CREDITS.md +++ /dev/null @@ -1,4 +0,0 @@ -The development of this module has been financially supported by: - -- ACSONE SA/NV (https://www.acsone.eu) -- Alcyon Belux \ No newline at end of file diff --git a/fs_attachment_azure/readme/DESCRIPTION.md b/fs_attachment_azure/readme/DESCRIPTION.md index c37359b415..e21f93c98c 100644 --- a/fs_attachment_azure/readme/DESCRIPTION.md +++ b/fs_attachment_azure/readme/DESCRIPTION.md @@ -1,7 +1,6 @@ This module extends the functionality of [fs_attachment](https://github.com/OCA/storage/tree/16.0/fs_attachment) -to better support Amazon S3 storage. It includes features such as: +to better support Azure storage. It includes features such as: -- Special handling of X-Accel-Redirect headers for S3 storages. -- Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private S3 bucket +- Special handling of X-Accel-Redirect headers for Azure storages. +- Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private Azure Blob Storage using X-Accel-Redirect without exposing the files publicly.) -- Enforcing the mimetype of files stored in S3. diff --git a/fs_attachment_azure/readme/HISTORY.md b/fs_attachment_azure/readme/HISTORY.md index b4c434c401..9ec3de838e 100644 --- a/fs_attachment_azure/readme/HISTORY.md +++ b/fs_attachment_azure/readme/HISTORY.md @@ -1,14 +1,3 @@ -## 17.0.1.2.1 (2026-05-27) +## 17.0.1.0.0 (2026-07-13) -### Bugfixes - -- Allow to use a prefix path and bucket in the directory_path on fs.storage - When the directory_path parameter is configured as / - the presigned url generation failed with a botocore error: "Invalid bucket name". ([#b17de9](https://github.com/OCA/storage/issues/b17de9)) - - -## 17.0.1.2.0 (2025-10-22) - -### Features - -- Adapt to handle {db_name} in directory_path. ([#db_name](https://github.com/OCA/storage/issues/db_name)) +- This module was "forked" from fs_attachment_s3 v17.0.1.2.1 diff --git a/fs_attachment_azure/static/description/index.html b/fs_attachment_azure/static/description/index.html index 233c7217f1..ee564e4e01 100644 --- a/fs_attachment_azure/static/description/index.html +++ b/fs_attachment_azure/static/description/index.html @@ -3,7 +3,7 @@ -README.rst +Fs Attachment Azure -
+
+

Fs Attachment Azure

- - -Odoo Community Association - -
-

Fs Attachment S3

-

Beta License: AGPL-3 OCA/storage Translate me on Weblate Try me on Runboat

+

Beta License: AGPL-3 OCA/storage Translate me on Weblate Try me on Runboat

This module extends the functionality of fs_attachment -to better support Amazon S3 storage. It includes features such as:

+to better support Azure storage. It includes features such as:

    -
  • Special handling of X-Accel-Redirect headers for S3 storages.
    • +
  • Special handling of X-Accel-Redirect headers for Azure storages.
  • Options for using signed URLs in X-Accel-Redirect. (This is required -to be able to serve files from a private S3 bucket using +to be able to serve files from a private Azure Blob Storage using X-Accel-Redirect without exposing the files publicly.)
    • -
  • Enforcing the mimetype of files stored in S3.

Table of contents

-

Configuration

+

Configuration

On the Odoo instance, go to Settings > Technical > Storage > File Storage.

-

When you create a new storage for s3 or modify an existing one, when you -activate the option “Use X-Sendfile To Serve Internal Url”, 2 additional -fields will appear:

+

When you create a new storage for Azure or modify an existing one, when +you activate the option “Use X-Sendfile To Serve Internal Url”, 2 +additional fields will appear:

    -
  • S3 Uses Signed URL For X-Accel-Redirect: If checked, the +
  • Azure Uses Signed URL For X-Accel-Redirect: If checked, the X-Accel-Redirect path will be a signed URL, which is useful for S3 storages that require signed URLs for access.
    • -
  • S3 Signed URL Expiration: The expiration time for the signed URL -in seconds. This field is only relevant if the previous option is +
  • Azure Signed URL Expiration: The expiration time for the signed +URL in seconds. This field is only relevant if the previous option is checked. By default, it is set to 30 seconds but it could be less since the url generated into the X-Accel-Redirect process is directly used by the web server to serve the file.
    • @@ -430,12 +416,12 @@

    Configuration

    The value of these fields can also be set in the server environment variables using the keys:

      -
    • s3_uses_signed_url_for_x_sendfile
      • -
    • s3_signed_url_expiration
      • +
    • azure_uses_signed_url_for_x_sendfile
      • +
    • azure_signed_url_expiration

    When the option “Use X-Sendfile To Serve Internal Url” is enabled, the system will generate an X-Accel-Redirect header in the response to a -request to get a file. In the case of S3 storages, it will follow the +request to get a file. In the case of Azure storages, it will follow the format:

     X-Accel-Redirect: /fs_x_sendfile/{scheme}/{host}/{path with query if any}
@@ -443,16 +429,16 @@ 
    Configuration
    
 
    Where:
    
 
      
 
    • {scheme}: The URL scheme (http or https).
      • 
-
    • {host}: The host of the S3 storage.
      • 
-
    • {path with query if any}: The path to the file in the S3 storage,
-including any query parameters. (Query parameters are set when the
-s3_uses_signed_url_for_x_sendfile option is enabled.)
      • 
+
    • {host}: The host of the Azure storage.
      • 
+
    • {path with query if any}: The path to the file in the Azure
+storage, including any query parameters. (Query parameters are set
+when the azure_uses_signed_url_for_x_sendfile option is enabled.)
      • 
 
    
 
    In order to serve files using X-Accel-Redirect, you must ensure that
 your web server is configured to handle these headers correctly. This
 typically involves setting up a location block in your web server
 configuration that matches the X-Accel-Redirect path and proxies the
-request to the S3 storage.
    
+request to the Azure storage.
    
 
    For example, if you are using Nginx, you would add a location block like
 this:
    
 
    @@ -469,70 +455,48 @@ 
    Configuration
    
 
 }
 
    
-
    Unlike the standard implementation of X-Accel-Redirect on non S3
-storages, the S3 implementation does not require a base URL to be set in
-the storage configuration. The X-Accel-Redirect path is constructed
-directly from the S3 storage’s URL defined for the connection, the
+
    Unlike the standard implementation of X-Accel-Redirect on non Azure
+storages, the Azure implementation does not require a base URL to be set
+in the storage configuration. The X-Accel-Redirect path is constructed
+directly from the Azure storage’s URL defined for the connection, the
 directory name as bucket name, and the file path.
-

Changelog

+

Changelog

-

17.0.1.2.1 (2026-05-27)

-
-

Bugfixes

+

17.0.1.0.0 (2026-07-13)

    -
  • Allow to use a prefix path and bucket in the directory_path on -fs.storage When the directory_path parameter is configured as / the -presigned url generation failed with a botocore error: “Invalid bucket -name”. (#b17de9)
    • +
  • This module was “forked” from fs_attachment_s3 v17.0.1.2.1
-
-

17.0.1.2.0 (2025-10-22)

-
-

Features

-
    -
  • Adapt to handle {db_name} in directory_path. -(#db_name)
    • -
-
-
-
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • ACSONE SA/NV
    • +
  • Camptocamp
-
-

Other credits

-

The development of this module has been financially supported by:

-
-

Maintainers

+

Maintainers

This module is maintained by the OCA.

Odoo Community Association @@ -541,12 +505,11 @@

Maintainers

mission is to support the collaborative development of Odoo features and promote its widespread use.

Current maintainer:

-

lmignon

-

This module is part of the OCA/storage project on GitHub.

+

grindtildeath

+

This module is part of the OCA/storage project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

-
diff --git a/fs_attachment_azure/tests/__init__.py b/fs_attachment_azure/tests/__init__.py index a788fc4772..fe85fc1d1c 100644 --- a/fs_attachment_azure/tests/__init__.py +++ b/fs_attachment_azure/tests/__init__.py @@ -1 +1 @@ -from . import test_fs_attachment_s3 +from . import test_fs_attachment_azure diff --git a/fs_attachment_azure/tests/common.py b/fs_attachment_azure/tests/common.py index 3f2c61b581..6d7c12454f 100644 --- a/fs_attachment_azure/tests/common.py +++ b/fs_attachment_azure/tests/common.py @@ -1,63 +1,47 @@ # Copyright 2025 ACSONE SA/NV (http://acsone.eu). +# Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). - from odoo.tests.common import TransactionCase from odoo.addons.base.tests.common import DISABLED_MAIL_CONTEXT -class TestFSAttachmentS3Common(TransactionCase): +class TestFSAttachmentAzureCommon(TransactionCase): @classmethod def setUpClass(cls): super().setUpClass() cls.env = cls.env(context=dict(cls.env.context, **DISABLED_MAIL_CONTEXT)) - cls.s3_backend_config = { - "name": "S3 Storage", - "protocol": "s3", - "code": "s3tst", - "directory_path": "test-bucket", - "json_options": { - "key": "aws-key", - "secret": "aws-secret", - "client_kwargs": { - "endpoint_url": "http://minio.minio/", - "region_name": "aws-region", - }, - }, - "base_url": False, # S3 does not use base_url for x-sendfile + cls.azure_backend_config = { + "name": "Azure Storage", + "protocol": "az", + "code": "azure", + "directory_path": "test-blob", } - cls.s3_backend = cls.env["fs.storage"].create(cls.s3_backend_config) + cls.azure_backend = cls.env["fs.storage"].create(cls.azure_backend_config) cls.ir_attachment_model = cls.env["ir.attachment"] - cls.fake_attachment_s3 = cls.env["ir.attachment"].create( + cls.fake_attachment_azure = cls.env["ir.attachment"].create( { - "name": "fake_s3_file.txt", - "fs_storage_id": cls.s3_backend.id, + "name": "fake_azure_file.txt", + "fs_storage_id": cls.azure_backend.id, } ) - cls.fake_attachment_s3.flush_recordset() - # update the attachment into database since we don't have a real S3 bucket - # and we can't use moto to mock S3 since s3fs rely on aiobotocore + cls.fake_attachment_azure.flush_recordset() + # update the attachment into database since we don't have a real blob storage cls.env.cr.execute( """ UPDATE ir_attachment SET - store_fname = 's3tst://dir/sub/fake_s3_file.txt', - fs_filename = 'fake_s3_file.txt', - fs_storage_code = 's3tst', + store_fname = 'azure://dir/sub/fake_azure_file.txt', + fs_filename = 'fake_azure_file.txt', + fs_storage_code = 'azure', checksum = 234, file_size = 1234, fs_storage_id = %s WHERE id = %s """, - (cls.s3_backend.id, cls.fake_attachment_s3.id), + (cls.azure_backend.id, cls.fake_attachment_azure.id), ) - cls.fake_attachment_s3.invalidate_recordset() - - def setUp(self): - super().setUp() - # enforce backend_config fields since it seems that they are reset on - # savepoint rollback when managed by server_environment -> TO Be investigated - self.s3_backend.write(self.s3_backend_config) + cls.fake_attachment_azure.invalidate_recordset() diff --git a/fs_attachment_azure/tests/test_fs_attachment_azure.py b/fs_attachment_azure/tests/test_fs_attachment_azure.py new file mode 100644 index 0000000000..5c715c4018 --- /dev/null +++ b/fs_attachment_azure/tests/test_fs_attachment_azure.py @@ -0,0 +1,62 @@ +# Copyright 2025 ACSONE SA/NV (http://acsone.eu). +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from adlfs import AzureBlobFileSystem + +from .common import TestFSAttachmentAzureCommon + +PROTOCOL = "https" +ACCOUNT_NAME = "myaccount" +ACCOUNT_KEY = "123456789" +DOMAIN = "blob.core.windows.net" +CONTAINER = "test-blob" +PATH = "dir/sub" +FILENAME = "fake_azure_file.txt" +BASE_URL = f"{PROTOCOL}://{ACCOUNT_NAME}.{DOMAIN}/{CONTAINER}/{PATH}/{FILENAME}" +TOKEN = "1111-2222-3333-4444" +CONNECTION_STRING = f"DefaultEndpointsProtocol={PROTOCOL};AccountName={ACCOUNT_NAME};AccountKey={ACCOUNT_KEY};BlobEndpoint={PROTOCOL}://{DOMAIN}/{ACCOUNT_NAME};" + + +def _fake_do_connect(self): + mock_service_client = MagicMock() + mock_blob_client = MagicMock() + mock_blob_client.url = BASE_URL + mock_service_client.get_container_client.return_value = Mock() + mock_service_client.get_blob_client.return_value = mock_blob_client + mock_service_client.connection_string = CONNECTION_STRING + mock_service_client.url.return_value = "?".join([BASE_URL, TOKEN]) + mock_service_client.close = AsyncMock(return_value="ok") + self.service_client = mock_service_client + + +class TestFSAttachementAzure(TestFSAttachmentAzureCommon): + def test_get_x_sendfile_path_azure_signed(self): + """Test the X-Accel-Redirect path generation for azure storage.""" + self.azure_backend.write( + { + "azure_uses_signed_url_for_x_sendfile": True, + "azure_signed_url_expiration": 60, + } + ) + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect): + url = self.fake_attachment_azure._get_x_sendfile_path() + self.assertTrue( + url.startswith( + "/fs_x_sendfile/https/myaccount.blob.core.windows.net/test-blob/dir/sub/fake_azure_file.txt?1111-2222-3333-4444" + ), + "The end of the path should contain the path to the file " + f"name and query parameters. ({url})", + ) + + def test_get_x_sendfile_path_azure(self): + """Test the X-Accel-Redirect path generation.""" + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect): + url = self.fake_attachment_azure._get_x_sendfile_path() + + self.assertEqual( + url, + "/fs_x_sendfile/https/myaccount.blob.core.windows.net/test-blob/dir/sub/fake_azure_file.txt", + f"The X-Accel-Redirect path should match the expected format. ({url})", + ) diff --git a/fs_attachment_azure/tests/test_fs_attachment_s3.py b/fs_attachment_azure/tests/test_fs_attachment_s3.py deleted file mode 100644 index 67488c7fff..0000000000 --- a/fs_attachment_azure/tests/test_fs_attachment_s3.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2025 ACSONE SA/NV (http://acsone.eu). -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). - - -from .common import TestFSAttachmentS3Common - - -class TestFSAttachementS3(TestFSAttachmentS3Common): - def test_get_x_sendfile_path_s3_signed(self): - """Test the X-Accel-Redirect path generation for S3 storage.""" - self.s3_backend.write( - { - "s3_uses_signed_url_for_x_sendfile": True, - "s3_signed_url_expiration": 60, - } - ) - - url = self.fake_attachment_s3._get_x_sendfile_path() - self.assertTrue( - url.startswith( - "/fs_x_sendfile/http/minio.minio/" - "test-bucket/dir/sub/fake_s3_file.txt?" - ), - "The end of the path should contain the path to the file " - f"name and query parameters. ({url})", - ) - - def test_get_x_sendfile_path(self): - """Test the X-Accel-Redirect path generation.""" - url = self.fake_attachment_s3._get_x_sendfile_path() - self.assertEqual( - url, - "/fs_x_sendfile/http/minio.minio/test-bucket/dir/sub/fake_s3_file.txt", - f"The X-Accel-Redirect path should match the expected format. ({url})", - ) - - def test_get_x_sendfile_path_s3_signed_with_prefix(self): - """Test the path generation when the directory path has a prefix.""" - self.s3_backend.write( - { - "s3_uses_signed_url_for_x_sendfile": True, - "s3_signed_url_expiration": 60, - "directory_path": "test-bucket/test-prefix", - } - ) - - url = self.fake_attachment_s3._get_x_sendfile_path() - self.assertTrue( - url.startswith( - "/fs_x_sendfile/http/minio.minio/" - "test-bucket/test-prefix/dir/sub/fake_s3_file.txt?" - ), - f"The URL should include the prefix. ({url})", - ) - - def test_get_x_sendfile_path_with_prefix(self): - """Test the unsigned path generation with a prefix.""" - self.s3_backend.write( - { - "s3_uses_signed_url_for_x_sendfile": False, - "directory_path": "test-bucket/test-prefix", - } - ) - url = self.fake_attachment_s3._get_x_sendfile_path() - self.assertEqual( - url, - "/fs_x_sendfile/http/minio.minio/test-bucket/test-prefix/dir/sub/fake_s3_file.txt", - f"The URL should include the prefix. ({url})", - ) diff --git a/fs_attachment_azure/views/fs_storage.xml b/fs_attachment_azure/views/fs_storage.xml index 8bd9160413..d1265a3884 100644 --- a/fs_attachment_azure/views/fs_storage.xml +++ b/fs_attachment_azure/views/fs_storage.xml @@ -1,5 +1,6 @@ @@ -9,12 +10,12 @@ diff --git a/requirements.txt b/requirements.txt index 275c9dfb40..0bc35526f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # generated from manifests external_dependencies +adlfs fsspec>=2024.5.0 fsspec>=2025.3.0 fsspec[s3] From 541a91e9eed19eca536a4a163aaeab575ad2f9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Todorovich?= Date: Tue, 18 Aug 2026 14:28:26 -0300 Subject: [PATCH 3/6] fixup! fs_attachment_azure: Adapt to azure --- fs_attachment_azure/models/ir_attachment.py | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/fs_attachment_azure/models/ir_attachment.py b/fs_attachment_azure/models/ir_attachment.py index c171bae118..ac39cd2d90 100644 --- a/fs_attachment_azure/models/ir_attachment.py +++ b/fs_attachment_azure/models/ir_attachment.py @@ -58,20 +58,22 @@ def _get_azure_x_sendfile_path(self): container_name = storage.get_directory_path() blob_client = azure_client.get_blob_client(container_name, file_path) if storage.azure_uses_signed_url_for_x_sendfile: - if ( - azure_client.connection_string - or azure_client.account_name - and azure_client.account_key + if root_fs.connection_string or ( + root_fs.account_name and root_fs.account_key ): - file_url = azure_client.url( - file_path, expires=storage.azure_signed_url_expiration + # AzureBlobFileSystem.url() signs with the shared key. + file_url = root_fs.url( + f"{container_name}/{file_path}", + expires=storage.azure_signed_url_expiration, ) else: - # Ideally we would be able to call azure_client.url() as it is calling + # Ideally we would be able to call root_fs.url() as it is calling # generate_blob_sas. However, it expects to use an account shared key # (i.e either a connection string or account name/key pair). - # For this we need to get a delegation key first - now = datetime.datetime.now() + # For this we need to get a delegation key first. The identity + # needs the "Storage Blob Delegator" role on the storage account, + # on top of a data plane role. + now = datetime.datetime.now(datetime.timezone.utc) expiry_time = now + datetime.timedelta( seconds=storage.azure_signed_url_expiration ) @@ -81,13 +83,13 @@ def _get_azure_x_sendfile_path(self): key_expiry_time=expiry_time, ) # Then we can call generate_blob_sas - sas_token = storage._azure_call_synchronous( - generate_blob_sas, - fs.fs.account_name, - fs.path, - file_path, + sas_token = generate_blob_sas( + account_name=blob_client.account_name, + container_name=container_name, + blob_name=file_path, user_delegation_key=delegation_key, permission=BlobSasPermissions(read=True), + start=now, expiry=expiry_time, ) file_url = f"{blob_client.url}?{sas_token}" From 83fff82a33f85b10a1820891041b286babca9d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Todorovich?= Date: Wed, 9 Sep 2026 13:44:59 -0300 Subject: [PATCH 4/6] fixup! fs_attachment_azure: Adapt to azure Cache the user delegation key and tolerate clock skew when signing. --- fs_attachment_azure/README.rst | 79 ++++-- fs_attachment_azure/models/fs_storage.py | 82 +++++- fs_attachment_azure/models/ir_attachment.py | 23 +- fs_attachment_azure/readme/CONFIGURE.md | 31 ++- .../static/description/index.html | 64 +++-- .../tests/test_fs_attachment_azure.py | 263 ++++++++++++++++-- fs_attachment_azure/tools.py | 83 ++++++ fs_attachment_azure/views/fs_storage.xml | 8 +- 8 files changed, 551 insertions(+), 82 deletions(-) create mode 100644 fs_attachment_azure/tools.py diff --git a/fs_attachment_azure/README.rst b/fs_attachment_azure/README.rst index 8c3bd487b5..9885c9668e 100644 --- a/fs_attachment_azure/README.rst +++ b/fs_attachment_azure/README.rst @@ -32,10 +32,10 @@ This module extends the functionality of `fs_attachment `__ to better support Azure storage. It includes features such as: -- Special handling of X-Accel-Redirect headers for Azure storages. -- Options for using signed URLs in X-Accel-Redirect. (This is required - to be able to serve files from a private Azure Blob Storage using - X-Accel-Redirect without exposing the files publicly.) +- Special handling of X-Accel-Redirect headers for Azure storages. +- Options for using signed URLs in X-Accel-Redirect. (This is required + to be able to serve files from a private Azure Blob Storage using + X-Accel-Redirect without exposing the files publicly.) **Table of contents** @@ -49,23 +49,28 @@ On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File Storage*. When you create a new storage for Azure or modify an existing one, when -you activate the option "Use X-Sendfile To Serve Internal Url", 2 +you activate the option "Use X-Sendfile To Serve Internal Url", 3 additional fields will appear: -- **Azure Uses Signed URL For X-Accel-Redirect**: If checked, the - X-Accel-Redirect path will be a signed URL, which is useful for S3 - storages that require signed URLs for access. -- **Azure Signed URL Expiration**: The expiration time for the signed - URL in seconds. This field is only relevant if the previous option is - checked. By default, it is set to 30 seconds but it could be less - since the url generated into the X-Accel-Redirect process is directly - used by the web server to serve the file. +- **Azure Uses Signed URL For X-Accel-Redirect**: If checked, the + X-Accel-Redirect path will be a signed URL, which is useful for Azure + storages that require signed URLs for access. +- **Azure Signed URL Expiration**: The expiration time for the signed + URL in seconds. This field is only relevant if the previous option is + checked. By default, it is set to 30 seconds but it could be less + since the url generated into the X-Accel-Redirect process is directly + used by the web server to serve the file. +- **Azure Delegation Key Expiration**: The lifetime of the user + delegation key, in seconds. This field is only relevant when the + storage authenticates with an identity (see below). By default it is + set to 1 hour, and Azure does not allow more than 7 days. The value of these fields can also be set in the server environment variables using the keys: -- *azure_uses_signed_url_for_x_sendfile* -- *azure_signed_url_expiration* +- *azure_uses_signed_url_for_x_sendfile* +- *azure_signed_url_expiration* +- *azure_delegation_key_expiration* When the option "Use X-Sendfile To Serve Internal Url" is enabled, the system will generate an X-Accel-Redirect header in the response to a @@ -78,11 +83,11 @@ format: Where: -- ``{scheme}``: The URL scheme (http or https). -- ``{host}``: The host of the Azure storage. -- ``{path with query if any}``: The path to the file in the Azure - storage, including any query parameters. (Query parameters are set - when the ``azure_uses_signed_url_for_x_sendfile`` option is enabled.) +- ``{scheme}``: The URL scheme (http or https). +- ``{host}``: The host of the Azure storage. +- ``{path with query if any}``: The path to the file in the Azure + storage, including any query parameters. (Query parameters are set + when the ``azure_uses_signed_url_for_x_sendfile`` option is enabled.) In order to serve files using X-Accel-Redirect, you must ensure that your web server is configured to handle these headers correctly. This @@ -115,13 +120,39 @@ in the storage configuration. The X-Accel-Redirect path is constructed directly from the Azure storage's URL defined for the connection, the directory name as bucket name, and the file path. +Signing with an identity +------------------------ + +Signed URLs are generated with the account shared key when the storage +is configured with a connection string or an account name/key pair. +Otherwise (managed identity, workload identity, service principal, ...) +they are signed with a *user delegation key*, which requires the +identity to have the **Storage Blob Delegator** role on the storage +account, on top of a data plane role such as *Storage Blob Data Reader*. + +Delegation keys are obtained from Azure with an extra request, so they +are kept in the cache of the Odoo registry for **Azure Delegation Key +Expiration** seconds. They are requested from Azure for slightly longer +than that, so that a key served from the cache still covers the URLs +signed with it. + +A higher value means fewer requests to Azure, but also a longer window +during which the key of a revoked identity remains usable. Note that a +key already issued by Azure stays valid until it expires anyway, +whatever Odoo does with its copy. + +The cache is only kept in memory, so it is never shared between +processes nor persisted in the database, and it is dropped whenever Odoo +clears the cache of the registry, which modifying a storage does. +Reconfiguring a storage is therefore applied right away. + Changelog ========= 17.0.1.0.0 (2026-07-13) ----------------------- -- This module was "forked" from fs_attachment_s3 v17.0.1.2.1 +- This module was "forked" from fs_attachment_s3 v17.0.1.2.1 Bug Tracker =========== @@ -145,9 +176,9 @@ Authors Contributors ------------ -- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) -- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) -- Akim Juillerat akim.juillerat@camptocamp.com +- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) +- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) +- Akim Juillerat akim.juillerat@camptocamp.com Maintainers ----------- diff --git a/fs_attachment_azure/models/fs_storage.py b/fs_attachment_azure/models/fs_storage.py index 31c0acb2d7..b9a64eb510 100644 --- a/fs_attachment_azure/models/fs_storage.py +++ b/fs_attachment_azure/models/fs_storage.py @@ -2,9 +2,21 @@ # Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import datetime + import fsspec.asyn -from odoo import api, fields, models +from odoo import _, api, fields, models +from odoo.exceptions import ValidationError + +from ..tools import ormcache_expiring + +# Start times are backdated by this many seconds, to tolerate clock skew +# between this host and Azure. +AZURE_CLOCK_SKEW_TOLERANCE = 300 + +# Azure refuses to issue a user delegation key valid for more than 7 days. +AZURE_MAX_DELEGATION_KEY_EXPIRATION = 7 * 24 * 3600 class FsStorage(models.Model): @@ -22,6 +34,16 @@ class FsStorage(models.Model): help="The expiration time for the signed URL in seconds. " "Default is 30 seconds.", ) + azure_delegation_key_expiration = fields.Integer( + string="Delegation Key Expiration (seconds)", + default=3600, + help="How long the user delegation key used to sign URLs is cached, when " + "the storage authenticates with an identity instead of a shared key. A " + "higher value means fewer calls to Azure, but a longer window during " + "which the key of a revoked identity remains usable. Azure does not " + "issue keys valid for more than 7 days, which this and the signed URL " + "expiration must leave room for.", + ) @property def _server_env_fields(self): @@ -31,10 +53,37 @@ def _server_env_fields(self): { "azure_uses_signed_url_for_x_sendfile": {}, "azure_signed_url_expiration": {}, + "azure_delegation_key_expiration": {}, } ) return fields + @api.constrains("azure_delegation_key_expiration", "azure_signed_url_expiration") + def _check_azure_delegation_key_expiration(self): + for rec in self: + if rec.azure_delegation_key_expiration <= 0: + raise ValidationError( + _("The delegation key expiration must be at least 1 second.") + ) + # See _azure_get_user_delegation_key for the lifetime the key is + # requested for. + requested = ( + rec.azure_delegation_key_expiration + + rec.azure_signed_url_expiration + + AZURE_CLOCK_SKEW_TOLERANCE + ) + if requested > AZURE_MAX_DELEGATION_KEY_EXPIRATION: + raise ValidationError( + _( + "Azure does not issue delegation keys valid for more than " + "7 days. The delegation key expiration, the signed URL " + "expiration and a %(skew)s seconds clock skew tolerance " + "must not add up to more than %(max)s seconds.", + skew=AZURE_CLOCK_SKEW_TOLERANCE, + max=AZURE_MAX_DELEGATION_KEY_EXPIRATION, + ) + ) + @property def is_azure_storage(self): """Check if the storage is an Azure storage.""" @@ -54,3 +103,34 @@ def _azure_call_synchronous(self, azure_client_function, *args, **kwargs): timeout=None, **kwargs, ) + + @ormcache_expiring( + "self.id", + "service_client.url", + expiration="self.azure_delegation_key_expiration", + ) + def _azure_get_user_delegation_key(self, service_client): + """Return a user delegation key to sign URLs for this storage. + + The key is cached for ``azure_delegation_key_expiration`` seconds and + shared by all the attachments of the storage. It is requested for a + bit longer than that, so that it still covers the signatures generated + by the very last call served from the cache. + + Getting such a key requires the identity to have the "Storage Blob + Delegator" role on the storage account, on top of a data plane role. + """ + self.ensure_one() + now = datetime.datetime.now(datetime.timezone.utc) + skew = datetime.timedelta(seconds=AZURE_CLOCK_SKEW_TOLERANCE) + expiry_time = ( + now + + datetime.timedelta(seconds=self.azure_delegation_key_expiration) + + datetime.timedelta(seconds=self.azure_signed_url_expiration) + + skew + ) + return self._azure_call_synchronous( + service_client.get_user_delegation_key, + key_start_time=now - skew, + key_expiry_time=expiry_time, + ) diff --git a/fs_attachment_azure/models/ir_attachment.py b/fs_attachment_azure/models/ir_attachment.py index ac39cd2d90..4b90298f06 100644 --- a/fs_attachment_azure/models/ir_attachment.py +++ b/fs_attachment_azure/models/ir_attachment.py @@ -70,26 +70,21 @@ def _get_azure_x_sendfile_path(self): # Ideally we would be able to call root_fs.url() as it is calling # generate_blob_sas. However, it expects to use an account shared key # (i.e either a connection string or account name/key pair). - # For this we need to get a delegation key first. The identity - # needs the "Storage Blob Delegator" role on the storage account, - # on top of a data plane role. - now = datetime.datetime.now(datetime.timezone.utc) - expiry_time = now + datetime.timedelta( - seconds=storage.azure_signed_url_expiration - ) - delegation_key = storage._azure_call_synchronous( - azure_client.get_user_delegation_key, - key_start_time=now, - key_expiry_time=expiry_time, - ) - # Then we can call generate_blob_sas + # For this we need a user delegation key first. + delegation_key = storage._azure_get_user_delegation_key(azure_client) + # Then we can call generate_blob_sas. No start time is given, so + # that the signature is valid as soon as Azure receives it + # whatever the clock skew between this host and Azure. This is + # also what adlfs does when signing with a shared key. + expiry_time = datetime.datetime.now( + datetime.timezone.utc + ) + datetime.timedelta(seconds=storage.azure_signed_url_expiration) sas_token = generate_blob_sas( account_name=blob_client.account_name, container_name=container_name, blob_name=file_path, user_delegation_key=delegation_key, permission=BlobSasPermissions(read=True), - start=now, expiry=expiry_time, ) file_url = f"{blob_client.url}?{sas_token}" diff --git a/fs_attachment_azure/readme/CONFIGURE.md b/fs_attachment_azure/readme/CONFIGURE.md index 84842eef89..f6b8fa5a30 100644 --- a/fs_attachment_azure/readme/CONFIGURE.md +++ b/fs_attachment_azure/readme/CONFIGURE.md @@ -1,21 +1,26 @@ On the Odoo instance, go to *Settings* > *Technical* > *Storage* > *File Storage*. When you create a new storage for Azure or modify an existing one, when you activate -the option "Use X-Sendfile To Serve Internal Url", 2 additional fields will appear: +the option "Use X-Sendfile To Serve Internal Url", 3 additional fields will appear: - **Azure Uses Signed URL For X-Accel-Redirect**: If checked, the X-Accel-Redirect - path will be a signed URL, which is useful for S3 storages that require + path will be a signed URL, which is useful for Azure storages that require signed URLs for access. - **Azure Signed URL Expiration**: The expiration time for the signed URL in seconds. This field is only relevant if the previous option is checked. By default, it is set to 30 seconds but it could be less since the url generated into the X-Accel-Redirect process is directly used by the web server to serve the file. +- **Azure Delegation Key Expiration**: The lifetime of the user delegation key, in + seconds. This field is only relevant when the storage authenticates with an + identity (see below). By default it is set to 1 hour, and Azure does not allow + more than 7 days. The value of these fields can also be set in the server environment variables using the keys: - *azure_uses_signed_url_for_x_sendfile* - *azure_signed_url_expiration* +- *azure_delegation_key_expiration* When the option "Use X-Sendfile To Serve Internal Url" is enabled, the system will generate an X-Accel-Redirect header in the response to a request to get a file. @@ -62,3 +67,25 @@ the Azure implementation does not require a base URL to be set in the storage configuration. The X-Accel-Redirect path is constructed directly from the Azure storage's URL defined for the connection, the directory name as bucket name, and the file path. + +## Signing with an identity + +Signed URLs are generated with the account shared key when the storage is configured +with a connection string or an account name/key pair. Otherwise (managed identity, +workload identity, service principal, ...) they are signed with a *user delegation +key*, which requires the identity to have the **Storage Blob Delegator** role on the +storage account, on top of a data plane role such as *Storage Blob Data Reader*. + +Delegation keys are obtained from Azure with an extra request, so they are kept in +the cache of the Odoo registry for **Azure Delegation Key Expiration** seconds. They +are requested from Azure for slightly longer than that, so that a key served from the +cache still covers the URLs signed with it. + +A higher value means fewer requests to Azure, but also a longer window during which +the key of a revoked identity remains usable. Note that a key already issued by Azure +stays valid until it expires anyway, whatever Odoo does with its copy. + +The cache is only kept in memory, so it is never shared between processes nor +persisted in the database, and it is dropped whenever Odoo clears the cache of the +registry, which modifying a storage does. Reconfiguring a storage is therefore +applied right away. diff --git a/fs_attachment_azure/static/description/index.html b/fs_attachment_azure/static/description/index.html index ee564e4e01..044c39ab2c 100644 --- a/fs_attachment_azure/static/description/index.html +++ b/fs_attachment_azure/static/description/index.html @@ -382,16 +382,19 @@

Fs Attachment Azure

Table of contents

    -
  • Configuration
    • -
  • Changelog
      -
    • 17.0.1.0.0 (2026-07-13)
      • +
    • Configuration
      • -
    • Bug Tracker
      • -
    • Credits @@ -401,23 +404,28 @@

      Configuration

      On the Odoo instance, go to Settings > Technical > Storage > File Storage.

      When you create a new storage for Azure or modify an existing one, when -you activate the option “Use X-Sendfile To Serve Internal Url”, 2 +you activate the option “Use X-Sendfile To Serve Internal Url”, 3 additional fields will appear:

      • Azure Uses Signed URL For X-Accel-Redirect: If checked, the -X-Accel-Redirect path will be a signed URL, which is useful for S3 +X-Accel-Redirect path will be a signed URL, which is useful for Azure storages that require signed URLs for access.
      • Azure Signed URL Expiration: The expiration time for the signed URL in seconds. This field is only relevant if the previous option is checked. By default, it is set to 30 seconds but it could be less since the url generated into the X-Accel-Redirect process is directly used by the web server to serve the file.
        • +
      • Azure Delegation Key Expiration: The lifetime of the user +delegation key, in seconds. This field is only relevant when the +storage authenticates with an identity (see below). By default it is +set to 1 hour, and Azure does not allow more than 7 days.

      The value of these fields can also be set in the server environment variables using the keys:

      • azure_uses_signed_url_for_x_sendfile
      • azure_signed_url_expiration
        • +
      • azure_delegation_key_expiration

      When the option “Use X-Sendfile To Serve Internal Url” is enabled, the system will generate an X-Accel-Redirect header in the response to a @@ -460,18 +468,40 @@

      Configuration

      in the storage configuration. The X-Accel-Redirect path is constructed directly from the Azure storage’s URL defined for the connection, the directory name as bucket name, and the file path.

      +
      +

      Signing with an identity

      +

      Signed URLs are generated with the account shared key when the storage +is configured with a connection string or an account name/key pair. +Otherwise (managed identity, workload identity, service principal, …) +they are signed with a user delegation key, which requires the +identity to have the Storage Blob Delegator role on the storage +account, on top of a data plane role such as Storage Blob Data Reader.

      +

      Delegation keys are obtained from Azure with an extra request, so they +are kept in the cache of the Odoo registry for Azure Delegation Key +Expiration seconds. They are requested from Azure for slightly longer +than that, so that a key served from the cache still covers the URLs +signed with it.

      +

      A higher value means fewer requests to Azure, but also a longer window +during which the key of a revoked identity remains usable. Note that a +key already issued by Azure stays valid until it expires anyway, +whatever Odoo does with its copy.

      +

      The cache is only kept in memory, so it is never shared between +processes nor persisted in the database, and it is dropped whenever Odoo +clears the cache of the registry, which modifying a storage does. +Reconfiguring a storage is therefore applied right away.

      +
-

Changelog

+

Changelog

-

17.0.1.0.0 (2026-07-13)

+

17.0.1.0.0 (2026-07-13)

  • This module was “forked” from fs_attachment_s3 v17.0.1.2.1
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed @@ -479,16 +509,16 @@

Bug Tracker

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • ACSONE SA/NV
  • Camptocamp
-

Maintainers

+

Maintainers

This module is maintained by the OCA.

Odoo Community Association diff --git a/fs_attachment_azure/tests/test_fs_attachment_azure.py b/fs_attachment_azure/tests/test_fs_attachment_azure.py index 5c715c4018..f638428bdd 100644 --- a/fs_attachment_azure/tests/test_fs_attachment_azure.py +++ b/fs_attachment_azure/tests/test_fs_attachment_azure.py @@ -1,9 +1,24 @@ # Copyright 2025 ACSONE SA/NV (http://acsone.eu). # Copyright 2026 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -from unittest.mock import AsyncMock, MagicMock, Mock, patch +import base64 +import datetime +import time +from contextlib import contextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch +from urllib.parse import parse_qsl from adlfs import AzureBlobFileSystem +from azure.storage.blob import UserDelegationKey + +from odoo.exceptions import ValidationError + +from odoo.addons.fs_attachment_azure import tools +from odoo.addons.fs_attachment_azure.models.fs_storage import ( + AZURE_CLOCK_SKEW_TOLERANCE, + AZURE_MAX_DELEGATION_KEY_EXPIRATION, +) from .common import TestFSAttachmentAzureCommon @@ -14,49 +29,253 @@ CONTAINER = "test-blob" PATH = "dir/sub" FILENAME = "fake_azure_file.txt" -BASE_URL = f"{PROTOCOL}://{ACCOUNT_NAME}.{DOMAIN}/{CONTAINER}/{PATH}/{FILENAME}" +ACCOUNT_URL = f"{PROTOCOL}://{ACCOUNT_NAME}.{DOMAIN}" +FILE_PATH = f"{PATH}/{FILENAME}" +BASE_URL = f"{ACCOUNT_URL}/{CONTAINER}/{FILE_PATH}" +REDIRECT_PATH = ( + f"/fs_x_sendfile/{PROTOCOL}/{ACCOUNT_NAME}.{DOMAIN}/{CONTAINER}/{FILE_PATH}" +) TOKEN = "1111-2222-3333-4444" CONNECTION_STRING = f"DefaultEndpointsProtocol={PROTOCOL};AccountName={ACCOUNT_NAME};AccountKey={ACCOUNT_KEY};BlobEndpoint={PROTOCOL}://{DOMAIN}/{ACCOUNT_NAME};" +DELEGATION_KEY_OID = "00000000-0000-0000-0000-000000000001" -def _fake_do_connect(self): - mock_service_client = MagicMock() +def _fake_delegation_key(): + key = UserDelegationKey() + key.signed_oid = DELEGATION_KEY_OID + key.signed_tid = "00000000-0000-0000-0000-000000000002" + key.signed_start = "2026-01-01T00:00:00Z" + key.signed_expiry = "2026-01-02T00:00:00Z" + key.signed_service = "b" + key.signed_version = "2023-11-03" + # The value is used as an HMAC key, so it must be base64 decodable. + key.value = base64.b64encode(b"delegation-key-secret").decode() + return key + + +def _install_service_client(fs): mock_blob_client = MagicMock() mock_blob_client.url = BASE_URL - mock_service_client.get_container_client.return_value = Mock() + mock_blob_client.account_name = ACCOUNT_NAME + mock_service_client = MagicMock() + mock_service_client.url = ACCOUNT_URL + mock_service_client.get_container_client.return_value = MagicMock() mock_service_client.get_blob_client.return_value = mock_blob_client - mock_service_client.connection_string = CONNECTION_STRING - mock_service_client.url.return_value = "?".join([BASE_URL, TOKEN]) mock_service_client.close = AsyncMock(return_value="ok") - self.service_client = mock_service_client + mock_service_client.get_user_delegation_key = AsyncMock( + return_value=_fake_delegation_key() + ) + fs.service_client = mock_service_client + + +def _fake_do_connect_shared_key(self): + """Connect with an account shared key: adlfs can sign URLs by itself.""" + self.connection_string = CONNECTION_STRING + _install_service_client(self) + + +def _fake_do_connect_identity(self): + """Connect with an identity: signing requires a user delegation key.""" + # Reset whatever the environment variables may have provided, so that the + # tests do not depend on the host configuration. + self.connection_string = None + self.account_name = None + self.account_key = None + _install_service_client(self) class TestFSAttachementAzure(TestFSAttachmentAzureCommon): - def test_get_x_sendfile_path_azure_signed(self): - """Test the X-Accel-Redirect path generation for azure storage.""" + def setUp(self): + super().setUp() + # The filesystem instances and the delegation keys are cached process + # wide: reset them so that each test connects with its own mock and + # doesn't leak into the next one. + self._clear_caches() + self.addCleanup(self._clear_caches) + + def _clear_caches(self): + # The delegation keys live in the registry cache, the filesystem + # instances in the fsspec one. + self.env.registry.clear_cache() + AzureBlobFileSystem.clear_instance_cache() + + @contextmanager + def _time_advanced_by(self, seconds): + """Run the block as if ``seconds`` had passed, for the caches. + + Only the clock the cache reads is moved: patching time.monotonic + itself would also move the one the asyncio event loop runs on. + """ + later = time.monotonic() + seconds + with patch.object(tools, "time", SimpleNamespace(monotonic=lambda: later)): + yield + + def _get_service_client(self): + """Return the mocked service client used by the storage. + + Must be called while ``do_connect`` is still patched. + """ + fs_storage = self.env["fs.storage"] + fs = fs_storage.get_fs_by_code(self.azure_backend.code) + return fs_storage._get_root_filesystem(fs).service_client + + def _enable_signed_url(self, expiration=60, **vals): self.azure_backend.write( { "azure_uses_signed_url_for_x_sendfile": True, - "azure_signed_url_expiration": 60, + "azure_signed_url_expiration": expiration, + **vals, } ) - with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect): - url = self.fake_attachment_azure._get_x_sendfile_path() - self.assertTrue( - url.startswith( - "/fs_x_sendfile/https/myaccount.blob.core.windows.net/test-blob/dir/sub/fake_azure_file.txt?1111-2222-3333-4444" - ), - "The end of the path should contain the path to the file " - f"name and query parameters. ({url})", - ) def test_get_x_sendfile_path_azure(self): """Test the X-Accel-Redirect path generation.""" - with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect): + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect_identity): url = self.fake_attachment_azure._get_x_sendfile_path() + service_client = self._get_service_client() self.assertEqual( url, - "/fs_x_sendfile/https/myaccount.blob.core.windows.net/test-blob/dir/sub/fake_azure_file.txt", + REDIRECT_PATH, f"The X-Accel-Redirect path should match the expected format. ({url})", ) + service_client.get_user_delegation_key.assert_not_awaited() + + def test_get_x_sendfile_path_azure_signed_shared_key(self): + """With a shared key, the URL is signed by adlfs itself.""" + self._enable_signed_url() + with patch.object( + AzureBlobFileSystem, "do_connect", _fake_do_connect_shared_key + ), patch.object( + AzureBlobFileSystem, "url", return_value=f"{BASE_URL}?{TOKEN}" + ) as mock_url: + url = self.fake_attachment_azure._get_x_sendfile_path() + service_client = self._get_service_client() + + mock_url.assert_called_once_with(f"{CONTAINER}/{FILE_PATH}", expires=60) + self.assertEqual(url, f"{REDIRECT_PATH}?{TOKEN}") + # No delegation key is needed to sign with a shared key. + service_client.get_user_delegation_key.assert_not_awaited() + + def test_get_x_sendfile_path_azure_signed_delegation_key(self): + """Without a shared key, the URL is signed with a delegation key.""" + self._enable_signed_url() + before = datetime.datetime.now(datetime.timezone.utc) + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect_identity): + url = self.fake_attachment_azure._get_x_sendfile_path() + service_client = self._get_service_client() + + path, _sep, query = url.partition("?") + self.assertEqual(path, REDIRECT_PATH) + params = dict(parse_qsl(query)) + self.assertEqual( + params.get("skoid"), + DELEGATION_KEY_OID, + f"The signature should be built from the delegation key. ({url})", + ) + self.assertIn("sig", params) + self.assertIn("se", params) + self.assertNotIn( + "st", + params, + "The signature should not have a start time, so that it is valid " + f"as soon as Azure receives it. ({url})", + ) + + key_args = service_client.get_user_delegation_key.await_args.kwargs + self.assertLess( + key_args["key_start_time"], + before, + "The delegation key start time should be backdated to tolerate " + "clock skew.", + ) + self.assertEqual( + key_args["key_expiry_time"] - key_args["key_start_time"], + datetime.timedelta( + seconds=self.azure_backend.azure_delegation_key_expiration + + self.azure_backend.azure_signed_url_expiration + + 2 * AZURE_CLOCK_SKEW_TOLERANCE + ), + "The key must cover the whole time it is cached, plus the " + "signatures generated by the last call served from the cache.", + ) + + def test_delegation_key_is_cached(self): + """The delegation key is fetched once and reused.""" + self._enable_signed_url() + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect_identity): + urls = [ + self.fake_attachment_azure._get_x_sendfile_path() for _i in range(3) + ] + service_client = self._get_service_client() + + self.assertEqual(service_client.get_user_delegation_key.await_count, 1) + for url in urls: + self.assertIn("skoid", url) + + def test_delegation_key_refreshed_once_expired(self): + """The key is fetched again by the first call made after it expired.""" + self._enable_signed_url() + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect_identity): + self.fake_attachment_azure._get_x_sendfile_path() + service_client = self._get_service_client() + self.assertEqual(service_client.get_user_delegation_key.await_count, 1) + # Right before the end of the caching duration, the key is reused. + with self._time_advanced_by( + self.azure_backend.azure_delegation_key_expiration - 1 + ): + self.fake_attachment_azure._get_x_sendfile_path() + self.assertEqual(service_client.get_user_delegation_key.await_count, 1) + # Past it, a new one is requested. + with self._time_advanced_by( + self.azure_backend.azure_delegation_key_expiration + 1 + ): + self.fake_attachment_azure._get_x_sendfile_path() + + self.assertEqual(service_client.get_user_delegation_key.await_count, 2) + + def test_delegation_key_dropped_when_storage_is_written(self): + """Reconfiguring the storage drops the key cached for it. + + This is what makes a new configuration effective right away instead of + at the end of the caching duration. + """ + self._enable_signed_url() + with patch.object(AzureBlobFileSystem, "do_connect", _fake_do_connect_identity): + self.fake_attachment_azure._get_x_sendfile_path() + service_client = self._get_service_client() + self.assertEqual(service_client.get_user_delegation_key.await_count, 1) + self.azure_backend.azure_delegation_key_expiration = 7200 + self.fake_attachment_azure._get_x_sendfile_path() + + self.assertEqual(service_client.get_user_delegation_key.await_count, 2) + + def test_delegation_key_expiration_constraint(self): + """Azure only issues delegation keys valid for up to 7 days.""" + for expiration in (0, -1, AZURE_MAX_DELEGATION_KEY_EXPIRATION): + with self.subTest(expiration=expiration), self.assertRaises( + ValidationError + ), self.env.cr.savepoint(): + self.azure_backend.azure_delegation_key_expiration = expiration + # The signed URL expiration and the clock skew tolerance are part of + # the lifetime the key is requested for, so they leave less room. + with self.assertRaises(ValidationError), self.env.cr.savepoint(): + self.azure_backend.write( + { + "azure_delegation_key_expiration": ( + AZURE_MAX_DELEGATION_KEY_EXPIRATION - AZURE_CLOCK_SKEW_TOLERANCE + ), + "azure_signed_url_expiration": 1, + } + ) + # The largest configuration Azure accepts. + self.azure_backend.write( + { + "azure_delegation_key_expiration": ( + AZURE_MAX_DELEGATION_KEY_EXPIRATION + - AZURE_CLOCK_SKEW_TOLERANCE + - 30 + ), + "azure_signed_url_expiration": 30, + } + ) diff --git a/fs_attachment_azure/tools.py b/fs_attachment_azure/tools.py new file mode 100644 index 0000000000..2308dfafb8 --- /dev/null +++ b/fs_attachment_azure/tools.py @@ -0,0 +1,83 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import logging +import time +from inspect import Parameter, signature + +from odoo.tools import ormcache + +_logger = logging.getLogger(__name__) + +unsafe_eval = eval + + +class ormcache_expiring(ormcache): + """An :class:`~odoo.tools.ormcache` whose entries expire. + + ``odoo.tools.ormcache`` keeps an entry until something clears the registry + caches, which is not enough for a value that is only valid for a while, + such as a credential obtained from a remote service. + + ``expiration`` is the number of seconds an entry remains usable, given + either as a number or, like the cache key parameters, as an expression + evaluated against the signature of the decorated method:: + + @ormcache_expiring("self.id", expiration="self.token_lifetime") + def _get_token(self): + ... + + Expiring is not invalidating: an outdated entry is simply treated as a + miss, so the method is called again by the first caller that needs it + after the entry expired. + """ + + def __init__(self, *args, expiration, **kwargs): + super().__init__(*args, **kwargs) + self.expiration = expiration + + def __call__(self, method): + lookup = super().__call__(method) + self.determine_expiration() + return lookup + + def determine_expiration(self): + """Determine the function that computes the lifetime of an entry.""" + if not isinstance(self.expiration, str): + expiration = self.expiration + self.compute_expiration = lambda *args, **kwargs: expiration + return + # Same approach as ormcache.determine_key: build a lambda over the + # signature of the decorated method and evaluate the expression in it. + args = ", ".join( + str(param.replace(annotation=Parameter.empty, default=Parameter.empty)) + for param in signature(self.method).parameters.values() + ) + self.compute_expiration = unsafe_eval(f"lambda {args}: {self.expiration}") + + def lookup(self, method, *args, **kwargs): + d, key0, counter = self.lru(args[0]) + key = key0 + self.key(*args, **kwargs) + now = time.monotonic() + try: + expiry, value = d[key] + if now < expiry: + counter.hit += 1 + return value + # The entry outlived its expiration: recompute it, below. + counter.miss += 1 + except KeyError: + counter.miss += 1 + except TypeError: + _logger.warning("cache lookup error on %r", key, exc_info=True) + counter.err += 1 + return self.method(*args, **kwargs) + value = self.method(*args, **kwargs) + d[key] = (now + self.compute_expiration(*args, **kwargs), value) + return value + + def add_value(self, *args, cache_value=None, **kwargs): + """Override to store the expiry along with the value.""" + d, key0, _counter = self.lru(args[0]) + key = key0 + self.key(*args, **kwargs) + expiry = time.monotonic() + self.compute_expiration(*args, **kwargs) + d[key] = (expiry, cache_value) diff --git a/fs_attachment_azure/views/fs_storage.xml b/fs_attachment_azure/views/fs_storage.xml index d1265a3884..deb724de62 100644 --- a/fs_attachment_azure/views/fs_storage.xml +++ b/fs_attachment_azure/views/fs_storage.xml @@ -11,11 +11,15 @@ + From 533091bc28907491b8c37bd9908f1c9c0cf392bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Todorovich?= Date: Wed, 9 Sep 2026 13:44:59 -0300 Subject: [PATCH 5/6] fixup! fs_attachment_azure: Adapt to azure Move the server environment support to fs_attachment_azure_environment. --- fs_attachment_azure/README.rst | 5 +- fs_attachment_azure/models/fs_storage.py | 13 - fs_attachment_azure/readme/CONFIGURE.md | 4 +- .../static/description/index.html | 5 +- fs_attachment_azure_environment/README.rst | 87 ++++ fs_attachment_azure_environment/__init__.py | 2 + .../__manifest__.py | 18 + fs_attachment_azure_environment/hooks.py | 23 + .../models/__init__.py | 1 + .../models/fs_storage.py | 21 + .../pyproject.toml | 3 + .../readme/CONTRIBUTORS.md | 3 + .../readme/DESCRIPTION.md | 1 + .../static/description/icon.png | Bin 0 -> 9455 bytes .../static/description/index.html | 428 ++++++++++++++++++ 15 files changed, 595 insertions(+), 19 deletions(-) create mode 100644 fs_attachment_azure_environment/README.rst create mode 100644 fs_attachment_azure_environment/__init__.py create mode 100644 fs_attachment_azure_environment/__manifest__.py create mode 100644 fs_attachment_azure_environment/hooks.py create mode 100644 fs_attachment_azure_environment/models/__init__.py create mode 100644 fs_attachment_azure_environment/models/fs_storage.py create mode 100644 fs_attachment_azure_environment/pyproject.toml create mode 100644 fs_attachment_azure_environment/readme/CONTRIBUTORS.md create mode 100644 fs_attachment_azure_environment/readme/DESCRIPTION.md create mode 100644 fs_attachment_azure_environment/static/description/icon.png create mode 100644 fs_attachment_azure_environment/static/description/index.html diff --git a/fs_attachment_azure/README.rst b/fs_attachment_azure/README.rst index 9885c9668e..3d791d3caf 100644 --- a/fs_attachment_azure/README.rst +++ b/fs_attachment_azure/README.rst @@ -65,8 +65,9 @@ additional fields will appear: storage authenticates with an identity (see below). By default it is set to 1 hour, and Azure does not allow more than 7 days. -The value of these fields can also be set in the server environment -variables using the keys: +The value of these fields can also be set in the server environment, by +installing the *fs_attachment_azure_environment* glue module and using +the keys: - *azure_uses_signed_url_for_x_sendfile* - *azure_signed_url_expiration* diff --git a/fs_attachment_azure/models/fs_storage.py b/fs_attachment_azure/models/fs_storage.py index b9a64eb510..7a8b1673e5 100644 --- a/fs_attachment_azure/models/fs_storage.py +++ b/fs_attachment_azure/models/fs_storage.py @@ -45,19 +45,6 @@ class FsStorage(models.Model): "expiration must leave room for.", ) - @property - def _server_env_fields(self): - """Override to include Azure specific fields.""" - fields = super()._server_env_fields - fields.update( - { - "azure_uses_signed_url_for_x_sendfile": {}, - "azure_signed_url_expiration": {}, - "azure_delegation_key_expiration": {}, - } - ) - return fields - @api.constrains("azure_delegation_key_expiration", "azure_signed_url_expiration") def _check_azure_delegation_key_expiration(self): for rec in self: diff --git a/fs_attachment_azure/readme/CONFIGURE.md b/fs_attachment_azure/readme/CONFIGURE.md index f6b8fa5a30..6c9b3f2d62 100644 --- a/fs_attachment_azure/readme/CONFIGURE.md +++ b/fs_attachment_azure/readme/CONFIGURE.md @@ -15,8 +15,8 @@ the option "Use X-Sendfile To Serve Internal Url", 3 additional fields will appe identity (see below). By default it is set to 1 hour, and Azure does not allow more than 7 days. -The value of these fields can also be set in the server environment variables using -the keys: +The value of these fields can also be set in the server environment, by installing +the *fs_attachment_azure_environment* glue module and using the keys: - *azure_uses_signed_url_for_x_sendfile* - *azure_signed_url_expiration* diff --git a/fs_attachment_azure/static/description/index.html b/fs_attachment_azure/static/description/index.html index 044c39ab2c..597a4c25cf 100644 --- a/fs_attachment_azure/static/description/index.html +++ b/fs_attachment_azure/static/description/index.html @@ -420,8 +420,9 @@

Configuration

storage authenticates with an identity (see below). By default it is set to 1 hour, and Azure does not allow more than 7 days. -

The value of these fields can also be set in the server environment -variables using the keys:

+

The value of these fields can also be set in the server environment, by +installing the fs_attachment_azure_environment glue module and using +the keys:

  • azure_uses_signed_url_for_x_sendfile
  • azure_signed_url_expiration
    • diff --git a/fs_attachment_azure_environment/README.rst b/fs_attachment_azure_environment/README.rst new file mode 100644 index 0000000000..f5dd396dfb --- /dev/null +++ b/fs_attachment_azure_environment/README.rst @@ -0,0 +1,87 @@ +=================================== +Filesystem Attachment Backend Azure +=================================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:64046497ed3ebd366cf8ceba5795bc4ef7fe763aecb0aa4a81b56f8cd9590f72 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fstorage-lightgray.png?logo=github + :target: https://github.com/OCA/storage/tree/17.0/fs_attachment_azure_environment + :alt: OCA/storage +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/storage-17-0/storage-17-0-fs_attachment_azure_environment + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/storage&target_branch=17.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Glue module for fs_attachment_azure to use fs_storage_environment + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ACSONE SA/NV +* Camptocamp + +Contributors +------------ + +- Laurent Mignon laurent.mignon@acsone.eu (https://www.acsone.eu) +- Stéphane Bidoul stephane.bidoul@acsone.eu (https://www.acsone.eu) +- Akim Juillerat akim.juillerat@camptocamp.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-grindtildeath| image:: https://github.com/grindtildeath.png?size=40px + :target: https://github.com/grindtildeath + :alt: grindtildeath + +Current `maintainer `__: + +|maintainer-grindtildeath| + +This module is part of the `OCA/storage `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/fs_attachment_azure_environment/__init__.py b/fs_attachment_azure_environment/__init__.py new file mode 100644 index 0000000000..1a9a001cf7 --- /dev/null +++ b/fs_attachment_azure_environment/__init__.py @@ -0,0 +1,2 @@ +from . import models +from .hooks import post_init_hook, uninstall_hook diff --git a/fs_attachment_azure_environment/__manifest__.py b/fs_attachment_azure_environment/__manifest__.py new file mode 100644 index 0000000000..9f9a6304e1 --- /dev/null +++ b/fs_attachment_azure_environment/__manifest__.py @@ -0,0 +1,18 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Filesystem Attachment Backend Azure", + "summary": "Allows to use server environment with fs storage attachment Azure", + "version": "17.0.1.0.0", + "category": "FS Storage", + "website": "https://github.com/OCA/storage", + "author": "ACSONE SA/NV,Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "development_status": "Beta", + "installable": True, + "depends": ["fs_attachment_environment", "fs_attachment_azure"], + "post_init_hook": "post_init_hook", + "uninstall_hook": "uninstall_hook", + "maintainers": ["grindtildeath"], +} diff --git a/fs_attachment_azure_environment/hooks.py b/fs_attachment_azure_environment/hooks.py new file mode 100644 index 0000000000..883f3b9c11 --- /dev/null +++ b/fs_attachment_azure_environment/hooks.py @@ -0,0 +1,23 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.server_environment.uninstall import restore_env_managed_columns + +ENV_MANAGED_FIELDS = [ + "azure_delegation_key_expiration", + "azure_signed_url_expiration", + "azure_uses_signed_url_for_x_sendfile", +] + + +def post_init_hook(env): + env["fs.storage"]._preserve_not_env_managed_data(ENV_MANAGED_FIELDS) + + +def uninstall_hook(env): + """Restore database columns dropped by server.env.mixin.""" + restore_env_managed_columns( + env, + "fs.storage", + ENV_MANAGED_FIELDS, + ) diff --git a/fs_attachment_azure_environment/models/__init__.py b/fs_attachment_azure_environment/models/__init__.py new file mode 100644 index 0000000000..349bb0495a --- /dev/null +++ b/fs_attachment_azure_environment/models/__init__.py @@ -0,0 +1 @@ +from . import fs_storage diff --git a/fs_attachment_azure_environment/models/fs_storage.py b/fs_attachment_azure_environment/models/fs_storage.py new file mode 100644 index 0000000000..0db6acea44 --- /dev/null +++ b/fs_attachment_azure_environment/models/fs_storage.py @@ -0,0 +1,21 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import models + + +class FsStorage(models.Model): + _inherit = "fs.storage" + + @property + def _server_env_fields(self): + """Override to include Azure specific fields.""" + fields = super()._server_env_fields + fields.update( + { + "azure_uses_signed_url_for_x_sendfile": {}, + "azure_signed_url_expiration": {}, + "azure_delegation_key_expiration": {}, + } + ) + return fields diff --git a/fs_attachment_azure_environment/pyproject.toml b/fs_attachment_azure_environment/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/fs_attachment_azure_environment/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/fs_attachment_azure_environment/readme/CONTRIBUTORS.md b/fs_attachment_azure_environment/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..cb7b22f28c --- /dev/null +++ b/fs_attachment_azure_environment/readme/CONTRIBUTORS.md @@ -0,0 +1,3 @@ +- Laurent Mignon (https://www.acsone.eu) +- Stéphane Bidoul (https://www.acsone.eu) +- Akim Juillerat \ No newline at end of file diff --git a/fs_attachment_azure_environment/readme/DESCRIPTION.md b/fs_attachment_azure_environment/readme/DESCRIPTION.md new file mode 100644 index 0000000000..6940407d26 --- /dev/null +++ b/fs_attachment_azure_environment/readme/DESCRIPTION.md @@ -0,0 +1 @@ +Glue module for fs_attachment_azure to use fs_storage_environment diff --git a/fs_attachment_azure_environment/static/description/icon.png b/fs_attachment_azure_environment/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/fs_attachment_azure_environment/static/description/index.html b/fs_attachment_azure_environment/static/description/index.html new file mode 100644 index 0000000000..85e89b449e --- /dev/null +++ b/fs_attachment_azure_environment/static/description/index.html @@ -0,0 +1,428 @@ + + + + + +Filesystem Attachment Backend Azure + + + +
    +

    Filesystem Attachment Backend Azure

    + + +

    Beta License: AGPL-3 OCA/storage Translate me on Weblate Try me on Runboat

    +

    Glue module for fs_attachment_azure to use fs_storage_environment

    +

    Table of contents

    + +
    +

    Bug Tracker

    +

    Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

    +

    Do not contact contributors directly about support or help with technical issues.

    +
    +
    +

    Credits

    +
    +

    Authors

    +
      +
    • ACSONE SA/NV
      • +
    • Camptocamp
      • +
    +
    + +
    +

    Maintainers

    +

    This module is maintained by the OCA.

    + +Odoo Community Association + +

    OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

    +

    Current maintainer:

    +

    grindtildeath

    +

    This module is part of the OCA/storage project on GitHub.

    +

    You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

    +
    +
    +
    + + From 1405953aebad0367f2d529ca8c16c2a66cb6da01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Todorovich?= Date: Wed, 9 Sep 2026 13:44:59 -0300 Subject: [PATCH 6/6] fixup! fs_attachment_azure: Adapt to azure Delete the orphaned blobs by batch during the garbage collection. --- fs_attachment_azure/README.rst | 2 + fs_attachment_azure/models/__init__.py | 1 + fs_attachment_azure/models/fs_file_gc.py | 106 ++++++++++ fs_attachment_azure/models/fs_storage.py | 50 +++++ fs_attachment_azure/readme/DESCRIPTION.md | 2 + .../static/description/index.html | 2 + fs_attachment_azure/tests/__init__.py | 1 + fs_attachment_azure/tests/test_fs_file_gc.py | 184 ++++++++++++++++++ 8 files changed, 348 insertions(+) create mode 100644 fs_attachment_azure/models/fs_file_gc.py create mode 100644 fs_attachment_azure/tests/test_fs_file_gc.py diff --git a/fs_attachment_azure/README.rst b/fs_attachment_azure/README.rst index 3d791d3caf..81213372d7 100644 --- a/fs_attachment_azure/README.rst +++ b/fs_attachment_azure/README.rst @@ -36,6 +36,8 @@ to better support Azure storage. It includes features such as: - Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private Azure Blob Storage using X-Accel-Redirect without exposing the files publicly.) +- Bulk deletion of the orphaned files during the garbage collection, + using Azure blob batch requests instead of one request per file. **Table of contents** diff --git a/fs_attachment_azure/models/__init__.py b/fs_attachment_azure/models/__init__.py index 45a28cbdca..2dbbbdd091 100644 --- a/fs_attachment_azure/models/__init__.py +++ b/fs_attachment_azure/models/__init__.py @@ -1,2 +1,3 @@ from . import fs_storage +from . import fs_file_gc from . import ir_attachment diff --git a/fs_attachment_azure/models/fs_file_gc.py b/fs_attachment_azure/models/fs_file_gc.py new file mode 100644 index 0000000000..e1d945489e --- /dev/null +++ b/fs_attachment_azure/models/fs_file_gc.py @@ -0,0 +1,106 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). +import logging + +from odoo import models + +from .fs_storage import AZURE_MAX_BLOBS_PER_BATCH + +_logger = logging.getLogger(__name__) + + +class FsFileGc(models.Model): + _inherit = "fs.file.gc" + + # One batch request is sent per iteration, so Azure's own limit is the + # most files we can handle at once. + _GC_BATCH_SIZE = AZURE_MAX_BLOBS_PER_BATCH + + def _gc_files_unsafe(self) -> None: + """Collect the Azure storages by batch before the file by file cleanup. + + Deleting blobs one by one is one request per file, which does not + scale to a large backlog. Azure deletes up to 256 blobs per request, + so the Azure storages are collected that way first and ``super()`` is + left with the rest: the other storages, and the blobs the batch could + not delete, which it retries one by one. + """ + self._gc_azure_bulk_delete() + return super()._gc_files_unsafe() + + def _gc_azure_bulk_delete(self) -> None: + """Delete the orphaned blobs of every Azure storage, by batch.""" + # autovacuum_gc is not a stored field, so the storages are filtered + # in memory. + storages = ( + self.env["fs.storage"] + .search([]) + .filtered( + lambda storage: storage.autovacuum_gc and storage.is_azure_storage + ) + ) + for storage in storages: + try: + self._gc_azure_bulk_delete_storage(storage) + except Exception: + _logger.exception( + "GC: could not batch delete the blobs of the storage %s", + storage.code, + ) + + def _gc_azure_bulk_delete_storage(self, storage) -> None: + """Delete the orphaned blobs of one Azure storage, one batch at a time.""" + while True: + self._cr.execute( + """ + SELECT + store_fname + FROM + fs_file_gc + WHERE + fs_storage_code = %s + AND NOT EXISTS ( + SELECT 1 + FROM ir_attachment + WHERE store_fname = fs_file_gc.store_fname + ) + LIMIT %s + """, + (storage.code, self._GC_BATCH_SIZE), + ) + store_fnames = [row[0] for row in self._cr.fetchall()] + if not store_fnames: + return + blob_names = [ + store_fname.partition("://")[2] for store_fname in store_fnames + ] + _logger.info( + "GC: batch deleting %s blobs of the storage %s", + len(blob_names), + storage.code, + ) + collected = set(storage._azure_delete_blobs(blob_names)) + deleted = [ + store_fname + for store_fname, blob_name in zip(store_fnames, blob_names, strict=True) + if blob_name in collected + ] + if deleted: + self._cr.execute( + """ + DELETE FROM + fs_file_gc + WHERE + store_fname = ANY(%s) + """, + (deleted,), + ) + if not self._is_test_mode(): + # Commit each batch, so that the progress is kept even if a + # later batch fails, and the locks taken by _gc_files are not + # held for the whole backlog. + self._cr.commit() # pylint: disable=invalid-commit + if len(deleted) < len(store_fnames): + # The blobs that could not be deleted would be selected again + # by the query above: leave them to the file by file cleanup. + return diff --git a/fs_attachment_azure/models/fs_storage.py b/fs_attachment_azure/models/fs_storage.py index 7a8b1673e5..5d9c7bac97 100644 --- a/fs_attachment_azure/models/fs_storage.py +++ b/fs_attachment_azure/models/fs_storage.py @@ -18,6 +18,25 @@ # Azure refuses to issue a user delegation key valid for more than 7 days. AZURE_MAX_DELEGATION_KEY_EXPIRATION = 7 * 24 * 3600 +# Azure accepts at most 256 subrequests in a single blob batch request. +AZURE_MAX_BLOBS_PER_BATCH = 256 + + +async def _azure_delete_blobs_batch(service_client, container_name, blob_names): + """Delete blobs in one batch request, returning a status code per blob. + + The container client is used as a context manager, the same way adlfs + does when it needs a blob client. + """ + async with service_client.get_container_client(container_name) as container_client: + responses = await container_client.delete_blobs( + *blob_names, + # A single blob that cannot be deleted must not discard the whole + # batch: the status of each subrequest is inspected instead. + raise_on_any_failure=False, + ) + return [response.status_code async for response in responses] + class FsStorage(models.Model): _inherit = "fs.storage" @@ -121,3 +140,34 @@ def _azure_get_user_delegation_key(self, service_client): key_start_time=now - skew, key_expiry_time=expiry_time, ) + + def _azure_delete_blobs(self, blob_names): + """Delete the given blobs from this storage's container. + + The deletion is sent as a single batch request, so the caller is the + one splitting larger lists, one batch at a time. + + :return: the blobs that are gone, either because they were deleted or + because they were already missing, which is as good as deleted. + :raise ValueError: if more blobs than a batch can hold are given. + """ + self.ensure_one() + if not blob_names: + return [] + if len(blob_names) > AZURE_MAX_BLOBS_PER_BATCH: + raise ValueError( + f"A blob batch request holds at most " + f"{AZURE_MAX_BLOBS_PER_BATCH} blobs, got {len(blob_names)}." + ) + root_fs = self._get_root_filesystem() + status_codes = self._azure_call_synchronous( + _azure_delete_blobs_batch, + root_fs.service_client, + self.get_directory_path(), + blob_names, + ) + return [ + blob_name + for blob_name, status_code in zip(blob_names, status_codes, strict=True) + if 200 <= status_code < 300 or status_code == 404 + ] diff --git a/fs_attachment_azure/readme/DESCRIPTION.md b/fs_attachment_azure/readme/DESCRIPTION.md index e21f93c98c..2f24666552 100644 --- a/fs_attachment_azure/readme/DESCRIPTION.md +++ b/fs_attachment_azure/readme/DESCRIPTION.md @@ -4,3 +4,5 @@ to better support Azure storage. It includes features such as: - Special handling of X-Accel-Redirect headers for Azure storages. - Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private Azure Blob Storage using X-Accel-Redirect without exposing the files publicly.) +- Bulk deletion of the orphaned files during the garbage collection, using Azure + blob batch requests instead of one request per file. diff --git a/fs_attachment_azure/static/description/index.html b/fs_attachment_azure/static/description/index.html index 597a4c25cf..75ac480e96 100644 --- a/fs_attachment_azure/static/description/index.html +++ b/fs_attachment_azure/static/description/index.html @@ -378,6 +378,8 @@

    Fs Attachment Azure

  • Options for using signed URLs in X-Accel-Redirect. (This is required to be able to serve files from a private Azure Blob Storage using X-Accel-Redirect without exposing the files publicly.)
    • +
  • Bulk deletion of the orphaned files during the garbage collection, +using Azure blob batch requests instead of one request per file.

Table of contents

diff --git a/fs_attachment_azure/tests/__init__.py b/fs_attachment_azure/tests/__init__.py index fe85fc1d1c..a5227618d9 100644 --- a/fs_attachment_azure/tests/__init__.py +++ b/fs_attachment_azure/tests/__init__.py @@ -1 +1,2 @@ from . import test_fs_attachment_azure +from . import test_fs_file_gc diff --git a/fs_attachment_azure/tests/test_fs_file_gc.py b/fs_attachment_azure/tests/test_fs_file_gc.py new file mode 100644 index 0000000000..b33210ebba --- /dev/null +++ b/fs_attachment_azure/tests/test_fs_file_gc.py @@ -0,0 +1,184 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from types import SimpleNamespace +from unittest.mock import patch + +from odoo.addons.fs_attachment_azure.models.fs_storage import ( + AZURE_MAX_BLOBS_PER_BATCH, +) + +from .common import TestFSAttachmentAzureCommon + +CONTAINER = "test-blob" + + +class FakeBatchResponses: + """Async iterator of subresponses, as ``delete_blobs`` returns one.""" + + def __init__(self, status_codes): + self._status_codes = list(status_codes) + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._status_codes: + raise StopAsyncIteration + return SimpleNamespace(status_code=self._status_codes.pop(0)) + + +class FakeContainerClient: + """Minimal stand-in for an asynchronous ContainerClient.""" + + def __init__(self, status_codes=None, error=None): + # blob name -> status code to answer, anything else is deleted (202) + self.status_codes = status_codes or {} + self.error = error + self.calls = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + return None + + async def delete_blobs(self, *blobs, **kwargs): + self.calls.append((blobs, kwargs)) + if self.error: + raise self.error + return FakeBatchResponses([self.status_codes.get(blob, 202) for blob in blobs]) + + +class FakeServiceClient: + def __init__(self, container_client): + self.container_client = container_client + self.container_names = [] + + def get_container_client(self, container_name): + self.container_names.append(container_name) + return self.container_client + + +class TestFsFileGcAzure(TestFSAttachmentAzureCommon): + def setUp(self): + super().setUp() + self.gc_file_model = self.env["fs.file.gc"] + + def _mark_for_gc(self, *store_fnames): + for store_fname in store_fnames: + self.gc_file_model._mark_for_gc(store_fname) + + def _gc_azure_bulk_delete(self, container_client): + """Run the bulk delete against a fake Azure container.""" + root_fs = SimpleNamespace(service_client=FakeServiceClient(container_client)) + storage_class = type(self.azure_backend) + code = self.azure_backend.code + with patch.object( + storage_class, + "is_azure_storage", + new=property(lambda storage: storage.code == code), + ), patch.object(storage_class, "_get_root_filesystem", return_value=root_fs): + self.gc_file_model._gc_azure_bulk_delete() + return root_fs.service_client + + def _gc_rows(self, *store_fnames): + return self.gc_file_model.search( + [("store_fname", "in", list(store_fnames))] + ).mapped("store_fname") + + def test_batch_size_is_azure_limit(self): + """A batch cannot hold more blobs than Azure accepts.""" + self.assertEqual(self.gc_file_model._GC_BATCH_SIZE, AZURE_MAX_BLOBS_PER_BATCH) + self.assertEqual(AZURE_MAX_BLOBS_PER_BATCH, 256) + + def test_delete_blobs_rejects_oversized_batch(self): + """The storage refuses more blobs than a batch request can hold.""" + blob_names = [f"blob_{i}" for i in range(AZURE_MAX_BLOBS_PER_BATCH + 1)] + with self.assertRaises(ValueError): + self.azure_backend._azure_delete_blobs(blob_names) + + def test_delete_blobs_without_blobs_sends_no_request(self): + """Nothing to delete means no request at all.""" + self.assertEqual(self.azure_backend._azure_delete_blobs([]), []) + + def test_bulk_delete_removes_orphaned_files(self): + """Orphaned blobs are deleted in one request, referenced ones are kept.""" + orphan_1 = "azure://dir/sub/orphan_1.txt" + orphan_2 = "azure://dir/sub/orphan_2.txt" + referenced = self.fake_attachment_azure.store_fname + self._mark_for_gc(orphan_1, orphan_2, referenced) + + container_client = FakeContainerClient() + service_client = self._gc_azure_bulk_delete(container_client) + + self.assertEqual(len(container_client.calls), 1) + blobs, kwargs = container_client.calls[0] + self.assertCountEqual(blobs, ["dir/sub/orphan_1.txt", "dir/sub/orphan_2.txt"]) + self.assertFalse( + kwargs["raise_on_any_failure"], + "One blob that cannot be deleted must not discard the batch.", + ) + self.assertEqual(service_client.container_names, [CONTAINER]) + self.assertEqual(self._gc_rows(orphan_1, orphan_2, referenced), [referenced]) + + def test_bulk_delete_collects_missing_blobs(self): + """A blob that is already gone is collected as if it was deleted.""" + orphan = "azure://dir/sub/orphan.txt" + self._mark_for_gc(orphan) + + container_client = FakeContainerClient(status_codes={"dir/sub/orphan.txt": 404}) + self._gc_azure_bulk_delete(container_client) + + self.assertFalse(self._gc_rows(orphan)) + + def test_bulk_delete_keeps_rows_of_failed_blobs(self): + """A blob that Azure refuses to delete is left to the file by file pass.""" + deleted = "azure://dir/sub/deleted.txt" + failed = "azure://dir/sub/failed.txt" + self._mark_for_gc(deleted, failed) + + container_client = FakeContainerClient(status_codes={"dir/sub/failed.txt": 500}) + self._gc_azure_bulk_delete(container_client) + + # A partial failure must not loop on the rows it cannot delete. + self.assertEqual(len(container_client.calls), 1) + self.assertEqual(self._gc_rows(deleted, failed), [failed]) + + def test_bulk_delete_keeps_rows_when_request_fails(self): + """A failing request is logged and leaves every row in place.""" + orphan = "azure://dir/sub/orphan.txt" + self._mark_for_gc(orphan) + + container_client = FakeContainerClient(error=Exception("Azure is unavailable")) + with self.assertLogs( + "odoo.addons.fs_attachment_azure.models.fs_file_gc", level="ERROR" + ): + self._gc_azure_bulk_delete(container_client) + + self.assertEqual(self._gc_rows(orphan), [orphan]) + + def test_bulk_delete_sends_one_request_per_batch(self): + """More orphans than a batch holds are deleted in several requests.""" + orphans = [f"azure://dir/sub/orphan_{i}.txt" for i in range(3)] + self._mark_for_gc(*orphans) + + container_client = FakeContainerClient() + with patch.object(type(self.gc_file_model), "_GC_BATCH_SIZE", 2): + self._gc_azure_bulk_delete(container_client) + + self.assertEqual( + [len(blobs) for blobs, _kwargs in container_client.calls], [2, 1] + ) + self.assertFalse(self._gc_rows(*orphans)) + + def test_bulk_delete_ignores_storages_without_autovacuum(self): + """A storage with the autovacuum disabled is not collected.""" + self.azure_backend.autovacuum_gc = False + orphan = "azure://dir/sub/orphan.txt" + self._mark_for_gc(orphan) + + container_client = FakeContainerClient() + self._gc_azure_bulk_delete(container_client) + + self.assertFalse(container_client.calls) + self.assertEqual(self._gc_rows(orphan), [orphan])