From eb3403032c9f7cfa43715d136a0d670a88e494a4 Mon Sep 17 00:00:00 2001 From: Dibik Date: Fri, 28 Jun 2024 22:16:31 +0530 Subject: [PATCH 1/4] G2P-2557: Mimetype issue in storage file --- g2p_documents/__manifest__.py | 2 +- g2p_documents/models/document_file.py | 45 ++++++++++++++++++++- g2p_documents/static/description/index.html | 7 ++-- g2p_documents/views/g2p_document_files.xml | 1 + requirements.txt | 1 + 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/g2p_documents/__manifest__.py b/g2p_documents/__manifest__.py index c739464..364f4a8 100644 --- a/g2p_documents/__manifest__.py +++ b/g2p_documents/__manifest__.py @@ -21,7 +21,7 @@ "views/g2p_document_tags.xml", "data/storage_backend.xml", ], - "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify"]}, + "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify", "python-magic"]}, "assets": {}, "demo": [], "images": [], diff --git a/g2p_documents/models/document_file.py b/g2p_documents/models/document_file.py index 3be5771..74b452e 100644 --- a/g2p_documents/models/document_file.py +++ b/g2p_documents/models/document_file.py @@ -1,6 +1,12 @@ +import base64 +import binascii import logging +import mimetypes +import os -from odoo import _, fields, models +import magic + +from odoo import _, api, fields, models from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -34,6 +40,42 @@ def _compute_file_type(self): else: file.file_type = False + def _inverse_data(self): + for record in self: + record.write(record._prepare_meta_for_file()) + if not record.mimetype: + binary_data = base64.b64decode(self.data) + magic_data = magic.Magic(mime=True) + record.mimetype = magic_data.from_buffer(binary_data) + + record.backend_id.sudo().add( + record.relative_path, + record.data, + mimetype=record.mimetype, + binary=False, + ) + + @api.depends("name") + @api.constrains("name") + def _compute_extract_filename(self): + for rec in self: + if rec.name: + rec.filename, rec.extension = os.path.splitext(rec.name) + mime, __ = mimetypes.guess_type(rec.name) + else: + rec.filename = rec.extension = mime = False + + if mime is None and rec.data: + try: + binary_data = base64.b64decode(rec.data) + magic_data = magic.Magic(mime=True) + mime = magic_data.from_buffer(binary_data) + + except binascii.Error as e: + _logger.info(f"Base64 decoding error: {e}") + + rec.mimetype = mime + def _compute_data(self): # Handled key error for rec in self: @@ -44,6 +86,7 @@ def _compute_data(self): rec.data = rec.backend_id.sudo().get(rec.relative_path, binary=False) else: rec.data = None + except Exception as e: if "NoSuchKey" in str(e): err_msg = "The file with the given name is not present on the s3." diff --git a/g2p_documents/static/description/index.html b/g2p_documents/static/description/index.html index 25d168c..eb50fcb 100644 --- a/g2p_documents/static/description/index.html +++ b/g2p_documents/static/description/index.html @@ -8,10 +8,11 @@ /* :Author: David Goodger (goodger@python.org) -:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $ +:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $ :Copyright: This stylesheet has been placed in the public domain. Default cascading style sheet for the HTML output of Docutils. +Despite the name, some widely supported CSS2 features are used. See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to customize this style sheet. @@ -274,7 +275,7 @@ margin-left: 2em ; margin-right: 2em } -pre.code .ln { color: grey; } /* line numbers */ +pre.code .ln { color: gray; } /* line numbers */ pre.code, code { background-color: #eeeeee } pre.code .comment, code .comment { color: #5C6576 } pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold } @@ -300,7 +301,7 @@ span.pre { white-space: pre } -span.problematic { +span.problematic, pre.problematic { color: red } span.section-subtitle { diff --git a/g2p_documents/views/g2p_document_files.xml b/g2p_documents/views/g2p_document_files.xml index c6d6359..d979692 100644 --- a/g2p_documents/views/g2p_document_files.xml +++ b/g2p_documents/views/g2p_document_files.xml @@ -22,6 +22,7 @@ view_g2p_document_files_form storage.file + diff --git a/requirements.txt b/requirements.txt index 9c0cf37..8f10b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies boto3<=1.15.18 +python-magic python_slugify From 020ec76bdfca4043fc854ee7efc95ca430970406 Mon Sep 17 00:00:00 2001 From: Dibik Date: Tue, 2 Jul 2024 15:29:07 +0530 Subject: [PATCH 2/4] Python library added for test run failing --- g2p_documents/__manifest__.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/g2p_documents/__manifest__.py b/g2p_documents/__manifest__.py index 364f4a8..41fec9b 100644 --- a/g2p_documents/__manifest__.py +++ b/g2p_documents/__manifest__.py @@ -21,7 +21,7 @@ "views/g2p_document_tags.xml", "data/storage_backend.xml", ], - "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify", "python-magic"]}, + "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify", "python-magic", "libmagic"]}, "assets": {}, "demo": [], "images": [], diff --git a/requirements.txt b/requirements.txt index 8f10b2f..c6a8a68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # generated from manifests external_dependencies boto3<=1.15.18 +libmagic python-magic python_slugify From 77f7c860530ba0c7f9aebe4103b36c98f9955f29 Mon Sep 17 00:00:00 2001 From: Dibik Date: Tue, 30 Jul 2024 18:21:10 +0530 Subject: [PATCH 3/4] python library added --- g2p_documents/__manifest__.py | 5 ++++- requirements.txt | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/g2p_documents/__manifest__.py b/g2p_documents/__manifest__.py index 41fec9b..c7bbada 100644 --- a/g2p_documents/__manifest__.py +++ b/g2p_documents/__manifest__.py @@ -21,7 +21,10 @@ "views/g2p_document_tags.xml", "data/storage_backend.xml", ], - "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify", "python-magic", "libmagic"]}, + "external_dependencies": { + "python": ["boto3<=1.15.18", "python_slugify", "python-magic"], + "deb": ["libmagic1"], + }, "assets": {}, "demo": [], "images": [], diff --git a/requirements.txt b/requirements.txt index c6a8a68..8f10b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ # generated from manifests external_dependencies boto3<=1.15.18 -libmagic python-magic python_slugify From 3167225695296d388b267b697b3e7ba1f184c85c Mon Sep 17 00:00:00 2001 From: Dibik Date: Wed, 31 Jul 2024 13:16:27 +0530 Subject: [PATCH 4/4] G2P-2496: Fixed preview mimetype issue in document using PILL python package --- g2p_documents/__manifest__.py | 5 +---- g2p_documents/models/document_file.py | 22 +++++++++++++++------- requirements.txt | 1 - 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/g2p_documents/__manifest__.py b/g2p_documents/__manifest__.py index c7bbada..c739464 100644 --- a/g2p_documents/__manifest__.py +++ b/g2p_documents/__manifest__.py @@ -21,10 +21,7 @@ "views/g2p_document_tags.xml", "data/storage_backend.xml", ], - "external_dependencies": { - "python": ["boto3<=1.15.18", "python_slugify", "python-magic"], - "deb": ["libmagic1"], - }, + "external_dependencies": {"python": ["boto3<=1.15.18", "python_slugify"]}, "assets": {}, "demo": [], "images": [], diff --git a/g2p_documents/models/document_file.py b/g2p_documents/models/document_file.py index 74b452e..3ef890e 100644 --- a/g2p_documents/models/document_file.py +++ b/g2p_documents/models/document_file.py @@ -1,10 +1,11 @@ import base64 import binascii +import io import logging import mimetypes import os -import magic +from PIL import Image from odoo import _, api, fields, models from odoo.exceptions import UserError @@ -44,9 +45,9 @@ def _inverse_data(self): for record in self: record.write(record._prepare_meta_for_file()) if not record.mimetype: - binary_data = base64.b64decode(self.data) - magic_data = magic.Magic(mime=True) - record.mimetype = magic_data.from_buffer(binary_data) + binary_data = base64.b64decode(record.data) + mime = self._get_mime_type(binary_data) + record.mimetype = mime record.backend_id.sudo().add( record.relative_path, @@ -68,14 +69,21 @@ def _compute_extract_filename(self): if mime is None and rec.data: try: binary_data = base64.b64decode(rec.data) - magic_data = magic.Magic(mime=True) - mime = magic_data.from_buffer(binary_data) - + mime = self._get_mime_type(binary_data) except binascii.Error as e: _logger.info(f"Base64 decoding error: {e}") rec.mimetype = mime + def _get_mime_type(self, binary_data): + try: + image = Image.open(io.BytesIO(binary_data)) + mime_type = Image.MIME[image.format] + return mime_type + except OSError as e: + _logger.info(f"Image processing error: {e}") + return None + def _compute_data(self): # Handled key error for rec in self: diff --git a/requirements.txt b/requirements.txt index 8f10b2f..9c0cf37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ # generated from manifests external_dependencies boto3<=1.15.18 -python-magic python_slugify