From 77d63152e074e5c6307f65a2dc142be6d0581ed9 Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Wed, 2 Feb 2022 01:15:59 +0000 Subject: [PATCH 1/2] Add FIPS support --- S3/S3.py | 15 ++++++++++++--- S3/Utils.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/S3/S3.py b/S3/S3.py index 04e06a63..0e9502d3 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -1647,7 +1647,10 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0, raise S3UploadError("Upload failed for: %s" % resource['uri']) if buffer == '': stream.seek(offset) - md5_hash = md5() + try: + md5_hash = md5() + except Exception: + md5_hash = md5(usedforsecurity=False) try: http_response = None @@ -1950,7 +1953,10 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_ if start_position == 0: # Only compute MD5 on the fly if we're downloading from beginning # Otherwise we'd get a nonsense. - md5_hash = md5() + try: + md5_hash = md5() + except Exception: + md5_hash = md5(usedforsecurity=False) size_left = int(response["headers"]["content-length"]) size_total = start_position + size_left current_position = start_position @@ -2069,7 +2075,10 @@ def parse_attrs_header(attrs_header): return attrs def compute_content_md5(body): - m = md5(encode_to_s3(body)) + try: + m = md5(encode_to_s3(body)) + except Exception: + m = md5(encode_to_s3(body), usedforsecurity=False) base64md5 = encodestring(m.digest()) base64md5 = decode_from_s3(base64md5) if base64md5[-1] == '\n': diff --git a/S3/Utils.py b/S3/Utils.py index a40f439c..5b3d55be 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -102,7 +102,10 @@ def mktmpfile(prefix = os.getenv('TMP','/tmp') + "/tmpfile-", randchars = 20): def hash_file_md5(filename): - h = md5() + try: + h = md5() + except Exception: + h = md5(usedforsecurity=False) with open(deunicodise(filename), "rb") as fp: while True: # Hash 32kB chunks @@ -310,7 +313,10 @@ def getHostnameFromBucket(bucket): def calculateChecksum(buffer, mfile, offset, chunk_size, send_chunk): - md5_hash = md5() + try: + md5_hash = md5() + except Exception: + md5_hash = md5(usedforsecurity=False) size_left = chunk_size if buffer == '': mfile.seek(offset) From 35e4cc72b9aaea7ca3a80ab4786356c8da750686 Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Fri, 4 Mar 2022 01:38:34 +0000 Subject: [PATCH 2/2] Move md5 to BaseUtils --- S3/BaseUtils.py | 15 +++++++++++++++ S3/S3.py | 22 ++++------------------ S3/Utils.py | 13 +++---------- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/S3/BaseUtils.py b/S3/BaseUtils.py index fd9e3cab..a8ccddfc 100644 --- a/S3/BaseUtils.py +++ b/S3/BaseUtils.py @@ -8,6 +8,8 @@ from __future__ import absolute_import, division +import functools +import hashlib import re import sys @@ -50,6 +52,19 @@ unicode = str +try: + md5 = hashlib.md5() +except Exception as exc: + try: + # md5 is disabled for FIPS-compliant Python builds. + # Since s3cmd does not use md5 in a security context, + # it is safe to allow the use of it by setting useforsecurity to False. + hashlib.md5(usedforsecurity=False) + md5 = functools.partial(hashlib.md5, usedforsecurity=False) + except: + raise exc + + __all__ = [] diff --git a/S3/S3.py b/S3/S3.py index 0e9502d3..c77ca901 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -33,13 +33,8 @@ import select -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - from .BaseUtils import (getListFromXml, getTextFromXml, getRootTagName, - decode_from_s3, encode_to_s3, s3_quote) + decode_from_s3, encode_to_s3, md5, s3_quote) from .Utils import (convertHeaderTupleListToDict, hash_file_md5, unicodise, deunicodise, check_bucket_name, check_bucket_name_dns_support, getHostnameFromBucket, @@ -1647,10 +1642,7 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0, raise S3UploadError("Upload failed for: %s" % resource['uri']) if buffer == '': stream.seek(offset) - try: - md5_hash = md5() - except Exception: - md5_hash = md5(usedforsecurity=False) + md5_hash = md5() try: http_response = None @@ -1953,10 +1945,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_ if start_position == 0: # Only compute MD5 on the fly if we're downloading from beginning # Otherwise we'd get a nonsense. - try: - md5_hash = md5() - except Exception: - md5_hash = md5(usedforsecurity=False) + md5_hash = md5() size_left = int(response["headers"]["content-length"]) size_total = start_position + size_left current_position = start_position @@ -2075,10 +2064,7 @@ def parse_attrs_header(attrs_header): return attrs def compute_content_md5(body): - try: - m = md5(encode_to_s3(body)) - except Exception: - m = md5(encode_to_s3(body), usedforsecurity=False) + m = md5(encode_to_s3(body)) base64md5 = encodestring(m.digest()) base64md5 = decode_from_s3(base64md5) if base64md5[-1] == '\n': diff --git a/S3/Utils.py b/S3/Utils.py index 5b3d55be..9e6a6f8a 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -14,7 +14,6 @@ import string as string_mod import random import errno -from hashlib import md5 from logging import debug @@ -30,7 +29,7 @@ import S3.Exceptions from S3.BaseUtils import (base_urlencode_string, base_replace_nonprintables, - base_unicodise, base_deunicodise) + base_unicodise, base_deunicodise, md5) __all__ = [] @@ -102,10 +101,7 @@ def mktmpfile(prefix = os.getenv('TMP','/tmp') + "/tmpfile-", randchars = 20): def hash_file_md5(filename): - try: - h = md5() - except Exception: - h = md5(usedforsecurity=False) + h = md5() with open(deunicodise(filename), "rb") as fp: while True: # Hash 32kB chunks @@ -313,10 +309,7 @@ def getHostnameFromBucket(bucket): def calculateChecksum(buffer, mfile, offset, chunk_size, send_chunk): - try: - md5_hash = md5() - except Exception: - md5_hash = md5(usedforsecurity=False) + md5_hash = md5() size_left = chunk_size if buffer == '': mfile.seek(offset)