From 074b753e6cea4a20efd94926d3708a2bac084347 Mon Sep 17 00:00:00 2001 From: The Magician Date: Tue, 27 Nov 2018 12:29:50 -0800 Subject: [PATCH] Ansible Storage Upload/Download --- lib/ansible/module_utils/gcp_utils.py | 14 +- .../cloud/google/gcp_storage_object.py | 285 ++++++++++++++++++ 2 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 lib/ansible/modules/cloud/google/gcp_storage_object.py diff --git a/lib/ansible/module_utils/gcp_utils.py b/lib/ansible/module_utils/gcp_utils.py index 6ae28598fe53b1..6982bd15301b5b 100644 --- a/lib/ansible/module_utils/gcp_utils.py +++ b/lib/ansible/module_utils/gcp_utils.py @@ -77,12 +77,19 @@ def get(self, url, body=None, **kwargs): except getattr(requests.exceptions, 'RequestException') as inst: self.module.fail_json(msg=inst.message) - def post(self, url, body=None): + def post(self, url, body=None, headers={}, **kwargs): + kwargs.update({'json': body, 'headers': self._merge_dictionaries(headers, self._headers())}) try: return self.session().post(url, json=body, headers=self._headers()) except getattr(requests.exceptions, 'RequestException') as inst: self.module.fail_json(msg=inst.message) + def post_contents(self, url, file_contents=None, headers={}, **kwargs): + try: + return self.session().post(url, data=file_contents, headers=self._headers()) + except getattr(requests.exceptions, 'RequestException') as inst: + self.module.fail_json(msg=inst.message) + def delete(self, url, body=None): try: return self.session().delete(url, json=body, headers=self._headers()) @@ -142,6 +149,11 @@ def _headers(self): 'User-Agent': "Google-Ansible-MM-{0}".format(self.product) } + def _merge_dictionaries(self, a, b): + new = a.copy() + new.update(b) + return new + class GcpModule(AnsibleModule): def __init__(self, *args, **kwargs): diff --git a/lib/ansible/modules/cloud/google/gcp_storage_object.py b/lib/ansible/modules/cloud/google/gcp_storage_object.py new file mode 100644 index 00000000000000..ee110a81bc1725 --- /dev/null +++ b/lib/ansible/modules/cloud/google/gcp_storage_object.py @@ -0,0 +1,285 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2017 Google +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# ---------------------------------------------------------------------------- +# +# *** AUTO GENERATED CODE *** AUTO GENERATED CODE *** +# +# ---------------------------------------------------------------------------- +# +# This file is automatically generated by Magic Modules and manual +# changes will be clobbered when the file is regenerated. +# +# Please read more about how to change this file at +# https://www.github.com/GoogleCloudPlatform/magic-modules +# +# ---------------------------------------------------------------------------- + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +################################################################################ +# Documentation +################################################################################ + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ["preview"], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: gcp_storage_object +description: +- Upload or download a file from a GCS bucket. +short_description: Creates a GCP Object +version_added: 2.8 +author: Google Inc. (@googlecloudplatform) +requirements: +- python >= 2.6 +- requests >= 2.18.4 +- google-auth >= 1.3.0 +options: + state: + description: + - Whether the given object should exist in GCP + choices: + - present + - absent + default: present + action: + description: + - Upload or download from the bucket. + required: false + choices: + - download + - upload + overwrite: + description: + - "'Overwrite the file on the bucket/local machine. If overwrite is false and + a difference exists between GCS + local, module will fail with error' ." + required: false + type: bool + src: + description: + - Source location of file (may be local machine or cloud depending on action). + required: false + dest: + description: + - Destination location of file (may be local machine or cloud depending on action). + required: false + bucket: + description: + - The name of the bucket. + required: false +extends_documentation_fragment: gcp +''' + +EXAMPLES = ''' +- name: create a object + gcp_storage_object: + name: ansible-storage-module + action: download + bucket: ansible-bucket + src: modules.zip + dest: "~/modules.zip" + project: "test_project" + auth_kind: "serviceaccount" + service_account_file: "/tmp/auth.pem" + state: present +''' + +RETURN = ''' +action: + description: + - Upload or download from the bucket. + returned: success + type: str +overwrite: + description: + - "'Overwrite the file on the bucket/local machine. If overwrite is false and a + difference exists between GCS + local, module will fail with error' ." + returned: success + type: bool +src: + description: + - Source location of file (may be local machine or cloud depending on action). + returned: success + type: str +dest: + description: + - Destination location of file (may be local machine or cloud depending on action). + returned: success + type: str +bucket: + description: + - The name of the bucket. + returned: success + type: str +''' + +################################################################################ +# Imports +################################################################################ + +from ansible.module_utils.gcp_utils import navigate_hash, GcpSession, GcpModule, GcpRequest, replace_resource_dict +import json +import os +import mimetypes +import hashlib +import base64 + +################################################################################ +# Main +################################################################################ + + +def main(): + """Main function""" + + module = GcpModule( + argument_spec=dict( + state=dict(default='present', choices=['present', 'absent'], type='str'), + action=dict(type='str', choices=['download', 'upload']), + overwrite=dict(type='bool'), + src=dict(type='path'), + dest=dict(type='path'), + bucket=dict(type='str') + ) + ) + + if not module.params['scopes']: + module.params['scopes'] = ['https://www.googleapis.com/auth/devstorage.full_control'] + + remote_object = fetch_resource(module, self_link(module)) + local_file_exists = os.path.isfile(local_file_path(module)) + + # Check if files exist. + if module.params['action'] == 'download' and not remote_object: + module.fail_json(msg="File does not exist in bucket") + + if module.params['action'] == 'upload' and not local_file_exists: + module.fail_json(msg="File does not exist on disk") + + # Check if we'll be overwriting files. + if not module.params['overwrite']: + remote_object['changed'] = False + if module.params['action'] == 'download' and local_file_exists: + # If files differ, throw an error + if get_md5_local(local_file_path(module)) != remote_object['md5Hash']: + module.fail_json(msg="Local file is different than remote file") + # If files are the same, module is done running. + else: + module.exit_json(**remote_object) + + elif module.params['action'] == 'upload' and remote_object: + # If files differ, throw an error + if get_md5_local(local_file_path(module)) != remote_object['md5Hash']: + module.fail_json(msg="Local file is different than remote file") + # If files are the same, module is done running. + else: + module.exit_json(**remote_object) + + # Upload/download the files + auth = GcpSession(module, 'storage') + if module.params['action'] == 'download': + results = download_file(module) + else: + results = upload_file(module) + + module.exit_json(**results) + + +def download_file(module): + auth = GcpSession(module, 'storage') + data = auth.get(media_link(module)) + with open(module.params['dest'], 'w') as f: + f.write(data.text.encode('utf8')) + return fetch_resource(module, self_link(module)) + + +def upload_file(module): + auth = GcpSession(module, 'storage') + with open(module.params['src'], 'r') as f: + results = return_if_object(module, auth.post_contents(upload_link(module), f, object_headers(module))) + results['changed'] = True + return results + + +def get_md5_local(path): + md5 = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + md5.update(chunk) + return base64.b64encode(md5.digest()) + + +def get_md5_remote(module): + resource = fetch_resource(module, self_link(module)) + return resource.get('md5Hash') + + +def fetch_resource(module, link, allow_not_found=True): + auth = GcpSession(module, 'storage') + return return_if_object(module, auth.get(link), allow_not_found) + + +def self_link(module): + if module.params['action'] == 'download': + return "https://www.googleapis.com/storage/v1/b/{bucket}/o/{src}".format(**module.params) + else: + return "https://www.googleapis.com/storage/v1/b/{bucket}/o/{dest}".format(**module.params) + + +def local_file_path(module): + if module.params['action'] == 'download': + return module.params['dest'] + else: + return module.params['src'] + + +def media_link(module): + if module.params['action'] == 'download': + return "https://www.googleapis.com/storage/v1/b/{bucket}/o/{src}?alt=media".format(**module.params) + else: + return "https://www.googleapis.com/storage/v1/b/{bucket}/o/{dest}?alt=media".format(**module.params) + + +def upload_link(module): + return "https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?uploadType=media&name={dest}".format(**module.params) + + +def return_if_object(module, response, allow_not_found=False): + # If not found, return nothing. + if allow_not_found and response.status_code == 404: + return None + + # If no content, return nothing. + if response.status_code == 204: + return None + + try: + module.raise_for_status(response) + result = response.json() + except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst: + module.fail_json(msg="Invalid JSON response with error: %s" % inst) + + if navigate_hash(result, ['error', 'errors']): + module.fail_json(msg=navigate_hash(result, ['error', 'errors'])) + + return result + + +# Remove unnecessary properties from the response. +# This is for doing comparisons with Ansible's current parameters. +def object_headers(module): + return { + "name": module.params['dest'], + "Content-Type": mimetypes.guess_type(module.params['src'])[0], + "Content-Length": str(os.path.getsize(module.params['src'])) + } + + +if __name__ == '__main__': + main()