ENCODE_update.py

#!/usr/bin/env python
# -*- coding: latin-1 -*-
'''Take a comma-delimited, double quote-quoted csv and update ENCODE objects appropriately'''

import requests, json, jsonschema
import sys, os, pdb, csv, subprocess, time
import logging
import common
from encodedcc import ENC_Key, ENC_Connection, ENC_Item
from StringIO import StringIO

logger = logging.getLogger(__name__)

EPILOG = '''Notes:
	Requires comma-delimited, double quote-quoted (for values containing commas or newlines)
	csv's should have a header as below and data rows in the form:

	accession,property1:array,property2,property3:int,property4:float ...
	ENCBSxyzabc,"[""elementwithcomma,andstuff"",""second element""]",stringproperty,3,4.1 ...
	...
	Note that array elements are comma-delimited within a double-quote quoted string
	Excel will export to csv in exactly that format automatically.  Following the example, the
	cell should contain exactly the string ["elementwithcomma,andstuff","second element"]
	Only arrays of strings are supported
	Int and float are supported as property:int and property:float

	To update an existing object, the accession (or uuid or other valid identifier) must be given,
	must exist, and the given properties must be in its schema.
	If the property has a value and does not exist in the object it is added with the specified value.
	If the property has a value and exists in the object, its value is over-written.
	If the property exists, there is no new value, and --put has been given, the property will be removed altogether.

	To create new objects, omit uuid and accession, but add @type, which must be the object schema
	name (like human_donor, not human-donors).

	New file object creation and uploading are supported.
	Give the path and file name in the submmitted_file_name property.  Only the base filename will be posted to submitted_file_name
	md5sum and file_size will be calculated.
	Upload to AWS requires your AWS credentials to be set up.

	Each object's identifier is echo'ed to stdout as the script works on it.

Examples:

	%(prog)s --key www --infile note2notes.csv

'''

def upload_file(credentials, f_path):
	#upload to S3
	env = os.environ.copy()
	env.update({
		'AWS_ACCESS_KEY_ID': credentials['access_key'],
		'AWS_SECRET_ACCESS_KEY': credentials['secret_key'],
		'AWS_SECURITY_TOKEN': credentials['session_token'],
	})

	logger.info("Uploading file.")
	start = time.time()
	try:
		subprocess.check_call(['aws', 's3', 'cp', f_path, credentials['upload_url'], '--quiet'], env=env)
	except subprocess.CalledProcessError as e:
		# The aws command returns a non-zero exit code on error.
		logger.error("Upload failed with exit code %d" % e.returncode)
		upload_returncode = e.returncode
	else:
		upload_returncode = 0
		end = time.time()
		duration = end - start
		logger.info("Uploaded in %.2f seconds" % duration)

	return upload_returncode

def get_upload_credentials(server, keypair, accession):
	r = requests.get()

def main():

	import argparse
	parser = argparse.ArgumentParser(
		description=__doc__, epilog=EPILOG,
		formatter_class=argparse.RawDescriptionHelpFormatter,
	)
	parser.add_argument('--key', default='default',
		help="The keypair identifier from the keyfile for the server.  Default is --key=default")

	parser.add_argument('--keyfile', default=os.path.expanduser("~/keypairs.json"),
		help="The keypair file.  Default is --keyfile=%s" %(os.path.expanduser("~/keypairs.json")))

	parser.add_argument('--infile', '-i',
		help="CSV file with metadata to update")

	parser.add_argument('--dryrun',	default=False, action='store_true',
		help="Do everything except save changes")

	parser.add_argument('--debug', default=False, action='store_true',
		help="Print debug messages.  Default is False.")

	parser.add_argument('--put', default=False, action='store_true',
		help="If property in the input is blank, remove that property entirely from the existing object")

	args = parser.parse_args()

	if args.debug:
		logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
	else:
		logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)

	key = ENC_Key(args.keyfile, args.key) #get the keypair
	connection = ENC_Connection(key) #initialize the connection object
	#biosample_collection = ENC_Collection(connection,'biosamples',frame='object')

	with open(args.infile,'rU') as f:
		reader = csv.DictReader(f, delimiter=',', quotechar='"')
		for new_metadata in reader:
			uuid = new_metadata.pop('uuid',None)
			accession = new_metadata.pop('accession',None)
			if uuid: #use the uuid if there is one
				obj_id = uuid
			elif accession: #if no uuid then use the accession
				obj_id = accession
			else: #if neither uuid or accession, assume this is a new object
				obj_id = None
			enc_object = ENC_Item(connection, obj_id)
			#print "Got accessioned object %s with status %s" %(enc_object.get('accession'), enc_object.get('status'))
			submit_new_file = False
			for prop in new_metadata:
				if new_metadata[prop].strip() == "":
					if args.put: #if empty, pop out the old property from the object
						old_value = enc_object.properties.pop(prop,None)
					continue #skip properties with no value for post or patch
				else: #new property or new value for old property
					new_metadata_string = new_metadata[prop]
					if ':' in prop:
						prop_name,sep,prop_type = prop.partition(':')
					else:
						prop_name = prop
						prop_type = 'string'
					if prop_type == 'array':
						# subreader = csv.reader(StringIO(new_metadata_string), delimiter=',', quotechar='"')
						# array_items = []
						# for line in subreader:
						# 	for s in line:
						# 		array_items.append(s)
						logger.debug("new_metadata_string is %s" %(new_metadata_string))
						array_items = json.loads(new_metadata_string)
						logger.debug("array_items is %s" %(array_items))
						json_obj = {prop_name: array_items}
					elif prop_type == 'int' or prop_type == 'integer':
						json_obj = {prop_name: int(new_metadata_string)}
					elif prop_type == 'float':
						json_obj = {prop_name: float(new_metadata_string)}
					else:
						json_obj = {prop_name: new_metadata_string} #default is string
					if prop == 'submitted_file_name':
						new_filename = new_metadata_string
						old_filename = enc_object.properties['submitted_file_name']
						if new_filename != old_filename:
							submit_new_file = True
					enc_object.properties.update(json_obj)
			if submit_new_file:
				path = os.path.expanduser(enc_object.get('submitted_file_name'))
				path = os.path.abspath(path)
				basename = os.path.basename(path)
				enc_object.properties.update({
					'submitted_file_name': basename,
					'md5sum': common.md5(path),
					'file_size': os.path.getsize(path)})
			if obj_id:
				logger.info('Syncing %s' %(obj_id))
			else:
				logger.info('Syncing new object')
			logger.debug('%s' %(json.dumps(enc_object.properties, sort_keys=True, indent=4, separators=(',', ': '))))
			result = enc_object.sync(args.dryrun)
			if not args.dryrun:
				try:
					assert result['status'] == 'success'
				except:
					logger.error('New object sync failed ... Skipping. %s' %(result))
				else:
					new_object = result['@graph'][0]
					if 'accession' in new_object:
						new_id = new_object['accession']
					else:
						new_id = new_object['uuid']
					logger.info("New object: %s" %(new_id))
					if enc_object.type == 'file' and 'submitted_file_name' in json_obj:
						upload_credentials = enc_object.new_creds()
						logger.debug(upload_credentials)
						rc = upload_file(upload_credentials,path)
						logger.info("Upload rc: %d" %(rc))

if __name__ == '__main__':
	main()