githelper

#!/usr/bin/python
#################################################################
##\file
#
# \note
#   Copyright (c) 2010 \n
#   Fraunhofer Institute for Manufacturing Engineering
#   and Automation (IPA) \n\n
#
#################################################################
#
# \note
#   Project name: care-o-bot
# \note
#   ROS stack name: setup
# \note
#   ROS package name: setup
#
# \author
#   Author: Florian Weisshardt, email:florian.weisshardt@ipa.fhg.de
# \author
#   Supervised by: Florian Weisshardt, email:florian.weisshardt@ipa.fhg.de
#
# \date Date of creation: Oct 2011
#
# \brief
#   Implements helper script for working with git and the care-o-bot stacks.
#
#################################################################
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     - Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer. \n
#     - Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution. \n
#     - Neither the name of the Fraunhofer Institute for Manufacturing
#       Engineering and Automation (IPA) nor the names of its
#       contributors may be used to endorse or promote products derived from
#       this software without specific prior written permission. \n
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License LGPL as 
# published by the Free Software Foundation, either version 3 of the 
# License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License LGPL for more details.
# 
# You should have received a copy of the GNU Lesser General Public 
# License LGPL along with this program. 
# If not, see <http://www.gnu.org/licenses/>.
#
#################################################################

import os
import pwd
import sys
import shlex
import time
import platform
from optparse import OptionParser
import subprocess
import re
import socket
import getpass
import json

# Check for filesystem
if platform.system() == 'Linux':
	# write to log file instead of /dev/null
	_output = "~/.githelper/log.txt"
	# check if log file exists, if not create dir and file, if yes delete old file
	if not os.path.exists(os.path.expanduser(os.path.dirname(_output))):
		os.makedirs(os.path.expanduser(os.path.dirname(_output)))
		os.system("touch " + _output)
	else:
		os.system("rm -r " + _output)
	
	# start new log with time information
	os.system("echo new log from " + time.strftime("%Y/%m/%d %H:%M:%S", time.localtime()) + " > " + _output + " 2>&1")

	# make logfile editable by everyone
	os.system("chmod a+rw -R " + _output)

elif platform.system() == 'Windows':
	_output = "nul"
else:
	_output = "/dev/null"

_usage = """usage: %prog [options] <command>

Commands:

%prog setup
  Once user has created an account on github.com, this sets up everything to get started working 

%prog clone -s STACK(s) [-r] [-d install_dir, default=~/git/care-o-bot]
  Clones the given list (comma-separated) of repositories from github (if necessary, forks from ipa320 before cloning)

%prog status [-s STACK(s), default=all stacks] [-d install_dir=~/git/care-o-bot]
  Executes 'git status -uno' 

%prog pull [-s STACK(s), default=all stacks] [-d install_dir=~/git/care-o-bot] [-b branch=groovy_dev]
  Pulls latest changes from your origin groovy_dev 

%prog push [-s STACK(s), default=all stacks] [-d install_dir=~/git/care-o-bot] [-b branch=groovy_dev] 
  Pushes local commits to origin groovy_dev 

%prog merge [-u GITHUBUSER, default="ipa320"] [-s STACK(s), default=all stacks] [-d install_dir=~/git/care-o-bot] [-b branch=groovy_dev]
  Merges from a remote branch 

%prog fetch_all [-s STACK(s), default=all stacks]
  Executes 'git fetch -p --all'

githelper -h
  Further help on options
"""


## Main githelper script
#
# Parses arguments and starts helping. Everything is done regarding the local groovy_dev branch
#TODO: check for latest version before executing command
def main():
	# check for existence of curl; abort if not installed
	curl_check = subprocess.Popen('which curl', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
	if len(curl_check)==0:
		print "Please install curl first: sudo apt-get install curl"
		print "Aborting."
		sys.exit(1)

	#print "main ..."
	parser = OptionParser(usage=_usage, prog=os.path.basename(sys.argv[0]))
	
	parser.add_option("-u", "--user",
		dest="githubuser", default="ipa320",
		help="Username on http://www.github.com. Default: ipa320")
	
	parser.add_option("-s", "--stacks",
		dest="stacks", metavar="STACK_NAMES_LIST,COMMA_SEPERATED", default="",
		help="Comma separated list of stacks to operate on. Default: all stacks in ~/git/care-o-bot")
	
	parser.add_option("-d", "--directory",
		dest="install_dir", default="~/git/care-o-bot",
		help="Installation directory of care-o-bot stacks. Default: ~/git/care-o-bot")

	parser.add_option("-r", "--readonly",
		action="store_true", dest="readonly",
		help="Access github repositories read-only")
	
	parser.add_option("-b", "--branch",
		dest="branch", default="groovy_dev",
		help="Remote branch . Default = groovy_dev")

	(options, args) = parser.parse_args()

	# check for input parameters
	if len(args) == 0:
		parser.error("You must supply a command to githelper.")
	if len(args) != 1:
		parser.error("githelper only accepts a single command.")

	# start helping
	#print 'DEBUG: githelper starting with args %s, options %s'%(args, options)
	print ""

	if options.stacks == "":
		stacks = get_all_stacks(options)
	else:
		# convert comma separated string to list of stacks
		splitter = shlex.shlex(options.stacks, posix=True)
		splitter.whitespace += ','
		splitter.whitespace_split = True
		stacks = list(splitter)

	if args[0] == "setup":
		print 'githelper setup: helps you to set up everything to get started working with the Care-O-bot stacks on github\n'
		print 'Your github and .gitconfig user names will be overwritten to match your github account name. Your email addresses stored in github and .gitconfig will also be overwritten to match each other.'
		print '\nIf you do not have an account on github yet, please interrupt and do one the following:'
		print '(1) Clone repositories from github read-only (no github account necessary):'
		print '    githelper clone -r -s STACK(s)'
		print '(2) Create an account on github.com (allows forking, read/write cloning and pushing changes to your forked remote repository)'
		print '    (Suggestion: choose ipa-xyz as github user name, xyz being your IPA account name)\n'
		
		github_user = subprocess.Popen('git config --global github.user', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
		if len(github_user)==0:
			github_user = raw_input('Please enter your github account name: ')
		else:
			print 'Found the following github account in .gitconfig: ' + github_user
			ans = raw_input('Press enter to use this account, or type in another github account to use: ')
			if len(ans)>0:
				github_user = ans

		print 'Please enter your github password: '
		github_pw = getpass.getpass()
		# try authentication on github:
		s = 'curl -u "' + github_user + ':' + github_pw + '" https://api.github.com'
		answer = subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
		m = re.search('Bad credentials', answer)
		if m:
			print 'Login on github failed; wrong password? - aborting.\n'
			sys.exit(1)
		else:
			print 'Login on github successful.\n'
			# if login correct, make sure all corresponding entries in .gitconfig are identical to the github account name:
			os.system('git config --global github.user ' + github_user)
			os.system('git config --global github.name ' + github_user)
			os.system('git config --global user.name ' + github_user)

		# fetch email from github:
		ans = subprocess.Popen('curl https://api.github.com/users/' + github_user, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
		userdata = json.loads(ans) 
		if "email" not in userdata or len(userdata['email'])==0:
			user_email = subprocess.Popen('git config --global user.email', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
			if len(user_email) > 0:
				print "No email address found on github. Uploading email address found in .gitconfig: " + user_email
				os.system('git config --global github.email "' + user_email + '"')
				os.system('curl -s -u "' + github_user + ':' + github_pw + '" -i -X PATCH -d \'{"email":"' + user_email + '"}\' https://api.github.com/user > /dev/null')
			else:
				print "No email address found on github and in .gitconfig."
				user_email = raw_input("Please enter your email address: ")
				os.system('git config --global user.email "' + user_email + '"')
				os.system('git config --global github.email "' + user_email + '"')
				os.system('curl -s -u "' + github_user + ':' + github_pw + '" -i -X PATCH -d \'{"email":"' + user_email + '"}\' https://api.github.com/user > /dev/null')
		else:
			print "Adding email address found on github to .gitconfig: " + userdata['email']
			os.system('git config --global user.email "' + userdata['email'] + '"')
			os.system('git config --global github.email "' + userdata['email'] + '"')

		# upload public ssh key to github (and if necessary, create it before that):
		if not(os.path.exists(os.path.expanduser('~/.ssh/id_rsa'))):
			answer = raw_input('\nNo id_rsa key found; generate one? [<any key> (recommended) / n] : ')
			if answer == 'n':
				print 'Need ssh-key, aborting...'
				sys.exit(1)
			else:
				os.system('ssh-keygen -t rsa -f ~/.ssh/id_rsa')

		print '\nFound public RSA key; trying to upload to github...'
		rsaTitle = pwd.getpwuid(os.getuid())[0] + '@' + socket.gethostname()
		f = open(os.path.expanduser('~/.ssh/id_rsa.pub'), 'r')
		rsaKey = f.read().strip()
		f.close()
		tt = 'curl -i -H "Accept: application/json" -H "Content-Type: application/json" -u "' + github_user + ':' + github_pw + '" -X POST -d \'{"title":"' + rsaTitle + '","key":"' + rsaKey + '"}\' https://api.github.com/user/keys'
		output = subprocess.Popen(tt, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
		if re.search('key is already in use', output):
			print 'Key is already uploaded to github; done.'
		elif re.search('Validation Failed', output):
			print 'Key upload was unsuccessful: Validation failed. Aborting...'
			sys.exit(1)
		else:
			print 'Successfully uploaded the key ' + rsaTitle

		# display info for user to be added to his ~/.bashrc file
		print ""
		print "Setup completed."
		print ""
		print "Please update your ROS_PACKAGE_PATH to include ~/git/care-o-bot: Either run the following line"
		print "on each terminal or add the line at the end of your ~/.bashrc file and source it again."
		print ""
		print '    export ROS_PACKAGE_PATH=~/git/care-o-bot:$ROS_PACKAGE_PATH'
		print ""

	elif args[0] == "clone":
		print 'githelper clone: clones the given list of repositories\n'

		if len(options.stacks)==0:
			print 'Please provide a comma-separated list of stacks to clone; example:'
			print 'githelper clone -s cob_command_tools,cob_robots'
			print 'Optional arguments: -r (read-only), -d target_directory'
			print 'Aborting.\n'
			sys.exit(1)

		stacks = options.stacks.split(',')
	
		if options.readonly==True: # no github account needed for readonly access
			github_user = raw_input("Please enter github user from which to clone read-only (Enter = ipa320): ")
			if len(github_user)==0:
				github_user = "ipa320"
			install_dir = os.path.expanduser(options.install_dir)
			if not(os.path.exists( install_dir )):
				os.makedirs( install_dir )
			print 'Cloning into ' + install_dir

			for stack in stacks:
				print 'Stack: ' + stack
				if os.path.exists( os.path.join(install_dir, stack) ):
					print "Already cloned into the specified directory; ignoring. Consider 'githelper pull' to get the latest changes."
				else:
					s = 'git clone git@github.com:' + github_user + '/' + stack + ' ' + os.path.join(install_dir, stack)
					print s
					answer = subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
					if re.search('fatal', answer[1]):
						print 'Repository ' + stack + ' could not be cloned. git error message:'
						print answer[1] + '\n'
				print "Done."
		else:
			# if git user.name or user.email are not set, prompt user to run 'githelper setup first' and abort.
			git_username = subprocess.Popen('git config --global user.name', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
			git_useremail = subprocess.Popen('git config --global user.email', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
			github_user = subprocess.Popen('git config --global github.user', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
			if len(git_username)==0 or len(git_useremail)==0 or len(github_user)==0:
				print "Missing git/github user information. Please run 'githelper setup' first, or clone read-only: 'githelper clone -r -s STACK(s)'."
				print 'Aborting.\n'
				sys.exit(1)

			github_user = subprocess.Popen('git config --global github.user', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].strip()
			if len(github_user) > 0:
				answer = raw_input('Is "' + github_user + '" your github user name? [<Return>=Yes; or enter another user name]: ')
				if len(answer) > 0:
					github_user = answer
			else:
				github_user = raw_input('Please enter your github user name: ')

			# github password
			print 'Please enter your github password: '
			github_pw = getpass.getpass()

			# try authentication on github
			s = 'curl -u "' + github_user + ':' + github_pw + '" https://api.github.com'
			answer = subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
			m = re.search('Bad credentials', answer)
			if m:
				print 'Login on github failed; check password or consider accessing remote repository read-only (githelper clone -r).'
				print 'Aborting.\n'
				sys.exit(1)
			else:
				print 'Login on github successful.\n'

			install_dir = os.path.expanduser(options.install_dir)
			if not(os.path.exists( install_dir )):
				os.makedirs( install_dir )
			print 'Cloning into ' + install_dir + '\n'
			for stack in stacks:
				print 'Stack: ' + stack
				if os.path.exists( os.path.join(install_dir, stack) ):
					print "Already cloned into the specified directory; ignoring. Consider 'githelper pull' to get the latest changes."
				else:
					s = 'git clone git@github.com:' + github_user + '/' + stack + ' ' + os.path.join(install_dir, stack)
					answer = subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
					if re.search('fatal', answer[1]):
						print 'Repository "' + stack + '" not found in repositories of github user "' + github_user + '".'
						print 'Now trying to fork from ipa320...'
						s = 'curl -u "' + github_user + ':' + github_pw + '" -X POST https://api.github.com/repos/ipa320/' + stack + '/forks'
						subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
						print 'Now trying to clone...'
						s = 'git clone git@github.com:' + github_user + '/' + stack + ' ' + os.path.join(install_dir, stack)
						answer2 = subprocess.Popen(s, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
						if re.search('fatal', answer2[1]):
							print 'Repository ' + stack + ' could not be cloned. git error message:'
							print answer2[1] + '\n'
				print 'Done.\n'


# ./githelper fork -s blub,cob_command_tools,bla,cob_navigation,cob_manipulation,cob_common
# ./githelper clone -s, default

	else:
		# operate git command on selected stacks
		for stack in stacks:
			# change environment to stack path
			full_dir = os.path.expanduser(options.install_dir + "/" + stack)
			if not os.path.exists(full_dir):
				print "Selected stack " + stack + " not found in " + options.install_dir + ", aborting..."
				write_to_log("Selected stack " + stack + " not found in " + options.install_dir + ", aborting...")
				write_to_log("")
				continue

			# set working directory to stack directory
			os.chdir(full_dir)
			print "-------------------------------------------"
			print "==> operating on " + os.getcwd()
			print "-------------------------------------------"

			if args[0] == "status":
				write_to_log("status of stack " + stack + "...")
				git_status()
			elif args[0] == "fetch_all":
				write_to_log("fetch upstream info " + stack + "...")
				git_fetch_all()
			elif args[0] == "pull":
				print "pulling..."
				write_to_log("pulling stack " + stack + "...")
				check_for_update()
				git_pull(options)
			elif args[0] == "push":
				print "pushing..."
				write_to_log("pushing stack " + stack + "...")
				check_for_update()
				git_push(options)
			elif args[0] == "merge":
				print "merging..."
				write_to_log("merging stack " + stack + "...")
				check_for_update()
				git_merge(stack, options)
			elif args[0] == "mergeto":
				print "merging to..."
				write_to_log("merging to for stack " + stack + "...")
				check_for_update()
				git_mergeto(stack, options)
			else:
				write_to_log("command \"" + args[0] + "\" not supported.")
				parser.error("command \"" + args[0] + "\" not supported.")
			print "...done."
			write_to_log("...done")
			print ""
			write_to_log("")




def write_to_log(message):
	os.system("echo " + message + " >> " + _output + " 2>&1")


def git_status():
	#print "git_status ... "
	os.system("git status -uno")

def git_fetch_all():
	#print "git_fetch_all ... "
	os.system("git fetch -p --all")


def git_pull(options):
	#print "git_pull ... "
	# check for uncommited changes
	check_for_changes()
	branch = os.path.expanduser(options.branch)
	# operate on groovy_dev branch
	os.system("git checkout" + branch + " >> " + _output + " 2>&1")

	# fetch from origin
	os.system("git fetch origin >> " + _output + " 2>&1")
	
	# merge own origin/groovy_dev
	os.system("git merge origin/" + branch + " >> " + _output)
	if has_conflicts():
		print "ERROR: Conflicts found. Please pull manually and resolve conflicts! Resetting changes and aborting..."
		os.system("git reset --hard HEAD")
		exit(1)


def git_push(options):
	#print "git_push ... "
	# first pull from origin groovy_dev
	git_pull(options)
	branch = os.path.expanduser(options.branch)
	# push to origin groovy_dev
	os.system("git push origin " + branch + " >> " + _output + " 2>&1")


def git_merge(stack, options):
	#print "git_merge ... "
	git_push(options)
	branch = os.path.expanduser(options.branch)
	# create remote origin if not exists
	p = os.popen("git remote")
	status = p.read()
	p.close()
	if status.count("origin-" + options.githubuser) <= 0:
		os.system("git remote add origin-" + options.githubuser + " git@github.com:" + options.githubuser + "/" + stack)

	# change to automerge branch
	os.system("git checkout -b automerge >> " + _output + " 2>&1")
	
	# fetch from origin-githubuser and merge and push
	os.system("git fetch origin-" + options.githubuser + " >> " + _output + " 2>&1")
	os.system("git merge origin-" + options.githubuser +  branch + " >> " + _output + " 2>&1")
	if has_conflicts():
		print "ERROR: conflicts found. Please merge manually and resolve conflicts! Resetting changes and aborting..."
		os.system("git reset --hard HEAD >> " + _output + " 2>&1")
		os.system("git checkout " + branch + " >> " + _output + " 2>&1")
		os.system("git branch -D automerge >> " + _output + " 2>&1")
		exit(1)
	
	os.system("git checkout " + branch + " >> " + _output + " 2>&1")
	os.system("git merge automerge >> " + _output + " 2>&1")
	os.system("git branch -D automerge >> " + _output + " 2>&1")
	git_push(options)

def git_mergeto(stack, options):
	#print "git_mergeto ... "
	branch = os.path.expanduser(options.branch)
	git_merge(stack, options)
	os.system("git push origin-" + options.githubuser + branch + " >> " + _output + " 2>&1")

def check_for_update():#TODO check for new version, use timeout if no internet connection available
	#print "check_for_update ... "
	#print "There is a new version of githelper available. Please use the following commands to update:\n"
	#print "wget http://www.github.com/ipa320/setup... "
	#print "chmod +x githelper"
	pass
	

## returns a list of all stacks in options.install_dir
def get_all_stacks(options):
	stacks = []

	# check if install_dir exists, otherwise return empty list of stacks
	if not os.path.exists(os.path.expanduser(options.install_dir)):
		return stacks

	# get all directories
	dirs = os.walk(os.path.expanduser(options.install_dir)).next()[1]
	
	# check if directory is a git repository
	for directory in dirs:
		if os.path.exists(os.path.expanduser(options.install_dir) + "/" + directory + "/.git/config"):
			#print directory, "is a stack"
			stacks.append(directory)
		else:
			#print directory, "is not a stack"
			pass
	return stacks


## checks for uncommited changes
def check_for_changes():
	#print "check_for_changes ..."
	p = os.popen('git status -uno --porcelain')
	status = p.read()
	p.close()
	if status != "":
		print "ERROR: found local changes in " + os.getcwd() + ": please commit or reset them first before continuing, aborting..."
		print "Status: \n" + status
		return


## check if a merge is conflicting
def has_conflicts():
	#print "has_conflicts ..."
	p = os.popen("git status -uno")
	status = p.read()
	p.close()
	
	if status.count("Unmerged paths") <= 0:
		# no conflict found
		return False
	else: 
		# conflict found
		return True
	


## Main routine for running the githelper script
#
if __name__ == '__main__':
	main()