#!/usr/bin/env python
# $Id: cpush 186 2011-01-21 23:07:00Z tjn $

import c3_config
import c3_version

try:
	import sys, os

	sys.path.append( c3_config.def_path )
	import c3_com_obj, c3_file_obj, c3_sock, socket, re

	######## constants ####################################
	help_info = """Usage: cpush [OPTIONS] [MACHINE DEFINTIONS] source [target]

The transfer will fail if the target contains any wild cards. 
If target is not specified then the source is placed in the
directory from which "source" came from.  The source can not have 
any wildcards if no target was specified.


	-h, --help	= Display help message.
	-f, --file	= File containing list of ip's.  If one is not 	
			  supplied then /etc/c3.conf will be used.
	-l, --list	= List of files to push (includes destination
			in file).
	-p, --pushlist	= List of files to push (does not include
			destination in file, must have on command line).
	-i		= Interactive mode.  Ask once before executing.
	--head		= Execute command on head node.  Does not
			  execute on compute nodes if specified.
	--nolocal	= The source file or directory lies on the head 
			  node of the remote cluster
	-b, --blind	= Pushes the entire file (cpush uses rsync to 
			  push)
	--all           = Execute command on all the nodes on all the
			  clusters in the configuration file.  Ignores
			  the [MACHINE_DEFINITONS]
	-n, --dryrun	= Does not send commands to remote machines.
	-v, --version 	= Display the version number

	Machine definitions are in the form of 
		clusterName: start-end, node number"""

	backslash = re.compile( r"\\" )
	#######################################################


	######## check for arguments  #########################
	if len(sys.argv) == 1:
		print help_info
		sys.exit()
	#######################################################


	######### et default options  #########################
	to_print = 0			#not used in parallel version
	interactive = 0			#prompt before execution
	head_node = 0			#execute only on head node
	filename = "/etc/c3.conf"	#default config file
	options = ""			#options to be passed to a remote cluster
	options_to_pass = ""		#if a remote cluster, options to command
	list_option = ""			#file with list of files to be transfered
	list_of_files = []		#list object of files to transfer
	exclude_pattern = ""		#regex to exclude from push
	pidlist = []                    #list if pid's to wait on   
	blind_set = 0			#if set do a blind push
	nolocal = 0			#default is to push the file from the current machine
	defusername = ""		#default user name
	defc3path = ""			#default path to the C3 files
	isdir = ""
	#######################################################

	## verbose output #####################################
	try:
		verbose = os.environ[ 'C3_VERBOSE' ]
	except KeyError:
		verbose = ""
	if verbose:
		rsyncv = "-v"
	else:
		rsyncv = ""
	#######################################################

	######### internal variables ##########################
	cluster_from_file = {}
	all = 0
	file_set = 0
	list_option_set = 0
	filelist_option_set = 0
	exclude_set = 0
	blind_set = 0
	dryrun = 0
	returncode = 0
	
	import pprint
	pp = pprint.PrettyPrinter(indent=4)
	#######################################################



	######### parse command line ##########################

	# first create one large string of the command 
	command_line_list = sys.argv[1:]
	command_line_string = ''
	for item in command_line_list:
		command_line_string = '%s %s' % (command_line_string, item)

	# object used to parse command line
	c3_command = c3_com_obj.c3_command_line( command_line_string )
	 
	# get first option
	try:
		option = c3_command.get_opt()
		while option: # while more options
			if option == '-l' or option == '--list': # list of file to transfer with destinations
				if not list_option_set and not filelist_option_set:
					list_option = c3_command.get_opt_string()
					list_option_set = 1
				else:
					print "only one file list may be specified"
					sys.exit(c3_config.C3_ERRLOCAL)
			elif option == '-p' or option == '--pushlist': # list of files to transfer with dest on command line
				if not filelist_option_set and not list_option_set:
					list_option = c3_command.get_opt_string()
					filelist_option_set = 1
				else:
					print "only one file list may be specified"
					sys.exit(c3_config.C3_ERRLOCAL)
			elif (option == '-f') or (option == '--file'): # alternate config file
				if not file_set:
					filename = c3_command.get_opt_string()
					file_set = 1
				else:
					print "only one file name can be specified."
					sys.exit(c3_config.C3_ERRLOCAL)
			elif option == '-h' or option == "--help": # print help info
				print help_info
				sys.exit()
			elif option == '-i': # ask once before executing command
				interactive = 1
			elif option == '--head': # execute only the head node
				head_node = 1
			elif option == '-x' or option == '--exclude': # exlude a pattern from the push
				if blind_set:
					print "can not do a blind push with an exclude"
					sys.exit(c3_config.C3_ERRLOCAL)
				exclude_set = 1	
				exclude_pattern = c3_command.get_opt_string()
				options_to_pass = options_to_pass + " -x \"" + exclude_pattern + "\""
				options = options + " --exclude \'" + exclude_pattern + "\'"
			elif option == '-d' or option == '--delete':
				options_to_pass = options_to_pass + " -d"
				options = options + " --delete"
			elif option == '--nolocal':
				nolocal = 1
			elif option == '-b' or option == '--blind':
				if exclude_set:
					print "can not do a blind push with an exclude"
					sys.exit(c3_config.C3_ERRLOCAL)
				blind_set = 1
			elif option == '--isdir':
				isdir = c3_command.get_opt_string()
			elif option == '--all':
				all = 1
			elif option == '-n' or option == '--dryrun':
				dryrun = 1
			elif option == '--version' or option == '-v':
				# Get C3's version
				print c3_version.c3_version
				sys.exit(c3_config.C3_OKEXIT)
			else: # a catch all, option supplied not valid
				print "option ", option, " is not valid"
				sys.exit(c3_config.C3_ERRLOCAL)
			option = c3_command.get_opt()
	except c3_com_obj.end_of_option: #quit parsing options
		pass

	if isdir != "":
		sock = c3_sock.client_sock()
		if os.path.isdir( isdir ):
			sock.send( 'yes' )
		else:
			sock.send( 'no' )
		sys.exit(c3_config.C3_OKEXIT)

	# create cluster object from command line 
	clusters = c3_com_obj.c3_cluster_list()
	 
	if all == 0:
		clusters = c3_command.get_clusters()
	else:
		c3_command.get_clusters()

	if list_option_set: # If the --list option is active
		templistoffiles = open( list_option, "r" ).readlines()
		counter = 0 # Because there will be more than two files per line now, we will need to have another dimension in the array
		for line in templistoffiles:
			temp_list = line.strip().split()
			if len( temp_list ) > 2:
				print "error in file list ", list_option, " more than source and destination files on one line."
				sys.exit(c3_config.C3_ERRLOCAL)
			if len( temp_list ) == 0:
				continue
			list_of_files.append( temp_list )
#			list_of_files[counter].append( temp_list )
			try:
				list_of_files[-1][1]
			except IndexError:
				list_of_files[-1].append( list_of_files[-1][0] )
			if list_of_files[-1][0][0] != '/':
				list_of_files[-1][0] = os.getcwd() + "/"
			if list_of_files[-1][-1][0] != '/':
				list_of_files[-1][1] = os.getcwd() + "/"
			counter = counter + 1
			
	elif filelist_option_set: # If the files are listed with the --filelist option
		#Get the sources from the file provided by the filelist
		templistoffiles = open( list_option, "r" ).readlines()
		list_of_files.append( [] )
		for line in templistoffiles:
			temp_list = line.strip().split()
			if len( temp_list ) == 0:
				continue
			for fn in temp_list:
				list_of_files[0].append(fn)
		#Get the target file from the command line
		try:
			target = c3_command.rest_of_command().strip().split()
		except c3_com_obj.bad_string: # Make sure there is something there
			print "No target file."
			sys.exit(c3_config.C3_ERRLOCAL)
		if len( target ) > 1:
			print "Extra command line arguments."
			sys.exit(c3_config.C3_ERRLOCAL)
		if len( target ) == 0:
			print "No target file."
			sys.exit(c3_config.C3_ERRLOCAL)
		list_of_files[0].append(target[0])
			
	else:	# If the files are listed on the command line
		
		list_of_files.append( [] )
		templistoffiles = []
		try:
			temp_list = c3_command.rest_of_command().strip().split()
		except c3_com_obj.bad_string: # Make sure there is something there
			print "No source or target file."
			sys.exit(c3_config.C3_ERRLOCAL)
		if len( temp_list ) < 2: # Make sure there is an explicit source and target
			print "Need an explicit source and target."
			sys.exit(c3_config.C3_ERRLOCAL)
		for temp_filename in temp_list: # Append all the file/folder names on to the file list
			list_of_files[0].append(temp_filename)
			
#	print pp.pprint(list_of_files)

	#######################################################


	######### test if ssh or rsh ##########################
	try:
		transport = os.environ[ 'C3_RSH' ]
	except KeyError:
		transport = 'ssh'
	#######################################################

	######### et default username #########################
	try:
		defusername = os.environ[ 'C3_USER' ]
	except KeyError:
		defusername = os.environ[ 'USER' ]
	#######################################################
        ######### set filename  ##########################
        if not file_set:
                try:
                        filename = os.environ[ 'C3_CONF' ]
                except KeyError:
                        filename = '/etc/c3.conf'
        #######################################################


	######### make cluster list object from file ##########
	try: # open file & initialize file parser
		file = c3_file_obj.cluster_def( filename )
	except IOError:
		print "error opening file: ", filename
		sys.exit(c3_config.C3_ERRLOCAL)

	# note, someday need to change to only get clusters needed, need changes to the
	# file parser object for that, will be done in later versions

	try:
		file.get_next_cluster() # set the default cluster (first cluster in list)
		try:
			if clusters.clusters[0] ==  "/default": # if default cluster set correct cluster name
				clusters.clusters[0] = file.get_cluster_name()
				clusters.node[file.get_cluster_name()] = clusters.node["/default"]
				del clusters.node["/default"]
				clusters.username[file.get_cluster_name()] = clusters.username["/default"]
		except IndexError: #if all is specified will throw
			pass
		while(1): #throws exception when no more clusters
			c_name = file.get_cluster_name() #name of cluster being parsed
			if all == 1:
				clusters.clusters.append( c_name )
				clusters.node[c_name] = []
				clusters.node[c_name].append( "" )
			if c_name in clusters.clusters:
				cluster_from_file[c_name] = {}
				cluster_from_file[c_name]['external'] = file.get_external_head_node()
				cluster_from_file[c_name]['internal'] = file.get_internal_head_node()
				cluster_from_file[c_name]['nodes'] = [] #list of node names from file
				if cluster_from_file[c_name]['external']: #if a direct cluster
					index = 0
					try:
						while(1): # build list of nodes
							node_obj = file.get_next_node()
							cluster_from_file[c_name]['nodes'].append( c3_file_obj.node_obj() )
							cluster_from_file[c_name]['nodes'][index] = node_obj
							index = index + 1
					except c3_file_obj.end_of_cluster:
						pass
			file.get_next_cluster() #repeat untill no more clusters
	except c3_file_obj.no_more_clusters:
		pass
	except c3_file_obj.parse_error, error:
		print error.description
		print "somewhere around ", error.last
		sys.exit(c3_config.C3_ERRLOCAL)

	#######################################################


	######### execute command on each node in cluster



	# there are two main groups, local and remote clusters
	# in each of those groups there are direct and indirect
	# modes, that is every node specified or a "link". A link
	# on a local cluster is of course invalid.

	# right now the only way I know how to check if a cluster
	# is local is to use a "gethostbyname" and compare it
	# to the head node names (both internel and externel).
	# right now that is acceptable as many tools require 
	# the function to work correctly (ssh being one of them)
	# my want to think about a better way.

	#host_ip = socket.gethostbyname( socket.gethostname() )



        while interactive: # if interactive execution, prompt once before executing
                answer = raw_input( "push " + source + " to " + target + " :" )
                if re.compile( r".*n(o)?.*", re.IGNORECASE ).match( answer ):
                        sys.exit()
                if re.compile( r".*y(es)?.*", re.IGNORECASE).match( answer ):
                        interactive = 0
		

	pid_list_outer = []

	for cluster in clusters.clusters:

		pid = os.fork()
		if pid == 0:

			############ get machine names #############################
			try: 
				local_ip = socket.gethostbyname( socket.gethostname() )
			except socket.error:
				print "Can not resolve local hostname"
				os._exit(c3_config.C3_ERRLOCAL)
			try:
				int_ip = socket.gethostbyname( cluster_from_file[cluster]['internal'] )
			except socket.error:
				int_ip = ""
			except KeyError:
				print "Cluster ", cluster, " is not in you configuration file."
				continue
	
			try:
				ext_ip = socket.gethostbyname( cluster_from_file[cluster]['external'] )
			except socket.error:
				ext_ip = ""
			except TypeError:
				ext_ip = int_ip
			############################################################
			
			try:
				if clusters.username[cluster]=="/default":
					username = defusername
				else:
					username = clusters.username[cluster]
			except KeyError: #if --all is specified
				username = defusername
					
			if head_node:   #if only execute on head node , do so
					#will execute on local cluster with ssh also
				for row in list_of_files:
					if blind_set:
						string_to_execute = transport + " " + "-l " + username + " " + node_name + " \'/bin/rm -f " + row[-1] + "\'"
						if dryrun:
							print string_to_execute
						else:
							code = os.system( string_to_execute )>>8
							if( code != 0 ): returncode = code
					string_to_execute = "rsync -az " + rsyncv + " --rsh=" + transport + " " + options + " " + " ".join(row[:-1]) + " " + username + "@" + ext_ip + ":\'" + row[-1] + "\'"
					if verbose:
						print "headnode cmd: " + string_to_execute
					if dryrun:
						print string_to_execute
					else:
						code = os.system( string_to_execute )>>8
						if( code != 0 ): returncode = code

			elif cluster_from_file[cluster]['external']: # if a direct cluster
				if ext_ip == local_ip or int_ip == local_ip: # if a local cluster

					if ext_ip == "": # error conditions (just don't execute current cluster)
						print "Can not resolve ", cluster_from_file[cluster]['external']
					elif int_ip == "":
						print "can not resolve ", cluster_from_file[cluster]['internal']

					elif clusters.node[cluster][0] != "" : #range specified on command line
						for node in clusters.node[cluster]: #for each cluster specified on the command line
							try:
								if not cluster_from_file[cluster]['nodes'][node].dead: #if machine is not dead
									node_name = cluster_from_file[cluster]['nodes'][node].name
									pid = os.fork() # execute command on each node in it's own process
									if pid == 0:
										node_name = cluster_from_file[cluster]['nodes'][node].name
										for row in list_of_files:
											if blind_set:
												string_to_execute = transport + " " + "-l " + username + " " + node_name + " \'/bin/rm -f " + row[-1] + "\'"
												if dryrun:
													print string_to_execute
												else:
													code = os.system( string_to_execute )>>8
													if( code != 0 ): returncode = code
											string_to_execute = "rsync -az " + rsyncv + " --rsh=" + transport + " " + options + " " + " ".join(row[:-1]) + " " + username + "@" + node_name + ":" + row[-1] + ""
											if verbose:
												print "dir cmd: " + string_to_execute
											if dryrun:
												print string_to_execute
											else:
												code = os.system( string_to_execute )>>8
												if( code != 0 ): returncode = code
										os._exit(returncode)
									pidlist.append(pid)
							except IndexError:
								pass
					else:  # no range specified on command line, do all nodes
						for node in cluster_from_file[cluster]['nodes']:  # for each node in cluster
							if not node.dead: #if node not dead
								pid = os.fork() # execute command in own process
								if pid == 0:
									node_name = node.name
									for row in list_of_files:
										if blind_set:
											string_to_execute = transport + " " + "-l " + username + " " + node_name + " \'/bin/rm -f " + row[-1] + "\'"
											if dryrun:
												print string_to_execute
											else:
												code = os.system( string_to_execute )>>8
												if( code != 0 ): returncode = code
										string_to_execute = "rsync -az " + rsyncv + " --rsh=" + transport + " " + options + " " + " ".join(row[:-1]) + " " + username + "@" + node_name + ":" + row[-1] + ""
										if verbose:
											print "dirall cmd: " + string_to_execute
										if dryrun:
											print string_to_execute
										else:
											code = os.system( string_to_execute )>>8
											if( code != 0 ): returncode = code
									os._exit(returncode)
								pidlist.append(pid)
				else: # remote cluster
					if ext_ip == "": # error condition
						print "Can not resolve ", cluster_from_file[cluster]['external']
						sys.stdout.flush()
					else:
						# generate temprorary config file
						cluster_def_string = "cluster " + cluster + " {\n"
						cluster_def_string = cluster_def_string + cluster_from_file[cluster]['external'] + ":" + cluster_from_file[cluster]['internal'] + "\n"

						if clusters.node[cluster][0] != "" : #range specified on command line
							try:
								for node in clusters.node[cluster]: #for each node specified on command line
									if not cluster_from_file[cluster]['nodes'][node].dead: # if cluster node not dead
										node_name = cluster_from_file[cluster]['nodes'][node].name #add cluster to list
										cluster_def_string = cluster_def_string + node_name + "\n"
							except IndexError:
								pass
						else:  # no range specified on command line, do all nodes
							for node in cluster_from_file[cluster]['nodes']: # for each node in cluster
								if not node.dead: #if node not dead, add to list
										cluster_def_string = cluster_def_string +  node.name + "\n"
						cluster_def_string = cluster_def_string + "}" # close list
										
						#this is an attempt to generate a unique file name. As there is no easy way
						#for me to see the remote machine I mangle a group of relativly unique
						#identifiers. since i prepend the machine ip address if the file does not reside localy
						#it should not remotely, at the very least it should be safe to rewrite the file
						
						tempfolderbase = local_ip + "%d" % os.getuid() + "%d" % os.getpid()
						
						tempfoldername = "/tmp/" + tempfolderbase
							
						string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + tempfoldername
						if dryrun:
							print string_to_execute
						else:
							sock = c3_sock.server_sock( string_to_execute )
							answer = sock.recieve()
							sock.close()
							while answer == 'good': #make sure file name is unique on remote machine
								tempfoldername = tempfoldername + "1"
								string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + tempfoldername
								sock.__init__( string_to_execute )
								answer = sock.recieve()
								sock.close()
						for row in list_of_files:
							go_nogo = "go"
							remotefoldername = ""
							if not nolocal:
								remotefoldername = tempfoldername + "/"
							else:
								for remotefilename in row[:-1]:
									string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + remotefilename
									if dryrun:
										print string_to_execute
									else:
										sock = c3_sock.server_sock( string_to_execute )
										answer = sock.recieve()
										sock.close()
									if answer == 'bad': #make sure file name is unique on remote machine
										print "file " + remotefilename + " must exist on remote machine"
										go_nogo = "nogo"
							
							if go_nogo == 'go':	
								FILE = open( tempfolderbase + "conf", 'w' )
								FILE.write( cluster_def_string )
								FILE.close()
								# push file to remote machine
								if not nolocal:
									string_to_execute = "rsync -az " + rsyncv + " --rsh=" + transport + " " + " ".join(row[:-1]) + " " + username + "@" + ext_ip + ":" + remotefoldername
									if verbose:
										print "indir cmd: " + string_to_execute
									if dryrun:
										print string_to_execute
									else:
										code = os.system( string_to_execute )>>8
										if( code != 0 ): returncode = code
							
								string_to_execute = "rsync " + rsyncv + " --rsh=" + transport + " " + tempfolderbase + "conf " + username + "@" + ext_ip + ":" + "/tmp/"
								if verbose:
									print "indir2 cmd: " + string_to_execute
								if dryrun:
									print string_to_execute
								else:
									code = os.system( string_to_execute )>>8
									if( code != 0 ): returncode = code

								# are we pushing a directory?
#									if os.path.isdir( row[0] ):
#										remotefoldername = remotefoldername + os.path.basename( row[0] )
#									else:
#										string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cpush --isdir " + line[1]
#										if verbose:
#											print "isdir cmd: " + string_to_execute
#										sock = c3_sock.server_sock( string_to_execute )
#										answer = sock.recieve()
#										if answer == 'yes':
#											row[1] = row[1] + '/' + os.path.basename(row[0])
#											sock.close()
								
								if not nolocal:
									string_to_execute = transport + " -l " + username + " " + ext_ip + "  " + c3_config.def_path + "/cpush " + options_to_pass + " -f /tmp/" + tempfolderbase + "conf " + remotefoldername + "* " + row[-1]
								else:
									string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cpush " + options_to_pass + " -f /tmp/" + tempfolderbase + "conf " + " ".join(row[:-1]) + " " + row[-1]
								if verbose:
									print "indir3 cmd: " + string_to_execute
								if dryrun:
									print string_to_execute
								else:
									code = os.system( string_to_execute )>>8
									if( code != 0 ): returncode = code

								# remove temporary file
								string_to_execute = transport + " -l " + username + " " + ext_ip + " /bin/rm -f /tmp/" + tempfolderbase + "conf"
								if verbose:
									print "rmconfig cmd: " + string_to_execute
								if dryrun:
									print string_to_execute								
								else:
									os.system( string_to_execute )
								if not nolocal:
									string_to_execute = transport + " -l " + username + " " + ext_ip + " /bin/rm -rf " + tempfoldername
									if dryrun:
										print string_to_execute
									else:
										code = os.system( string_to_execute )>>8
										if( code != 0 ): returncode = code
						if os.path.exists( tempfolderbase + "conf"):
							os.unlink( tempfolderbase + "conf" ) # remove local temp file
					
			else: # indirect clusters
				if int_ip == local_ip:  # can not have a indirect local cluster since if your default cluster
							# is local you would generate a circular reference
					print "error local cluster"
					sys.stdout.flush()
				else: # remote indirect cluster
					if int_ip == "": # error condition
						print "Can not resolve hostname ", cluster_from_file[cluster]['internal']
						sys.stdout.flush()
					else:
						for row in list_of_files:
							go_nogo = "go"
							if not nolocal:
								tempfoldername = "/tmp/" + local_ip + "%d" % os.getuid() + "%d" % os.getpid()

								string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + tempfoldername + "/"
								if dryrun:
									print string_to_execute
								else:
									sock = c3_sock.server_sock( string_to_execute )
									answer = sock.recieve()
									sock.close()
									while answer == 'good': #make sure file name is unique on local machine
										tempfoldername = tempfoldername + "1"
										string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + tempfoldername + "/"
										sock.__init__( string_to_execute )
										answer = sock.recieve()
										sock.close()
							
							else:
								for tempfilename in row[:-1]:
									string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cget --head --non_interactive " + tempfilename
									if verbose:
										print "filecheck: " + string_to_execute
									if dryrun:
										print string_to_execute
									else:
										sock = c3_sock.server_sock( string_to_execute )
										answer = sock.recieve()
										sock.close()
										if answer == 'bad': #make sure file name is unique on local machine
											go_nogo = "nogo"
											print "file must exist on remote machine"
							
							if go_nogo == 'go':
								# push file to remote machine
								if not nolocal:
									string_to_execute = "rsync -az " + rsyncv + " --rsh=" + transport + " " + " ".join(row[:-1]) + " " + username + "@" + int_ip + ":" + tempfoldername + "/"
									if verbose:
										print "cmd: " + string_to_execute
									if dryrun:
										print string_to_execute
									else:
										code = os.system( string_to_execute )>>8
										if( code != 0 ): returncode = code
																
								node_list = "" # generate new node list from the command line
								if clusters.node[cluster][0] != "" : #range specified on command line
									node_list = ":%d" % clusters.node[cluster].pop(0)

									for node in clusters.node[cluster]:
										node_list = node_list + ", %d" % node	
								# execute command on remote machine
#								string_to_execute = transport + " -l " + username + " " + ext_ip + " " + c3_config.def_path + "/cpush --isdir " + row[-1]
#								if dryrun:
#									print string_to_execute
#								else:
#									sock = c3_sock.server_sock( string_to_execute )
#									answer = sock.recieve()
#									if answer == 'yes':
#										row[-1] = row[-1] + '/' + os.path.basename(tempfoldername)
#									sock.close()

								if not nolocal:
									string_to_execute = transport + " -l " + username + " " + int_ip + " " + c3_config.def_path + "/cpush " + options_to_pass + " " + node_list + " \'" + tempfoldername + "/*\' " + row[-1]
								else:
									string_to_execute = transport + " -l " + username + " " + int_ip + " " + c3_config.def_path + "/cpush " + options_to_pass + " " + node_list + " " + " ".join(row[:-1]) + " " + row[-1] 
								if dryrun:
									print string_to_execute
								else:
									code = os.system( string_to_execute )>>8
									if( code != 0 ): returncode = code

								if not nolocal:
									# remove temporary file
									string_to_execute = transport + " -l " + username + " " + int_ip + " /bin/rm -rf " + tempfoldername
									if dryrun:
										print string_to_execute
									else:
										code = os.system( string_to_execute )>>8
										if( code != 0 ): returncode = code

			for pid in pidlist: # wait for all processes spawned to finish
				pid, code = os.wait()
				if(code>>8 != 0): returncode = code>>8
			os._exit(returncode)
		pid_list_outer.append(pid)
	for pid in pid_list_outer:
		pid, code = os.wait()
		if(code>>8 != 0): returncode = code>>8
except KeyboardInterrupt:
        print "Keyboard interrupt\n"

sys.exit(returncode)

# vim:tabstop=4:shiftwidth=4:noexpandtab:textwidth=76
