#!/usr/bin/python
import sys, string
import os.path
import unique
import math
import copy
import time
import shutil
import gene_associations
import BuildAffymetrixEntrezAssociations
import OBO_import
import mappfinder
dirfile = unique
###### File Import Functions ######

def filepath(filename):
    dir=os.path.dirname(dirfile.__file__)       #directory file is input as a variable under the main            
    fn=os.path.join(dir,filename)
    fn = string.replace(fn,'\\library.zip','') ###py2exe on some systems, searches for all files in the library file, eroneously
    return fn

def refDir():
    reference_dir=os.path.dirname(dirfile.__file__)       #directory file is input as a variable under the main            
    reference_dir = string.replace(reference_dir,'\\library.zip','') ###py2exe on some systems, searches for all files in the library file, eroneously
    return reference_dir

def read_directory(sub_dir):
    dir=os.path.dirname(dirfile.__file__)
    dir = string.replace(dir,'\\library.zip','') ###py2exe on some systems, searches for all files in the library file, eroneously
    dir_list = os.listdir(dir + sub_dir)
    #add in code to prevent folder names from being included
    dir_list2 = [] 
    for entry in dir_list:
        if entry[-4:] == ".txt" or entry[-4:] == ".csv" or entry[-4:] == ".TXT":
            dir_list2.append(entry)
    return dir_list2

def eliminate_redundant_dict_values(database):
    db1={}
    for key in database: list = unique.unique(database[key]); list.sort(); db1[key] = list
    return db1

###### Classes ######
class GrabFiles:
    def setdirectory(self,value):
        self.data = value
    def display(self):
        print self.data
    def searchdirectory(self,search_term):
        #self is an instance while self.data is the value of the instance
        file = getDirectoryFiles(self.data,str(search_term))
        if len(file)<1: print search_term,'not found'
        return file
    def searchdirectory_start(self,search_term):
        #self is an instance while self.data is the value of the instance
        files = getAllDirectoryFiles(self.data,str(search_term))
        match = ''
        for file in files:
            split_strs = string.split(file,'.')
            split_strs = string.split(split_strs[0],'/')
            if search_term  == split_strs[-1]: match = file
        return match
    
def getDirectoryFiles(import_dir, search_term):
    exact_file = ''
    dir_list = read_directory(import_dir)  #send a sub_directory to a function to identify all files in a directory
    for data in dir_list:    #loop through each file in the directory to output results
        affy_data_dir = import_dir[1:]+'/'+data
        if search_term in affy_data_dir: exact_file = affy_data_dir
    return exact_file

def getAllDirectoryFiles(import_dir, search_term):
    exact_files = []
    dir_list = read_directory(import_dir)  #send a sub_directory to a function to identify all files in a directory
    for data in dir_list:    #loop through each file in the directory to output results
        affy_data_dir = import_dir[1:]+'/'+data
        if search_term in affy_data_dir: exact_files.append(affy_data_dir)
    return exact_files

###### GO-Elite Functions ######

def import_path_index():
    global path_id_to_goid;  global full_path_db; global all_nested_paths
    built_go_paths, path_goid_db, path_dictionary = OBO_import.buildNestedGOTree('no','no')
    
    full_path_db = built_go_paths
    path_id_to_goid = path_goid_db
    #all_nested_paths = path_dictionary 
    print len(full_path_db), 'GO paths imported'
    
class MAPPFinderResults:
    def __init__(self,goid,go_name,zscore,num_changed,onto_type,permute_p,result_line):
        self._goid = goid; self._zscore = zscore; self._num_changed = num_changed
        self._onto_type = onto_type; self._permute_p = permute_p; self._go_name= go_name
        self._result_line = result_line
    def GOID(self): return self._goid
    def GOName(self): return self._go_name
    def ZScore(self): return self._zscore
    def NumChanged(self): return self._num_changed
    def OntoType(self): return self._onto_type
    def PermuteP(self): return self._permute_p
    def ResultLine(self): return self._result_line
    def Report(self):
        output = self.GOID()+'|'+self.GOName()
        return output
    def __repr__(self): return self.Report()

def cleanUpLine(line):
    line = string.replace(line,'\n','')
    line = string.replace(line,'\c','')
    data = string.replace(line,'\r','')
    data = string.replace(data,'"','')
    return data

def moveMAPPFinderFiles():
    ###Create new archive directory
    input_dir = '/input/MAPPFinder'; dir_list = read_directory(input_dir)
    if len(dir_list)>0: ###if there are recently run files in the directory
        time_stamp = string.replace(time.ctime(),':','.')
        time_stamp = time_stamp[4:] ###Use a time-stamp as the output dir (minus the day)
        archive_dir = string.replace(input_dir[1:],'MAPPFinder','MAPPFinder/archived-'+time_stamp)
        fn = filepath(archive_dir)
        os.mkdir(fn) ###create new directory
        ###Read all files in old directory and copy to new directory
        m = GrabFiles(); m.setdirectory(input_dir)
        print 'Moving MAPPFinder results to the archived directory',archive_dir
        for mappfinder_input in dir_list: 
            mappfinder_input_dir = m.searchdirectory(mappfinder_input)
            fn = filepath(mappfinder_input_dir)
            destination_dir = string.replace(fn,'/MAPPFinder/','/MAPPFinder/archived-'+time_stamp+'/'); shutil.copyfile(fn, destination_dir)
            proceed = 'no'
            while proceed == 'no':
                try: os.remove(fn); proceed = 'yes'
                except OSError:
                        print 'Tried to move the file',mappfinder_input,'to an archived folder, but it is currently open.'
                        print 'Please close this file and hit return or quit GO-Elite'
                        inp = sys.stdin.readline()
        
def importMAPPFinderResults(filename,p):
    zscore_changed_path_db = {}; go_full = {}; zscore_goid = {}
    global dummy_db; dummy_db = {}; global file_headers; file_headers = []
    global full_go_name_db; full_go_name_db = {}; filtered_mapps=[]; filtered_mapp_list = []
    run_mod = ''; run_source=''; gene_identifier_file = ''
    fn=filepath(filename); x=0; y=0
    print "Importing MAPPFinder data for:",filename
    for line in open(fn,'rU').readlines():
        data = cleanUpLine(line)
        if x == 0 and y == 0:
            if 'GOID' in data: x = 1; go_titles = data + '\n'; input_file_type = 'GO'
            elif 'MAPP Name' in data: y = 1; mapp_titles = data + '\n'; input_file_type = 'local'
            elif data in species_list: ###Check species from results file
                if data in species_list: species = data; species_code = species_codes[species]
            elif 'source identifiers supplied' in data:
                space_delimited = string.split(data,' ')
                gene_input_file_data = string.split(data,'supplied in the input file:')
                putative_source_type = space_delimited[1]  ###Extract out uid from GO/MAPP results file
                if putative_source_type in source_types: run_source = putative_source_type
                elif putative_source_type in mod_types: run_source = putative_source_type
                if len(gene_input_file_data)>1: gene_identifier_file = gene_input_file_data[-1]
            elif 'source identifiers linked to' in data:
                space_delimited = string.split(data,' ')
                putative_mod_type = space_delimited[-2] ###Extract out MOD from GO/MAPP results file
                if putative_mod_type in mod_types: run_mod = putative_mod_type
            elif 'Probes linked to a ' in data:
                space_delimited = string.split(data,' ')
                putative_mod_type = space_delimited[-2] ###Extract out MOD from GO/MAPP results file (derived from GenMAPP MAPPFinder results)
                if putative_mod_type in mod_types: run_mod = putative_mod_type
                else: print "WARNING!!! The MOD",putative_mod_type,"used by GenMAPP MAPPFinder, is not available in GO-Elite. Errors may result in deriving propper gene associations since a different MOD has to be used."
        elif x == 1:
            try: goid, go_name, go_type, local_changed, local_measured, local_go, percent_changed_local, percent_present_local, number_changed, number_measured, number_go, percent_changed, percent_present, z_score, permutep, adjustedp = string.split(data,'\t')
            except ValueError:
                t = string.split(data,'\t')
                print 'WARNING...GeneOntology results file has too few or too many columns'
                print 'Expected 16 columns, but read', len(t); print t; print 'Please correct and re-run (hit return to exit)'
                inp = sys.stdin.readline(); sys.exit()                
            if goid != 'GO': ###Exclude the top level of the GO tree (not an ID number MAPPFinder output)
                #go_id_full = 'GO:'+goid
                ###Check to make sure that the GOID is the right length (usually proceeded by zeros which can be removed)
                if len(goid)<7:
                    extended_goid=goid
                    while len(extended_goid)< 7: extended_goid = '0'+extended_goid
                    if extended_goid in full_path_db: goid = extended_goid
                if goid in full_path_db:
                    for path in full_path_db[goid]:
                        #goid = int(goid); path = path_data.PathTuple() ###Path is a tuple of integers
                        z_score = float(z_score); permutep = float(permutep); number_changed = int(number_changed)
                        #path_list = string.split(path,'.')#; path_list=[]; full_tree_index[path] = path_list ###used to created nested gene associations to GO-elite terms
                        full_go_name_db[goid] = go_name
                        if z_score > z_threshold and number_changed > change_threshold and permutep < p_val_threshold:
                            #tree_index[path] = path_list; zscore_changed_path_db[path] = z_score, number_changed; zscore_goid[goid] = z_score, go_type; go_terms[path] = goid
                            z = MAPPFinderResults(goid,go_name,z_score,number_changed,input_file_type,permutep,data)
                            zscore_changed_path_db[path] = z; zscore_goid[goid] = z
                            go_full[goid]= data + '\n'
            ### for all entries, save a dummy entry.  This is needed if we want to re-examine in MAPPFinder but don't want to visualize non-GO-elite selected terms
            dummy_data = str(goid) +"\t"+ go_name +"\t"+ go_type
            dummy_data = dummy_data+ "\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"0"+"\t"+"1"+"\t"+"1" +"\n"
            dummy_db[goid] = dummy_data    
        elif y == 1:
                ###CODE USED TO PROCESS LOCAL RESULT FILES FOR MAPPS
                try:
                    try:
                        t = string.split(data,'\t')
                        mapp_name = t[0]; num_changed = t[1]; zscore = t[6]; permutep = t[7]
                        num_changed = int(num_changed); zscore = float(zscore); permutep = float(permutep)
                        #print [mapp_name, num_changed, zscore,permutep]
                        if zscore > z_threshold and num_changed > change_threshold and permutep < p_val_threshold:
                            filtered_mapps.append((zscore, line, mapp_name))
                            #zscore_goid[mapp_name] = zscore, 'MAPP'
                            z = MAPPFinderResults(mapp_name,mapp_name,zscore,num_changed,input_file_type,permutep,data)
                            zscore_goid[mapp_name] = z
                            filtered_mapp_list.append(mapp_name) ###Use this for downstream analyses
                    except ValueError: continue
                except IndexError: continue
    if x == 1:
        try: species_code = species_code ###Test to see if there is a proper species name found
        except UnboundLocalError:
            #print 'No proper species name found in GO results. Please change and re-run'
            #inp = sys.stdin.readline(); sys.exit()
            ###Pick arbitrary species code
            for species in species_codes: species_code = species_codes[species]
        return run_mod,run_source,zscore_changed_path_db, go_full, go_titles, zscore_goid, input_file_type, gene_identifier_file, species_code
    if y == 1:
        filtered_mapps.sort(); filtered_mapps.reverse()
        return run_mod,run_source,filtered_mapp_list, filtered_mapps, mapp_titles, zscore_goid, input_file_type, gene_identifier_file, species_code

def buildFullTreePath(go_index_list):
    """Find common parent nodes among children and parents and collapse into a single branch node"""
    global path_dictionary; path_dictionary = {}; path_dictionary_filtered={}
    for tuple_index in go_index_list: path_dictionary[tuple_index] = [] ###store the original top level path-indeces
    for tuple_index in go_index_list:
        path_len = len(tuple_index); i=-1
        while path_len+i > 0:
            parent_path = tuple_index[:i]
            if parent_path in path_dictionary:
                path_dictionary[parent_path].append(tuple_index) ###if a parent path of a
            i-=1
    for parent_path in path_dictionary:
        if len(path_dictionary[parent_path])>0:
            path_dictionary[parent_path].sort()
            path_dictionary_filtered[parent_path] = path_dictionary[parent_path]
    return path_dictionary_filtered

def comparison(zscore_changed_path_db):
    """This function performs the following operations:
    1) builds a database of all possible parent and child paths (e.g. nodes) from the list of filtered GO terms.
    2) identifies the most top-level nodes from 1 that will contain all nodes (where there's at one parent and one child).
    3) identifies all node without a parent or child in zscore_changed_path_db (paths filtered upon import)"""
    
    ###For each node in the hierarchy that has a child, list the parent as the key and any and all children (not specific branches) as the values.
    ###Each parent could be a child if it has a parent in another dictionary.  Nodes not in path_dictionary have no parents or children regulated.
    path_dictionary = buildFullTreePath(zscore_changed_path_db)  

    """Find the top parent nodes generated in the above path_dict query that are not in other dictionary
    items.  These represent unique top level parent nodes... the rest are redundant, since they will be children of the top parent"""
    path_dictionary_exclude = {}
    for redundant_path in path_dictionary:
        for unique_path in path_dictionary:
            if redundant_path!= unique_path:  ###If not looking at the same path
                if redundant_path in path_dictionary[unique_path]:
                    ####Therefore unique_path is the actual parent of redundant_path, example: redundant_path_index = (0, 3, 24, 30, 8) and unique_path_index = (0, 3)
                    redundant_children_paths = path_dictionary[redundant_path]
                    ###Store these so we can remove both the parent and child paths
                    path_dictionary_exclude[redundant_path] = redundant_children_paths  ###Probably don't need to store the redundant_children_paths (see next paragraph)
                    
    path_dictionary_include = {}; path_dictionary_include2 = {}
    ###Find the top level parents by removeing the rendundant entries from the path_dictionary (excluding these entries from a new dictionary)
    for parent_path in path_dictionary:
        if parent_path not in path_dictionary_exclude:
            unique_children_paths = path_dictionary[parent_path]
            path_dictionary_include['node',parent_path] = unique_children_paths ### The node annotations is used later on to destinguish this as a known top level parent node
            path_dictionary_include2[parent_path] = unique_children_paths  

    ###Now, identify those top-level nodes not found in the above paragraph, since they are regulated but have no other significant parents or
    ###child terms in the zscore_changed_path_db dictionary.  This will be likely listed in the output file.
    exclude = {}; zscore_changed_path_db_unique = {}
    for filtered_path in zscore_changed_path_db:
        for parent_path in path_dictionary_include2:
            if filtered_path in path_dictionary_include2[parent_path]: exclude[filtered_path]=filtered_path   
        if filtered_path in path_dictionary_include2: exclude[filtered_path]=filtered_path         
    for filtered_path in zscore_changed_path_db:
        if filtered_path not in exclude: zscore_changed_path_db_unique['node',filtered_path] = ['null']

    print "\nRound 1: length of original GO index:",len(zscore_changed_path_db)
    print "Non-nested GO terms (unique):",len(zscore_changed_path_db_unique)
    print "Length of initial GO tree build:", len(path_dictionary)
    print "Length of GO tree node dictionary excluded:",len(path_dictionary_exclude)
    print "Length of GO tree node dictionary included:",len(path_dictionary_include),'\n'

    iteration_count = 0  ###The next function iteratures many of the queries in this function, until optimized tree-hierarchies of nodes are created.  This states that no interations have occured yet

    return path_dictionary_include,zscore_changed_path_db_unique,iteration_count, path_dictionary
    
def comparison_iterate(dbase, finished_node_dbase,iteration_count):
    """This function is almost identical to "comparison" but handles two input databases from comparison and
    allows for multiple items in the key field of the input and output tables. NOTE: comparison1 could be
    adapted to serve this purpose"""

    #This code finds common parent nodes among children and parents and collapses into a single branch
    #node (i.e. dictionary).    
    path_dictionary = {}
    path_dictionary_temp = {}
    for item in dbase:
        for entry in dbase[item]:   #unlike the 1st comparison function, query the dictionary definitions
            path_char = len(entry)  #get the number of characters to compare
            if item in dbase:        #compare back to the same entry
                for entry2 in dbase[item]:
                    if entry != entry2:  #looking for children of entry
                        base_path = entry2[0:path_char]
                        #print entry, "base_path:",base_path
                        if base_path == entry:
                            new_node_set=[]
                            for object in item:      #grab all individual key entries 
                                new_node_set.append(object)  #append the next child node in the new key
                            new_node_set.append(entry)
                            new_node_set = tuple(new_node_set)   #need to convert the list to a tuple to be a dictionary key
                            path_dictionary_temp[item[-1]] = ['temp']
                            try: path_dictionary[new_node_set].append(entry2) #append child of entry
                            except KeyError: path_dictionary[new_node_set] = [entry2]

    #need to only look at the last entry of dbase to find last unique entry
    dbase_temp = {}
    for entry in dbase:
        dbase_temp[entry[-1]] = ['temp']

    #find those nodes that had no more children this run and append to unique entries from last run
    dbase_unique = {}
    for entry in dbase:
        entry2 = entry[-1]   #we added the word "node" into the key at [0]
        #this adds entries that had no more nesting(one entry) to the finished dictionary as well
        #as parent entries added to path_dictionary
        if entry2 not in path_dictionary_temp:
            if len(dbase[entry]) == 1: #this should have no sibling entries below the parent
                finished_node_dbase[entry] = dbase[entry]
            else:  #therefore, there are siblings that need to be separated into distinct entries
                for node in dbase[entry]:
                    entry3 = entry[1:]
                    temp_list=[]
                    temp_list.append('sibling1')
                    for item in entry3:
                        temp_list.append(item)
                    temp_list.append(node)
                    new_key = tuple(temp_list)
                    finished_node_dbase[new_key] = ['siblings_type1']   #type1 specifies that the parent is unique, therefore no nested siblings of the parent term, just loners=> can analyze in calculate_score_for_children functio
                    entry3 = entry[1:]
                    temp_list=[]
                    temp_list.append('sibling2')
                    for item in entry3:
                        temp_list.append(item)
                    new_key = tuple(temp_list)
                    finished_node_dbase[new_key] = dbase[entry] #also add an entry which contains both children under the true parent: note: instead of node, sibling is listed now to distinguish
                    
    #find parent nodes generated in the above path_dict query that are not in other dictionary
    #items of path_dict.  These represent unique top level parent nodes... the rest are redundant
    path_dictionary_exclude = {}
    for item in path_dictionary:
        #item_parent = item[-2]
        item_child = item[-1]
        for item2 in path_dictionary:
            #item2_parent = item2[-2]
            #item2_child = item2[-1]
            if item != item2:
                if item_child in path_dictionary[item2]: #if the parent node is in the list of another entry, then it was prematurely added and should be excluded
                    parent_path = path_dictionary[item] #not really necessary to add the parent_path?
                    path_dictionary_exclude[item] = parent_path #exclude the parent item, prematurely added
    path_dictionary_include = {}
    for item in path_dictionary:
        if item not in path_dictionary_exclude:
            parent_path = path_dictionary[item]
            path_dictionary_include[item] = parent_path
    
    #print "\nRound 2: length of original GO tree:",len(dbase)
    #print "length of initial GO tree node dictionary:", len(path_dictionary)
    #print "number of non-nested GO terms:",len(finished_node_dbase)
    #print "length of GO tree node dictionary excluded:",len(path_dictionary_exclude)
    #print "length of GO tree node dictionary included:",len(path_dictionary_include)
            
    ### added this code to account for nodes that have a parent, no children, but other siblings with and without children
    ### previously these would have been eroneously eliminated
    nodes_examined = []
    for entry in path_dictionary_include:
        for node in path_dictionary_include[entry]:
             nodes_examined.append(node)
        for node in entry:
             if node != 'node':
                nodes_examined.append(node)
    for entry in finished_node_dbase:
        for node in finished_node_dbase[entry]:
             nodes_examined.append(node)
        for node in entry:
             if node != 'node':
                nodes_examined.append(node)
    nodes_examined = unique.unique(nodes_examined)
    for nodes in dbase: ###This is where it switched
        for entry in dbase[nodes]:
            if entry not in nodes_examined:
                temp_list = []
                for item in nodes:
                    temp_list.append(item)
                temp_list.append(entry)
                new_key = tuple(temp_list)
                #print "new_key",new_key
                finished_node_dbase[new_key] = ['null']
                
    if len(path_dictionary_include) > 0:
        iteration_count +=1
        """print "path_dictionary_include"
        for entry in path_dictionary_include:
            print entry, ":", path_dictionary_include[entry]
        print "finished_node_dbase"
        for entry in finished_node_dbase:
            print entry, ":", finished_node_dbase[entry] """
        return comparison_iterate(path_dictionary_include, finished_node_dbase, iteration_count)
    else:
        """for i in finished_node_dbase:
            if (0, 3, 24) in i: print i, finished_node_dbase[i]
        kill"""
        print "Total number of iterations:",iteration_count
        return chew_back(finished_node_dbase)

def chew_back(dbase):
    """This function takes a database of go nodes (e.g. [0.1.2,0.1.2.1,0.1.2.1.2]:[0.1.2.1.2.23]) and checks
    to see if the second to last parent node on is found in any other dictionary keys.  If it is, then there
    is no need to transfer parent nodes to the children list, if no, then we must chew back."""
    iterate_dbase ={}; finished_dbase = {}
    #for entry in dbase: print entry,":", dbase[entry]
    
    original_dbase = copy.deepcopy(dbase)
    print "Lenth of input chewback database:",len(dbase)
    loop = 0;count = 0
    while len(dbase) > 0:
        loop +=1
        for entry in dbase:
            count = 0; parent_node = entry[-2]; unique_branch_node = entry[-1]
            if len(entry)<3: #if there is only one parent node
                finished_dbase[entry] = dbase[entry]; count = 1
            else:
                for entry2 in original_dbase:
                    if entry != entry2 and unique_branch_node not in entry2:
                        if parent_node in entry2: finished_dbase[entry] = dbase[entry]; count = 1
            if count == 0:   #if the parent entry was not found in any of the dictionary keys, it's unique and thus, we need to chew back
                new_entry = entry[0:-1]
                children = dbase[entry]; last_entry = entry[-1]
                updated_children = [last_entry]+children
                iterate_dbase[new_entry] = updated_children
        dbase = iterate_dbase
        iterate_dbase = {}
        #print "number of chew-back iterations:",loop
        
    print "Length of output chewback database:",len(finished_dbase)
    """for entry in finished_dbase: print entry,':',finished_dbase[entry] """
    return finished_dbase             

def link_score_to_all_paths(all_paths,zscore_changed_path_db):
    temp_list = []
    parent_highest_score = {}
    for entry in all_paths:
        if entry in zscore_changed_path_db:
            if filter_method == 'z-score': new_entry = zscore_changed_path_db[entry].ZScore(), entry
            elif filter_method == 'gene number':new_entry = zscore_changed_path_db[entry].NumChanged(), entry
            elif filter_method == 'combination':
                z_score_val = zscore_changed_path_db[entry].ZScore()
                gene_num = math.log(zscore_changed_path_db[entry].NumChanged(),2)
                new_entry = gene_num*z_score_val, entry
            else:
                print 'error, no filter method selected!!'
                break
        for item in all_paths[entry]:
            if filter_method == 'z-score':
                new_item = zscore_changed_path_db[item].ZScore(), item
                temp_list.append(new_item)
            elif filter_method == 'gene number':
                new_item = zscore_changed_path_db[item].NumChanged(), item
                temp_list.append(new_item)
            elif filter_method == 'combination':
                z_score_val = zscore_changed_path_db[item].ZScore()
                gene_num = math.log(zscore_changed_path_db[item].NumChanged(),2)
                new_item = gene_num*z_score_val, item
                temp_list.append(new_item)
            else:
                print 'error, no filter method selected!!'
                break
        max_child_score = max(temp_list)[0]
        max_child_path = max(temp_list)[1]
        parent_score = new_entry[0]
        parent_path = new_entry[1]
        if max_child_score <= parent_score:
            parent_highest_score[parent_path] = all_paths[entry]   #include the elite parent and it's children
            #print "parent_path",parent_path,parent_score,"max_child_path", max_child_path, max_child_score
        temp_list=[]
    """for entry in parent_highest_score:
        print entry,':',parent_highest_score[entry]"""
    print "Number of parents > child in tree:",len(parent_highest_score)
    return parent_highest_score

def calculate_score_for_children(tree, zscore_changed_path_db):
    temp_list = []
    child_highest_score = {}
    for entry in tree:  ###The tree is a key with one or more nodes representing a branch, chewed back, with or without children
        for item in tree[entry]: #get the children of the tree
            if item in zscore_changed_path_db:
                if filter_method == 'z-score':
                    new_item = zscore_changed_path_db[item].ZScore(), item
                    temp_list.append(new_item)
                elif filter_method == 'gene number':
                    new_item = zscore_changed_path_db[item].NumChanged(), item
                    temp_list.append(new_item)
                elif filter_method == 'combination':
                    z_score_val = zscore_changed_path_db[item].ZScore()
                    gene_num = math.log(zscore_changed_path_db[item].NumChanged(),2)
                    new_item = gene_num*z_score_val, item
                    temp_list.append(new_item)
                    #print new_item,z_score_val
                else:
                    print 'error, no filter method selected!!'
                    break
            """elif item == 'siblings_type1': #this will only occur if an parent had multiple children, none of them with nested children
                 if filter_method == 'z-score':
                    parent_item = entry[-1]
                    new_item = zscore_changed_path_db[parent_item][0], parent_item
                    temp_list.append(new_item)
                elif filter_method == 'gene number':
                    new_item = zscore_changed_path_db[parent_item][1], parent_item
                    temp_list.append(new_item)
                elif filter_method == 'combination':
                    z_score_val = zscore_changed_path_db[parent_item][0]
                    gene_num = math.log(zscore_changed_path_db[parent_item][1],2)
                    new_item = gene_num*z_score_val, parent_item
                    temp_list.append(new_item)"""
        #print new_item,z_score_val               
        if len(temp_list)> 0:  #if there is at least one nested go_path
            max_child_score = max(temp_list)[0]
            max_child_path = max(temp_list)[1]
            child_highest_score[entry]=max_child_path
            temp_list = []
    """for entry in child_highest_score:
        print entry,':',child_highest_score[entry]"""
    return child_highest_score

def collapse_tree(parent_highest_score,child_highest_score,tree):
    collapsed = {}
    collapsed_parents = {}
    collapsed_children = {}
    for entry in tree:
        count=0 #see next comment
        for item in entry:
            if item in parent_highest_score and count==0: #ensures that only the top parent is added
                collapsed_parents[entry] = item, parent_highest_score[item] #include the children of the elite parent to exclude later
                ### aka elite_parents[parent_paths] = elite_parent_path, non_elite_children
                count +=1  #see last comment
                break
    for entry in tree:
        if entry not in collapsed_parents:  ###if the highest score does not occur for one of the parent terms in the key of the tree-branch
            if entry in child_highest_score: ###Entry is the parent path stored and the value is the child path with the max score (greater than parent)
                max_child_path = child_highest_score[entry]
                collapsed_children[entry] = max_child_path, tree[entry] #include the children to exclude from other entries later (includes the max_child_path though)
    """for entry in collapsed_children:
        print entry,':',collapsed_children[entry]"""
    temp_list = []
    for entry in collapsed_parents:
        node = collapsed_parents[entry][0]
        children = collapsed_parents[entry][1]
        collapsed['parent',node] = children
        """if entry in tree:
            for item in entry:
                temp_list.append(item)
            for item in tree[entry]:
                temp_list.append(item)
            temp_list.sort()
            temp_list2 = unique.unique(temp_list)
            collapsed['parent',node] = temp_list2
            temp_list=[]"""
    for entry in collapsed_children:
        #previously we included the blocked out code which also
        #would later exclude entries that contain key entry items in the elite entries
        node = collapsed_children[entry][0]
        siblings = collapsed_children[entry][1]
        collapsed['child',node] = siblings
        """if entry in tree:
            for item in entry: temp_list.append(item)
            for item in tree[entry]: temp_list.append(item)
            temp_list.sort()
            temp_list2 = unique.unique(temp_list)
            collapsed['child',node] = temp_list2
            temp_list=[]"""
    for entry in tree:
        nested_path = tree[entry]
        if nested_path == ['null']:
            collapsed['unique',entry[-1]] = 'unique',entry[-1]
    print "The length of the collapsed list is:",len(collapsed)
    ###This code is used to check whether or not a tree with multiple siblings listed as children was added to collapsed because the parent term(s) had a higher score
    ###if not, we must individually add the child sibling terms
    for entry in tree:
        if tree[entry] == ['siblings_type1']:
            parent_root_node = 'parent',entry[-2]
            if parent_root_node not in collapsed:
                #use 'child' since we have to over-write the child entry created if the 'parent' was'nt more significant (thus syblings would eroneously be considered children)
                 collapsed['child',entry[-1]] = 'node',entry[-1] 
    #for entry in collapsed:
        #print entry,collapsed[entry]
    return collapsed


def link_goid(tree,zscore_changed_path_db,all_paths):
    """Starting with the collapsed tree, convert to goids"""
    temp1=[]; temp2=[]; new_tree = {}; count = 0

    for entry in tree:
        for top_node in entry:
            try: goid = zscore_changed_path_db[top_node].GOID(); temp1.append(goid)
            except KeyError: not_valid_goid = ''
        for child_node in tree[entry]:
            try: goid= zscore_changed_path_db[child_node].GOID(); temp2.append(goid)
            except KeyError: not_valid_goid = ''
        temp2.sort()
        new_top = tuple(temp1) #one elite go-term in goid form
        new_child = temp2   #list of children (value items) in dictionary, in goid form
        count +=1; temp_list=[]
        if new_top in new_tree:  #if the new dictionary, new_tree already has this key added
            top_node = new_top[0]
            for node in new_tree[new_top]:
                if node != top_node: temp_list.append(node)
            for node in new_child:
                if node != top_node: temp_list.append(node)
            temp_list.sort(); new_list = unique.unique(temp_list)
            new_tree[new_top] = new_list; temp_list=[]
        else:
            #print new_child[0], new_top[0]
            if new_child[0] == new_top[0]:  #if the first dictionary (which should be the only) value equals the key
                new_tree[new_top] = []
            else:  ###note: if two parents, both more significant than all children, share a child, both will still be reported. This is fine, since we don't know the parent's relationship
                temp_list2 = []
                for node in new_child:  
                    if node != new_top[0]: #get rid of values that equal the key!!!
                        temp_list2.append(node)
                temp_list2.sort()
                new_child = temp_list2
                new_tree[new_top] = new_child #this should be the unique term
                #print new_top, new_child
        temp1 = []; temp2 = []
    ###remove entries that are parents or children of the most 'elite' entry, in dictionary
    ###note: parents of the entry shouldn't be selected, since collapsing chooses
    ###selects the highest scoring node.
    if exclude_related == 'yes':
        exclude = {}
        for entry in new_tree:
            for item in new_tree[entry]:
                exclude[item] = ''
                #print item, entry, new_tree[entry]
        new_tree2={}               
        for entry in new_tree:
            if entry[0] not in exclude:
                new_tree2[entry[0]] = ''
        new_tree = new_tree2
    
    #for entry in tree: print entry,':',tree[entry]
    #for entry in new_tree: print entry,':',new_tree[entry] 
    
    print "Length of input tree/output tree",len(tree),'/',len(new_tree),'count:',count
    return new_tree
                            
def exportGOResults(go_full,go_titles,collapsed_go_tree,zscore_goid,go_gene_annotation_db,go_values_db,value_headers,goids_with_redundant_genes):
    if include_headers_in_output == 'yes': output_results = "output//"+mappfinder_input[0:-4]+ "_" +filter_method +"_elite-headers.txt"
    else: output_results = "output//"+mappfinder_input[0:-4]+ "_" +filter_method +"_elite.txt"
    entries_added=[]  #keep track of added GOIDs - later, add back un-added for MAPPFinder examination
    fn=filepath(output_results)
    
    try: raw = open(fn,'w')
    except IOError:
        try:
            reference_dir = refDir()
            os.mkdir(reference_dir+'//output') ###Re-Create directory if deleted
            os.mkdir(reference_dir+'//output//gene_associations') ###Re-Create directory if deleted
            raw = open(fn,'w')
        except OSError:
            ###Occurs if a result file is open (IOError)
            print 'Results file:',output_results, 'is open...can not re-write. Please close file and re-run'
            inp = sys.stdin.readline(); sys.exit()
            
    if include_headers_in_output == 'yes':
        for row in file_headers:
            row = row + '\n'
            raw.write(row)
    if export_user_summary_values == 'yes' and len(value_headers)>0: value_headers = '\t'+string.join(value_headers,'\t')
    else: value_headers = ''
    if len(go_gene_annotation_db)>0: go_titles = go_titles[:-1]+'\t'+'redundant with terms'+'\t'+'inverse redundant'+'\t'+'gene symbols'+value_headers+'\n' ###If re-outputing results after building gene_associations
    raw.write(go_titles)
    combined_results[output_results] = [('go',mappfinder_input,go_titles)]
    
    #append data to a list, to sort it by go_cateogry and z-score
    collapsed_go_list = []; collapsed_go_list2 = []
    for entry in collapsed_go_tree:
        goid = entry
        entries_added.append(goid)
        if goid in zscore_goid:
            z_score_val = zscore_goid[goid].ZScore()
            go_type = zscore_goid[goid].OntoType()
            if sort_only_by_zscore == 'yes': info = z_score_val,z_score_val,goid
            else: info = go_type,z_score_val,goid
            collapsed_go_list.append(info); collapsed_go_list2.append(goid)
    collapsed_go_list.sort()
    collapsed_go_list.reverse()
    for (go_type,z_score_val,goid) in collapsed_go_list:
        if goid in go_full:
            data = go_full[goid]
            if len(go_gene_annotation_db)>0:
                symbol_ls = []; goid_vals = ''
                if goid in go_gene_annotation_db:
                    for s in go_gene_annotation_db[goid]: symbol_ls.append(s.Symbol()) #s.GeneID()
                if goid in go_values_db: goid_vals = string.join(go_values_db[goid],'\t') ###mean of values from input file, for each GO term
                symbol_ls = unique.unique(symbol_ls); symbol_ls.sort()
                symbols = string.join(symbol_ls,'|')
                try:
                    rr = goids_with_redundant_genes[goid]
                    redundant_with_terms = rr.RedundantNames(); inverse_redundant = rr.InverseNames()
                except KeyError: redundant_with_terms = ' '; inverse_redundant =' '
                if export_user_summary_values == 'yes' and len(goid_vals)>0: goid_vals = '\t'+goid_vals
                else: goid_vals = ''
                data = data[:-1]+'\t'+redundant_with_terms+'\t'+inverse_redundant+'\t'+symbols+goid_vals+'\n'  ###If re-outputing results after building gene_associations
            raw.write(data)
            combined_results[output_results].append(('go',mappfinder_input,data))
    if include_headers_in_output == 'yes':
        for goid in dummy_db:
            if goid not in entries_added:
                dummy_data = dummy_db[goid]
                raw.write(dummy_data)
    raw.close()

    combined_results[output_results].append(('','',''))
    print "New MAPPFinder Elite file", output_results, "written...."
    return collapsed_go_list2

def exportLocalResults(filtered_mapps,mapp_titles,mapp_gene_annotation_db,mapp_values_db,mapp_value_headers,mapps_with_redundant_genes):
    output_results = "output//"+mappfinder_input[0:-4]+ "_" +filter_method +"_elite.txt"
    fn=filepath(output_results)
    raw = open(fn,'w')

    if export_user_summary_values == 'yes' and len(mapp_value_headers)>0: mapp_value_headers = '\t'+string.join(mapp_value_headers,'\t')
    else: mapp_value_headers = ''
    if len(mapp_gene_annotation_db)>0: mapp_titles = mapp_titles[:-1]+'\t'+'redundant with terms'+'\t'+'inverse redundant'+'\t'+'gene symbols'+mapp_value_headers+'\n'
    raw.write(mapp_titles)
    combined_results[output_results] = [('mapp',mappfinder_input,mapp_titles)]

    filtered_mapp_list = []
    for (zscore,line,mapp_name) in filtered_mapps:
        if len(mapp_gene_annotation_db)>0:
            symbol_ls = []; mapp_vals=''
            if mapp_name in mapp_gene_annotation_db:
                for s in mapp_gene_annotation_db[mapp_name]: symbol_ls.append(s.Symbol()) #s.GeneID()
                symbol_ls = unique.unique(symbol_ls); symbol_ls.sort()
                symbols = string.join(symbol_ls,'|')
                if mapp_name in mapp_values_db: mapp_vals = string.join(mapp_values_db[mapp_name],'\t') ###mean of values from input file, for each MAPP
                try:
                    rr = mapps_with_redundant_genes[mapp_name]
                    redundant_with_terms = rr.RedundantNames(); inverse_redundant = rr.InverseNames()
                except KeyError: redundant_with_terms = ' '; inverse_redundant = ' '

                if export_user_summary_values == 'yes' and len(mapp_vals)>0: mapp_vals = '\t'+mapp_vals
                else: mapp_vals = ''                
                line = line[:-1]+'\t'+redundant_with_terms+'\t'+inverse_redundant+'\t'+symbols+mapp_vals+'\n'  ###If re-outputing results after building gene_associations
        raw.write(line)
        combined_results[output_results].append(('mapp',mappfinder_input,line))
    raw.close()
    combined_results[output_results].append(('','',''))
    print "Local Filtered MAPPFinder file", output_results, "written...."

def output_combined_results(combined_results):
    output_results = "output//"+"combined-results_" +filter_method +"_elite.txt"
    fn=filepath(output_results)
    try: raw = open(fn,'w'); filename_list = []
    except IOError:
        ###Occurs if a result file is open (IOError)
        print 'Results file:',output_results, 'is open...can not re-write. Please close file and re-run'
        inp = sys.stdin.readline(); sys.exit()
    for filename in combined_results: filename_list.append(filename)
    filename_list.sort() ### rank the results alphebetically so local and GO results are seqeuntial
    for filename in filename_list:
        for (pathway_type,mapp_name,line) in combined_results[filename]:
            data = cleanUpLine(line)
            t = string.split(data,'\t')
            if pathway_type == 'go':
                del t[0]; del t[2:7]
                go_type = t[1]; del t[1]; specific_pathway_type = go_type
            elif pathway_type == 'mapp': specific_pathway_type = 'local'
            else: specific_pathway_type = ''
            t.reverse(); t.append(specific_pathway_type); t.append(mapp_name); t.reverse()
            vals = string.join(t,'\t'); vals = vals + '\n'
            raw.write(vals)
    raw.close()

def identifyGeneFiles(import_dir, mappfinder_input):
    e = GrabFiles(); e.setdirectory(import_dir)
    if len(gene_identifier_file)>0: ###global variable set upon MAPPFinder result import
        gene_file_dir = e.searchdirectory(gene_identifier_file)
        return gene_file_dir
    else:
        mappfinder_input = string.replace(mappfinder_input,'-GO.txt','') ###the prefix in most cases will be the same for the MAPPFinder results and gene input filename
        mappfinder_input = string.replace(mappfinder_input,'-local.txt','')
        split_name = string.split(mappfinder_input,'.')
        
        gene_file_dir = e.searchdirectory(mappfinder_input)
        if len(gene_file_dir)>0: return gene_file_dir
        else:
            try:
                index = int(split_name[0])
                index_str = str(index)
                gene_file_dir = e.searchdirectory_start(index_str)
            except ValueError: gene_file_dir =''
            return gene_file_dir

def grabAllNestedGOIDs(collapsed_go_tree,all_paths):
    ###Updated all_paths contains all possible paths listed in GO
    nested_collapsed_go_tree={}
    for goid_tuple in collapsed_go_tree:
        for goid in goid_tuple:
            child_goids=[]
            path_id_list = full_path_db[goid]
            for path_id in path_id_list: ###examine all possible paths for that goid (and different possible children)
                #path_id = path_id_data.PathTuple()
                if path_id in all_paths:
                    child_path_id_list = all_paths[path_id]
                    for child_path_id in child_path_id_list:
                        child_goid = path_id_to_goid[child_path_id]
                        child_goids.append(child_goid) ###append all possible children terms to a new list
        nested_collapsed_go_tree[goid] = child_goids
    nested_collapsed_go_tree = eliminate_redundant_dict_values(nested_collapsed_go_tree)
    return nested_collapsed_go_tree

def speciesData():
    filename = 'Databases//species.txt'
    fn=filepath(filename); global species_list; species_list=[]; global species_codes; species_codes={}
    for line in open(fn,'rU').readlines():             
        data = cleanUpLine(line)
        abrev,species = string.split(data,'\t')
        species_list.append(species)
        species_codes[species] = abrev
        
def sourceData():
    filename = 'Databases//source_data.txt'
    fn=filepath(filename)
    global source_types; source_types=[]
    global system_codes; system_codes={}
    global mod_types; mod_types=[]
    for line in open(fn,'rU').readlines():             
        data = cleanUpLine(line)
        t = string.split(data,'\t'); source=t[0]
        try: system_code=t[1]
        except IndexError: system_code = 'NuLL'
        if len(t)>2: ### Therefore, this ID system is a potential MOD
            if t[2] == 'MOD': mod_types.append(source)
        if source not in mod_types: source_types.append(source) 
        system_codes[system_code] = source ###Used when users include system code data in their input file
        
def getParametersFromUsers():
  global include_headers_in_output; global filter_method; global z_threshold; global p_val_threshold
  global sort_only_by_zscore; global run_mappfinder
  include_headers_in_output=''; filter_method=''; z_threshold=''; p_val_threshold=''
  speciesData(); sourceData(); source_data = ''; mod = mod_types[0]
  
  a = 'z-score'; b = 'gene number'; c = 'combination'; 
  sort_only_by_zscore = 'yes'

  print "GO-Elite version Beta 1.18, compiled 6-25-08\n"

  proceed = 'no'
  while proceed == 'no':
      print "Choose Analysis Parameters"
      print "1) Run GO-Elite on pre-run MAPPFinder results"
      print "2) Calculate MAPPFinder results from input gene lists prior to GO-Elite"
      print "3) Build new gene-association files (must replace existing files when finished)"
      inp = sys.stdin.readline(); inp = inp.strip()
      if inp == "1": run_mappfinder = 'no'; proceed = 'yes'
      elif inp == "2": run_mappfinder = 'yes'; proceed = 'yes'
      elif inp == "3": BuildAffymetrixEntrezAssociations.runBuildAffymetrixAssociations(); sys.exit()
      else:
          print "Sorry... that command is not an option\n"

  x=1;print "Choose Primary Relational Gene Database (aka the MOD)"
  for mod_name in mod_types: print str(x)+')',mod_name; x+=1
  if run_mappfinder == 'no': print str(x)+') NA'
  inp = sys.stdin.readline(); inp = int(inp.strip())
  try: mod = mod_types[inp-1]
  except IndexError: mod = mod_types[0] ###Occurs when running GO-Elite with option 1

  if run_mappfinder == 'no':

      """###Currently not used, but may be unmasked if problems with recongnizing IDs occur
      define_ids = 'yes'
      print "Gene Identifier Identification"
      print "note: gene identifiers can be supplied in the input file (second column), predicted by program or specified below"
      print "1) Automatically determine or supplied in input file"
      print "2) Manually specify one system for all files"
      inp = sys.stdin.readline(); inp = inp.strip()
      if inp == "1": define_ids = 'no'
      elif inp == "2": define_ids = 'yes'

      if define_ids == 'yes':       
          x = 1; print "Select Type of Gene Input Data"
          for source_type in source_types: print str(x)+')',source_type; x+=1
          print str(x)+') '+mod; x+=1
          print str(x)+') Not Applicable'
          inp = sys.stdin.readline(); inp = int(inp.strip())
          if inp == (x-1): source_data = mod
          elif inp == x: source_data = ''
          else:source_data = source_types[inp-1]
  """
  proceed = 'no'
  while proceed == 'no':          
      print "Choose GO-Elite Filtering Parameters"
      print "1) default\n2) custom"
      print "\nDEFAULT: \n  Statistical Filtering Method = Z-score\n  Formatted for MAPPFinder = no\n  Z-score > 1.96\n  Permutation p-value < 0.05\n"
      inp = sys.stdin.readline(); inp = inp.strip()
      if inp == "1": analysis_type = 'default'; proceed = 'yes'
      elif inp == "2": analysis_type = 'custom'; proceed = 'yes'
      else:
          print "Sorry... that command is not an option\n"
          
  if analysis_type == 'default':
      filter_method=a
      z_threshold = 1.96
      p_val_threshold = 0.05
      include_headers_in_output='no'
  else:
      while z_threshold == '':
          print "Enter your desired Z-score cutoff (e.g. numeric value > 0)?"
          inp = sys.stdin.readline(); inp = inp.strip()
          try: z_threshold = float(inp)
          except ValueError: print "Requires a numeric value > 0"
      while p_val_threshold == '':          
          print "Enter your desired permuted p-value cutoff (numeric value between 0-1)?"
          print "(note: if no permuted p-value calculated, type in 1)"
          inp = sys.stdin.readline(); inp = inp.strip()
          try:
              p_val_threshold = float(inp)
              if p_val_threshold>1: p_val_threshold=''
          except ValueError: print "Requires a numeric value between 0-1" 
      while len(filter_method)<1:
          print "Which filtering statistic should GO-elite use(type number below)?"
          print "1) z-score\n2) gene number\n3) combination"
          inp = sys.stdin.readline(); inp = inp.strip()
          if inp == "1": filter_method = a
          elif inp == "2": filter_method = b
          elif inp == "3": filter_method = c
      while len(include_headers_in_output)<1:
          print "Format GO-elite results for further analysis in MAPPFinder?"
          print "1) no\n2) yes"
          inp = sys.stdin.readline(); inp = inp.strip()
          if inp == "2" or inp == "y" or inp == 'Y': include_headers_in_output = 'yes'
          elif inp == "1" or inp == "n" or inp == 'N': include_headers_in_output = 'no'
  return source_data,mod
      
def buildFullPathForCollapsedTree(collapsed_tree):
    filtered_tree_index=[]
    for (type,path) in collapsed_tree: filtered_tree_index.append(path)
    all_nested_paths = buildFullTreePath(filtered_tree_index)
    return all_nested_paths

def getNonEliteTerms(collapsed_goid_list,full_goid_db):
    non_elite_db=[]
    for goid in full_goid_db:
        if goid not in collapsed_goid_list: non_elite_db.append(goid)
        #non_elite_db.append(goid)
    print "Out of",len(collapsed_goid_list),"GO-Elite terms for",len(full_goid_db),"terms examined,",len(non_elite_db),"terms are not-elite"
    return non_elite_db

def runGOElite(source_data,mod):
  global change_threshold; global exclude_related; global mappfinder_input; global nested_collapsed_go_tree; global uid_to_go; global collapsed_go_tree
  global path_dictionary_include; global organized_tree; global parent_highest_score; global gene_identifier_file
  change_threshold = 2
  exclude_related = 'yes'
  nested_paths_stored = '' ###Variable used to determine if gene databases for GO analysis are imported for that species

  global combined_results; combined_results={}

  ###Import all paths to GO terms (note: must be updated with GenMAPP-databases)
  import_dir = '/paths'
  global combined_associations; combined_associations={}; global combined_gene_ranking; combined_gene_ranking={}
  p = GrabFiles(); p.setdirectory(import_dir)
  try:
      if run_mappfinder=='yes':
          x = 1; print "Select Species for MAPPFinder Analysis"
          for species_name in species_list: print str(x)+')',species_name; x+=1
          inp = sys.stdin.readline(); inp = int(inp.strip())
          species = species_list[inp-1]; species_code = species_codes[species]
          
          """ ###Currently not used, but may be unmasked if problems with recongnizing IDs occur
          define_ids = 'yes'; source_data = mod  ###default ID
          print "Gene Identifier Identification"
          print "note: gene identifiers can be supplied in the input file (second column), predicted by program or specified below"
          print "1) Automatically determine or supplied in input file"
          print "2) Manually specify one system for all files"
          inp = sys.stdin.readline(); inp = inp.strip()
          if inp == "1": define_ids = 'no'
          elif inp == "2": define_ids = 'yes'

          if define_ids == 'yes':
              x = 1; print "Select Type of Gene Input Data"
              for source_type in source_types: print str(x)+')',source_type; x+=1
              print str(x)+') '+ mod
              inp = sys.stdin.readline(); inp = int(inp.strip())
              if inp == x: source_data = mod
              else: source_data = source_types[inp-1]
          """
          
          permutations = -1
          while permutations < 0:
              print "Enter Number of permutations to run by mappfinder (default is 2000)"
              print "note: the number of permutations is the largest determinant of run-time"
              inp = sys.stdin.readline(); inp = int(inp.strip())
              permutations = inp
          global path_id_to_goid;  global full_path_db; global all_nested_paths; global collapsed_tree; export_databases = 'no'
          genmapp_mod = 'Ensembl' ### default
          full_path_db,path_id_to_goid,null = OBO_import.buildNestedGOAssociations(species_code,export_databases,mod_types,genmapp_mod)
          mappfinder.generateMAPPFinderScores(species,species_code,source_data,mod,system_codes,permutations)
          
      elif len(full_path_db)<1: import_path_index()
  except NameError: import_path_index()

  global export_user_summary_values
  get_non_elite_terms_only = 'no'
  export_user_summary_values = 'yes'
  user_defined_source = source_data
  user_defined_mod = mod      
  import_dir = '/input/MAPPFinder'
  m = GrabFiles(); m.setdirectory(import_dir)
  dir_list = read_directory(import_dir)  #send a sub_directory to a function to identify all files in a directory
  for mappfinder_input in dir_list:    #loop through each file in the directory to output results
    source_data = user_defined_source ###Resets the variables back to the user defined when missing from the MAPPFinder files
    mod = user_defined_mod
    mappfinder_input_dir = m.searchdirectory(mappfinder_input)
    run_mod,run_source,zscore_changed_path_db,go_full,go_titles,zscore_goid,input_file_type,gene_identifier_file,species_code = importMAPPFinderResults(mappfinder_input_dir,p)
    if len(run_mod)>1: mod = run_mod
    if len(run_source)>1: source_data = run_source
    if input_file_type == 'GO':
        path_dictionary_include,zscore_changed_path_db_unique,iteration_count,all_paths = comparison(zscore_changed_path_db)
        organized_tree = comparison_iterate(path_dictionary_include,zscore_changed_path_db_unique,iteration_count)
        parent_highest_score = link_score_to_all_paths(all_paths,zscore_changed_path_db)
        child_highest_score = calculate_score_for_children(organized_tree,zscore_changed_path_db)
        collapsed_tree = collapse_tree(parent_highest_score,child_highest_score,organized_tree)
        collapsed_go_tree = link_goid(collapsed_tree,zscore_changed_path_db,all_paths)
        collapsed_go_list = exportGOResults(go_full,go_titles,collapsed_go_tree,zscore_goid,{},{},{},{})

        ###For testing purposes, we can export all non-Elite terms
        if get_non_elite_terms_only == 'yes': collapsed_go_list = getNonEliteTerms(collapsed_go_list,go_full)

        ###Identify gene lists for the corresponding GO-elite results and generate nested associations
        gene_file_dir = identifyGeneFiles('//input/GenesToQuery/'+species_code, mappfinder_input)
        if len(gene_file_dir) > 0:  
            if species_code != nested_paths_stored:  ###Otherwise these relationships are built already
                nested_paths_stored = species_code
                uid_to_go, uid_to_mapp, uid_system, gene_annotations = gene_associations.grabNestedGeneAssociations(species_code,mod,source_data)
            else:
                print 'Apply Existing Gene Association Databases to GO Files'  ### Relevant species databases already exist (uid_to_go,gene_annotations,all_nested_paths)
            combined_associations,combined_gene_ranking,go_gene_annotation_db,go_values_db,value_headers,goids_with_redundant_genes = gene_associations.matchInputIDsToGOEliteTerms(gene_file_dir,system_codes,mappfinder_input_dir,collapsed_go_list,uid_to_go,gene_annotations,full_go_name_db,uid_system,combined_associations,combined_gene_ranking)

            unique_genes={}
            for goid in go_gene_annotation_db:
                for s in go_gene_annotation_db[goid]: unique_genes[s.GeneID()]=[]
            print len(unique_genes), "unique genes associated with GO-Elite terms"
            
            ###Re-output results, now with gene annotation data
            collapsed_go_list = exportGOResults(go_full,go_titles,collapsed_go_list,zscore_goid,go_gene_annotation_db,go_values_db,value_headers,goids_with_redundant_genes)
    else:
        filtered_mapp_list = zscore_changed_path_db
        exportLocalResults(go_full,go_titles,{},{},{},{})
        ###Identify gene lists for the corresponding GO-elite results and generate nested associations
        gene_file_dir = identifyGeneFiles('//input/GenesToQuery/'+species_code, mappfinder_input)
        if len(gene_file_dir) > 0:
            if species_code != nested_paths_stored:  ###Otherwise these relationships are built already
                nested_paths_stored = species_code
                uid_to_go, uid_to_mapp, uid_system, gene_annotations = gene_associations.grabNestedGeneAssociations(species_code,mod,source_data)
            else:
                print 'Apply Existing Gene Association Databases to Local Files' ### Relevant species databases already exist (uid_to_go,gene_annotations,all_nested_paths)
            combined_associations,combined_gene_ranking,mapp_gene_annotation_db,mapp_values_db,mapp_value_headers,mapps_with_redundant_genes = gene_associations.matchInputIDsToMAPPEliteTerms(gene_file_dir,system_codes,mappfinder_input_dir,uid_to_mapp,filtered_mapp_list,gene_annotations,uid_system,combined_associations,combined_gene_ranking)
            exportLocalResults(go_full,go_titles,mapp_gene_annotation_db,mapp_values_db,mapp_value_headers,mapps_with_redundant_genes)
  output_combined_results(combined_results)
  gene_associations.exportCombinedAssociations(combined_associations,'gene-associations')
  gene_associations.exportCombinedAssociations(combined_gene_ranking,'gene-ranking')
  moveMAPPFinderFiles()
  print "Analysis Complete (hit the return key to exit)"; sys.stdin.readline()
  
def importGOEliteParameters(parameters):
    global include_headers_in_output; global filter_method; global z_threshold; global p_val_threshold
    global sort_only_by_zscore
    global run_mappfinder
    include_headers_in_output, filter_method, z_threshold, p_val_threshold, sort_only_by_zscore, mod, run_mappfinder, source_data = parameters
    runGOElite(source_data,mod)

if __name__ == '__main__':
    source_data,mod = getParametersFromUsers()
    runGOElite(source_data,mod)

###########################
#Program:	GO-elite.py
#Author:	Nathan Salomonis
#Date:		12/12/06
#Website:	http://www.genmapp.org
#Email:	    nsalomonis@gmail.com
###########################    