added multiprocessing

bayespairing/src/parse_sequences.py

@@ -10,16 +10,31 @@ import time as timer
 import collections
 from functools import reduce
 from Bio import AlignIO
-
+import os.path
 from operator import itemgetter
 
+def unpick(dataset,direc,typ):
+    file_path = "../"+direc+"/" + dataset + "_"+typ
+    if os.path.exists(file_path):
+        while True:
+          try:
+            nets = pickle.load(open(file_path, "rb"))
+            break
+          except:
+            pass
+        return nets
+    else:
+        print("Dataset not found!")
+        return None
+
 def run_BP(seq, ss, modules_to_parse, dataset, left_out, aln=False, t=-5, samplesize=20000, pretrained=False, Lambda=0.35, Theta=1,Delta=None, leave_out_sequence=False,left_out_sequence="",fuzzy=False,verbose=False, first_run=False, constraints = '',indexes=[]):
     if not (os.path.isfile("../models/" + dataset + "_models.pickle" )) or left_out!="NONE":
         if verbose:
             print("running cross-validation")
         nets = collections.OrderedDict()
     else:
-        nets = pickle.load(open("../models/" + dataset + "_models.pickle", "rb"))
+        #nets = pickle.load(open("../models/" + dataset + "_models.pickle", "rb"))
+        nets = unpick(dataset,"models","models.pickle")
     if not pretrained:
         if verbose and first_run:
             print("reseting models, training with Theta",str(Theta))
@@ -29,7 +44,8 @@ def run_BP(seq, ss, modules_to_parse, dataset, left_out, aln=False, t=-5, sample
         BNs = BayesPairing.setup_BN(modules_to_parse, nets, dataset, left_out,leave_out_sequence=leave_out_sequence,left_out_sequence=left_out_sequence, Lambda=Lambda,Theta=Theta, verbose=verbose,indexes=indexes)
         pickle.dump(BNs,open("../models/" + dataset + "_models.pickle", "wb"))
     else:
-        nets = pickle.load(open("../models/" + dataset + "_models.pickle", "rb"))
+        #nets = pickle.load(open("../models/" + dataset + "_models.pickle", "rb"))
+        nets = unpick(dataset,"models","models.pickle")
         if verbose:
             print('PRE-LOADED MODELS:',str(list(nets.keys())))
         BNs = nets
@@ -40,8 +56,9 @@ def run_BP(seq, ss, modules_to_parse, dataset, left_out, aln=False, t=-5, sample
         return_dict = BayesPairing.parse_sequence(seq, modules_to_parse, ss, dataset, BNs,t, samplesize, Lambda, Theta, Delta, fuzzy=fuzzy, verbose=verbose, constraints = constraints)
 
         
-    siblings = pickle.load(open("../models/" + dataset + "_siblings.pickle", "rb"))    
-    
+    #siblings = pickle.load(open("../models/" + dataset + "_siblings.pickle", "rb"))    
+    siblings = unpick(dataset,"models","siblings.pickle")
+
     noSiblings = process_siblings(return_dict,siblings)
     
     return noSiblings
@@ -643,7 +660,8 @@ if __name__ == "__main__":
     # the default dataset is rna3dmotif
 
     # we load the modules from the dataset to get the number of modules available.
-    graphs = pickle.load(open("../models/" + dataset + "_one_of_each_graph.cPickle", "rb"))
+    #graphs = pickle.load(open("../models/" + dataset + "_one_of_each_graph.cPickle", "rb"))
+    graphs = unpick(dataset,"models","one_of_each_graph.cPickle")
     if "mod" in arguments:
         modules_to_check = [int(input_number) for input_number in arguments["mod"]]
     else: