Finished

parallel/generator.py

@@ -5,10 +5,14 @@ from io import StringIO
 ppservers = ()
 #ppservers = ("10.0.0.1",)
 
+#note threads are automatically detected for performance
 
-
-
-
+#number of jobs to append each round
+parts = 1280000
+#number of times to loop jobs
+rounds = 1280000
+jobs = []
+current = 0
 
 
 def backtesting(ind):
@@ -33,10 +37,10 @@ def backtesting(ind):
 if len(sys.argv) > 1:
     ncpus = int(sys.argv[1])
     # Creates jobserver with ncpus workers
-    job_server = pp.Server(ncpus, ppservers=ppservers)
+    job_server = pp.Server(ncpus, ppservers=ppservers, restart=True)
 else:
     # Creates jobserver with automatically detected number of workers
-    job_server = pp.Server(ppservers=ppservers)
+    job_server = pp.Server(ppservers=ppservers, restart=True)
 
 print("Starting pp with", job_server.get_ncpus(), "workers")
 
@@ -45,15 +49,14 @@ start_time = time.time()
 
 # Since jobs are not equal in the execution time, division of the problem 
 # into a 100 of small subproblems leads to a better load balancing
-parts = 128
-rounds = 128
-jobs = []
-current = 0
+
 for index in range(rounds):
+    print('Please wait... it takes a few minutes to hit the while loop after all jobs are appended for the round.')
     for index in range(parts):
         jobs.append(job_server.submit(backtesting, (index,)))
-    job_server.wait()
+    while True:
         for job in jobs:
+            try:
                 res = job()
                 string = str(res)
                 params = re.search(r'~~~~(.*)~~~~', string).group(1)
@@ -84,12 +87,8 @@ for index in range(rounds):
                     if rsi:
                         print('~~~RSI~~~')
                         print(rsi.group(1))
+            except exception as e:
+                print(e)
 
-
-
-
-
-
-    jobs = []
 print("Time elapsed: ", time.time() - start_time, "s")
 job_server.print_stats()