Error running parallel

[shabnamkadir]

Traceback (most recent call last):
  File "parallel_global_script.py", line 294, in <module>
    supercluster_results = lbv.map(lambda channel: supercluster_info['kk_sub'][channel].cluster_mask_starts(),full_adjacency.keys())
  File "<string>", line 2, in map
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 55, in sync_results
    ret = f(self, *args, **kwargs)
  File "<string>", line 2, in map
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 40, in save_ids
    ret = f(self, *args, **kwargs)
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 1123, in map
    return pf.map(*sequences)
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 271, in map
    ret = self(*sequences)
  File "<string>", line 2, in __call__
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 78, in sync_view_results
    return f(self, *args, **kwargs)
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 254, in __call__
    return r.get()
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/asyncresult.py", line 104, in get
    raise self._exception
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/asyncresult.py", line 139, in wait
    results = error.collect_exceptions(results, self._fname)
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/error.py", line 233, in collect_exceptions
    raise e
  File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/error.py", line 231, in collect_exceptions
    raise CompositeError(msg, elist)
IPython.parallel.error.CompositeError: one or more exceptions from call to method: <lambda>
[0:apply]: NameError: name 'supercluster_info' is not defined

[rossant]

you need to import your functions on all nodes first

[shabnamkadir]

Appears to now run, but gives nonsensical results:
'''
INFO klustakwik: Number of spikes in data set: 4001
INFO klustakwik: Number of unique masks in data set: 2575
INFO klustakwik.initial_parameters: full_step_every = 1
INFO klustakwik.initial_parameters: penalty_k = 0.0
INFO klustakwik.initial_parameters: fast_split = False
INFO klustakwik.initial_parameters: split_every = 40
INFO klustakwik.initial_parameters: use_noise_cluster = True
INFO klustakwik.initial_parameters: subset_break_fraction = 0.01
INFO klustakwik.initial_parameters: mua_point = 2
INFO klustakwik.initial_parameters: max_split_iterations = None
INFO klustakwik.initial_parameters: max_possible_clusters = 1000
INFO klustakwik.initial_parameters: max_iterations = 1000
INFO klustakwik.initial_parameters: break_fraction = 0.0
INFO klustakwik.initial_parameters: prior_point = 1
INFO klustakwik.initial_parameters: num_changed_threshold = 0.05
INFO klustakwik.initial_parameters: always_split_bimodal = False
INFO klustakwik.initial_parameters: use_mua_cluster = True
INFO klustakwik.initial_parameters: split_first = 20
INFO klustakwik.initial_parameters: points_for_cluster_mask = 100
INFO klustakwik.initial_parameters: max_quick_step_candidates_fraction = 0.4
INFO klustakwik.initial_parameters: penalty_k_log_n = 1.0
INFO klustakwik.initial_parameters: max_quick_step_candidates = 100000000
INFO klustakwik.initial_parameters: noise_point = 1
INFO klustakwik.initial_parameters: num_starting_clusters = 500
INFO klustakwik.initial_parameters: consider_cluster_deletion = True
INFO klustakwik.initial_parameters: dist_thresh = 9.21034037198
INFO klustakwik.initial_parameters: use_noise_cluster = True
INFO klustakwik.initial_parameters: use_mua_cluster = True
Time taken for parallel clustering 156.44 s
[None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]
'''

[shabnamkadir]

I'm not sure why it is returning None objects instead of KK objects.

[shabnamkadir]

'''
Traceback (most recent call last):
File "parallel_global_script.py", line 304, in
supercluster_results = lbv.map(lambda channel: run_subset_KK(supercluster_info['kk_sub'][channel]),full_adjacency.keys())
File "", line 2, in map
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 55, in sync_results
ret = f(self, _args, *_kwargs)
File "", line 2, in map
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 40, in save_ids
ret = f(self, _args, *_kwargs)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 1123, in map
return pf.map(_sequences)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 271, in map
ret = self(_sequences)
File "", line 2, in call
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 78, in sync_view_results
return f(self, _args, *_kwargs)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py", line 254, in call
return r.get()
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/asyncresult.py", line 104, in get
raise self._exception
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/asyncresult.py", line 139, in wait
results = error.collect_exceptions(results, self._fname)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/error.py", line 233, in collect_exceptions
raise e
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/error.py", line 231, in collect_exceptions
raise CompositeError(msg, elist)
IPython.parallel.error.CompositeError: one or more exceptions from call to method:
[5:apply]: NameError: name 'run_subset_KK' is not defined
[1:apply]: NameError: name 'run_subset_KK' is not defined
[7:apply]: NameError: name 'run_subset_KK' is not defined
[3:apply]: NameError: name 'run_subset_KK' is not defined
.... 116 more exceptions ...

'''

[shabnamkadir]

importing run_subset_KK from parallel_global on engine(s)
[0:apply]: 
---------------------------------------------------------------------------ImportError                               Traceback (most recent call last)<string> in <module>()
/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py in remote_import(name, fromlist, level)
    439             import sys
    440             user_ns = globals()
--> 441             mod = __import__(name, fromlist=fromlist, level=level)
    442             if fromlist:
    443                 for key in fromlist:
ImportError: No module named 'parallel_global'

[1:apply]: 

[shabnamkadir]

This bug is now a Heisenbug. It sometimes parallelises fine.

About to parallelize
Time taken for parallel clustering 151.85 s

[shabnamkadir]

It always fails the first time it is launched, but if you keep the same engines running
and don't restart, and run the script again - it works! The second time, the clustering happens fine...

[shabnamkadir]

When changing the number of points without restarting the engines (yes, I know):

About to parallelize
Time taken for parallel clustering 587.06 s
Traceback (most recent call last):
  File "parallel_global_script_40000.py", line 180, in <module>
    superclusters[supercluster_info['sub_spikes'][channel],i] = supercluster_results[i]+1
ValueError: shape mismatch: value array of shape (304,) could not be broadcast to indexing result of shape (2898,)

[shabnamkadir]

Possibly related:
Traceback (most recent call last):
File "nickground_global_script_1280000.py", line 185, in
c[:]['supercluster_info'] = supercluster_info
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 806, in setitem
self.update({key:value})
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 687, in update
return self.push(ns, block=self.block, track=self.track)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 708, in push
return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
File "", line 2, in _really_apply
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 55, in sync_results
ret = f(self, _args, *_kwargs)
File "", line 2, in _really_apply
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 40, in save_ids
ret = f(self, _args, *_kwargs)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/view.py", line 562, in _really_apply
ident=ident)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/parallel/client/client.py", line 1280, in send_apply_request
metadata=metadata, track=track)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/IPython/kernel/zmq/session.py", line 660, in send
tracker = stream.send_multipart(to_send, copy=False, track=True)
File "/home/skadir/.conda/envs/globalphy/lib/python3.4/site-packages/zmq/sugar/socket.py", line 331, in send_multipart
return self.send(msg_parts[-1], flags, copy=copy, track=track)
File "zmq/backend/cython/socket.pyx", line 619, in zmq.backend.cython.socket.Socket.send (zmq/backend/cython/socket.c:6169)
File "zmq/backend/cython/socket.pyx", line 674, in zmq.backend.cython.socket.Socket.send (zmq/backend/cython/socket.c:6034)
File "zmq/backend/cython/socket.pyx", line 169, in zmq.backend.cython.socket._send_frame (zmq/backend/cython/socket.c:2118)
File "zmq/backend/cython/checkrc.pxd", line 19, in zmq.backend.cython.checkrc._check_rc (zmq/backend/cython/socket.c:6920)
zmq.error.Again: Resource temporarily unavailable
Bad address (stream_engine.cpp:788)
Aborted (core dumped)