python multiprocessing Can't pickle <type 'function'>
Python's multiprocessing
module can not deal with functions/methods which cannot be pickled, which means you cannot use class or instance methods without a lot of hassle. I would recommend to use multiprocess
, which uses dill
for serialization instead of pickle
, and can deal with class or instance methods.
As far as I know, the interface is exactly the same as the one used in multiprocessing
, so you can use it as a drop-in replacement.
See also https://stackoverflow.com/a/21345423/1170207
Vity Lin
Updated on August 21, 2022Comments
-
Vity Lin over 1 year
python2.7 in windows | i add the mysql connect into the class and use multiprocessing ,raise the error .
self.ispop and self.match_var return the dict
sprawn_self_calcu() and unwrap_self_f() is a proxy for the function of Map_class
the function of Map_class need self var.
my code like this:
from analysis_conf.pop_config import pop_config import datetime import multiprocessing from functools import partial from sqlalchemy import create_engine from multiprocessing import Pool as threadpool def sprawn_self_calcu(arg, **kwarg): return Map.mapCin(*arg, **kwarg) def unwrap_self_f(arg, **kwarg): return Map.mappalg(*arg, **kwarg) partial_unwrap = partial(unwrap_self_f) partial_sprawn = partial(sprawn_self_calcu) class Map: def __init__(self): self.ispop = pop_config() self.match_var = self.ispop.pop_match_var() def CreateSqlalchemyEngine(self,config): sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'], config['ipaddr'],config['port'],config['dbname'] ) return create_engine(sigma,pool_recycle=10,pool_timeout=10800) def Mapping(self,conSet): self.baseCon = conSet self.mappalg() Time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") def IUCMapping(self,i,con): print i print self.conf l = con.execute('show tables;') def mappalg(self): mt_val = [1,2,3,4,5] pool = threadpool(4) result = pool.map(partial_sprawn,zip([self]*5,mt_val)) # result = pool.map(partial_sprawn,zip([self]*mtlen,mt_val)) pool.close() pool.join() return True def mapCin(self,i): pid_val = multiprocessing.current_process().pid%4 con = self.baseCon[pid_val] print i self.IUCMapping(i,con) return True class Create_MultiCon: def __init__(self): self.adapter = pop_config() self.conf = self.adapter.pop_baseDB() self.match_var = self.adapter.pop_match_var() def CreateSqlalchemyEngine(self,config): sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'], config['ipaddr'],config['port'],config['dbname'] ) return create_engine(sigma,pool_recycle=10,pool_timeout=10800) def RdictXcon(self,x): t = {} engine = self.CreateSqlalchemyEngine(self.conf) for i in xrange(x): t[i] = engine.connect() return t if __name__ == '__main__': l = Create_MultiCon() conSet = l.RdictXcon(4) ScMap = Map() ScMap.Mapping(conSet)
the Error :
result = pool.map(partial_sprawn,zip([self]*5,mt_val)) File "C:\Python27\lib\multiprocessing\pool.py", line 251, in map return self.map_async(func, iterable, chunksize).get() File "C:\Python27\lib\multiprocessing\pool.py", line 567, in get raise self._value cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
how to solv the ERROR