Databricks error with dill module. It appears that you are attempting to reference SparkContext from a broadcast variable, action, or transformation - save

Question

I am trying to a scikit-learnpipeline on Databricks. However, I encounter the following issue:
import dill
f = open("/dbfs/HR_pipe.p", mode='wb')
dill.dump(fitted,f)
where:
fitted = main_pipeline.fit(X_train, y_train)
Below you can find the error traceback, I am actually using scikit-learn version 0.24.2 and the class of the fitted object is a sklearn pipeline.
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<command-97102145541341> in <module>
1 import dill
2 f = open("/dbfs/HR_pipe.p", mode='wb')
----> 3 dill.dump(fitted,f)
/local_disk0/.ephemeral_nfs/envs/pythonEnv-1f8e9f3e-114d-4f9b-915f-25ba440655c4/lib/python3.7/site-packages/dill/_dill.py in dump(obj, file, protocol, byref, fmode, recurse, **kwds)
334 _kwds = kwds.copy()
335 _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
--> 336 Pickler(file, protocol, **_kwds).dump(obj)
337 return
338
/local_disk0/.ephemeral_nfs/envs/pythonEnv-1f8e9f3e-114d-4f9b-915f-25ba440655c4/lib/python3.7/site-packages/dill/_dill.py in dump(self, obj)
618 raise PicklingError(msg)
619 else:
--> 620 StockPickler.dump(self, obj)
621 return
622 dump.__doc__ = StockPickler.dump.__doc__
/usr/lib/python3.7/pickle.py in dump(self, obj)
435 if self.proto >= 4:
436 self.framer.start_framing()
--> 437 self.save(obj)
438 self.write(STOP)
439 self.framer.end_framing()
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
547
548 # Save the reduce() output and finally memoize the object
--> 549 self.save_reduce(obj=obj, *rv)
550
551 def persistent_id(self, obj):
/usr/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
660
661 if state is not None:
--> 662 save(state)
663 write(BUILD)
664
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/local_disk0/.ephemeral_nfs/envs/pythonEnv-1f8e9f3e-114d-4f9b-915f-25ba440655c4/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
1249 # we only care about session the first pass thru
1250 pickler._first_pass = False
-> 1251 StockPickler.save_dict(pickler, obj)
1252 log.info("# D2")
1253 return
/usr/lib/python3.7/pickle.py in save_dict(self, obj)
857
858 self.memoize(obj)
--> 859 self._batch_setitems(obj.items())
860
861 dispatch[dict] = save_dict
/usr/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/usr/lib/python3.7/pickle.py in save_list(self, obj)
817
818 self.memoize(obj)
--> 819 self._batch_appends(obj)
820
821 dispatch[list] = save_list
/usr/lib/python3.7/pickle.py in _batch_appends(self, items)
841 write(MARK)
842 for x in tmp:
--> 843 save(x)
844 write(APPENDS)
845 elif n:
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/usr/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
547
548 # Save the reduce() output and finally memoize the object
--> 549 self.save_reduce(obj=obj, *rv)
550
551 def persistent_id(self, obj):
/usr/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
631 "args[0] from __newobj__ args has the wrong class")
632 args = args[1:]
--> 633 save(cls)
634 save(args)
635 write(NEWOBJ)
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/local_disk0/.ephemeral_nfs/envs/pythonEnv-1f8e9f3e-114d-4f9b-915f-25ba440655c4/lib/python3.7/site-packages/dill/_dill.py in save_type(pickler, obj, postproc_list)
1838 _save_with_postproc(pickler, (_create_type, (
1839 type(obj), obj.__name__, obj.__bases__, _dict
-> 1840 )), obj=obj, postproc_list=postproc_list)
1841 log.info("# %s" % _t)
1842 else:
/local_disk0/.ephemeral_nfs/envs/pythonEnv-1f8e9f3e-114d-4f9b-915f-25ba440655c4/lib/python3.7/site-packages/dill/_dill.py in _save_with_postproc(pickler, reduction, is_pickler_dill, obj, postproc_list)
1152 if source:
1153 pickler.write(pickler.get(pickler.memo[id(dest)][0]))
-> 1154 pickler._batch_setitems(iter(source.items()))
1155 else:
1156 # Updating with an empty dictionary. Same as doing nothing.
/usr/lib/python3.7/pickle.py in _batch_setitems(self, items)
883 for k, v in tmp:
884 save(k)
--> 885 save(v)
886 write(SETITEMS)
887 elif n:
/usr/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
522 reduce = getattr(obj, "__reduce_ex__", None)
523 if reduce is not None:
--> 524 rv = reduce(self.proto)
525 else:
526 reduce = getattr(obj, "__reduce__", None)
/databricks/spark/python/pyspark/context.py in __getnewargs__(self)
354 # This method is called when attempting to pickle SparkContext, which is always an error:
355 raise Exception(
--> 356 "It appears that you are attempting to reference SparkContext from a broadcast "
357 "variable, action, or transformation. SparkContext can only be used on the driver, "
358 "not in code that it run on workers. For more information, see SPARK-5063."
Exception: It appears that you are attempting to reference SparkContext from a broadcast variable, action, or transformation. SparkContext can only be used on the driver, not in code that it run on workers. For more information, see SPARK-5063.
Do you happen to know how to solve such issue?
details about the questions ..........................................................................