Page MenuHomePhabricator
Paste P11630

pyspark / py4j invoke java methods via reflection
ActivePublic

Authored by EBernhardson on Mon, Jun 22, 8:42 PM.
# Usefull when debugging out own java classes on the live cluster, making protected methods accessible
# and allowing to view dataframe contents mid-computation
def invoke(j_obj, method_str, *args):
gateway = SparkContext._gateway
jvm = SparkContext.getOrCreate()._jvm
methods = [x for x in j_obj.getClass().getDeclaredMethods() if x.getName() == method_str]
assert len(methods) == 1, repl([x.toString() for x in methods])
methods[0].setAccessible(True)
obj_arr = gateway.new_array(jvm.java.lang.Object, len(args))
for i, arg in enumerate(args):
obj_arr[i] = arg
return methods[0].invoke(j_obj, obj_arr)

Event Timeline