NotSerializableException while executing streams via JavaScript in a cluster in DIST mode ----------------------------------------------------------------------------------------- Key: ISPN-6173 URL: https://issues.jboss.org/browse/ISPN-6173 Project: Infinispan Issue Type: Bug Reporter: Anna Manukyan Assignee: Galder Zamarreño Fix For: 9.0.0.Alpha3, 9.0.0.Final Attachments: wordCountStream_Exec.js The following exception appears when I try to execute a JS script which uses Java Stream API on 4 node cluster in DIST_SYNC mode. No matter which mode is set in JS metadata (local or distributed) the test ends with exception. You can find the test and the script here: https://github.com/andyuk1986/infinispan/blob/2e19b65a00d962201c81216ca3b... https://github.com/andyuk1986/infinispan/blob/2e19b65a00d962201c81216ca3b... When the mode=local, the exception is: {code} org.infinispan.commons.marshall.NotSerializableException: java.lang.invoke.BoundMethodHandle$Species_LL Caused by: an exception which occurred: in field apply in object jdk.nashorn.javaadapters.java.util.function.Function$$Serializable@5e828108 -> toString = jdk.nashorn.javaadapters.java.util.function.Function$$Serializable@5e828108 in object java.util.ArrayDeque@7074aba7 -> toString = [org.infinispan.stream.impl.intops.object.MapOperation@11cc7142] in object org.infinispan.stream.impl.termop.SegmentRetryingOperation@70414f65 -> toString = org.infinispan.stream.impl.termop.SegmentRetryingOperation@70414f65 in object org.infinispan.stream.impl.StreamRequestCommand@38614d0c -> toString = StreamRequestCommand{cacheName='___defaultcache'} {code} When the mode=distributed, the exception is: {code} java.util.concurrent.ExecutionException: java.util.concurrent.CancellationException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895) at org.infinispan.scripting.DistributedCacheScriptingTest.testDistributedMapReduceStream(DistributedCacheScriptingTest.java:61) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeMethod(Invoker.java:714) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.access$000(SuiteRunner.java:38) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382) at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.util.concurrent.CancellationException at java.util.concurrent.CompletableFuture.cancel(CompletableFuture.java:2263) at org.infinispan.util.concurrent.CompletableFutures.lambda$toCompletableFuture$16(CompletableFutures.java:49) at org.infinispan.commons.util.concurrent.BaseNotifyingFuture.fireListeners(BaseNotifyingFuture.java:37) at org.infinispan.commons.util.concurrent.NotifyingFutureImpl.notifyException(NotifyingFutureImpl.java:105) at org.infinispan.distexec.DefaultExecutorService$LocalDistributedTaskPart$1.doLocalInvoke(DefaultExecutorService.java:1117) at org.infinispan.distexec.DefaultExecutorService$LocalDistributedTaskPart$1.call(DefaultExecutorService.java:1103) ... 4 more {code} But for both cases, from the TRACE logs the real exception is: {code} java.io.NotSerializableException: java.lang.invoke.BoundMethodHandle$Species_LL at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:860) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.river.RiverMarshaller.doWriteFields(RiverMarshaller.java:1032) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.river.RiverMarshaller.doWriteSerializableObject(RiverMarshaller.java:988) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:854) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.infinispan.stream.impl.intops.IntermediateOperationExternalizer.writeObject(IntermediateOperationExternalizer.java:194) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.intops.IntermediateOperationExternalizer.writeObject(IntermediateOperationExternalizer.java:23) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:481) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:138) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:569) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.infinispan.stream.impl.termop.TerminalOperationExternalizer.writeObject(TerminalOperationExternalizer.java:83) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.termop.TerminalOperationExternalizer.writeObject(TerminalOperationExternalizer.java:34) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:481) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:138) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeCommandParameters(ReplicableCommandExternalizer.java:66) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.marshallParameters(CacheRpcCommandExternalizer.java:128) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:112) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:68) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:481) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:138) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) ~[jboss-marshalling-osgi-1.4.10.Final.jar:1.4.10.Final] at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectToObjectStream(AbstractJBossMarshaller.java:72) ~[infinispan-commons-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.marshall.core.VersionAwareMarshaller.objectToBuffer(VersionAwareMarshaller.java:77) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.commons.marshall.AbstractMarshaller.objectToBuffer(AbstractMarshaller.java:41) [infinispan-commons-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectToBuffer(AbstractDelegatingMarshaller.java:85) [infinispan-commons-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectToBuffer(MarshallerAdapter.java:23) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:265) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:121) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotelyAsync(JGroupsTransport.java:578) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotelyAsync(RpcManagerImpl.java:162) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.ClusterStreamManagerImpl.submitAsyncTasks(ClusterStreamManagerImpl.java:169) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.ClusterStreamManagerImpl.commonRemoteStreamOperation(ClusterStreamManagerImpl.java:86) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.ClusterStreamManagerImpl.remoteStreamOperationRehashAware(ClusterStreamManagerImpl.java:70) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.AbstractCacheStream.performOperationRehashAware(AbstractCacheStream.java:306) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.AbstractCacheStream.performOperation(AbstractCacheStream.java:258) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.stream.impl.DistributedCacheStream.collect(DistributedCacheStream.java:329) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at jdk.nashorn.internal.scripts.Script$\^eval\_.:program(<eval>:23) [?:?] at jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:640) [nashorn.jar:?] at jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:228) [nashorn.jar:?] at jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:393) [nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.evalImpl(NashornScriptEngine.java:418) [nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.access$300(NashornScriptEngine.java:73) [nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine$3.eval(NashornScriptEngine.java:511) [nashorn.jar:?] at javax.script.CompiledScript.eval(CompiledScript.java:92) [?:1.8.0_66] at org.infinispan.scripting.impl.ScriptingManagerImpl.execute(ScriptingManagerImpl.java:182) [classes/:?] at org.infinispan.scripting.impl.DistributedScript.call(DistributedScript.java:30) [classes/:?] at org.infinispan.commands.read.DistributedExecuteCommand.perform(DistributedExecuteCommand.java:103) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.distexec.DefaultExecutorService$LocalDistributedTaskPart$1.doLocalInvoke(DefaultExecutorService.java:1113) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.distexec.DefaultExecutorService$LocalDistributedTaskPart$1.call(DefaultExecutorService.java:1103) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_66] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_66] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_66] at java.lang.Thread.run(Thread.java:745) [?:1.8.0_66] Caused by: org.jboss.marshalling.TraceInformation {code} Seems that this is general Nashorn thing, as complex object serialization also gives the same exception. E.g the following script execution throws same exception with a bit different stack trace: {code} // mode=distributed,language=javascript var Function = Java.type("java.util.function.Function") var Serializable = Java.type("java.io.Serializable") var SerializableFunction = Java.extend(Function, Serializable); var CollectionsAndFiles = new JavaImporter( java.util, java.io, java.nio); var e = new SerializableFunction( { apply: function(object) { return object.getValue().toLowerCase().split(/[\\W]+/) } }) with (CollectionsAndFiles) { var bos = new ByteArrayOutputStream(); var out = null; try { out = new ObjectOutputStream(bos); out.writeObject(e); var yourBytes = bos.toByteArray(); } finally { try { if (out != null) { out.close(); } } finally { } } } {code} the stacktrace for the code above is: {code} java.lang.RuntimeException: java.io.NotSerializableException: java.lang.invoke.BoundMethodHandle$Species_LL at jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:397) ~[nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.evalImpl(NashornScriptEngine.java:446) ~[nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.evalImpl(NashornScriptEngine.java:403) ~[nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.evalImpl(NashornScriptEngine.java:399) ~[nashorn.jar:?] at jdk.nashorn.api.scripting.NashornScriptEngine.eval(NashornScriptEngine.java:155) ~[nashorn.jar:?] at javax.script.AbstractScriptEngine.eval(AbstractScriptEngine.java:233) ~[?:1.8.0_66] at org.infinispan.scripting.impl.ScriptingManagerImpl.execute(ScriptingManagerImpl.java:186) ~[classes/:?] at org.infinispan.scripting.impl.DistributedScript.call(DistributedScript.java:30) ~[classes/:?] at org.infinispan.commands.read.DistributedExecuteCommand.perform(DistributedExecuteCommand.java:103) ~[infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34) [infinispan-core-8.2.0-SNAPSHOT.jar:8.2.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_66] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_66] at java.lang.Thread.run(Thread.java:745) [?:1.8.0_66] Caused by: java.io.NotSerializableException: java.lang.invoke.BoundMethodHandle$Species_LL at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184) ~[?:1.8.0_66] at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) ~[?:1.8.0_66] at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) ~[?:1.8.0_66] at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) ~[?:1.8.0_66] at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) ~[?:1.8.0_66] at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348) ~[?:1.8.0_66] at jdk.nashorn.internal.scripts.Script$2$\^eval\_.:program(<eval>:21) ~[?:?] at jdk.nashorn.internal.runtime.ScriptFunctionData.invoke(ScriptFunctionData.java:640) ~[nashorn.jar:?] at jdk.nashorn.internal.runtime.ScriptFunction.invoke(ScriptFunction.java:228) ~[nashorn.jar:?] at jdk.nashorn.internal.runtime.ScriptRuntime.apply(ScriptRuntime.java:393) ~[nashorn.jar:?] ... 13 more {code} UPDATE: In case when the test tries to execute javascript where the mode is set to local, and the map/reduce is executed using submitConsumer(...) method of the cacheManager's executor, the following exception is thrown: {code} java.util.concurrent.CompletionException: java.lang.ClassCastException: Cannot cast org.infinispan.manager.DefaultCacheManager to java.util.function.Function org.infinispan.commons.marshall.NotSerializableException: java.lang.invoke.BoundMethodHandle$Species_L3 Caused by: an exception which occurred: in field apply in object jdk.nashorn.javaadapters.java.util.function.Function$$Serializable@1ad8df52 -> toString = jdk.nashorn.javaadapters.java.util.function.Function$$Serializable@1ad8df52 in object org.infinispan.manager.impl.ReplicableCommandManagerFunction@a52ca2e -> toString = org.infinispan.manager.impl.ReplicableCommandManagerFunction@a52ca2e {code} The code is attached wordCountStream_Exec.js.