"Writes closed" when deploying application in domain ---------------------------------------------------- Key: WFCORE-965 URL: https://issues.jboss.org/browse/WFCORE-965 Project: WildFly Core Issue Type: Bug Components: Domain Management Affects Versions: 2.0.0.Beta6 Environment: domain profile=default 1x domain controller without servers 3x slave, 10 servers each Reporter: Martin Simka Assignee: Brian Stansberry Some of our tests for large domain sometimes fail with error: "Writes closed". It happens only with some higher number of servers per host. {noformat} Failed response: { "outcome" => "failed", "result" => { "step-1" => { "outcome" => "failed", "rolled-back" => true }, "step-2" => { "outcome" => "failed", "rolled-back" => true }, "step-3" => { "outcome" => "failed", "rolled-back" => true } }, "failure-description" => {"WFLYDC0074: Operation failed or was rolled back on all servers. Server failures:" => {"server-group" => { "server-group-A" => {"host" => { "perf31" => { "perf31-server-3" => "Writes closed", "perf31-server-5" => "Writes closed" }, "perf33" => {"perf33-server-3" => "Writes closed"} }}, "server-group-B" => {"host" => {"perf32" => { "perf32-server-2" => "Writes closed", "perf32-server-6" => "Writes closed" }}} }}}, "rolled-back" => true, "server-groups" => { "server-group-A" => {"host" => { "perf31" => { "perf31-server-1" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf31-server-3" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => "Writes closed", "rolled-back" => true }}, "perf31-server-5" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => "Writes closed", "rolled-back" => true }}, "perf31-server-7" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf31-server-9" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} }, "perf32" => { "perf32-server-1" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf32-server-3" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => undefined, "rolled-back" => true }}, "perf32-server-5" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf32-server-7" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf32-server-9" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} }, "perf33" => { "perf33-server-1" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-3" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => "Writes closed", "rolled-back" => true }}, "perf33-server-5" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-7" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-9" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} } }}, "server-group-B" => {"host" => { "perf31" => { "perf31-server-10" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf31-server-2" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => undefined, "rolled-back" => true }}, "perf31-server-4" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => undefined, "rolled-back" => true }}, "perf31-server-6" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf31-server-8" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} }, "perf32" => { "perf32-server-10" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf32-server-2" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => "Writes closed", "rolled-back" => true }}, "perf32-server-4" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => undefined, "rolled-back" => true }}, "perf32-server-6" => {"response" => { "outcome" => "failed", "result" => undefined, "failure-description" => "Writes closed", "rolled-back" => true }}, "perf32-server-8" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} }, "perf33" => { "perf33-server-10" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-2" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-4" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-6" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }}, "perf33-server-8" => {"response" => { "outcome" => "failed", "result" => {"step-1" => { "outcome" => "failed", "rolled-back" => true }}, "failure-description" => "WFLYCTL0063: Composite operation was rolled back", "rolled-back" => true }} } }} } } {noformat} I know this isn't detailed bug report, I didn't find easy way to reproduce it, other than running large domain tests. Tests are modified tests from https://github.com/wildfly/wildfly/blob/master/testsuite/domain/src/test/... . I can provide other logs, outputs ...., just let me know which log categories would help. Also I tried to play a bit with remoting's HEARTBEAT interval (through subsystem configuration), but that doesn't help.