<p dir="ltr">Are you using any entrypoint?<br>
Time or length based?<br>
It could be possible you have to retract manually the events.</p>
<div class="gmail_quote">On 24 Oct 2013 08:23, "Elran Dvir" <<a href="mailto:elrand@checkpoint.com">elrand@checkpoint.com</a>> wrote:<br type="attribution"><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">
<div lang="EN-US" link="blue" vlink="purple">
<div>
<p class="MsoNormal"><span style="color:#1f497d">I am sending this message again because maybe the last wasn’t sent because of the attached snapshots.</span><u></u><u></u></p>
<p class="MsoNormal"><span style="color:#1f497d">I removed them now.<u></u><u></u></span></p>
<p class="MsoNormal"><span style="color:#1f497d">Thanks.<u></u><u></u></span></p>
<p class="MsoNormal"><span style="color:#1f497d"><u></u> <u></u></span></p>
<p class="MsoNormal">Hi all,<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">I am using Drools Fusion. I am getting OutOfMemoryError rather fast. My JVM is running with –Xmx4g flag.<u></u><u></u></p>
<p class="MsoNormal">I have rules defined in another (not Drools) language.<u></u><u></u></p>
<p class="MsoNormal">Every rule is translated programmatically to a drl file. This is because the user can add and remove rules (in the other language) dynamically.<u></u><u></u></p>
<p class="MsoNormal">The default configuration contains 125 rules.<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">For example, one rule is supposed to identify a port scan event.<u></u><u></u></p>
<p>The basic fact is connection log. For each combination of src (source IP) and dst (destination IP) , detect a port scan event, if over 60 seconds there were at least 20 connection logs with different service and protocol.<u></u><u></u></p>
<p>The event will stay closed for 10 minute - no event will be sent during this time for this combination of src and dst. The event the connection logs’ ids (markers).<u></u><u></u></p>
<p class="MsoNormal">(other rules are very similar in structure, but different in logic, of course)
<u></u><u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">This is its programmatic drl file:<u></u><u></u></p>
</div>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">package com.checkpoint.correlation.impl.drools.package30;<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">import java.util.Date<u></u><u></u></p>
<p class="MsoNormal">import java.util.HashMap<u></u><u></u></p>
<p class="MsoNormal">import java.util.Set<u></u><u></u></p>
<p class="MsoNormal">import com.checkpoint.correlation.impl.drools.Log<u></u><u></u></p>
<p class="MsoNormal">import com.checkpoint.correlation.impl.drools.CorrelatedEvent<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">global com.checkpoint.correlation.server.EventsHandler externalEventsHandler;<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">import function com.checkpoint.correlation.impl.utils.UserDefinedFunctions.isInDayHourRange<u></u><u></u></p>
<p class="MsoNormal">import function com.checkpoint.correlation.impl.utils.UserDefinedFunctions.isInIpRange<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">function boolean filter(Log log) { <u></u><u></u></p>
<p class="MsoNormal"> return (!((log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "10.80.0.0", "10.80.255.255")) || (log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "124.0.0.0",
"124.255.255.255")) || (log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "192.168.0.0", "192.168.255.255")) || (log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "195.158.7.0", "195.158.7.255"))
|| (log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "11.25.0.0", "11.25.255.255")) || (log.fieldsMap.get("src")!= null && isInIpRange(log.fieldsMap.get("src").toString(), "128.157.0.0", "128.157.255.255")) || (log.fieldsMap.get("src")!=
null && isInIpRange(log.fieldsMap.get("src").toString(), "213.114.0.0", "213.114.255.255"))));<u></u><u></u></p>
<p class="MsoNormal">}<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">function String markersToString(Set markersSet) { <u></u><u></u></p>
<p class="MsoNormal"> int i = 0;<u></u><u></u></p>
<p class="MsoNormal"> String markersString = "";<u></u><u></u></p>
<p class="MsoNormal"> for (Object marker : markersSet) { <u></u><u></u></p>
<p class="MsoNormal"> if (i == 25) break;<u></u><u></u></p>
<p class="MsoNormal"> String markerStr = marker.toString();<u></u><u></u></p>
<p class="MsoNormal"> if (i > 0) markersString += "\n";<u></u><u></u></p>
<p class="MsoNormal"> markersString += markerStr;<u></u><u></u></p>
<p class="MsoNormal"> }<u></u><u></u></p>
<p class="MsoNormal"> return markersString;<u></u><u></u></p>
<p class="MsoNormal">}<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">function String calcSeverity(Log log) { <u></u><u></u></p>
<p class="MsoNormal"> return "High";<u></u><u></u></p>
<p class="MsoNormal">}<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">function String getUniqueId(Log log) { <u></u><u></u></p>
<p class="MsoNormal"> String uniqueId="";<u></u><u></u></p>
<p class="MsoNormal"> uniqueId += (log.fieldsMap.get("service") != null ? log.fieldsMap.get("service").toString() : "null");<u></u><u></u></p>
<p class="MsoNormal"> uniqueId += (log.fieldsMap.get("proto") != null ? log.fieldsMap.get("proto").toString() : "null");<u></u><u></u></p>
<p class="MsoNormal"> return uniqueId;<u></u><u></u></p>
<p class="MsoNormal">}<u></u><u></u></p>
<p class="MsoNormal">
<u></u><u></u></p>
<p class="MsoNormal">declare Log<u></u><u></u></p>
<p class="MsoNormal"> @role(event)<u></u><u></u></p>
<p class="MsoNormal">end<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">declare CorrelatedEvent<u></u><u></u></p>
<p class="MsoNormal"> @role(event)<u></u><u></u></p>
<p class="MsoNormal"> @expires(600s)<u></u><u></u></p>
<p class="MsoNormal">end<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">rule "Port scan from external network"<u></u><u></u></p>
<p class="MsoNormal">enabled true<u></u><u></u></p>
<p class="MsoNormal">dialect "java"<u></u><u></u></p>
<p class="MsoNormal">no-loop<u></u><u></u></p>
<p class="MsoNormal">when<u></u><u></u></p>
<p class="MsoNormal"> $log : Log(eval(filter($log)))<u></u><u></u></p>
<p class="MsoNormal"> not CorrelatedEvent(getId() == "{8AC52BA8-1EE8-4f18-9BB4-54492116501C}", groupByFieldsMap.get("src") == $log.fieldsMap.get("src"), groupByFieldsMap.get("dst") == $log.fieldsMap.get("dst"))<u></u><u></u></p>
<p class="MsoNormal"> accumulate($accumulatedLog : Log(eval(filter($accumulatedLog)), this after[0s,60s] $log, fieldsMap.get("src") == $log.fieldsMap.get("src"), fieldsMap.get("dst") == $log.fieldsMap.get("dst"), $id : getUniqueId(this));<u></u><u></u></p>
<p class="MsoNormal"> $idSet : collectSet($id);<u></u><u></u></p>
<p class="MsoNormal"> $idSet.size > 19)<u></u><u></u></p>
<p class="MsoNormal"> accumulate($accumulatedLog : Log(eval(filter($accumulatedLog)), this after[0s,60s] $log, fieldsMap.get("src") == $log.fieldsMap.get("src"), fieldsMap.get("dst") == $log.fieldsMap.get("dst"), $idSet.contains(getUniqueId(this)),
$marker : fieldsMap.get("marker"));<u></u><u></u></p>
<p class="MsoNormal"> $markerSet : collectSet($marker))<u></u><u></u></p>
<p class="MsoNormal"> then<u></u><u></u></p>
<p class="MsoNormal"> CorrelatedEvent $ce = new CorrelatedEvent("{8AC52BA8-1EE8-4f18-9BB4-54492116501C}");<u></u><u></u></p>
<p class="MsoNormal"> $ce.groupByFieldsMap.put("src", $log.fieldsMap.get("src"));<u></u><u></u></p>
<p class="MsoNormal"> $ce.groupByFieldsMap.put("dst", $log.fieldsMap.get("dst"));<u></u><u></u></p>
<p class="MsoNormal"> insert($ce);<u></u><u></u></p>
<p class="MsoNormal"> HashMap<String,Object> fieldsMap = new HashMap<String,Object>();<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("cu_rule_id", "{8AC52BA8-1EE8-4f18-9BB4-54492116501C}");<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("event_name", "Port scan from external network");<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("cu_rule_severity", calcSeverity($log));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("cu_rule_category", "Scans");<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("cu_log_count", $markerSet.size());<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("time", new Date());<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("cu_markers_list", markersToString($markerSet));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("src", $log.fieldsMap.get("src"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("src_machine_name", $log.fieldsMap.get("src_machine_name"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("src_user_name", $log.fieldsMap.get("src_user_name"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("dst", $log.fieldsMap.get("dst"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("dst_machine_name", $log.fieldsMap.get("dst_machine_name"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("dst_user_name", $log.fieldsMap.get("dst_user_name"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("service", $log.fieldsMap.get("service"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("proto", $log.fieldsMap.get("proto"));<u></u><u></u></p>
<p class="MsoNormal"> fieldsMap.put("product", $log.fieldsMap.get("product"));<u></u><u></u></p>
<p class="MsoNormal"> externalEventsHandler.handleEvent(fieldsMap);<u></u><u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal">end <u></u><u></u></p>
</div>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">I am sending logs in a rate of up to 200 logs/sec. After about 3 minutes, my application starts to be unresponsive.
<u></u><u></u></p>
<p class="MsoNormal">I monitored the JVM with VisualVM. Two snapshots of VisualVM are attached.<u></u><u></u></p>
<p class="MsoNormal">I found out that the class consuming most memory is FromNodeLeftTuple of drools (as can be seen in “instances.png”).<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p><u></u><span>1)<span style="font:7.0pt "Times New Roman"">
</span></span><u></u><span dir="LTR"></span>Is my inserting rate is too high?<u></u><u></u></p>
<p><u></u><span>2)<span style="font:7.0pt "Times New Roman"">
</span></span><u></u><span dir="LTR"></span>Is There a way I can make my rules more memory efficient?<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">Thanks.<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal">Inserting logs:<u></u><u></u></p>
</div>
<p class="MsoNormal">public void insertEvents(Collection<Map<String, Object>> logs)<u></u><u></u></p>
<p class="MsoNormal">{<u></u><u></u></p>
<p class="MsoNormal"> for (Map<String, Object> map : logs) {<u></u><u></u></p>
<p class="MsoNormal"> Log log = new Log();<u></u><u></u></p>
<p class="MsoNormal"> Log.fieldsMap.putAll(map);<u></u><u></u></p>
<p class="MsoNormal"> session.insert(log);<u></u><u></u></p>
<p class="MsoNormal"> session.fireAllRules();<u></u><u></u></p>
<p class="MsoNormal"> }<u></u><u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal">}<u></u><u></u></p>
</div>
<p class="MsoNormal"><u></u> <u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal">Log class:<u></u><u></u></p>
</div>
<p class="MsoNormal">public class Log<u></u><u></u></p>
<p class="MsoNormal">{<u></u><u></u></p>
<p class="MsoNormal"> public HashMap<String, Object> fieldsMap = new HashMap<>();<u></u><u></u></p>
<p class="MsoNormal">}<u></u><u></u></p>
<div style="border-top:solid windowtext 1.0pt;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:none;padding:1.0pt 0in 1.0pt 0in">
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">CorrelatedEvent class:<u></u><u></u></p>
</div>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal">public class CorrelatedEvent<u></u><u></u></p>
<p class="MsoNormal">{<u></u><u></u></p>
<p class="MsoNormal"> public Map<String, Object> groupByFieldsMap;<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal"> private String id;<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal"> public CorrelatedEvent(String id)<u></u><u></u></p>
<p class="MsoNormal"> {<u></u><u></u></p>
<p class="MsoNormal"> groupByFieldsMap = new HashMap<>();<u></u><u></u></p>
<p class="MsoNormal"> <a href="http://this.id" target="_blank">this.id</a> = id;<u></u><u></u></p>
<p class="MsoNormal"> }<u></u><u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal"> public String getId()<u></u><u></u></p>
<p class="MsoNormal"> {<u></u><u></u></p>
<p class="MsoNormal"> return id;<u></u><u></u></p>
<p class="MsoNormal"> }<u></u><u></u></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0in 0in 1.0pt 0in">
<p class="MsoNormal">}<u></u><u></u></p>
</div>
<p class="MsoNormal"><u></u> <u></u></p>
<p class="MsoNormal"><u></u> <u></u></p>
</div>
</div>
<br>_______________________________________________<br>
rules-users mailing list<br>
<a href="mailto:rules-users@lists.jboss.org">rules-users@lists.jboss.org</a><br>
<a href="https://lists.jboss.org/mailman/listinfo/rules-users" target="_blank">https://lists.jboss.org/mailman/listinfo/rules-users</a><br></blockquote></div>