<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<base href="https://hibernate.atlassian.net">
<title>Message Title</title>
</head>
<body class="jira" style="color: #333333; font-family: Arial, sans-serif; font-size: 14px; line-height: 1.429">
<table id="background-table" cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0; background-color: #f5f5f5; border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0" bgcolor="#f5f5f5">
<!-- header here -->
<tbody>
<tr>
<td id="header-pattern-container" style="padding: 0; border-collapse: collapse; padding: 10px 20px">
<table id="header-pattern" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<td id="header-avatar-image-container" valign="top" style="padding: 0; border-collapse: collapse; vertical-align: top; width: 32px; padding-right: 8px" width="32"> <img id="header-avatar-image" class="image_fix" src="https://secure.gravatar.com/avatar/2a8bdd4ffd282b7185c74b52ab452617?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FYR-6.png&size=48&s=48" height="32" width="32" border="0" style="border-radius: 3px; vertical-align: top"> </td>
<td id="header-text-container" valign="middle" style="padding: 0; border-collapse: collapse; vertical-align: middle; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px"> <a class="user-hover" rel="yrodiere" style="color:#6c797f;; color: #3b73af; text-decoration: none" id="email_yrodiere" href="https://hibernate.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A58fa1ced-171a-4c00-97e8-5d70d442cc4b"> Yoann Rodière </a> <strong>created</strong> an issue </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="email-content-container" style="padding: 0; border-collapse: collapse; padding: 0 20px">
<table id="email-content-table" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0; border-spacing: 0; border-collapse: separate">
<tbody>
<tr>
<!-- there needs to be content in the cell for it to render in some clients -->
<td class="email-content-rounded-top mobile-expand" style="padding: 0; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 15px; background-color: #ffffff; border-left: 1px solid #cccccc; border-top: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom: 0; border-top-right-radius: 5px; border-top-left-radius: 5px; height: 10px; line-height: 10px; padding: 0 15px 0 16px; mso-line-height-rule: exactly" height="10" bgcolor="#ffffff"> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand " style="padding: 0; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff">
<table class="page-title-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<td class="page-title-pattern-first-line " style="padding: 0; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; padding-top: 10px"> <a href="https://hibernate.atlassian.net/browse/HSEARCH?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" style="color: #3b73af; text-decoration: none">Hibernate Search</a> / <a href="https://hibernate.atlassian.net/browse/HSEARCH-3878?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" style="color: #3b73af; text-decoration: none"><img src="cid:jira-generated-image-avatar-5f14778e-a506-453f-9a70-55a992d14077" height="16" width="16" border="0" align="absmiddle" alt="Improvement" style="vertical-align: text-bottom"></a> <a href="https://hibernate.atlassian.net/browse/HSEARCH-3878?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" style="color: #3b73af; text-decoration: none">HSEARCH-3878</a> </td>
</tr>
<tr>
<td style="vertical-align: top;; padding: 0; border-collapse: collapse; padding-right: 5px; font-size: 20px; line-height: 30px; mso-line-height-rule: exactly" class="page-title-pattern-header-container"> <span class="page-title-pattern-header" style="font-family: Arial, sans-serif; padding: 0; font-size: 20px; line-height: 30px; mso-text-raise: 2px; mso-line-height-rule: exactly; vertical-align: middle"> <a href="https://hibernate.atlassian.net/browse/HSEARCH-3878?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" style="color: #3b73af; text-decoration: none">Maximize utilization of database connections during mass indexing</a> </span> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand wrapper-special-margin" style="padding: 0; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff; padding-top: 10px; padding-bottom: 5px" bgcolor="#ffffff">
<table class="keyvalue-table" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Issue Type:</th>
<td class="has-icon" style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <img src="cid:jira-generated-image-avatar-5f14778e-a506-453f-9a70-55a992d14077" height="16" width="16" border="0" align="absmiddle" alt="Improvement" style="vertical-align: text-bottom"> Improvement </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Assignee:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> Unassigned </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Attachments:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> queueCount10_bulkSize250.svg </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Components:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> mapper-orm </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Created:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> 01/Apr/2020 08:19 AM </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Fix Versions:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> 6.0.0.Beta-backlog-low-priority </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Priority:</th>
<td class="has-icon" style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <img src="cid:jira-generated-image-static-major-b4597d43-4505-450b-970e-05f4fc913449" height="16" width="16" border="0" align="absmiddle" alt="Major" style="vertical-align: text-bottom"> Major </td>
</tr>
<tr>
<th style="color: #707070; font: normal 14px/20px Arial, sans-serif; text-align: left; vertical-align: top; padding: 2px 0">Reporter:</th>
<td style="padding: 0; border-collapse: collapse; font: normal 14px/20px Arial, sans-serif; padding: 2px 0 2px 5px; vertical-align: top"> <a class="user-hover" rel="yrodiere" style="color:#6c797f;; color: #3b73af; text-decoration: none" id="email_yrodiere" href="https://hibernate.atlassian.net/secure/ViewProfile.jspa?accountId=557058%3A58fa1ced-171a-4c00-97e8-5d70d442cc4b"> Yoann Rodière </a> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand issue-description-container" style="padding: 0; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff; padding-top: 5px; padding-bottom: 10px" bgcolor="#ffffff">
<table class="text-paragraph-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 2px">
<tbody>
<tr>
<td class="text-paragraph-pattern-container mobile-resize-text " style="padding: 0; border-collapse: collapse; padding: 0 0 10px"> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0; margin-top: 0">The mass indexer threads responsible for loading entities from the database currently have a loop that looks like this:</p>
<ul>
<li>load X entities from the database</li>
<li>submit indexing requests for each entity</li>
<li>wait for indexing to finish for these entities</li>
<li>repeat</li>
</ul> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">The "wait" step means that, while the backend is busy indexing, the thread will not load anything from the database. This effectively means that it will hold a reference to a database connection for nothing during that time.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">This means that indexing doesn't execute in parallel of database loading, and as a result the execution time of mass indexing is probably close to the <b>sum</b> of the time spent loading entities and the time spent indexing. Ideally, we'd want those two operations to happen in parallel, so that the execution time of mass indexing is close to the <b>maximum</b> instead of the sum.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">You can see the effect quite clearly in <span class="nobr"><a href="https://hibernate.atlassian.net/secure/attachment/49646/49646_queueCount10_bulkSize250.svg" title="queueCount10_bulkSize250.svg attached to HSEARCH-3878" data-attachment-type="file" data-attachment-name="queueCount10_bulkSize250.svg" data-media-services-type="file" data-media-services-id="82cc2834-4e9d-44f3-9f82-0130feb2b69f" rel="noreferrer" style="color: #3b73af; text-decoration: none">queueCount10_bulkSize250.svg<sup><img class="rendericon" src="cid:jira-generated-image-static-link_attachment_7-15d8dbc4-4284-4a5e-b372-73b51be30f7c" height="7" width="7" align="absmiddle" alt="" border="0"></sup></a></span>, which is a gantt chart of the indexing tasks in an Elasticsearch backend during mass indexing. From time to time, almost all executors are idle, because entities have not been submitted yet... If entity loading happened in parallel, the indexing executes would be less likely to stay idle.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">One solution to this problem would be to move to a loop like this:</p>
<ul>
<li>load X entities from the database (load #1)</li>
<li>submit indexing requests for each entity</li>
<li>load X entities from the database (load #2)</li>
<li>submit indexing requests for each entity</li>
<li>wait for indexing to finish for load #1</li>
<li>load X entities from the database (load #3)</li>
<li>submit indexing requests for each entity</li>
<li>wait for indexing to finish for load #2</li>
<li>load X entities from the database (load #4)</li>
<li>submit indexing requests for each entity</li>
<li>wait for indexing to finish for load #3</li>
<li>...</li>
</ul> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">This should greatly reduce the amount of waiting in the loading thread, since we will be loading the next batch of entities while the previous batch is being indexed. And this preserves the characteristics of the previous algorithm: if database loading is much faster than indexing (who knows...), we won't flood indexing queues with indexing requests, and will only ever have at most two batches of entities pending at any given time.</p> <p style="margin-top:0;margin-bottom:10px;; margin: 10px 0 0">This would probably be rather easy to implement, since the completion of indexing is modelled by <tt>CompletableFutures</tt>: we just have to store the future of the last batch somewhere, and wait for it to finish.</p> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td class="email-content-main mobile-expand " style="padding: 0; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff"> <script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "EmailMessage",
"description": "View Issue",
"potentialAction": {
"@type": "ViewAction",
"target": "https://hibernate.atlassian.net/browse/HSEARCH-3878?inbox=true&",
"name": "View Issue"
},
"publisher": {
"@type": "Organization",
"name": "Atlassian",
"url": "https://www.atlassian.com"
}
}
</script>
<table id="actions-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px">
<tbody>
<tr>
<td id="actions-pattern-container" valign="middle" style="padding: 0; border-collapse: collapse; padding: 10px 0 10px 24px; vertical-align: middle; padding-left: 0">
<table align="left" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<td class="actions-pattern-action-icon-container" style="padding: 0; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0; vertical-align: middle"> <a href="https://hibernate.atlassian.net/browse/HSEARCH-3878#add-comment?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none"> <img class="actions-pattern-action-icon-image" src="cid:jira-generated-image-static-comment-icon-59df938c-d885-497f-bb7e-99ac4ece402a" alt="Add Comment" title="Add Comment" height="16" width="16" border="0" style="vertical-align: middle"> </a> </td>
<td class="actions-pattern-action-text-container" style="padding: 0; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px"> <a href="https://hibernate.atlassian.net/browse/HSEARCH-3878#add-comment?atlOrigin=eyJpIjoiMmJjZGFmMWU0NjViNDBhMTk5MTg4MGJlZWMwYTZhZDYiLCJwIjoiaiJ9" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none">Add Comment</a> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table> </td>
</tr>
<!-- there needs to be content in the cell for it to render in some clients -->
<tr>
<td class="email-content-rounded-bottom mobile-expand" style="padding: 0; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 5px; line-height: 5px; background-color: #ffffff; border-top: 0; border-left: 1px solid #cccccc; border-bottom: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom-right-radius: 5px; border-bottom-left-radius: 5px; mso-line-height-rule: exactly" height="5" bgcolor="#ffffff"> </td>
</tr>
</tbody>
</table> </td>
</tr>
<tr>
<td id="footer-pattern" style="padding: 0; border-collapse: collapse; padding: 12px 20px">
<table id="footer-pattern-container" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<td id="footer-pattern-mobile-separated-links" class="mobile-resize-text" width="100%" colspan="2" style="padding: 0; border-collapse: collapse; color: #999999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px"> Get Jira notifications on your phone! Download the Jira Cloud app for <a href="https://play.google.com/store/apps/details?id=com.atlassian.android.jira.core&referrer=utm_source%3DNotificationLink%26utm_medium%3DEmail" style="color: #3b73af; text-decoration: none">Android</a> or <a href="https://itunes.apple.com/app/apple-store/id1006972087?pt=696495&ct=EmailNotificationLink&mt=8" style="color: #3b73af; text-decoration: none">iOS</a>
<hr> </td>
</tr>
<tr>
<td id="footer-pattern-text" class="mobile-resize-text" width="100%" style="padding: 0; border-collapse: collapse; color: #999999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px"> This message was sent by Atlassian Jira <span id="footer-build-information">(v1001.0.0-SNAPSHOT#100123-<span title="26275c0ff0aca8c17e6c3e023ea3e02c68410ed4" data-commit-id="26275c0ff0aca8c17e6c3e023ea3e02c68410ed4}">sha1:26275c0</span>)</span> </td>
<td id="footer-pattern-logo-desktop-container" valign="top" style="padding: 0; border-collapse: collapse; padding-left: 20px; vertical-align: top">
<table style="border-collapse: collapse; mso-table-lspace: 0; mso-table-rspace: 0">
<tbody>
<tr>
<td id="footer-pattern-logo-desktop-padding" style="padding: 0; border-collapse: collapse; padding-top: 3px; opacity: 0.150"> <img id="footer-pattern-logo-desktop" src="cid:jira-generated-image-static-footer-desktop-logo-fcf95e3b-75ac-459b-b1a5-ae8a64978500" alt="Atlassian logo" title="Atlassian logo" width="192" height="24" class="image_fix"> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table> </td>
</tr>
</tbody>
</table>
</body>
</html>