Author: bdaw Date: 2010-09-22 13:42:03 -0400 (Wed, 22 Sep 2010) New Revision: 415 Modified: idm/branches/1.1.0/parent/logging.properties idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreConfiguration.java idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreImpl.java idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/SimpleLDAPIdentityStoreConfiguration.java idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k3.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k8.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends12.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends20.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-redhatds.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-sunds.xml idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config.xml Log: - Support for "Simple Paged Results" LDAP extension to split big searches Modified: idm/branches/1.1.0/parent/logging.properties =================================================================== --- idm/branches/1.1.0/parent/logging.properties 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/parent/logging.properties 2010-09-22 17:42:03 UTC (rev 415) @@ -51,14 +51,14 @@ # Naming style for the output file: # (The output file is placed in the directory # defined by the "user.home" System property.) -java.util.logging.FileHandler.pattern=%h/java%u.log +java.util.logging.FileHandler.pattern=%h/jul-logs/java%u.log # Limiting size of output file in bytes: -java.util.logging.FileHandler.limit=50000 +java.util.logging.FileHandler.limit=5000000 # Number of output files to cycle through, by appending an # integer to the base file name: -java.util.logging.FileHandler.count=1 +java.util.logging.FileHandler.count=10000 # Style of output (Simple or XML): java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter \ No newline at end of file Modified: idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreConfiguration.java =================================================================== --- idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreConfiguration.java 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreConfiguration.java 2010-09-22 17:42:03 UTC (rev 415) @@ -86,4 +86,8 @@ boolean isCreateMissingContexts(); + boolean isPagedExtensionSupported(); + + int getPagedExtensionSize(); + } Modified: idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreImpl.java =================================================================== --- idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreImpl.java 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/LDAPIdentityStoreImpl.java 2010-09-22 17:42:03 UTC (rev 415) @@ -50,9 +50,11 @@ import org.picketlink.idm.spi.store.IdentityStoreInvocationContext; import org.picketlink.idm.spi.store.IdentityStoreSession; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -87,6 +89,8 @@ import javax.naming.ldap.InitialLdapContext; import javax.naming.ldap.LdapContext; import javax.naming.ldap.LdapName; +import javax.naming.ldap.PagedResultsControl; +import javax.naming.ldap.PagedResultsResponseControl; import javax.naming.ldap.SortControl; /** @@ -3316,13 +3320,11 @@ LdapContext ldapContext = getLDAPContext(ctx); - if (ldapContext != null) - { - ldapContext.setRequestControls(requestControls); - } + //Reset request controls + ldapContext.setRequestControls(null); + + List<SearchResult> results = null; - NamingEnumeration results = null; - List<SearchResult> finalResults; try @@ -3357,14 +3359,14 @@ if (filterArgs == null) { - results = ldapContext.search(jndiName, filter, searchControls); + results = searchLDAP(ldapContext, jndiName, filter, null, searchControls, requestControls); } else { - results = ldapContext.search(jndiName, filter, filterArgs, searchControls); + results = searchLDAP(ldapContext, jndiName, filter, filterArgs, searchControls, requestControls); } - List toReturn = Tools.toList(results); + List toReturn = results; if (log.isLoggable(Level.FINER) && toReturn != null) { @@ -3387,13 +3389,13 @@ if (filterArgs == null) { - results = ldapContext.search(jndiName, filter, searchControls); + results = searchLDAP(ldapContext, jndiName, filter, null, searchControls, requestControls); } else { - results = ldapContext.search(jndiName, filter, filterArgs, searchControls); + results = searchLDAP(ldapContext, jndiName, filter, filterArgs, searchControls, requestControls); } - List singleResult = Tools.toList(results); + List singleResult = results; if (log.isLoggable(Level.FINER) && merged != null) { @@ -3401,7 +3403,6 @@ } merged.addAll(singleResult); - results.close(); } @@ -3411,10 +3412,6 @@ } finally { - if (results != null) - { - results.close(); - } ldapContext.close(); } @@ -3434,6 +3431,162 @@ return finalResults; } + protected List<SearchResult> searchLDAP(LdapContext ldapContext, + Name jndiName, + String filter, + Object[] filterArgs, + SearchControls searchControls, + Control[] requestControls) throws NamingException + { + + // Add paged controls if needed + + int pageSize = configuration.getPagedExtensionSize(); + + if (configuration.isPagedExtensionSupported()) + { + if (log.isLoggable(Level.FINER)) + { + log.finer("Adding 'Simple Paged Results' extension to LDAP search with page size: " + pageSize); + } + + try + { + // Merge existing controls with PageResultscontrol + if (requestControls != null) + { + + List<Control> controls = new ArrayList<Control>(); + controls.addAll(Arrays.asList(requestControls)); + controls.add(new PagedResultsControl(pageSize, Control.CRITICAL)); + + ldapContext.setRequestControls(controls.toArray(new Control[controls.size()])); + } + else + { + ldapContext.setRequestControls(new Control[]{ + new PagedResultsControl(pageSize, Control.CRITICAL) + }); + } + } + catch (IOException e) + { + if (log.isLoggable(Level.INFO)) + { + log.log(Level.INFO, "Could not add 'Simple Paged Results' extension: ", e); + } + } + } + else + { + ldapContext.setRequestControls(requestControls); + } + + int count = 0; + byte[] cookie = null; + int total = -1; + + List<SearchResult> results = new LinkedList<SearchResult>(); + + + while(true) + { + count++; + + //System.err.println("Search loop count = " + count); + + // Perform the search + + NamingEnumeration resultsEnumeration = null; + + if (filterArgs == null) + { + resultsEnumeration = ldapContext.search(jndiName, filter, searchControls); + } + else + { + resultsEnumeration = ldapContext.search(jndiName, filter, filterArgs, searchControls); + } + + if (resultsEnumeration != null) + { + results.addAll(Tools.toList(resultsEnumeration)); + resultsEnumeration.close(); + } + + if (configuration.isPagedExtensionSupported()) + { + + + // Examine the paged results control response + Control[] controls = ldapContext.getResponseControls(); + if(controls!=null) + { + for(int k = 0; k<controls.length; k++) + { + if(controls[k] instanceof PagedResultsResponseControl) + { + PagedResultsResponseControl prrc = + (PagedResultsResponseControl)controls[k]; + total = prrc.getResultSize(); + cookie = prrc.getCookie(); + } + } + } + + if (log.isLoggable(Level.FINER)) + { + log.finer("'Simple Paged Results' extension search pass: " + count + + "; pageSize: " + pageSize + + "; total: " + total); + } + } + + if(cookie == null) + { + break; + } + + if (configuration.isPagedExtensionSupported()) + { + // Re-activate paged results + try + { + if (requestControls != null) + { + List<Control> controls = new LinkedList<Control>(); + controls.addAll(Arrays.asList(requestControls)); + controls.add(new PagedResultsControl(pageSize, cookie, Control.CRITICAL)); + + ldapContext.setRequestControls(controls.toArray(new Control[controls.size()])); + + } + else + { + ldapContext.setRequestControls(new Control[]{ + new PagedResultsControl(pageSize, cookie, Control.CRITICAL)}); + } + } + catch (IOException e) + { + if (log.isLoggable(Level.INFO)) + { + log.log(Level.INFO, "Could not add 'Simple Paged Results' extension: ", e); + } + } + } + } + + if (log.isLoggable(Level.FINER) && results != null) + { + log.log(Level.FINER, "LDAP Search returned ", + results.size() + " results"); + } + + + return results; + + } + // HELPER private LDAPIdentityObjectImpl getSafeLDAPIO(IdentityStoreInvocationContext ctx, IdentityObject io) throws IdentityException Modified: idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/SimpleLDAPIdentityStoreConfiguration.java =================================================================== --- idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/SimpleLDAPIdentityStoreConfiguration.java 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-ldap/src/main/java/org/picketlink/idm/impl/store/ldap/SimpleLDAPIdentityStoreConfiguration.java 2010-09-22 17:42:03 UTC (rev 415) @@ -85,6 +85,10 @@ private final boolean sortExtensionSupported; + private final boolean pagedExtensionSupported; + + private final int pagedExtensionSize; + private final boolean createMissingContexts; @@ -132,6 +136,12 @@ public static final String SORT_EXTENSION_SUPPORTED = "sortExtensionSupported"; + public static final String PAGE_EXTENSION_SUPPORTED = "pagedResultsExtensionSupported"; + + public static final String PAGE_EXTENSION_SIZE = "pagedResultsExtensionSize"; + + public static final int PAGE_EXTENSION_SIZE_DEFAULT = 1000; + public static final String CREATE_MISSING_CONTEXTS = "createMissingContexts"; public SimpleLDAPIdentityStoreConfiguration(IdentityStoreConfigurationMetaData storeMD) @@ -186,6 +196,27 @@ this.sortExtensionSupported = true; } + String pagedExtension = storeMD.getOptionSingleValue(PAGE_EXTENSION_SUPPORTED); + if (pagedExtension != null && pagedExtension.equalsIgnoreCase("true")) + { + this.pagedExtensionSupported = true; + } + else + { + this.pagedExtensionSupported = false; + } + + String pageSize = storeMD.getOptionSingleValue(PAGE_EXTENSION_SIZE); + + if (pageSize != null) + { + this.pagedExtensionSize = Integer.valueOf(pageSize); + } + else + { + this.pagedExtensionSize = PAGE_EXTENSION_SIZE_DEFAULT; + } + String createMissingContexts = storeMD.getOptionSingleValue(CREATE_MISSING_CONTEXTS); if (createMissingContexts != null && createMissingContexts.equalsIgnoreCase("true")) { @@ -498,4 +529,14 @@ { return createMissingContexts; } + + public boolean isPagedExtensionSupported() + { + return pagedExtensionSupported; + } + + public int getPagedExtensionSize() + { + return pagedExtensionSize; + } } Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k3.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k3.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k3.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -889,6 +889,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k8.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k8.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-msad-2k8.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -889,6 +889,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends12.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends12.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends12.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -899,6 +899,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends20.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends20.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-opends20.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -899,6 +899,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-redhatds.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-redhatds.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-redhatds.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -888,6 +888,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-sunds.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-sunds.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config-sunds.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -888,6 +888,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores> Modified: idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config.xml =================================================================== --- idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config.xml 2010-09-20 21:08:57 UTC (rev 414) +++ idm/branches/1.1.0/picketlink-idm-testsuite/src/test/resources/test-identity-config.xml 2010-09-22 17:42:03 UTC (rev 415) @@ -937,6 +937,14 @@ <name>cache.configFile</name> <value>jboss-cache.xml</value> </option> + <option> + <name>pagedResultsExtensionSupported</name> + <value>true</value> + </option> + <option> + <name>pagedResultsExtensionSize</name> + <value>1</value> + </option> </options> </identity-store> </identity-stores>