You need to specify the module with your custom lm in the standalone.xml file:
<security-domain name="MyCustomSecurityDomain" cache-type="default">
<authentication>
<login-module code="com.my.CustomSecurityLoginModule" flag="required" module="deployment.MyEar.ear.SecurityModule.jar">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
</authentication>
</security-domain>