Author: mladen.turk(a)jboss.com Date: 2008-06-11 01:49:17 -0400 (Wed, 11 Jun 2008) New Revision: 1689 Added: sandbox/aloha/httpd/modules/manager/mm_connection.c Modified: sandbox/aloha/httpd/modules/manager/NMAKEmanager sandbox/aloha/httpd/modules/manager/mm_api.h sandbox/aloha/httpd/modules/manager/mm_app.c sandbox/aloha/httpd/modules/manager/mm_balancer.c sandbox/aloha/httpd/modules/manager/mm_host.c sandbox/aloha/httpd/modules/manager/mm_module.c sandbox/aloha/httpd/modules/manager/mm_util.c sandbox/aloha/httpd/modules/manager/modules.mk Log: Some native updates - Connection pool Modified: sandbox/aloha/httpd/modules/manager/NMAKEmanager =================================================================== --- sandbox/aloha/httpd/modules/manager/NMAKEmanager 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/NMAKEmanager 2008-06-11 05:49:17 UTC (rev 1689) @@ -56,6 +56,7 @@ $(WORKDIR)\mm_util.obj \ $(WORKDIR)\mm_app.obj \ $(WORKDIR)\mm_asmm.obj \ + $(WORKDIR)\mm_connection.obj \ $(WORKDIR)\mm_cookie.obj \ $(WORKDIR)\mm_balancer.obj \ $(WORKDIR)\mm_host.obj \ Modified: sandbox/aloha/httpd/modules/manager/mm_api.h =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_api.h 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_api.h 2008-06-11 05:49:17 UTC (rev 1689) @@ -55,6 +55,10 @@ #include "apr_optional.h" #include "apr_uuid.h" #include "apr_version.h" +#include "apr_atomic.h" +#include "apr_thread_cond.h" +#include "apr_ring.h" + #define APR_WANT_STRFUNC #include "apr_want.h" @@ -110,7 +114,8 @@ #define MM_DEFAULT_DBROOT "logs" #define MM_LOCK_NAME "/_mm_lock" #define MM_DEFAULT_HANDLE "/server-manager" -#define MM_HANDLER_NAME "Manager-Handler" +#define MM_MHANDLER_NAME "Manager-Handler" +#define MM_CHANDLER_NAME "Cluster-Handler" #define MM_CMDHEADER_NAME "X-Server-Command" #define MM_RESHEADER_NAME "X-Server-Resource" #define MM_PINGHEADER_NAME "X-Server-Ping" @@ -249,8 +254,8 @@ apr_cpystrn((t), (f), (s)) #define MM_MAYBEQ_RPUTS(r, p, s) \ - if ((p) && (strchr((s), ' ') || strchr((s), ';') || \ - strchr((s), ',') || strchr((s), '\t'))) \ + if ((p) && (ap_strchr_c((s), ' ') || ap_strchr_c((s), ';') || \ + ap_strchr_c((s), ',') || ap_strchr_c((s), '\t'))) \ ap_rvputs((r), "\"", (s), "\"", NULL); \ else ap_rvputs((r), (s), NULL) @@ -534,6 +539,11 @@ volatile apr_uint64_t transferred; }; +/* For external modules */ +extern module AP_MODULE_DECLARE_DATA manager_module; + +#define MM_REQUEST_REC_NOTE "mm-request-rec" + /** * Data structure representing a Manager request */ @@ -541,9 +551,61 @@ struct mm_request_rec { request_rec *r; mm_server_conf_t *sc; + apr_uint32_t app_id; + apr_uint32_t host_id; + mmdb_host_t *host; + mmdb_hostlist_t *hostlist; + mmdb_balancer_t *balancer; + mmdb_app_t *app; + mmdb_app_pool_t *app_pool; }; /** + * Connection pool data structure. + */ +typedef struct mm_connection_pool_t mm_connection_pool_t; + +/** + * Connection paraneters data structure. + */ +typedef struct mm_conn_parameters_t mm_conn_parameters_t; +struct mm_conn_parameters_t { + mm_protocol_e protocol; /** Connection protocol used */ + apr_sockaddr_t *addr; /** Preparsed remote address info */ + const char *hostname; /** Remote host */ + apr_port_t port; /** Remote protocol */ + apr_interval_time_t timeout; /** Connection timeout */ +}; + +/** + * Connection data structure. + */ +typedef struct mm_connection_t mm_connection_t; +struct mm_connection_t { + apr_pool_t *pool; + mm_connection_pool_t *conn_pool; + mm_conn_parameters_t *params; /** Connection parameters */ + apr_socket_t *sock; /** Backend connection socket */ + conn_rec *client; /** Client conn_rec associated with this + * connection. + */ + conn_rec *conn; /** Backend conn_rec created from client + * conn_rec and our socket. + */ + void *data; /** Per connection data */ + apr_uint32_t id; /** Our connection id */ + apr_uint32_t flags; /** Conection flags */ + int close; /** Close the connection and remove + * it from the connection pool. + */ + int reuse; /** Reuse the connection when returning + * to the connection pool. + */ + apr_time_t freed; + APR_RING_ENTRY(mm_connection_t) link; +}; + +/** * Manager private. * Initialize callback system. * @param pool The pool to use for callbacks. @@ -805,13 +867,13 @@ * Find registered resource provider. * @param name Resource provider name. */ -MANAGER_DECLARE(mm_resource_provider_t *)mm_lookup_resource_provider(const char *name); +MANAGER_DECLARE(mm_resource_provider_t *) mm_lookup_resource_provider(const char *name); /** * List registered resource provider. * @param pool The poll to allocate list. */ -MANAGER_DECLARE(mm_resource_list_t *)mm_list_resource_providers(apr_pool_t *pool); +MANAGER_DECLARE(mm_resource_list_t *) mm_list_resource_providers(apr_pool_t *pool); /** * Run registered resource providers. @@ -819,11 +881,71 @@ * @param op Resource operation. @see mm_resource_op_e * @param pool The pool to create any storage from */ -MANAGER_DECLARE(void)mm_run_resource_providers(mm_server_conf_t *cfg, - mm_resource_op_e op, - apr_pool_t *pool); +MANAGER_DECLARE(void )mm_run_resource_providers(mm_server_conf_t *cfg, + mm_resource_op_e op, + apr_pool_t *pool); /** + * Create connection pool. + * @param c The newly created connection pool descriptor. + * @param min_conn Minimum nuber of connections + * @param max_conn Maximum nuber of connections + * @param max_idle Number of connections that will always be + * kept in this connection pool. If zero no + * connection will be maintained + * @param ttl Time To Live for idle connections + * @param acquire Acquire timeout. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_create(mm_connection_pool_t **c, + int min_conn, + int max_conn, + int max_idle, + apr_interval_time_t ttl, + apr_interval_time_t acquire, + apr_pool_t *pool); + +/** + * Clear connection pool and destroy all the connections. + * @param c The connection pool descriptor to clear. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_clear(mm_connection_pool_t *c); + +/** + * Destroy connection pool. + * @param c The connection pool descriptor to destroy. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_destroy(mm_connection_pool_t *c); + +/** + * Maintain connection pool. + * @param c The connection pool descriptor to maintain. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_maintain(mm_connection_pool_t *c); + +/** + * Get the connection from the connection pool. + * @param c The connection pool to use. + * @param conn The newly acquired connection. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_pop(mm_connection_pool_t *c, + mm_connection_t **conn); + +/** + * Return the connection to the connection pool. + * @param c The connection pool to use. + * @param conn The connection to return. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_ret(mm_connection_t *conn); + +/** + * Remove the connection to the connection pool. + * @param c The connection pool to use. + * @param conn The connection to remove. This will destroy the connection + * as well. + */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_del(mm_connection_t *conn); + +/** * Manager private. * Register static Manager resource */ Modified: sandbox/aloha/httpd/modules/manager/mm_app.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_app.c 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_app.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -491,7 +491,7 @@ int wdone = 0; int cblind = 0; int cnt = 0; - int wcm; + int wcm = 0; const apr_array_header_t *ca = apr_table_elts(cmd_table); apr_table_entry_t *ce = (apr_table_entry_t *)ca->elts; apr_uint32_t next = 0; Modified: sandbox/aloha/httpd/modules/manager/mm_balancer.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_balancer.c 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_balancer.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -333,7 +333,6 @@ (*balancer)->method = mm_balance_r; (*balancer)->retries = 0; /* Unlimited */ (*balancer)->timeout = cfg->main_server->timeout; - return APR_SUCCESS; } @@ -513,10 +512,16 @@ if (!balancer_db) return APR_ENOMEM; /* Create _default_ balancer */ - return create_balancer(cfg, MM_DEFAULT_NAME, - balancer_db, - &balancer, - &balancer_id); + if (create_balancer(cfg, MM_DEFAULT_NAME, + balancer_db, + &balancer, + &balancer_id) == APR_SUCCESS) { + /* Activate default balancer at startup */ + balancer->status = mm_state_active; + return APR_SUCCESS; + } + else + return APR_ENOMEM; } static apr_status_t ini_balancer(mm_server_conf_t *cfg, Added: sandbox/aloha/httpd/modules/manager/mm_connection.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_connection.c (rev 0) +++ sandbox/aloha/httpd/modules/manager/mm_connection.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -0,0 +1,512 @@ +/* + * ModManager - JBoss Aloha + * + * Copyright(c) 2008 Red Hat Middleware, LLC, + * and individual contributors as indicated by the @authors tag. + * See the copyright.txt in the distribution for a + * full listing of individual contributors. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library in the file COPYING.LIB; + * if not, write to the Free Software Foundation, Inc., + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + * + * @author Mladen Turk + */ + +/* Connection and ConnectionPool utils */ +#include "mm_api.h" + +static volatile apr_uint32_t conn_id_counter = 0; + +/** + * A single resource element. + */ +typedef struct mm_res_t mm_res_t; +struct mm_res_t { + apr_time_t freed; + mm_connection_t *conn; + APR_RING_ENTRY(mm_res_t) link; +}; + +/** + * A ring of resources representing the list of available resources. + */ +APR_RING_HEAD(mm_conn_ring_t, mm_connection_t); +typedef struct mm_conn_ring_t mm_conn_ring_t; + +struct mm_connection_pool_t { + apr_pool_t *pool; + apr_reslist_t *lres; /* Connection resource list */ + int num_conn; /* Total number of managed connections */ + int num_idle; /* Number of available connections */ + int num_used; /* Number of used connections */ + int min_conn; + int max_conn; + int max_idle; + apr_interval_time_t ttl; /* TTL when we have too many resources */ + apr_interval_time_t acquire; /* Timeout for waiting on resource */ + apr_thread_mutex_t *listlock; + apr_thread_cond_t *avail; + mm_conn_ring_t ilist; + mm_conn_ring_t flist; + mm_conn_ring_t ulist; + +}; + +static apr_status_t connection_cleanup(void *slot) +{ + mm_connection_t *conn = (mm_connection_t *)slot; + + /* Do any additional cleanup for this connection. + */ + dbprintf("Connection %d cleanup %p -> %p (%p)", conn->id, conn, + conn->pool, conn->conn_pool); + if (conn->pool) { + /* Mark we have run the cleanup. + * connection pool cleanup will check for our pool and + * if NULL won't run the destroy second time. + */ + conn->pool = NULL; + } + return APR_SUCCESS; +} + +/** + * Create a new resource and return it. + * Assumes: that the reslist is locked. + */ +static void create_connection(mm_connection_pool_t *cp, + mm_connection_t **conn) +{ + if (!APR_RING_EMPTY(&cp->flist, mm_connection_t, link)) { + *conn = APR_RING_FIRST(&cp->flist); + APR_RING_REMOVE(*conn, link); + } + else { + *conn = apr_pcalloc(cp->pool, sizeof(mm_connection_t)); + apr_pool_create(&(*conn)->pool, cp->pool); + apr_pool_cleanup_register((*conn)->pool, *conn, connection_cleanup, + apr_pool_cleanup_null); + } + (*conn)->conn_pool = cp; + (*conn)->id = apr_atomic_inc32(&conn_id_counter); + if (!(*conn)->pool) { + apr_pool_create(&(*conn)->pool, cp->pool); + apr_pool_cleanup_register((*conn)->pool, *conn, connection_cleanup, + apr_pool_cleanup_null); + } +} + +/** + * Destroy a single idle resource. + * Assumes: that the reslist is locked. + */ +static void clear_connection(mm_connection_t *conn) +{ + dbprintf("Connection %d clear %p -> %p", conn->id, conn, conn->pool); + if (conn->pool) { + apr_pool_t *pool = conn->pool; + + conn->conn_pool = NULL; + apr_pool_cleanup_run(pool, conn, connection_cleanup); + apr_pool_clear(pool); + memset(conn, 0, sizeof(mm_connection_t)); + conn->pool = pool; + apr_pool_cleanup_register(conn->pool, conn, connection_cleanup, + apr_pool_cleanup_null); + } +} + + +static apr_status_t connection_pool_cleanup(void *context) +{ + apr_status_t rv = APR_SUCCESS; + mm_connection_pool_t *cp = (mm_connection_pool_t *)context; + mm_connection_t *conn; + + apr_thread_mutex_lock(cp->listlock); + dbprintf("Connection pool cleanup %d %d", cp->num_idle, cp->num_used); + + while (cp->num_idle > 0) { + conn = APR_RING_FIRST(&cp->ilist); + APR_RING_REMOVE(conn, link); + + if (conn->pool) { + conn->conn_pool = NULL; + apr_pool_cleanup_run(conn->pool, conn, connection_cleanup); + apr_pool_destroy(conn->pool); + } + cp->num_idle--; + cp->num_conn--; + } + + while (cp->num_used > 0) { + conn = APR_RING_FIRST(&cp->ulist); + APR_RING_REMOVE(conn, link); + + if (conn->pool) { + conn->conn_pool = NULL; + apr_pool_cleanup_run(conn->pool, conn, connection_cleanup); + apr_pool_destroy(conn->pool); + } + cp->num_used--; + cp->num_conn--; + } + + apr_thread_mutex_destroy(cp->listlock); + apr_thread_cond_destroy(cp->avail); + cp->pool = NULL; + + return rv; +} + +/** + * Perform routine maintenance on the resource list. This call + * may instantiate new resources or expire old resources. + */ +static apr_status_t run_maintenance(mm_connection_pool_t *cp) +{ + apr_time_t now; + apr_status_t rv; + mm_connection_t *conn; + int created = 0; + + apr_thread_mutex_lock(cp->listlock); + + /* Check if we need to create more resources, and if we are allowed to. */ + while (cp->num_idle < cp->min_conn && cp->num_conn < cp->max_conn) { + create_connection(cp, &conn); + APR_RING_INSERT_HEAD(&cp->ilist, conn, mm_connection_t, link); + conn->freed = apr_time_now(); + cp->num_idle++; + cp->num_conn++; + /* If someone is waiting on that guy, wake them up. */ + rv = apr_thread_cond_signal(cp->avail); + if (rv != APR_SUCCESS) { + apr_thread_mutex_unlock(cp->listlock); + return rv; + } + created++; + } + + /* We don't need to see if we're over the max if we were under it before */ + if (created) { + apr_thread_mutex_unlock(cp->listlock); + return APR_SUCCESS; + } + + /* Check if we need to expire old resources */ + now = apr_time_now(); + while (cp->num_idle > cp->max_idle && cp->num_idle > 0) { + /* Peak at the last resource in the list */ + conn = APR_RING_LAST(&cp->ilist); + /* See if the oldest entry should be expired */ + if ((now - conn->freed) < cp->ttl) { + /* If this entry is too young, none of the others + * will be ready to be expired either, so we are done. */ + break; + } + APR_RING_REMOVE(conn, link); + cp->num_idle--; + cp->num_conn--; + clear_connection(conn); + APR_RING_INSERT_TAIL(&cp->flist, conn, mm_connection_t, link); + } + + apr_thread_mutex_unlock(cp->listlock); + return APR_SUCCESS; +} + + +MANAGER_DECLARE(apr_status_t) mm_connection_pool_create(mm_connection_pool_t **c, + int min_conn, + int max_conn, + int max_idle, + apr_interval_time_t ttl, + apr_interval_time_t acquire, + apr_pool_t *pool) +{ + apr_status_t rv; + mm_connection_pool_t *cp; + mm_connection_t *conn; + + /* Parameter sanity check */ + if (max_conn < 1) + max_conn = 1; + if (max_idle == 0 || max_idle > max_conn) + max_idle = max_conn; + if (min_conn > max_conn) + min_conn = max_conn; + if (min_conn > max_idle) + max_idle = min_conn; + cp = (mm_connection_pool_t *)apr_pcalloc(pool, sizeof(mm_connection_pool_t)); + cp->pool = pool; + cp->min_conn = min_conn; + cp->max_conn = max_conn; + cp->max_idle = max_idle; + cp->ttl = ttl; + cp->acquire = acquire; + + APR_RING_INIT(&cp->ilist, mm_connection_t, link); + APR_RING_INIT(&cp->flist, mm_connection_t, link); + APR_RING_INIT(&cp->ulist, mm_connection_t, link); + + if ((rv = apr_thread_mutex_create(&cp->listlock, + APR_THREAD_MUTEX_DEFAULT, + pool)) != APR_SUCCESS) { + return rv; + } + if ((rv = apr_thread_cond_create(&cp->avail, pool)) != APR_SUCCESS) { + return rv; + } + + /* Create initial number of connection slots + */ + while (cp->num_idle < cp->min_conn && cp->num_conn < cp->max_conn) { + create_connection(cp, &conn); + APR_RING_INSERT_HEAD(&cp->ilist, conn, mm_connection_t, link); + conn->freed = apr_time_now(); + cp->num_idle++; + cp->num_conn++; + } + /* Register our cleanup. + */ + apr_pool_cleanup_register(cp->pool, cp, + connection_pool_cleanup, + apr_pool_cleanup_null); + *c = cp; + return rv; +} + +MANAGER_DECLARE(apr_status_t) mm_connection_pool_clear(mm_connection_pool_t *c) +{ + apr_status_t rv = APR_SUCCESS; + mm_connection_t *conn; + + apr_thread_mutex_lock(c->listlock); + + while (c->num_idle > 0) { + conn = APR_RING_FIRST(&c->ilist); + APR_RING_REMOVE(conn, link); + + clear_connection(conn); + c->num_idle--; + c->num_conn--; + APR_RING_INSERT_TAIL(&c->flist, conn, mm_connection_t, link); + + } + + while (c->num_used > 0) { + conn = APR_RING_FIRST(&c->ulist); + APR_RING_REMOVE(conn, link); + + clear_connection(conn); + c->num_used--; + c->num_conn--; + APR_RING_INSERT_TAIL(&c->flist, conn, mm_connection_t, link); + } + /* Create back initial idle connections */ + while (c->num_idle < c->min_conn && c->num_conn < c->max_conn) { + create_connection(c, &conn); + APR_RING_INSERT_HEAD(&c->ilist, conn, mm_connection_t, link); + conn->freed = apr_time_now(); + c->num_idle++; + c->num_conn++; + } + + apr_thread_mutex_unlock(c->listlock); + return rv; +} + +MANAGER_DECLARE(apr_status_t) mm_connection_pool_destroy(mm_connection_pool_t *c) +{ + return apr_pool_cleanup_run(c->pool, c, connection_pool_cleanup); +} + +MANAGER_DECLARE(apr_status_t) mm_connection_pool_maintain(mm_connection_pool_t *c) +{ + apr_time_t now; + apr_status_t rv; + mm_connection_t *conn; + int created = 0; + + apr_thread_mutex_lock(c->listlock); + + /* Check if we need to create more resources, and if we are allowed to. */ + while (c->num_idle < c->min_conn && c->num_conn < c->max_conn) { + create_connection(c, &conn); + APR_RING_INSERT_HEAD(&c->ilist, conn, mm_connection_t, link); + conn->freed = apr_time_now(); + c->num_idle++; + c->num_conn++; + /* If someone is waiting on that guy, wake them up. */ + rv = apr_thread_cond_signal(c->avail); + if (rv != APR_SUCCESS) { + apr_thread_mutex_unlock(c->listlock); + return rv; + } + created++; + } + + /* We don't need to see if we're over the max if we were under it before */ + if (created) { + apr_thread_mutex_unlock(c->listlock); + return APR_SUCCESS; + } + + now = apr_time_now(); + /* Check if we need to expire old resources */ + while (c->num_idle > c->max_idle && c->num_idle > 0) { + /* Peak at the last resource in the list */ + conn = APR_RING_LAST(&c->ilist); + /* See if the oldest entry should be expired */ + if ((now - conn->freed) < c->ttl) { + /* If this entry is too young, none of the others + * will be ready to be expired either, so we are done. */ + break; + } + APR_RING_REMOVE(conn, link); + c->num_idle--; + c->num_conn--; + clear_connection(conn); + APR_RING_INSERT_TAIL(&c->flist, conn, mm_connection_t, link); + } + + apr_thread_mutex_unlock(c->listlock); + return APR_SUCCESS; +} + +MANAGER_DECLARE(apr_status_t) mm_connection_pool_pop(mm_connection_pool_t *c, + mm_connection_t **conn) +{ + apr_status_t rv = APR_SUCCESS; + apr_time_t now; + + apr_thread_mutex_lock(c->listlock); + /* If there are idle connection on the available list, use + * them right away. + */ + now = apr_time_now(); + while (c->num_idle > 0) { + /* Pop off the first idle connection */ + *conn = APR_RING_FIRST(&c->ilist); + APR_RING_REMOVE(*conn, link); + c->num_idle--; + if (c->ttl && (now - (*conn)->freed >= c->ttl)) { + /* this connection is expired - kill it */ + clear_connection(*conn); + APR_RING_INSERT_TAIL(&c->flist, *conn, mm_connection_t, link); + c->num_conn--; + continue; + } + APR_RING_INSERT_TAIL(&c->ulist, *conn, mm_connection_t, link); + c->num_used++; + + apr_thread_mutex_unlock(c->listlock); + return APR_SUCCESS; + } + /* If we've hit our max, block until we're allowed to create + * a new one, or something becomes free. + */ + while (c->num_conn >= c->max_conn && c->num_idle <= 0) { + if (c->acquire) { + if ((rv = apr_thread_cond_timedwait(c->avail, + c->listlock, + c->acquire)) != APR_SUCCESS) { + apr_thread_mutex_unlock(c->listlock); + return rv; + } + } + else { + /* Wait infinite */ + apr_thread_cond_wait(c->avail, c->listlock); + } + } + /* If we popped out of the loop, first try to see if there + * are new resources available for immediate use. + */ + if (c->num_idle > 0) { + *conn = APR_RING_FIRST(&c->ilist); + APR_RING_REMOVE(*conn, link); + APR_RING_INSERT_TAIL(&c->ulist, *conn, mm_connection_t, link); + c->num_idle--; + c->num_used++; + apr_thread_mutex_unlock(c->listlock); + return APR_SUCCESS; + } + /* Otherwise the reason we dropped out of the loop + * was because there is a new slot available, so create + * a resource to fill the slot and use it. + */ + else { + create_connection(c, conn); + APR_RING_INSERT_TAIL(&c->ulist, *conn, mm_connection_t, link); + c->num_conn++; + c->num_used++; + apr_thread_mutex_unlock(c->listlock); + return rv; + } + + return rv; +} + +/* Put the connection to the idle list */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_ret(mm_connection_t *conn) +{ + mm_connection_pool_t *cp = conn->conn_pool; + apr_status_t rv = APR_SUCCESS; + + if (!cp) + return APR_SUCCESS; + apr_thread_mutex_lock(cp->listlock); + + APR_RING_REMOVE(conn, link); + cp->num_used--; + if (conn->reuse) { + APR_RING_INSERT_HEAD(&cp->ilist, conn, mm_connection_t, link); + conn->freed = apr_time_now(); + cp->num_idle++; + } + else { + clear_connection(conn); + cp->num_conn--; + APR_RING_INSERT_TAIL(&cp->flist, conn, mm_connection_t, link); + } + apr_thread_cond_signal(cp->avail); + apr_thread_mutex_unlock(cp->listlock); + + dbprintf("Connection %d reuse=%d %p", conn->id, conn->reuse, conn->pool); + return rv; +} + +/* Destroy the connection and put it to the free list */ +MANAGER_DECLARE(apr_status_t) mm_connection_pool_del(mm_connection_t *conn) +{ + mm_connection_pool_t *cp = conn->conn_pool; + apr_status_t rv = APR_SUCCESS; + + if (!cp) + return APR_SUCCESS; + apr_thread_mutex_lock(cp->listlock); + + clear_connection(conn); + APR_RING_REMOVE(conn, link); + cp->num_conn--; + cp->num_used--; + APR_RING_INSERT_TAIL(&cp->flist, conn, mm_connection_t, link); + apr_thread_cond_signal(cp->avail); + apr_thread_mutex_unlock(cp->listlock); + + return rv; +} Property changes on: sandbox/aloha/httpd/modules/manager/mm_connection.c ___________________________________________________________________ Name: svn:eol-style + native Modified: sandbox/aloha/httpd/modules/manager/mm_host.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_host.c 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_host.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -647,6 +647,8 @@ MM_SSAFE_COPY(alias->name, s->server_hostname); alias->host_id = host_id; alias->selfref = hostlist_id; + /* Activate default host at startup */ + host->status = mm_state_active; } /* Look for ServerAlias */ if (s->names) { Modified: sandbox/aloha/httpd/modules/manager/mm_module.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_module.c 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_module.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -50,6 +50,8 @@ module AP_MODULE_DECLARE_DATA manager_module; +mm_connection_pool_t *conn_pool; + /* * Server global data */ @@ -61,7 +63,9 @@ apr_pool_t *cpool; } mm_global_data_t; -static const char *mm_handler_name = MM_HANDLER_NAME; +static const char *mm_mhandler_name = MM_MHANDLER_NAME; +static const char *mm_chandler_name = MM_CHANDLER_NAME; + /* * Global data instance * For parent, registered in process pool @@ -87,6 +91,7 @@ static int mm_manager_strict = 0; static int mm_advertise_set = 0; static int mm_advertise_run = 0; +static int mm_advertise_stat = 0; static char *mm_database_root = NULL; static char *mm_lock_filename = NULL; static char *mm_advertise_adrs = NULL; @@ -298,7 +303,7 @@ else return "ServerAdvertise must be Off or On"; if (opt) { - const char *p = strstr(opt, "://"); + const char *p = ap_strstr_c(opt, "://"); if (p) { mm_advertise_srvm = apr_pstrndup(cmd->pool, opt, p - opt); opt = p + 3; @@ -373,7 +378,7 @@ if (errs != NULL) return errs; - if ((p = strchr(arg, '.')) || (p = strchr(arg, ','))) + if ((p = ap_strchr_c(arg, '.')) || (p = ap_strchr_c(arg, ','))) u = atoi(p + 1); s = atoi(arg); @@ -467,6 +472,11 @@ break; if (mm_advertise_run) { a_step += MM_TM_RESOLUTION; + if (mm_advertise_stat != mmdb_manager_rec->status) { + /* Force advertise on status change */ + mm_advertise_stat = mmdb_manager_rec->status; + f_time = 1; + } if (a_step >= mm_advertise_freq || f_time) { /* Run advertise */ mm_advertise_callback_run(mmdb_manager_rec->advertise_mode, @@ -475,9 +485,9 @@ a_step = 0; f_time = 0; } + if (!is_mp_running) + break; } - if (!is_mp_running) - break; /* Maintain database. * Add new memory segments if requested from client */ @@ -507,6 +517,7 @@ if (++step >= MM_TM_MAINTAIN_STEP) { mm_maintain_callback_run(mm_maintain_full); + mm_connection_pool_maintain(conn_pool); step = 0; } else @@ -739,7 +750,7 @@ } else ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, - "mod_manager: shared memory database created at %s", + "shared memory database created at %s", mmgd->database_path); #if !defined(WIN32) umask(mmode); @@ -975,8 +986,26 @@ " from parent process %" APR_PID_T_FMT, getpid(), mm_parent_pid); +#if 1 + { + mm_connection_pool_create(&conn_pool, 10, 10, 10, 0, 0, p); + } + +#endif } +static apr_status_t mm_request_cleanup(void *data) +{ + request_rec *r = (request_rec *)data; + mm_request_rec *mmrr = (mm_request_rec *)ap_get_module_config(r->request_config, + &manager_module); + dbprintf("Request cleanup %p for %s\n", mmrr, r->uri); + if (mmrr) { + mmrr->app_pool->busy--; + } + return APR_SUCCESS; +} + /*--------------------------------------------------------------------------*/ /* */ /* Post read request handler hook. */ @@ -989,7 +1018,9 @@ if (!is_mc_running) return DECLINED; - + apr_pool_cleanup_register(r->pool, r, mm_request_cleanup, + apr_pool_cleanup_null); + dbprintf("Request registered for %s\n", r->uri); return DECLINED; } @@ -1009,6 +1040,71 @@ return DECLINED; } +static int ccompare_app(const void *k, const void *a) +{ + const mm_request_rec *rr = k; + const mmdb_app_t *ap = a; + const char *request_uri = rr->r->uri; + const char *application = ap->url; + size_t ul, al; + int co; + + if (ap->host_id != rr->host_id) + return -1; + if (*request_uri == '/') + request_uri++; + if (*application == '~') { + /* Skip wildchar applications */ + return -1; + } + ul = strlen(request_uri); + al = strlen(application); + if (ul < al) + return -1; + if (mmdb_manager_rec->case_blind) { + co = strncasecmp(request_uri, application, al); + } + else { + co = strncmp(request_uri, application, al); + } + if (co == 0) { + /* Check if this is valid app uri. + * + */ + if (request_uri[al] == '\0' || + request_uri[al] == '/' || + request_uri[al] == '?' || + request_uri[al] == ';') { + return 0; + } + } + return 1; +} + +static int wcompare_app(const void *k, const void *a) +{ + const mm_request_rec *rr = k; + const mmdb_app_t *ap = a; + const char *request_uri = rr->r->uri; + const char *application = ap->url; + int mf = mmdb_manager_rec->case_blind ? APR_FNM_CASE_BLIND : 0; + + if (ap->host_id != rr->host_id) + return -1; + if (*request_uri == '/') + request_uri++; + if (*application != '~') { + /* Skip wildchar applications */ + return -1; + } + else + application++; + if (apr_fnmatch(application, request_uri, mf) == APR_SUCCESS) + return 0; + else + return 1; +} + /*--------------------------------------------------------------------------*/ /* */ /* Map to storage handler hook. */ @@ -1018,6 +1114,12 @@ { const char *path_cmd = NULL; mm_server_conf_t *mmsc = MM_SRVCONF(r); + mm_request_rec *mmrr; + asmm_slotmem_t *host_db; + asmm_slotmem_t *hostlist_db; + asmm_slotmem_t *balancer_db; + asmm_slotmem_t *app_db; + asmm_slotmem_t *app_pool_db; if (!is_mc_running) return DECLINED; @@ -1036,7 +1138,7 @@ /* Check if we have /foo or /foo/ */ if (r->uri[strlen(mmsc->handle)] == '\0' || r->uri[strlen(mmsc->handle)] == '/') { - r->handler = mm_handler_name; + r->handler = mm_mhandler_name; return OK; } } @@ -1044,18 +1146,179 @@ APR_UUID_FORMATTED_LENGTH)) { if (r->uri[APR_UUID_FORMATTED_LENGTH + 1] == '\0' || r->uri[APR_UUID_FORMATTED_LENGTH + 1] == '/') { - r->handler = mm_handler_name; + r->handler = mm_mhandler_name; return OK; } } + if (!(mmrr = apr_pcalloc(r->pool, sizeof(mm_request_rec)))) { + /* Memory error. Request will probably fail later on. + */ + return HTTP_INTERNAL_SERVER_ERROR; + } + mmrr->r = r; + mmrr->sc = mmsc; /* Stage 1: Find the Host */ + host_db = mmdb_table_getn(MMDB_HOST); + hostlist_db = mmdb_table_getn(MMDB_HOST_LIST); + if (!host_db || !hostlist_db) { + /* Corrupt database ? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Host table"); + return HTTP_INTERNAL_SERVER_ERROR; + } + if (!(mmrr->hostlist = asmm_slot_search(hostlist_db, + r->hostname, 1, + offsetof(mmdb_hostlist_t, name), + NULL))) { + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "Cannot find matching host %s", r->hostname); + if (r->server->is_virtual) { + /* VirtualHost without our Host record. + * Someone probably deleted configured + * VirtualHost. + * + * TODO: See if returning DECLINED should be default + * for all non matched Hosts + */ + return DECLINED; + } + /* Fallback to _default_ host */ + if (!(mmrr->hostlist = asmm_slot(hostlist_db, 0))) { + /* HostList _default_ record missing + * Corrupt database? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Default Host record"); + return HTTP_INTERNAL_SERVER_ERROR; + } + } + mmrr->host_id = mmrr->hostlist->host_id; + if (!(mmrr->host = asmm_slot(host_db, mmrr->host_id))) { + /* HostList without Host record + * Corrupt database? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Host record for %s", r->hostname); + return HTTP_INTERNAL_SERVER_ERROR; + } + switch (mmrr->host->status) { + case mm_state_busy: + return HTTP_SERVICE_UNAVAILABLE; + break; + case mm_state_error: + return HTTP_INTERNAL_SERVER_ERROR; + break; + case mm_state_unknown: + case mm_state_stopped: + return HTTP_NOT_FOUND; + break; + case mm_state_disabled: + return HTTP_GONE; + break; + } - /* Stage 2: Find the Application */ + /* Stage 2: Find the Balancer */ + if (!(balancer_db = mmdb_table_getn(MMDB_BALANCER))) { + /* Corrupt database ? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Balancer table"); + return HTTP_INTERNAL_SERVER_ERROR; + } + if (!(mmrr->balancer = asmm_slot(balancer_db, mmrr->host->balancer_id))) { + /* HostList without Host record + * Corrupt database? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Balancer record %d", + mmrr->host->balancer_id); + return HTTP_INTERNAL_SERVER_ERROR; + } + switch (mmrr->balancer->status) { + case mm_state_busy: + return HTTP_SERVICE_UNAVAILABLE; + break; + case mm_state_error: + return HTTP_INTERNAL_SERVER_ERROR; + break; + case mm_state_unknown: + case mm_state_stopped: + return HTTP_NOT_FOUND; + break; + case mm_state_disabled: + return HTTP_GONE; + break; + } - /* Stage 3: Find the Balancer */ + /* Stage 3: Find the Application for the Host */ + if (!(app_db = mmdb_table_getn(MMDB_APPLICATION))) { + /* Corrupt database ? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing Application table"); + return HTTP_INTERNAL_SERVER_ERROR; + } + if (!(mmrr->app = asmm_slot_find(app_db, mmrr, + ccompare_app, + &mmrr->app_id))) { + if (!(mmrr->app = asmm_slot_find(app_db, mmrr, + wcompare_app, + &mmrr->app_id))) { + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "No matching application found for %s", r->uri); + if (mmrr->balancer->sticky_session_remove) { + /* Try to remove the StickySessionPath from the url/filename + * This might be the request for static resource + */ + char *ssp; + if (r->uri) { + if ((ssp = ap_strstr(r->uri, + mmrr->balancer->sticky_session_path))) { + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "Removing session identifier %s for %s", + ssp, r->uri); + *ssp = '\0'; + } + } + if (r->filename) { + if ((ssp = ap_strstr(r->filename, + mmrr->balancer->sticky_session_path))) + *ssp = '\0'; + } + } + return DECLINED; + } + } + /* Stage 4: Find the ApplicationPool */ + if (!(app_pool_db = mmdb_table_getn(MMDB_APP_POOL))) { + /* Corrupt database ? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing ApplicationPool table"); + return HTTP_INTERNAL_SERVER_ERROR; + } + if (!(mmrr->app_pool = asmm_slot(app_pool_db, mmrr->app->app_pool_id))) { + /* Application without ApplicationPool record + * Corrupt database? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: missing ApplicationPool record %d", + mmrr->app->app_pool_id); + return HTTP_INTERNAL_SERVER_ERROR; + } + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "Found: Host=%s Application=%s ApplicationPool=%s" + " Balancer=%s for %s", + mmrr->hostlist->name, mmrr->app->url, + mmrr->app_pool->name, mmrr->balancer->name, + r->uri); + /* Setup gathered config and continue */ + ap_set_module_config(r->request_config, &manager_module, mmrr); + r->handler = mm_chandler_name; - return DECLINED; + return OK; } /*--------------------------------------------------------------------------*/ @@ -1265,8 +1528,115 @@ return status; } +static void dump_cluster_request(request_rec *r, mm_request_rec *mmrr) +{ + ap_set_content_type(r, "text/html"); + ap_rvputs(r, "<hr />\n<h2>Cluster request for: ", + r->unparsed_uri, "</h2>\n", NULL); + ap_rputs("<table border=\"0\">\n", r); + ap_rvputs(r, "<tr>", + "<th>ApplicationPool</th><th>Balancer</th>", + "<th>Host</th><th>Application</th>", + "</tr>\n", + "<tr>", NULL); + ap_rvputs(r, "<td>", + mmrr->app_pool->name, + "</td>", NULL); + ap_rvputs(r, "<td>", + mmrr->balancer->name, + "</td>", NULL); + ap_rvputs(r, "<td>", mmrr->hostlist->name, + "</td>", NULL); + ap_rvputs(r, "<td>", + mmrr->app->url, + "</td>", + "</tr>\n", NULL); + ap_rputs("</table>\n", r); + + ap_rvputs(r, "<hr />\n<h2>ApplicationPool: ", + mmrr->app_pool->name, "</h2>\n", NULL); + ap_rputs("<table border=\"0\">\n", r); + ap_rvputs(r, "<tr>", + "<th>Status</th><th>Busy</th>", + "<th>Concurrency</th><th>Applications</th>", + "</tr>\n", + "<tr>", NULL); + ap_rprintf(r, "<td>%s</td>", + mm_state_name_get(mmrr->app_pool->status)); + ap_rprintf(r, "<td>%d</td>", mmrr->app_pool->busy); + ap_rprintf(r, "<td>%d</td>", mmrr->app_pool->concurrency); + ap_rprintf(r, "<td>%d</td>", mmrr->app_pool->applications); + ap_rputs("</tr>\n</table>\n", r); + +} + /*--------------------------------------------------------------------------*/ /* */ +/* Cluster request handler hook. */ +/* */ +/*--------------------------------------------------------------------------*/ +static int mm_handler_cluster(request_rec *r) +{ + apr_status_t rc; + int status = OK; + asmm_slotmem_t *app_pool_db; + mm_server_conf_t *mmsc = MM_SRVCONF(r); + mm_request_rec *mmrr = (mm_request_rec *)ap_get_module_config(r->request_config, + &manager_module); + if (!mmrr) { + /* Something nasty is going on */ + return HTTP_INTERNAL_SERVER_ERROR; + } + if (!(app_pool_db = mmdb_table_getn(MMDB_APP_POOL))) { + /* Corrupt database ? + */ + ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, + "mod_manager: could not open ApplicationPool table"); + return HTTP_INTERNAL_SERVER_ERROR; + } + if ((rc = asmm_slotmem_lock(app_pool_db)) != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_ERR, rc, r->server, + "mod_manager: locking ApplicationPool %s", + mmrr->app_pool->name); + return HTTP_INTERNAL_SERVER_ERROR; + } + /* Stage 1: Check the ApplicationPool bussines */ + mmrr->app_pool->busy++; + if (mmrr->app_pool->busy > mmrr->app_pool->concurrency) { + apr_interval_time_t i; + for (i = 0; i < mmrr->app_pool->timeout; i += MM_TM_RESOLUTION) { + apr_sleep(MM_TM_RESOLUTION); + if (mmrr->app_pool->busy <= mmrr->app_pool->concurrency) + break; + } + if (mmrr->app_pool->busy > mmrr->app_pool->concurrency) { + asmm_slotmem_unlock(app_pool_db); + ap_log_error(APLOG_MARK, APLOG_INFO, 0, r->server, + "mod_manager: timeout for ApplicationPool %s", + mmrr->app_pool->name); + return HTTP_SERVICE_UNAVAILABLE; + } + } + asmm_slotmem_unlock(app_pool_db); + dump_cluster_request(r, mmrr); +#if 1 + { + mm_connection_t *c; + + mm_connection_pool_pop(conn_pool, &c); + dbprintf("Connection acquired %d", c->id); + + Sleep(30000); + dbprintf("Connection returned %d", c->id); + c->reuse = 0; + mm_connection_pool_ret(c); + } +#endif + return status; +} + +/*--------------------------------------------------------------------------*/ +/* */ /* Request handler hook. */ /* */ /*--------------------------------------------------------------------------*/ @@ -1281,8 +1651,10 @@ return DECLINED; } /* Is this for us? */ - if (!strcasecmp(r->handler, mm_handler_name)) + if (!strcasecmp(r->handler, mm_mhandler_name)) return mm_handler_manager(r); + else if (!strcasecmp(r->handler, mm_chandler_name)) + return mm_handler_cluster(r); return DECLINED; } @@ -1396,12 +1768,12 @@ NULL, APR_HOOK_MIDDLE); - /* Request handler hook + /* Post read request handler hook */ - ap_hook_handler(mm_handler_hook, - NULL, - NULL, - APR_HOOK_MIDDLE); + ap_hook_post_read_request(mm_post_read_request_hook, + NULL, + before_proxy, + APR_HOOK_FIRST); /* Translate name handler hook */ @@ -1410,13 +1782,6 @@ NULL, APR_HOOK_FIRST); - /* Post read request handler hook - */ - ap_hook_post_read_request(mm_post_read_request_hook, - NULL, - before_proxy, - APR_HOOK_FIRST); - /* Map to storage handler hook */ ap_hook_map_to_storage(mm_map_to_storage_hook, @@ -1424,6 +1789,11 @@ NULL, APR_HOOK_FIRST); + /* Request handler hook + */ + ap_hook_handler(mm_handler_hook, NULL, NULL, APR_HOOK_MIDDLE); + + /* Register resource objecst */ mm_register_resource_manager(p); Modified: sandbox/aloha/httpd/modules/manager/mm_util.c =================================================================== --- sandbox/aloha/httpd/modules/manager/mm_util.c 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/mm_util.c 2008-06-11 05:49:17 UTC (rev 1689) @@ -97,7 +97,7 @@ static apr_sockaddr_t *mma_listen_sa = NULL; /* Advertise sequence number */ -static volatile apr_uint64_t mma_sequence = 0; +static volatile apr_int64_t mma_sequence = 0; /* * --------------------------------------------------------------------- @@ -220,6 +220,7 @@ #define MM_ADVERTISE_SERVER_FMT \ "HTTP/1.0 %s" CRLF \ "Date: %s" CRLF \ + "Sequence: %" APR_INT64_T_FMT CRLF \ "Digest: %s" CRLF \ "Server: %s" CRLF @@ -241,15 +242,19 @@ mmdb_manager_t *m = (mmdb_manager_t *)cd; mma_sequence++; + if (mma_sequence < 1) + mma_sequence = 1; + sprintf(buf, "%" APR_INT64_T_FMT, mma_sequence); ap_recent_rfc822_date(dat, apr_time_now()); asl = ap_get_status_line(m->status); /* Create MD5 digest - * salt + date + srvid + * salt + date + sequence + srvid */ apr_md5_init(&md); apr_md5_update(&md, m->ssalt, APR_MD5_DIGESTSIZE); apr_md5_update(&md, dat, strlen(dat)); + apr_md5_update(&md, buf, strlen(buf)); apr_md5_update(&md, m->srvid + 1, strlen(m->srvid + 1)); apr_md5_final(msig, &md); /* Convert MD5 digest to hex string */ @@ -259,7 +264,7 @@ } ssig[c] = '\0'; n = apr_snprintf(p, l, MM_ADVERTISE_SERVER_FMT, - asl, dat, ssig, m->srvid + 1); + asl, dat, mma_sequence, ssig, m->srvid + 1); if (m->type == MM_ADVERTISE_SERVER) { l -= n; if (data && *data) { Modified: sandbox/aloha/httpd/modules/manager/modules.mk =================================================================== --- sandbox/aloha/httpd/modules/manager/modules.mk 2008-06-11 05:47:04 UTC (rev 1688) +++ sandbox/aloha/httpd/modules/manager/modules.mk 2008-06-11 05:49:17 UTC (rev 1689) @@ -1,5 +1,5 @@ -mod_manager.la: mm_app.slo mm_asmm.slo mm_balancer.slo mm_cookie.slo mm_host.slo mm_manager.slo mm_member.slo mm_module.slo mm_pool.slo mm_server.slo mm_util.slo - $(SH_LINK) -rpath $(libexecdir) -module -avoid-version mm_app.lo mm_asmm.lo mm_balancer.lo mm_cookie.lo mm_host.lo mm_manager.lo mm_member.lo mm_module.lo mm_pool.lo mm_server.lo mm_util.lo $(MOD_MANAGER_LDADD) +mod_manager.la: mm_app.slo mm_asmm.slo mm_balancer.slo mm_connection.slo mm_cookie.slo mm_host.slo mm_manager.slo mm_member.slo mm_module.slo mm_pool.slo mm_server.slo mm_util.slo + $(SH_LINK) -rpath $(libexecdir) -module -avoid-version mm_app.lo mm_asmm.lo mm_balancer.lo mm_connection.lo mm_cookie.lo mm_host.lo mm_manager.lo mm_member.lo mm_module.lo mm_pool.lo mm_server.lo mm_util.lo $(MOD_MANAGER_LDADD) DISTCLEAN_TARGETS = modules.mk static = shared = mod_manager.la