Author: jfrederic.clere(a)jboss.com Date: 2008-10-17 07:42:06 -0400 (Fri, 17 Oct 2008) New Revision: 1959 Modified: trunk/mod_cluster/native/include/mod_proxy_cluster.h trunk/mod_cluster/native/include/node.h trunk/mod_cluster/native/mod_manager/mod_manager.c trunk/mod_cluster/native/mod_proxy_cluster/mod_proxy_cluster.c Log: Add missing logic for the StickySessionRemove. Arrange node removal. Modified: trunk/mod_cluster/native/include/mod_proxy_cluster.h =================================================================== --- trunk/mod_cluster/native/include/mod_proxy_cluster.h 2008-10-16 08:15:45 UTC (rev 1958) +++ trunk/mod_cluster/native/include/mod_proxy_cluster.h 2008-10-17 11:42:06 UTC (rev 1959) @@ -27,6 +27,12 @@ #ifndef MOD_PROXY_CLUSTER_H #define MOD_PROXY_CLUSTER_H + +/* define the values for sticky_force */ +#define STSESSION 0x01 /* Use sticky session logic (first sessionid and then domain) */ +#define STSESSREM 0x02 /* Remove session information if the failover can't use sticky */ +#define STSESSFOR 0x04 /* Force sticky (return error if no worker corresponds to sessionid or domain) */ + struct balancer_method { /** * Check that the node is responding Modified: trunk/mod_cluster/native/include/node.h =================================================================== --- trunk/mod_cluster/native/include/node.h 2008-10-16 08:15:45 UTC (rev 1958) +++ trunk/mod_cluster/native/include/node.h 2008-10-17 11:42:06 UTC (rev 1959) @@ -69,6 +69,7 @@ int id; /* id in table and worker id */ apr_time_t updatetimelb; /* time of last update of the lbstatus value */ int oldelected; /* value of s->elected when calculating the lbstatus */ + apr_time_t lastcleantry; /* time of last unsuccessful try to clean the worker in proxy part */ }; typedef struct nodemess nodemess_t; @@ -203,7 +204,7 @@ /* * Find the node using the JVMRoute information */ -int (*find_node)(nodeinfo_t **node, const char *route); +apr_status_t (*find_node)(nodeinfo_t **node, const char *route); /* * Lock the whole node table */ Modified: trunk/mod_cluster/native/mod_manager/mod_manager.c =================================================================== --- trunk/mod_cluster/native/mod_manager/mod_manager.c 2008-10-16 08:15:45 UTC (rev 1958) +++ trunk/mod_cluster/native/mod_manager/mod_manager.c 2008-10-17 11:42:06 UTC (rev 1959) @@ -531,6 +531,7 @@ nodeinfo.mess.ttl = apr_time_from_sec(60); nodeinfo.mess.timeout = 0; nodeinfo.mess.id = 0; + nodeinfo.mess.lastcleantry = 0; /* Fill default balancer values */ memset(&balancerinfo, '\0', sizeof(balancerinfo)); Modified: trunk/mod_cluster/native/mod_proxy_cluster/mod_proxy_cluster.c =================================================================== --- trunk/mod_cluster/native/mod_proxy_cluster/mod_proxy_cluster.c 2008-10-16 08:15:45 UTC (rev 1958) +++ trunk/mod_cluster/native/mod_proxy_cluster/mod_proxy_cluster.c 2008-10-17 11:42:06 UTC (rev 1959) @@ -116,6 +116,7 @@ } reuse = 1; } else { + /* XXX: (*worker)->id ?= node->mess.id; */ return; /* Done Already existing */ } @@ -148,6 +149,7 @@ (*worker)->keepalive_set = 1; (*worker)->is_address_reusable = 1; (*worker)->acquire = apr_time_make(0, 2 * 1000); /* 2 ms */ + (*worker)->retry = apr_time_from_sec(PROXY_WORKER_DEFAULT_RETRY); /* * The Shared datastatus may already contains a valid information @@ -229,7 +231,13 @@ /* XXX: StickySession, StickySessionRemove not in */ balancer->sticky = apr_psprintf(conf->pool, "%s|%s", balan->StickySessionCookie, balan->StickySessionPath); - balancer->sticky_force = balan->StickySessionForce; + balancer->sticky_force = 0; + if (balan->StickySession) + balancer->sticky_force += STSESSION; + if (balan->StickySessionForce) + balancer->sticky_force += STSESSFOR; + if (balan->StickySessionRemove) + balancer->sticky_force += STSESSREM; balancer->timeout = balan->Timeout; balancer->max_attempts = balan->Maxattempts; @@ -310,8 +318,10 @@ worker->id = 0; /* mark it removed */ return (0); - } else + } else { + node->mess.lastcleantry = apr_time_now(); return (1); /* We should retry later */ + } } /* * Create/Remove workers corresponding to updated nodes. @@ -746,8 +756,8 @@ * stopped in one node but not in others. * We also try the domain. */ -static proxy_worker *find_best_byrequests(proxy_balancer *balancer, - request_rec *r) +static proxy_worker *internal_find_best_byrequests(proxy_balancer *balancer, + request_rec *r, char *domain, int failoverdomain) { int i; proxy_worker *worker; @@ -756,7 +766,6 @@ int checked_standby = 0; int checked_domain = 1; void *sconf = r->server->module_config; - const char *domain = apr_table_get(r->notes, "session-domain"); proxy_server_conf *conf = (proxy_server_conf *) ap_get_module_config(sconf, &proxy_module); @@ -770,7 +779,7 @@ /* First try to see if we have available candidate */ if (domain && strlen(domain)>0) checked_domain = 0; - while (!mycandidate && !checked_standby) { + while (!checked_standby) { worker = (proxy_worker *)balancer->workers->elts; for (i = 0; i < balancer->workers->nelts; i++, worker++) { nodeinfo_t *node; @@ -794,7 +803,9 @@ * not in error state or not disabled. * and that can map the context. */ - node_storage->read_node(worker->id, &node); + if (node_storage->read_node(worker->id, &node) != APR_SUCCESS) + continue; /* Can't read node */ + if (PROXY_WORKER_IS_USABLE(worker) && iscontext_host_ok(r, balancer, node)) { if (!checked_domain) { /* First try only nodes in the domain */ @@ -823,13 +834,20 @@ } } } + if (mycandidate) + break; if (checked_domain) { + if (failoverdomain) + break; /* We only failover in the domain */ checked_standby = checking_standby++; } checked_domain++; } if (mycandidate) { + /* Failover in domain */ + if (!checked_domain) + apr_table_setn(r->notes, "session-domain_over", "1"); mycandidate->s->elected++; ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, "proxy: byrequests balancer DONE (%s)", mycandidate->name); @@ -839,6 +857,11 @@ } return mycandidate; } +static proxy_worker *find_best_byrequests(proxy_balancer *balancer, + request_rec *r) +{ + return (internal_find_best_byrequests(balancer, r, NULL, 0)); +} static const proxy_balancer_method byrequests = { "byrequests", @@ -1109,7 +1132,7 @@ /* * Remove node that have beeen marked removed for more than 10 seconds. */ -static void remove_removed_node(apr_pool_t *pool) +static void remove_removed_node(apr_pool_t *pool, server_rec *server) { int *id, size, i; apr_time_t now = apr_time_now(); @@ -1120,7 +1143,8 @@ for (i=0; i<size; i++) { nodeinfo_t *ou; node_storage->read_node(id[i], &ou); - if (ou->mess.remove && (now - ou->updatetime) >= apr_time_from_sec(WAITFORREMOVE)) { + if (ou->mess.remove && (now - ou->updatetime) >= apr_time_from_sec(WAITFORREMOVE) && + (now - ou->mess.lastcleantry) >= apr_time_from_sec(WAITFORREMOVE)) { /* remove the node from the shared memory */ node_storage->remove_node(ou); } @@ -1140,7 +1164,7 @@ /* Create new workers if the shared memory changes */ update_workers_node(conf, pool, s); /* cleanup removed node in shared memory */ - remove_removed_node(pool); + remove_removed_node(pool, s); /* Calculate the lbstatus for each node */ update_workers_lbstatus(conf, pool, s); apr_pool_destroy(pool); @@ -1242,17 +1266,15 @@ * Find the worker that has the 'route' defined * (Should we also find the domain corresponding to it). */ -static proxy_worker *find_route_worker(proxy_balancer *balancer, - const char *route, request_rec *r) +static proxy_worker *find_route_worker(request_rec *r, + proxy_balancer *balancer, + const char *route) { int i; int checking_standby; int checked_standby; proxy_worker *worker; - nodeinfo_t *ou; - char *domain; - apr_status_t rv; checking_standby = checked_standby = 0; while (!checked_standby) { @@ -1289,7 +1311,7 @@ */ if (*worker->s->redirect) { proxy_worker *rworker = NULL; - rworker = find_route_worker(balancer, worker->s->redirect, r); + rworker = find_route_worker(r, balancer, worker->s->redirect); /* Check if the redirect worker is usable */ if (rworker && !PROXY_WORKER_IS_USABLE(rworker)) { /* @@ -1310,13 +1332,6 @@ } checked_standby = checking_standby++; } - /* We don't find a worker try to locate the domain for a fail-over */ - rv = node_storage->find_node(&ou, route); - if (rv == APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, - "proxy: Found domain %s for %s", ou->mess.Domain, route); - apr_table_setn(r->notes, "session-domain", ou->mess.Domain); - } return NULL; } @@ -1327,13 +1342,17 @@ request_rec *r, char **route, char **sticky_used, - char **url) + char **url, + char **domain) { proxy_worker *worker = NULL; char *sticky, *sticky_path, *path; if (!balancer->sticky) return NULL; + if (! (balancer->sticky_force & STSESSION)) + return NULL; + sticky = sticky_path = apr_pstrdup(r->pool, balancer->sticky); if ((path = strchr(sticky, '|'))) { *path++ = '\0'; @@ -1357,12 +1376,23 @@ if ((*route) && ((*route = strchr(*route, '.')) != NULL )) (*route)++; if ((*route) && (**route)) { + nodeinfo_t *ou; + apr_status_t rv; ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, "proxy: CLUSTER: Found route %s", *route); + /* Read the domain in case we have to make a failover */ + rv = node_storage->find_node(&ou, *route); + if (rv == APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server, + "proxy: Found domain %s for %s", ou->mess.Domain, *route); + apr_table_setn(r->notes, "session-domain", ou->mess.Domain); + if (domain) + *domain = ou->mess.Domain; + } /* We have a route in path or in cookie * Find the worker that has this route defined. */ - worker = find_route_worker(balancer, *route, r); + worker = find_route_worker(r, balancer, *route); if (worker && strcmp(*route, worker->s->route)) { /* * Notice that the route of the worker chosen is different from @@ -1380,7 +1410,7 @@ } static proxy_worker *find_best_worker(proxy_balancer *balancer, - request_rec *r) + request_rec *r, char *domain, int failoverdomain) { proxy_worker *candidate = NULL; apr_status_t rv; @@ -1391,7 +1421,8 @@ return NULL; } - candidate = (*balancer->lbmethod->finder)(balancer, r); + /* XXX: candidate = (*balancer->lbmethod->finder)(balancer, r); */ + candidate = internal_find_best_byrequests(balancer, r, domain, failoverdomain); if (candidate) { proxy_cluster_helper *helper; @@ -1429,7 +1460,7 @@ while (tval < timeout) { apr_sleep(step); /* Try again */ - if ((candidate = find_best_worker(balancer, r))) + if ((candidate = find_best_worker(balancer, r, domain, failoverdomain))) break; tval += step; } @@ -1463,6 +1494,76 @@ } /* + * Remove the session information + */ +void remove_session_route(request_rec *r, const char *name) +{ + char *path = NULL; + char *url = r->filename; + char *start = NULL; + char *cookies; + const char *readcookies; + char *start_cookie; + + /* First try to manipulate the url. */ + for (path = strstr(url, name); path; path = strstr(path + 1, name)) { + start = path; + if (*(start-1) == '&') + start--; + path += strlen(name); + if (*path == '=') { + ++path; + if (strlen(path)) { + char *filename = r->filename; + while (*path !='&' || *path !='\0') + path++; + /* We have it */ + *start = '\0'; + r->filename = apr_pstrcat(r->pool, filename, path, NULL); + return; + } + } + } + /* Second try to manipulate the cookie header... */ + + if ((readcookies = apr_table_get(r->headers_in, "Cookie"))) { + cookies = apr_pstrdup(r->pool, readcookies); + for (start_cookie = ap_strstr(cookies, name); start_cookie; + start_cookie = ap_strstr(start_cookie + 1, name)) { + if (start_cookie == cookies || + start_cookie[-1] == ';' || + start_cookie[-1] == ',' || + isspace(start_cookie[-1])) { + + start = start_cookie; + if (start_cookie != cookies && + (start_cookie[-1] == ';' || start_cookie[-1] == ',' || isspace(start_cookie[-1]))) { + start--; + } + start_cookie += strlen(name); + while(*start_cookie && isspace(*start_cookie)) + ++start_cookie; + if (*start_cookie == '=' && start_cookie[1]) { + /* + * Session cookie was found, get it's value + */ + char *end_cookie; + char *cookie; + ++start_cookie; + if ((end_cookie = ap_strchr(start_cookie, ';')) == NULL) + end_cookie = ap_strchr(start_cookie, ','); + + cookie = cookies; + *start = '\0'; + cookies = apr_pstrcat(r->pool, cookie , end_cookie, NULL); + apr_table_setn(r->headers_in, "Cookie", cookies); + } + } + } + } +} + +/* * Find a worker for mod_proxy logic */ static int proxy_cluster_pre_request(proxy_worker **worker, @@ -1474,6 +1575,8 @@ proxy_worker *runtime; char *route = NULL; char *sticky = NULL; + char *domain = NULL; + int failoverdomain = 0; apr_status_t rv; *worker = NULL; @@ -1482,6 +1585,23 @@ * If balancer is already provided skip the search * for balancer, because this is failover attempt. */ + if (*balancer) { + /* Adjust the helper->count corresponding to the previous try */ + const char *domain = apr_table_get(r->subprocess_env, "BALANCER_WORKER_NAME"); + if (domain && *domain) { + int i; + runtime = (proxy_worker *)(*balancer)->workers->elts; + for (i = 0; i < (*balancer)->workers->nelts; i++, runtime++) { + if (runtime->name && strcmp(domain, runtime->name) == 0) { + proxy_cluster_helper *helper; + helper = (proxy_cluster_helper *) (runtime)->opaque; + if (helper->count_active>0) + helper->count_active--; + } + } + } + } + apr_thread_mutex_lock(lock); if (!*balancer && !(*balancer = ap_proxy_get_balancer(r->pool, conf, *url))) { @@ -1498,7 +1618,7 @@ /* Step 2: find the session route */ - runtime = find_session_route(*balancer, r, &route, &sticky, url); + runtime = find_session_route(*balancer, r, &route, &sticky, url, &domain); apr_thread_mutex_unlock(lock); /* Lock the LoadBalancer @@ -1514,23 +1634,13 @@ runtime->s->elected++; *worker = runtime; } - else if (route && (*balancer)->sticky_force) { - int i, member_of = 0; - proxy_worker *workers; - /* - * We have a route provided that doesn't match the - * balancer name. See if the provider route is the - * member of the same balancer in which case return 503 - */ - workers = (proxy_worker *)(*balancer)->workers->elts; - for (i = 0; i < (*balancer)->workers->nelts; i++) { - if (*(workers->s->route) && strcmp(workers->s->route, route) == 0) { - member_of = 1; - break; - } - workers++; - } - if (member_of) { + else if (route && ((*balancer)->sticky_force & STSESSFOR)) { + if (domain == NULL) { + /* + * We have a route provided that doesn't match the + * balancer name. See if the provider route is the + * member of the same balancer in which case return 503 + */ ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "proxy: CLUSTER: (%s). All workers are in error state for route (%s)", (*balancer)->name, route); @@ -1539,7 +1649,10 @@ "proxy: CLUSTER: (%s). Unlock failed for pre_request", (*balancer)->name); } - return HTTP_SERVICE_UNAVAILABLE; + return HTTP_SERVICE_UNAVAILABLE; + } else { + /* We try to to failover using another node in the domain */ + failoverdomain = 1; } } @@ -1555,7 +1668,10 @@ (*balancer)->name); } if (!*worker) { - runtime = find_best_worker(*balancer, r); + /* + * We have to failover (in domain only may be) or we don't use sticky sessions + */ + runtime = find_best_worker(*balancer, r, domain, failoverdomain); if (!runtime) { ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server, "proxy: CLUSTER: (%s). All workers are in error state", @@ -1574,6 +1690,15 @@ */ apr_table_setn(r->subprocess_env, "BALANCER_ROUTE_CHANGED", "1"); } + if (route && ((*balancer)->sticky_force & STSESSREM)) { + /* + * Failover to another domain. Remove sessionid information. + */ + const char *domain = apr_table_get(r->notes, "session-domain-ok"); + if (!domain) { + remove_session_route(r, sticky); + } + } *worker = runtime; }