1 /* Licensed to the Apache Software Foundation (ASF) under one or more
2 * contributor license agreements. See the NOTICE file distributed with
3 * this work for additional information regarding copyright ownership.
4 * The ASF licenses this file to You under the Apache License, Version 2.0
5 * (the "License"); you may not use this file except in compliance with
6 * the License. You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 /* The purpose of this MPM is to fix the design flaws in the threaded
18 * model. Because of the way that pthreads and mutex locks interact,
19 * it is basically impossible to cleanly gracefully shutdown a child
20 * process if multiple threads are all blocked in accept. This model
21 * fixes those problems.
25 #include "apr_portable.h"
26 #include "apr_strings.h"
27 #include "apr_file_io.h"
28 #include "apr_thread_proc.h"
29 #include "apr_signal.h"
31 #include "apr_thread_mutex.h"
32 #include "apr_thread_cond.h"
33 #include "apr_proc_mutex.h"
34 #define APR_WANT_STRFUNC
40 #if APR_HAVE_SYS_SOCKET_H
41 #include <sys/socket.h>
43 #if APR_HAVE_SYS_WAIT_H
46 #ifdef HAVE_SYS_PROCESSOR_H
47 #include <sys/processor.h> /* for bindprocessor() */
51 #error The Worker MPM requires APR threads, but they are unavailable.
56 #include "ap_config.h"
58 #include "http_main.h"
60 #include "http_config.h" /* for read_config */
61 #include "http_core.h" /* for get_remote_host */
62 #include "http_connection.h"
65 #include "mpm_common.h"
66 #include "ap_listen.h"
67 #include "scoreboard.h"
68 #include "mpm_default.h"
71 #include <limits.h> /* for INT_MAX */
73 /* Limit on the total --- clients will be locked out if more servers than
74 * this are needed. It is intended solely to keep the server from crashing
75 * when things get out of hand.
77 * We keep a hard maximum number of servers, for two reasons --- first off,
78 * in case something goes seriously wrong, we want to stop the fork bomb
79 * short of actually crashing the machine we're running on by filling some
80 * kernel table. Secondly, it keeps the size of the scoreboard file small
81 * enough that we can read the whole thing without worrying too much about
84 #ifndef DEFAULT_SERVER_LIMIT
85 #define DEFAULT_SERVER_LIMIT 16
88 /* Admin can't tune ServerLimit beyond MAX_SERVER_LIMIT. We want
89 * some sort of compile-time limit to help catch typos.
91 #ifndef MAX_SERVER_LIMIT
92 #define MAX_SERVER_LIMIT 20000
95 /* Limit on the threads per process. Clients will be locked out if more than
96 * this * server_limit are needed.
98 * We keep this for one reason it keeps the size of the scoreboard file small
99 * enough that we can read the whole thing without worrying too much about
102 #ifndef DEFAULT_THREAD_LIMIT
103 #define DEFAULT_THREAD_LIMIT 64
106 /* Admin can't tune ThreadLimit beyond MAX_THREAD_LIMIT. We want
107 * some sort of compile-time limit to help catch typos.
109 #ifndef MAX_THREAD_LIMIT
110 #define MAX_THREAD_LIMIT 20000
114 * Actual definitions of config globals
117 int ap_threads_per_child = 0; /* Worker threads per child */
118 static int ap_daemons_to_start = 0;
119 static int min_spare_threads = 0;
120 static int max_spare_threads = 0;
121 static int ap_daemons_limit = 0;
122 static int server_limit = DEFAULT_SERVER_LIMIT;
123 static int first_server_limit;
124 static int thread_limit = DEFAULT_THREAD_LIMIT;
125 static int first_thread_limit;
126 static int changed_limit_at_restart;
127 static int dying = 0;
128 static int workers_may_exit = 0;
129 static int start_thread_may_exit = 0;
130 static int listener_may_exit = 0;
131 static int requests_this_child;
132 static int num_listensocks = 0;
133 static int resource_shortage = 0;
134 static int mpm_state = AP_MPMQ_STARTING;
136 /* The structure used to pass unique initialization info to each thread */
143 /* Structure used to pass information to the thread responsible for
144 * creating the rest of the threads.
147 apr_thread_t **threads;
148 apr_thread_t *listener;
150 apr_threadattr_t *threadattr;
153 #define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t)
156 * The max child slot ever assigned, preserved across restarts. Necessary
157 * to deal with MaxClients changes across AP_SIG_GRACEFUL restarts. We
158 * use this value to optimize routines that have to scan the entire
161 int ap_max_daemons_limit = -1;
163 static ap_pod_t *pod;
165 /* *Non*-shared http_main globals... */
167 server_rec *ap_server_conf;
169 /* The worker MPM respects a couple of runtime flags that can aid
170 * in debugging. Setting the -DNO_DETACH flag will prevent the root process
171 * from detaching from its controlling terminal. Additionally, setting
172 * the -DONE_PROCESS flag (which implies -DNO_DETACH) will get you the
173 * child_main loop running in the process which originally started up.
174 * This gives you a pretty nice debugging environment. (You'll get a SIGHUP
175 * early in standalone_main; just continue through. This is the server
176 * trying to kill off any child processes which it might have lying
177 * around --- Apache doesn't keep track of their pids, it just sends
178 * SIGHUP to the process group, ignoring it in the root process.
179 * Continue through and you'll be fine.).
182 static int one_process = 0;
185 int raise_sigstop_flags;
188 static apr_pool_t *pconf; /* Pool for config stuff */
189 static apr_pool_t *pchild; /* Pool for httpd child stuff */
191 static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main
192 thread. Use this instead */
193 static pid_t parent_pid;
194 static apr_os_thread_t *listener_os_thread;
196 /* Locks for accept serialization */
197 static apr_proc_mutex_t *accept_mutex;
199 #if APR_O_NONBLOCK_INHERITED
200 #undef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
201 #endif /* APR_O_NONBLOCK_INHERITED */
203 #ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT
204 #define SAFE_ACCEPT(stmt) (ap_listeners->next ? (stmt) : APR_SUCCESS)
206 #define SAFE_ACCEPT(stmt) (stmt)
209 /* The LISTENER_SIGNAL signal will be sent from the main thread to the
210 * listener thread to wake it up for graceful termination (what a child
211 * process from an old generation does when the admin does "apachectl
212 * graceful"). This signal will be blocked in all threads of a child
213 * process except for the listener thread.
215 #define LISTENER_SIGNAL SIGHUP
218 /* Possible states of a worker thread. */
225 /* Structure used to wake up an idle worker thread
230 worker_state_e state;
231 apr_thread_cond_t *cond;
232 apr_thread_mutex_t *mutex;
233 } worker_wakeup_info;
235 /* Structure used to hold a stack of idle worker threads
238 apr_thread_mutex_t *mutex;
239 apr_thread_cond_t *cond;
240 worker_wakeup_info **stack;
246 static worker_stack* worker_stack_create(apr_pool_t *pool, apr_size_t max)
249 worker_stack *stack = (worker_stack *)apr_palloc(pool, sizeof(*stack));
251 if ((rv = apr_thread_mutex_create(&stack->mutex, APR_THREAD_MUTEX_DEFAULT,
252 pool)) != APR_SUCCESS) {
255 if ((rv = apr_thread_cond_create(&stack->cond, pool)) != APR_SUCCESS) {
261 (worker_wakeup_info **)apr_palloc(pool, stack->nalloc *
262 sizeof(worker_wakeup_info *));
263 stack->terminated = 0;
267 static apr_status_t worker_stack_wait(worker_stack *stack,
268 worker_wakeup_info *wakeup)
272 wakeup->state = WORKER_IDLE;
274 if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
277 if (stack->terminated) {
278 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
283 if (stack->nelts == stack->nalloc) {
284 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
289 stack->stack[stack->nelts] = wakeup;
290 /* Signal a blocking listener thread only if we just made the
291 * stack non-empty. */
292 if (stack->nelts++ == 0) {
293 (void)apr_thread_cond_signal(stack->cond);
295 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
299 /* At this point we've already added this worker to the stack, now
300 * we just wait until the listener has accept()ed a connection
302 if ((rv = apr_thread_mutex_lock(wakeup->mutex)) != APR_SUCCESS) {
305 while (wakeup->state == WORKER_IDLE) {
306 if ((rv = apr_thread_cond_wait(wakeup->cond, wakeup->mutex)) !=
311 if ((rv = apr_thread_mutex_unlock(wakeup->mutex)) != APR_SUCCESS) {
317 static apr_status_t worker_stack_pop(worker_stack *stack,
318 worker_wakeup_info **worker)
321 if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
324 AP_DEBUG_ASSERT(stack->nelts >= 0);
325 while ((stack->nelts == 0) && (!stack->terminated)) {
326 rv = apr_thread_cond_wait(stack->cond, stack->mutex);
327 if (rv != APR_SUCCESS) {
329 rv2 = apr_thread_mutex_unlock(stack->mutex);
330 if (rv2 != APR_SUCCESS) {
336 if (stack->terminated) {
337 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
342 *worker = stack->stack[--stack->nelts];
343 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
349 static apr_status_t worker_stack_terminate(worker_stack *stack)
352 worker_wakeup_info *worker;
354 if ((rv = apr_thread_mutex_lock(stack->mutex)) != APR_SUCCESS) {
357 stack->terminated = 1;
358 /* Wake up the listener thread. Although there will never be
359 * more than one thread blocking on this condition, broadcast
361 apr_thread_cond_broadcast(stack->cond);
362 while (stack->nelts) {
363 worker = stack->stack[--stack->nelts];
364 apr_thread_mutex_lock(worker->mutex);
366 worker->state = WORKER_TERMINATED;
367 apr_thread_cond_signal(worker->cond);
368 apr_thread_mutex_unlock(worker->mutex);
370 if ((rv = apr_thread_mutex_unlock(stack->mutex)) != APR_SUCCESS) {
376 static worker_stack *idle_worker_stack;
378 static void wakeup_listener(void)
382 listener_may_exit = 1;
383 if (!idle_worker_stack) {
386 if ((rv = apr_thread_mutex_lock(idle_worker_stack->mutex)) != APR_SUCCESS) {
389 if ((rv = apr_thread_cond_signal(idle_worker_stack->cond)) !=
393 if ((rv = apr_thread_mutex_unlock(idle_worker_stack->mutex)) != APR_SUCCESS) {
396 if (!listener_os_thread) {
397 /* XXX there is an obscure path that this doesn't handle perfectly:
398 * right after listener thread is created but before
399 * listener_os_thread is set, the first worker thread hits an
400 * error and starts graceful termination
405 * we should just be able to "kill(ap_my_pid, LISTENER_SIGNAL)" on all
406 * platforms and wake up the listener thread since it is the only thread
407 * with SIGHUP unblocked, but that doesn't work on Linux
409 #ifdef HAVE_PTHREAD_KILL
410 pthread_kill(*listener_os_thread, LISTENER_SIGNAL);
412 kill(ap_my_pid, LISTENER_SIGNAL);
417 #define ST_GRACEFUL 1
418 #define ST_UNGRACEFUL 2
420 static int terminate_mode = ST_INIT;
422 static void signal_threads(int mode)
424 if (terminate_mode == mode) {
427 terminate_mode = mode;
428 mpm_state = AP_MPMQ_STOPPING;
430 /* in case we weren't called from the listener thread, wake up the
435 /* for ungraceful termination, let the workers exit now;
436 * for graceful termination, the listener thread will notify the
437 * workers to exit once it has stopped accepting new connections
439 if (mode == ST_UNGRACEFUL) {
440 workers_may_exit = 1;
441 worker_stack_terminate(idle_worker_stack);
445 AP_DECLARE(apr_status_t) ap_mpm_query(int query_code, int *result)
448 case AP_MPMQ_MAX_DAEMON_USED:
449 *result = ap_max_daemons_limit;
451 case AP_MPMQ_IS_THREADED:
452 *result = AP_MPMQ_STATIC;
454 case AP_MPMQ_IS_FORKED:
455 *result = AP_MPMQ_DYNAMIC;
457 case AP_MPMQ_HARD_LIMIT_DAEMONS:
458 *result = server_limit;
460 case AP_MPMQ_HARD_LIMIT_THREADS:
461 *result = thread_limit;
463 case AP_MPMQ_MAX_THREADS:
464 *result = ap_threads_per_child;
466 case AP_MPMQ_MIN_SPARE_DAEMONS:
469 case AP_MPMQ_MIN_SPARE_THREADS:
470 *result = min_spare_threads;
472 case AP_MPMQ_MAX_SPARE_DAEMONS:
475 case AP_MPMQ_MAX_SPARE_THREADS:
476 *result = max_spare_threads;
478 case AP_MPMQ_MAX_REQUESTS_DAEMON:
479 *result = ap_max_requests_per_child;
481 case AP_MPMQ_MAX_DAEMONS:
482 *result = ap_daemons_limit;
484 case AP_MPMQ_MPM_STATE:
491 /* a clean exit from a child with proper cleanup */
492 static void clean_child_exit(int code) __attribute__ ((noreturn));
493 static void clean_child_exit(int code)
495 mpm_state = AP_MPMQ_STOPPING;
497 apr_pool_destroy(pchild);
502 static void just_die(int sig)
507 /*****************************************************************
508 * Connection structures and accounting...
511 /* volatile just in case */
512 static int volatile shutdown_pending;
513 static int volatile restart_pending;
514 static int volatile is_graceful;
515 static volatile int child_fatal;
516 ap_generation_t volatile ap_my_generation;
519 * ap_start_shutdown() and ap_start_restart(), below, are a first stab at
520 * functions to initiate shutdown or restart without relying on signals.
521 * Previously this was initiated in sig_term() and restart() signal handlers,
522 * but we want to be able to start a shutdown/restart from other sources --
523 * e.g. on Win32, from the service manager. Now the service manager can
524 * call ap_start_shutdown() or ap_start_restart() as appropiate. Note that
525 * these functions can also be called by the child processes, since global
526 * variables are no longer used to pass on the required action to the parent.
528 * These should only be called from the parent process itself, since the
529 * parent process will use the shutdown_pending and restart_pending variables
530 * to determine whether to shutdown or restart. The child process should
531 * call signal_parent() directly to tell the parent to die -- this will
532 * cause neither of those variable to be set, which the parent will
533 * assume means something serious is wrong (which it will be, for the
534 * child to force an exit) and so do an exit anyway.
537 static void ap_start_shutdown(void)
539 mpm_state = AP_MPMQ_STOPPING;
540 if (shutdown_pending == 1) {
541 /* Um, is this _probably_ not an error, if the user has
542 * tried to do a shutdown twice quickly, so we won't
543 * worry about reporting it.
547 shutdown_pending = 1;
550 /* do a graceful restart if graceful == 1 */
551 static void ap_start_restart(int graceful)
553 mpm_state = AP_MPMQ_STOPPING;
554 if (restart_pending == 1) {
555 /* Probably not an error - don't bother reporting it */
559 is_graceful = graceful;
562 static void sig_term(int sig)
567 static void restart(int sig)
569 ap_start_restart(sig == AP_SIG_GRACEFUL);
572 static void set_signals(void)
574 #ifndef NO_USE_SIGACTION
579 ap_fatal_signal_setup(ap_server_conf, pconf);
582 #ifndef NO_USE_SIGACTION
583 sigemptyset(&sa.sa_mask);
586 sa.sa_handler = sig_term;
587 if (sigaction(SIGTERM, &sa, NULL) < 0)
588 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
589 "sigaction(SIGTERM)");
591 if (sigaction(SIGINT, &sa, NULL) < 0)
592 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
593 "sigaction(SIGINT)");
596 sa.sa_handler = SIG_DFL;
597 if (sigaction(SIGXCPU, &sa, NULL) < 0)
598 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
599 "sigaction(SIGXCPU)");
602 sa.sa_handler = SIG_DFL;
603 if (sigaction(SIGXFSZ, &sa, NULL) < 0)
604 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
605 "sigaction(SIGXFSZ)");
608 sa.sa_handler = SIG_IGN;
609 if (sigaction(SIGPIPE, &sa, NULL) < 0)
610 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
611 "sigaction(SIGPIPE)");
614 /* we want to ignore HUPs and AP_SIG_GRACEFUL while we're busy
616 sigaddset(&sa.sa_mask, SIGHUP);
617 sigaddset(&sa.sa_mask, AP_SIG_GRACEFUL);
618 sa.sa_handler = restart;
619 if (sigaction(SIGHUP, &sa, NULL) < 0)
620 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
621 "sigaction(SIGHUP)");
622 if (sigaction(AP_SIG_GRACEFUL, &sa, NULL) < 0)
623 ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
624 "sigaction(" AP_SIG_GRACEFUL_STRING ")");
628 apr_signal(SIGXCPU, SIG_DFL);
631 apr_signal(SIGXFSZ, SIG_DFL);
635 apr_signal(SIGTERM, sig_term);
637 apr_signal(SIGHUP, restart);
639 #ifdef AP_SIG_GRACEFUL
640 apr_signal(AP_SIG_GRACEFUL, restart);
641 #endif /* AP_SIG_GRACEFUL */
643 apr_signal(SIGPIPE, SIG_IGN);
649 /*****************************************************************
650 * Here follows a long bunch of generic server bookkeeping stuff...
653 int ap_graceful_stop_signalled(void)
654 /* XXX this is really a bad confusing obsolete name
655 * maybe it should be ap_mpm_process_exiting?
658 /* note: for a graceful termination, listener_may_exit will be set before
659 * workers_may_exit, so check listener_may_exit
661 return listener_may_exit;
664 /*****************************************************************
665 * Child process main loop.
668 static void process_socket(apr_pool_t *p, apr_socket_t *sock, int my_child_num,
669 int my_thread_num, apr_bucket_alloc_t *bucket_alloc)
671 conn_rec *current_conn;
672 long conn_id = ID_FROM_CHILD_THREAD(my_child_num, my_thread_num);
676 ap_create_sb_handle(&sbh, p, my_child_num, my_thread_num);
677 apr_os_sock_get(&csd, sock);
679 current_conn = ap_run_create_connection(p, ap_server_conf, sock,
680 conn_id, sbh, bucket_alloc);
682 ap_process_connection(current_conn, sock);
683 ap_lingering_close(current_conn);
687 /* requests_this_child has gone to zero or below. See if the admin coded
688 "MaxRequestsPerChild 0", and keep going in that case. Doing it this way
689 simplifies the hot path in worker_thread */
690 static void check_infinite_requests(void)
692 if (ap_max_requests_per_child) {
693 signal_threads(ST_GRACEFUL);
696 /* wow! if you're executing this code, you may have set a record.
697 * either this child process has served over 2 billion requests, or
698 * you're running a threaded 2.0 on a 16 bit machine.
700 * I'll buy pizza and beers at Apachecon for the first person to do
701 * the former without cheating (dorking with INT_MAX, or running with
702 * uncommitted performance patches, for example).
704 * for the latter case, you probably deserve a beer too. Greg Ames
707 requests_this_child = INT_MAX; /* keep going */
711 static void unblock_signal(int sig)
715 sigemptyset(&sig_mask);
716 sigaddset(&sig_mask, sig);
717 #if defined(SIGPROCMASK_SETS_THREAD_MASK)
718 sigprocmask(SIG_UNBLOCK, &sig_mask, NULL);
720 pthread_sigmask(SIG_UNBLOCK, &sig_mask, NULL);
724 static void dummy_signal_handler(int sig)
726 /* XXX If specifying SIG_IGN is guaranteed to unblock a syscall,
727 * then we don't need this goofy function.
731 static void *listener_thread(apr_thread_t *thd, void * dummy)
733 proc_info * ti = dummy;
734 int process_slot = ti->pid;
735 apr_pool_t *tpool = apr_thread_pool_get(thd);
737 apr_pool_t *ptrans; /* Pool for per-transaction stuff */
739 apr_pollfd_t *pollset;
741 ap_listen_rec *lr, *last_lr = ap_listeners;
742 worker_wakeup_info *worker = NULL;
746 apr_poll_setup(&pollset, num_listensocks, tpool);
747 for(lr = ap_listeners ; lr != NULL ; lr = lr->next)
748 apr_poll_socket_add(pollset, lr->sd, APR_POLLIN);
750 /* Unblock the signal used to wake this thread up, and set a handler for
753 unblock_signal(LISTENER_SIGNAL);
754 apr_signal(LISTENER_SIGNAL, dummy_signal_handler);
756 /* TODO: Switch to a system where threads reuse the results from earlier
757 poll calls - manoj */
759 /* TODO: requests_this_child should be synchronized - aaron */
760 if (requests_this_child <= 0) {
761 check_infinite_requests();
763 if (listener_may_exit) break;
765 if (worker == NULL) {
766 rv = worker_stack_pop(idle_worker_stack, &worker);
767 if (APR_STATUS_IS_EOF(rv)) {
770 else if (rv != APR_SUCCESS) {
771 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf,
772 "worker_stack_pop failed");
775 ptrans = worker->pool;
777 AP_DEBUG_ASSERT(worker->state == WORKER_IDLE);
779 if ((rv = SAFE_ACCEPT(apr_proc_mutex_lock(accept_mutex)))
781 int level = APLOG_EMERG;
783 if (listener_may_exit) {
786 if (ap_scoreboard_image->parent[process_slot].generation !=
787 ap_scoreboard_image->global->running_generation) {
788 level = APLOG_DEBUG; /* common to get these at restart time */
790 ap_log_error(APLOG_MARK, level, rv, ap_server_conf,
791 "apr_proc_mutex_lock failed. Attempting to shutdown "
792 "process gracefully.");
793 signal_threads(ST_GRACEFUL);
794 break; /* skip the lock release */
797 if (!APR_O_NONBLOCK_INHERITED && !ap_listeners->next) {
798 /* Only one listener, so skip the poll */
802 while (!listener_may_exit) {
806 ret = apr_poll(pollset, num_listensocks, &n, -1);
807 if (ret != APR_SUCCESS) {
808 if (APR_STATUS_IS_EINTR(ret)) {
812 /* apr_poll() will only return errors in catastrophic
813 * circumstances. Let's try exiting gracefully, for now. */
814 ap_log_error(APLOG_MARK, APLOG_ERR, ret, (const server_rec *)
815 ap_server_conf, "apr_poll: (listen)");
816 signal_threads(ST_GRACEFUL);
819 if (listener_may_exit) break;
821 /* find a listener */
828 /* XXX: Should we check for POLLERR? */
829 apr_poll_revents_get(&event, lr->sd, pollset);
830 if (event & APR_POLLIN) {
834 } while (lr != last_lr);
838 if (!listener_may_exit) {
839 rv = lr->accept_func(&csd, lr, ptrans);
840 /* later we trash rv and rely on csd to indicate success/failure */
841 AP_DEBUG_ASSERT(rv == APR_SUCCESS || !csd);
843 if (rv == APR_EGENERAL) {
844 /* E[NM]FILE, ENOMEM, etc */
845 resource_shortage = 1;
846 signal_threads(ST_GRACEFUL);
848 if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(accept_mutex)))
850 int level = APLOG_EMERG;
852 if (listener_may_exit) {
855 if (ap_scoreboard_image->parent[process_slot].generation !=
856 ap_scoreboard_image->global->running_generation) {
857 level = APLOG_DEBUG; /* common to get these at restart time */
859 ap_log_error(APLOG_MARK, level, rv, ap_server_conf,
860 "apr_proc_mutex_unlock failed. Attempting to "
861 "shutdown process gracefully.");
862 signal_threads(ST_GRACEFUL);
865 /* Wake up the sleeping worker. */
866 apr_thread_mutex_lock(worker->mutex);
867 worker->csd = (apr_socket_t *)csd;
868 worker->state = WORKER_BUSY;
869 /* Posix allows us to signal this condition without
870 * owning the associated mutex, but in that case it can
871 * not guarantee predictable scheduling. See
872 * _UNIX Network Programming: Interprocess Communication_
873 * by W. Richard Stevens, Vol 2, 2nd Ed, pp. 170-171. */
874 apr_thread_cond_signal(worker->cond);
875 apr_thread_mutex_unlock(worker->mutex);
880 if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(accept_mutex)))
882 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
883 "apr_proc_mutex_unlock failed. Attempting to "
884 "shutdown process gracefully.");
885 signal_threads(ST_GRACEFUL);
891 workers_may_exit = 1;
893 apr_thread_mutex_lock(worker->mutex);
894 worker->state = WORKER_TERMINATED;
895 /* Posix allows us to signal this condition without
896 * owning the associated mutex, but in that case it can
897 * not guarantee predictable scheduling. See
898 * _UNIX Network Programming: Interprocess Communication_
899 * by W. Richard Stevens, Vol 2, 2nd Ed, pp. 170-171. */
900 apr_thread_cond_signal(worker->cond);
901 apr_thread_mutex_unlock(worker->mutex);
903 worker_stack_terminate(idle_worker_stack);
905 ap_scoreboard_image->parent[process_slot].quiescing = 1;
907 /* wake up the main thread */
908 kill(ap_my_pid, SIGTERM);
910 apr_thread_exit(thd, APR_SUCCESS);
914 /* XXX For ungraceful termination/restart, we definitely don't want to
915 * wait for active connections to finish but we may want to wait
916 * for idle workers to get out of the queue code and release mutexes,
917 * since those mutexes are cleaned up pretty soon and some systems
918 * may not react favorably (i.e., segfault) if operations are attempted
919 * on cleaned-up mutexes.
921 static void * APR_THREAD_FUNC worker_thread(apr_thread_t *thd, void * dummy)
923 proc_info * ti = dummy;
924 int process_slot = ti->pid;
925 int thread_slot = ti->tid;
926 apr_bucket_alloc_t *bucket_alloc;
927 apr_pool_t *tpool = apr_thread_pool_get(thd);
928 apr_pool_t *ptrans; /* Pool for per-transaction stuff */
929 apr_allocator_t *allocator;
931 worker_wakeup_info *wakeup;
935 ap_update_child_status_from_indexes(process_slot, thread_slot, SERVER_STARTING, NULL);
937 apr_allocator_create(&allocator);
938 apr_allocator_max_free_set(allocator, ap_max_mem_free);
939 /* XXX: why is ptrans's parent not tpool? --jcw 08/2003 */
940 apr_pool_create_ex(&ptrans, NULL, NULL, allocator);
941 apr_allocator_owner_set(allocator, ptrans);
942 bucket_alloc = apr_bucket_alloc_create_ex(allocator);
944 wakeup = (worker_wakeup_info *)apr_palloc(tpool, sizeof(*wakeup));
945 wakeup->pool = ptrans;
946 if ((rv = apr_thread_cond_create(&wakeup->cond, tpool)) != APR_SUCCESS) {
947 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
948 "apr_thread_cond_create failed. Attempting to shutdown "
949 "process gracefully.");
950 signal_threads(ST_GRACEFUL);
951 apr_thread_exit(thd, rv);
953 if ((rv = apr_thread_mutex_create(&wakeup->mutex, APR_THREAD_MUTEX_DEFAULT,
954 tpool)) != APR_SUCCESS) {
955 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
956 "apr_thread_mutex_create failed. Attempting to shutdown "
957 "process gracefully.");
958 signal_threads(ST_GRACEFUL);
959 apr_thread_exit(thd, rv);
962 while (!workers_may_exit) {
963 ap_update_child_status_from_indexes(process_slot, thread_slot, SERVER_READY, NULL);
964 rv = worker_stack_wait(idle_worker_stack, wakeup);
965 if (APR_STATUS_IS_EOF(rv)) {
966 break; /* The queue has been terminated. */
968 else if (rv != APR_SUCCESS) {
969 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf,
970 "worker_stack_wait failed");
971 break; /* Treat all other errors as fatal. */
973 else if (wakeup->state == WORKER_TERMINATED) {
974 break; /* They told us to quit. */
976 AP_DEBUG_ASSERT(wakeup->state != WORKER_IDLE);
977 process_socket(ptrans, wakeup->csd,
978 process_slot, thread_slot, bucket_alloc);
979 requests_this_child--; /* FIXME: should be synchronized - aaron */
980 apr_pool_clear(ptrans);
983 ap_update_child_status_from_indexes(process_slot, thread_slot,
984 (dying) ? SERVER_DEAD : SERVER_GRACEFUL, (request_rec *) NULL);
986 apr_bucket_alloc_destroy(bucket_alloc);
988 apr_thread_exit(thd, APR_SUCCESS);
992 static int check_signal(int signum)
1002 static void create_listener_thread(thread_starter *ts)
1004 int my_child_num = ts->child_num_arg;
1005 apr_threadattr_t *thread_attr = ts->threadattr;
1009 my_info = (proc_info *)malloc(sizeof(proc_info));
1010 my_info->pid = my_child_num;
1011 my_info->tid = -1; /* listener thread doesn't have a thread slot */
1013 rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
1015 if (rv != APR_SUCCESS) {
1016 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,
1017 "apr_thread_create: unable to create listener thread");
1018 /* In case system resources are maxxed out, we don't want
1019 * Apache running away with the CPU trying to fork over and
1020 * over and over again if we exit.
1021 * XXX Jeff doesn't see how Apache is going to try to fork again since
1022 * the exit code is APEXIT_CHILDFATAL
1024 apr_sleep(10 * APR_USEC_PER_SEC);
1025 clean_child_exit(APEXIT_CHILDFATAL);
1027 apr_os_thread_get(&listener_os_thread, ts->listener);
1030 /* XXX under some circumstances not understood, children can get stuck
1031 * in start_threads forever trying to take over slots which will
1032 * never be cleaned up; for now there is an APLOG_DEBUG message issued
1033 * every so often when this condition occurs
1035 static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
1037 thread_starter *ts = dummy;
1038 apr_thread_t **threads = ts->threads;
1039 apr_threadattr_t *thread_attr = ts->threadattr;
1040 int child_num_arg = ts->child_num_arg;
1041 int my_child_num = child_num_arg;
1045 int threads_created = 0;
1047 int prev_threads_created;
1049 idle_worker_stack = worker_stack_create(pchild, ap_threads_per_child);
1050 if (idle_worker_stack == NULL) {
1051 ap_log_error(APLOG_MARK, APLOG_ALERT, 0, ap_server_conf,
1052 "worker_stack_create() failed");
1053 clean_child_exit(APEXIT_CHILDFATAL);
1056 loops = prev_threads_created = 0;
1058 /* ap_threads_per_child does not include the listener thread */
1059 for (i = 0; i < ap_threads_per_child; i++) {
1060 int status = ap_scoreboard_image->servers[child_num_arg][i].status;
1062 if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
1066 my_info = (proc_info *)malloc(sizeof(proc_info));
1067 if (my_info == NULL) {
1068 ap_log_error(APLOG_MARK, APLOG_ALERT, errno, ap_server_conf,
1069 "malloc: out of memory");
1070 clean_child_exit(APEXIT_CHILDFATAL);
1072 my_info->pid = my_child_num;
1076 /* We are creating threads right now */
1077 ap_update_child_status_from_indexes(my_child_num, i,
1078 SERVER_STARTING, NULL);
1079 /* We let each thread update its own scoreboard entry. This is
1080 * done because it lets us deal with tid better.
1082 rv = apr_thread_create(&threads[i], thread_attr,
1083 worker_thread, my_info, pchild);
1084 if (rv != APR_SUCCESS) {
1085 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,
1086 "apr_thread_create: unable to create worker thread");
1087 /* In case system resources are maxxed out, we don't want
1088 Apache running away with the CPU trying to fork over and
1089 over and over again if we exit. */
1090 apr_sleep(10 * APR_USEC_PER_SEC);
1091 clean_child_exit(APEXIT_CHILDFATAL);
1094 if (threads_created == 1) {
1095 /* now that we have a worker thread, it makes sense to create
1096 * a listener thread (we don't want a listener without a worker!)
1098 create_listener_thread(ts);
1101 if (start_thread_may_exit || threads_created == ap_threads_per_child) {
1104 /* wait for previous generation to clean up an entry */
1105 apr_sleep(1 * APR_USEC_PER_SEC);
1107 if (loops % 120 == 0) { /* every couple of minutes */
1108 if (prev_threads_created == threads_created) {
1109 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf,
1110 "child %" APR_PID_T_FMT " isn't taking over "
1111 "slots very quickly (%d of %d)",
1112 ap_my_pid, threads_created, ap_threads_per_child);
1114 prev_threads_created = threads_created;
1118 /* What state should this child_main process be listed as in the
1120 * ap_update_child_status_from_indexes(my_child_num, i, SERVER_STARTING,
1121 * (request_rec *) NULL);
1123 * This state should be listed separately in the scoreboard, in some kind
1124 * of process_status, not mixed in with the worker threads' status.
1125 * "life_status" is almost right, but it's in the worker's structure, and
1126 * the name could be clearer. gla
1128 apr_thread_exit(thd, APR_SUCCESS);
1132 static void join_workers(apr_thread_t *listener, apr_thread_t **threads)
1135 apr_status_t rv, thread_rv;
1140 /* deal with a rare timing window which affects waking up the
1141 * listener thread... if the signal sent to the listener thread
1142 * is delivered between the time it verifies that the
1143 * listener_may_exit flag is clear and the time it enters a
1144 * blocking syscall, the signal didn't do any good... work around
1145 * that by sleeping briefly and sending it again
1150 #ifdef HAVE_PTHREAD_KILL
1151 pthread_kill(*listener_os_thread, 0)
1156 /* listener not dead yet */
1157 apr_sleep(APR_USEC_PER_SEC / 2);
1162 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf,
1163 "the listener thread didn't exit");
1166 rv = apr_thread_join(&thread_rv, listener);
1167 if (rv != APR_SUCCESS) {
1168 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf,
1169 "apr_thread_join: unable to join listener thread");
1174 for (i = 0; i < ap_threads_per_child; i++) {
1175 if (threads[i]) { /* if we ever created this thread */
1176 rv = apr_thread_join(&thread_rv, threads[i]);
1177 if (rv != APR_SUCCESS) {
1178 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf,
1179 "apr_thread_join: unable to join worker "
1187 static void join_start_thread(apr_thread_t *start_thread_id)
1189 apr_status_t rv, thread_rv;
1191 start_thread_may_exit = 1; /* tell it to give up in case it is still
1192 * trying to take over slots from a
1193 * previous generation
1195 rv = apr_thread_join(&thread_rv, start_thread_id);
1196 if (rv != APR_SUCCESS) {
1197 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf,
1198 "apr_thread_join: unable to join the start "
1203 static void child_main(int child_num_arg)
1205 apr_thread_t **threads;
1208 apr_threadattr_t *thread_attr;
1209 apr_thread_t *start_thread_id;
1211 mpm_state = AP_MPMQ_STARTING; /* for benefit of any hooks that run as this
1214 ap_my_pid = getpid();
1215 ap_fatal_signal_child_setup(ap_server_conf);
1216 apr_pool_create(&pchild, pconf);
1218 /*stuff to do before we switch id's, so we have permissions.*/
1219 ap_reopen_scoreboard(pchild, NULL, 0);
1221 rv = SAFE_ACCEPT(apr_proc_mutex_child_init(&accept_mutex, ap_lock_fname,
1223 if (rv != APR_SUCCESS) {
1224 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
1225 "Couldn't initialize cross-process lock in child");
1226 clean_child_exit(APEXIT_CHILDFATAL);
1229 if (unixd_setup_child()) {
1230 clean_child_exit(APEXIT_CHILDFATAL);
1233 ap_run_child_init(pchild, ap_server_conf);
1235 /* done with init critical section */
1237 /* Just use the standard apr_setup_signal_thread to block all signals
1238 * from being received. The child processes no longer use signals for
1239 * any communication with the parent process.
1241 rv = apr_setup_signal_thread();
1242 if (rv != APR_SUCCESS) {
1243 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
1244 "Couldn't initialize signal thread");
1245 clean_child_exit(APEXIT_CHILDFATAL);
1248 if (ap_max_requests_per_child) {
1249 requests_this_child = ap_max_requests_per_child;
1252 /* coding a value of zero means infinity */
1253 requests_this_child = INT_MAX;
1256 /* Setup worker threads */
1258 /* clear the storage; we may not create all our threads immediately,
1259 * and we want a 0 entry to indicate a thread which was not created
1261 threads = (apr_thread_t **)calloc(1,
1262 sizeof(apr_thread_t *) * ap_threads_per_child);
1263 if (threads == NULL) {
1264 ap_log_error(APLOG_MARK, APLOG_ALERT, errno, ap_server_conf,
1265 "malloc: out of memory");
1266 clean_child_exit(APEXIT_CHILDFATAL);
1269 ts = (thread_starter *)apr_palloc(pchild, sizeof(*ts));
1271 apr_threadattr_create(&thread_attr, pchild);
1272 /* 0 means PTHREAD_CREATE_JOINABLE */
1273 apr_threadattr_detach_set(thread_attr, 0);
1275 ts->threads = threads;
1276 ts->listener = NULL;
1277 ts->child_num_arg = child_num_arg;
1278 ts->threadattr = thread_attr;
1280 rv = apr_thread_create(&start_thread_id, thread_attr, start_threads,
1282 if (rv != APR_SUCCESS) {
1283 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,
1284 "apr_thread_create: unable to create worker thread");
1285 /* In case system resources are maxxed out, we don't want
1286 Apache running away with the CPU trying to fork over and
1287 over and over again if we exit. */
1288 apr_sleep(10 * APR_USEC_PER_SEC);
1289 clean_child_exit(APEXIT_CHILDFATAL);
1292 mpm_state = AP_MPMQ_RUNNING;
1294 /* If we are only running in one_process mode, we will want to
1295 * still handle signals. */
1297 /* Block until we get a terminating signal. */
1298 apr_signal_thread(check_signal);
1299 /* make sure the start thread has finished; signal_threads()
1300 * and join_workers() depend on that
1302 /* XXX join_start_thread() won't be awakened if one of our
1303 * threads encounters a critical error and attempts to
1304 * shutdown this child
1306 join_start_thread(start_thread_id);
1307 signal_threads(ST_UNGRACEFUL); /* helps us terminate a little more
1308 * quickly than the dispatch of the signal thread
1309 * beats the Pipe of Death and the browsers
1311 /* A terminating signal was received. Now join each of the
1312 * workers to clean them up.
1313 * If the worker already exited, then the join frees
1314 * their resources and returns.
1315 * If the worker hasn't exited, then this blocks until
1316 * they have (then cleans up).
1318 join_workers(ts->listener, threads);
1320 else { /* !one_process */
1321 /* remove SIGTERM from the set of blocked signals... if one of
1322 * the other threads in the process needs to take us down
1323 * (e.g., for MaxRequestsPerChild) it will send us SIGTERM
1325 unblock_signal(SIGTERM);
1326 apr_signal(SIGTERM, dummy_signal_handler);
1327 /* Watch for any messages from the parent over the POD */
1329 rv = ap_mpm_pod_check(pod);
1330 if (rv == AP_NORESTART) {
1331 /* see if termination was triggered while we slept */
1332 switch(terminate_mode) {
1341 if (rv == AP_GRACEFUL || rv == AP_RESTART) {
1342 /* make sure the start thread has finished;
1343 * signal_threads() and join_workers depend on that
1345 join_start_thread(start_thread_id);
1346 signal_threads(rv == AP_GRACEFUL ? ST_GRACEFUL : ST_UNGRACEFUL);
1351 if (rv == AP_GRACEFUL) {
1352 /* A terminating signal was received. Now join each of the
1353 * workers to clean them up.
1354 * If the worker already exited, then the join frees
1355 * their resources and returns.
1356 * If the worker hasn't exited, then this blocks until
1357 * they have (then cleans up).
1359 join_workers(ts->listener, threads);
1365 clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0);
1368 static int make_child(server_rec *s, int slot)
1372 if (slot + 1 > ap_max_daemons_limit) {
1373 ap_max_daemons_limit = slot + 1;
1378 ap_scoreboard_image->parent[slot].pid = getpid();
1382 if ((pid = fork()) == -1) {
1383 ap_log_error(APLOG_MARK, APLOG_ERR, errno, s,
1384 "fork: Unable to fork new process");
1386 /* fork didn't succeed. Fix the scoreboard or else
1387 * it will say SERVER_STARTING forever and ever
1389 ap_update_child_status_from_indexes(slot, 0, SERVER_DEAD, NULL);
1391 /* In case system resources are maxxed out, we don't want
1392 Apache running away with the CPU trying to fork over and
1393 over and over again. */
1394 apr_sleep(10 * APR_USEC_PER_SEC);
1400 #ifdef HAVE_BINDPROCESSOR
1401 /* By default, AIX binds to a single processor. This bit unbinds
1402 * children which will then bind to another CPU.
1404 int status = bindprocessor(BINDPROCESS, (int)getpid(),
1405 PROCESSOR_CLASS_ANY);
1407 ap_log_error(APLOG_MARK, APLOG_WARNING, errno,
1409 "processor unbind failed %d", status);
1411 RAISE_SIGSTOP(MAKE_CHILD);
1413 apr_signal(SIGTERM, just_die);
1416 clean_child_exit(0);
1419 ap_scoreboard_image->parent[slot].quiescing = 0;
1420 ap_scoreboard_image->parent[slot].pid = pid;
1424 /* start up a bunch of children */
1425 static void startup_children(int number_to_start)
1429 for (i = 0; number_to_start && i < ap_daemons_limit; ++i) {
1430 if (ap_scoreboard_image->parent[i].pid != 0) {
1433 if (make_child(ap_server_conf, i) < 0) {
1442 * idle_spawn_rate is the number of children that will be spawned on the
1443 * next maintenance cycle if there aren't enough idle servers. It is
1444 * doubled up to MAX_SPAWN_RATE, and reset only when a cycle goes by
1445 * without the need to spawn.
1447 static int idle_spawn_rate = 1;
1448 #ifndef MAX_SPAWN_RATE
1449 #define MAX_SPAWN_RATE (32)
1451 static int hold_off_on_exponential_spawning;
1453 static void perform_idle_server_maintenance(void)
1456 int idle_thread_count;
1460 int totally_free_length = 0;
1461 int free_slots[MAX_SPAWN_RATE];
1465 /* initialize the free_list */
1468 idle_thread_count = 0;
1472 for (i = 0; i < ap_daemons_limit; ++i) {
1473 /* Initialization to satisfy the compiler. It doesn't know
1474 * that ap_threads_per_child is always > 0 */
1475 int status = SERVER_DEAD;
1476 int any_dying_threads = 0;
1477 int any_dead_threads = 0;
1478 int all_dead_threads = 1;
1480 if (i >= ap_max_daemons_limit && totally_free_length == idle_spawn_rate)
1482 ps = &ap_scoreboard_image->parent[i];
1483 for (j = 0; j < ap_threads_per_child; j++) {
1484 ws = &ap_scoreboard_image->servers[i][j];
1485 status = ws->status;
1487 /* XXX any_dying_threads is probably no longer needed GLA */
1488 any_dying_threads = any_dying_threads ||
1489 (status == SERVER_GRACEFUL);
1490 any_dead_threads = any_dead_threads || (status == SERVER_DEAD);
1491 all_dead_threads = all_dead_threads &&
1492 (status == SERVER_DEAD ||
1493 status == SERVER_GRACEFUL);
1495 /* We consider a starting server as idle because we started it
1496 * at least a cycle ago, and if it still hasn't finished starting
1497 * then we're just going to swamp things worse by forking more.
1498 * So we hopefully won't need to fork more if we count it.
1499 * This depends on the ordering of SERVER_READY and SERVER_STARTING.
1501 if (status <= SERVER_READY && status != SERVER_DEAD &&
1503 ps->generation == ap_my_generation &&
1504 /* XXX the following shouldn't be necessary if we clean up
1505 * properly after seg faults, but we're not yet GLA
1508 ++idle_thread_count;
1511 if (any_dead_threads && totally_free_length < idle_spawn_rate
1512 && (!ps->pid /* no process in the slot */
1513 || ps->quiescing)) { /* or at least one is going away */
1514 if (all_dead_threads) {
1515 /* great! we prefer these, because the new process can
1516 * start more threads sooner. So prioritize this slot
1517 * by putting it ahead of any slots with active threads.
1519 * first, make room by moving a slot that's potentially still
1520 * in use to the end of the array
1522 free_slots[free_length] = free_slots[totally_free_length];
1523 free_slots[totally_free_length++] = i;
1526 /* slot is still in use - back of the bus
1528 free_slots[free_length] = i;
1532 /* XXX if (!ps->quiescing) is probably more reliable GLA */
1533 if (!any_dying_threads) {
1538 ap_max_daemons_limit = last_non_dead + 1;
1540 if (idle_thread_count > max_spare_threads) {
1541 /* Kill off one child */
1542 ap_mpm_pod_signal(pod, TRUE);
1543 idle_spawn_rate = 1;
1545 else if (idle_thread_count < min_spare_threads) {
1546 /* terminate the free list */
1547 if (free_length == 0) {
1548 /* only report this condition once */
1549 static int reported = 0;
1552 ap_log_error(APLOG_MARK, APLOG_ERR, 0,
1554 "server reached MaxClients setting, consider"
1555 " raising the MaxClients setting");
1558 idle_spawn_rate = 1;
1561 if (free_length > idle_spawn_rate) {
1562 free_length = idle_spawn_rate;
1564 if (idle_spawn_rate >= 8) {
1565 ap_log_error(APLOG_MARK, APLOG_INFO, 0,
1567 "server seems busy, (you may need "
1568 "to increase StartServers, ThreadsPerChild "
1569 "or Min/MaxSpareThreads), "
1570 "spawning %d children, there are around %d idle "
1571 "threads, and %d total children", free_length,
1572 idle_thread_count, total_non_dead);
1574 for (i = 0; i < free_length; ++i) {
1575 make_child(ap_server_conf, free_slots[i]);
1577 /* the next time around we want to spawn twice as many if this
1578 * wasn't good enough, but not if we've just done a graceful
1580 if (hold_off_on_exponential_spawning) {
1581 --hold_off_on_exponential_spawning;
1583 else if (idle_spawn_rate < MAX_SPAWN_RATE) {
1584 idle_spawn_rate *= 2;
1589 idle_spawn_rate = 1;
1593 static void server_main_loop(int remaining_children_to_start)
1596 apr_exit_why_e exitwhy;
1597 int status, processed_status;
1601 while (!restart_pending && !shutdown_pending) {
1602 ap_wait_or_timeout(&exitwhy, &status, &pid, pconf);
1604 if (pid.pid != -1) {
1605 processed_status = ap_process_child_status(&pid, exitwhy, status);
1606 if (processed_status == APEXIT_CHILDFATAL) {
1607 shutdown_pending = 1;
1611 /* non-fatal death... note that it's gone in the scoreboard. */
1612 child_slot = find_child_by_pid(&pid);
1613 if (child_slot >= 0) {
1614 for (i = 0; i < ap_threads_per_child; i++)
1615 ap_update_child_status_from_indexes(child_slot, i, SERVER_DEAD,
1616 (request_rec *) NULL);
1618 ap_scoreboard_image->parent[child_slot].pid = 0;
1619 ap_scoreboard_image->parent[child_slot].quiescing = 0;
1620 if (processed_status == APEXIT_CHILDSICK) {
1621 /* resource shortage, minimize the fork rate */
1622 idle_spawn_rate = 1;
1624 else if (remaining_children_to_start
1625 && child_slot < ap_daemons_limit) {
1626 /* we're still doing a 1-for-1 replacement of dead
1627 * children with new children
1629 make_child(ap_server_conf, child_slot);
1630 --remaining_children_to_start;
1632 #if APR_HAS_OTHER_CHILD
1634 else if (apr_proc_other_child_read(&pid, status) == 0) {
1638 else if (is_graceful) {
1639 /* Great, we've probably just lost a slot in the
1640 * scoreboard. Somehow we don't know about this child.
1642 ap_log_error(APLOG_MARK, APLOG_WARNING, 0,
1644 "long lost child came home! (pid %ld)",
1647 /* Don't perform idle maintenance when a child dies,
1648 * only do it when there's a timeout. Remember only a
1649 * finite number of children can die, and it's pretty
1650 * pathological for a lot to die suddenly.
1654 else if (remaining_children_to_start) {
1655 /* we hit a 1 second timeout in which none of the previous
1656 * generation of children needed to be reaped... so assume
1657 * they're all done, and pick up the slack if any is left.
1659 startup_children(remaining_children_to_start);
1660 remaining_children_to_start = 0;
1661 /* In any event we really shouldn't do the code below because
1662 * few of the servers we just started are in the IDLE state
1663 * yet, so we'd mistakenly create an extra server.
1668 perform_idle_server_maintenance();
1672 int ap_mpm_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s)
1674 int remaining_children_to_start;
1677 ap_log_pid(pconf, ap_pid_fname);
1679 first_server_limit = server_limit;
1680 first_thread_limit = thread_limit;
1681 if (changed_limit_at_restart) {
1682 ap_log_error(APLOG_MARK, APLOG_WARNING, 0, s,
1683 "WARNING: Attempt to change ServerLimit or ThreadLimit "
1684 "ignored during restart");
1685 changed_limit_at_restart = 0;
1688 /* Initialize cross-process accept lock */
1689 ap_lock_fname = apr_psprintf(_pconf, "%s.%" APR_PID_T_FMT,
1690 ap_server_root_relative(_pconf, ap_lock_fname),
1693 rv = apr_proc_mutex_create(&accept_mutex, ap_lock_fname,
1694 ap_accept_lock_mech, _pconf);
1695 if (rv != APR_SUCCESS) {
1696 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, s,
1697 "Couldn't create accept lock");
1698 mpm_state = AP_MPMQ_STOPPING;
1702 #if APR_USE_SYSVSEM_SERIALIZE
1703 if (ap_accept_lock_mech == APR_LOCK_DEFAULT ||
1704 ap_accept_lock_mech == APR_LOCK_SYSVSEM) {
1706 if (ap_accept_lock_mech == APR_LOCK_SYSVSEM) {
1708 rv = unixd_set_proc_mutex_perms(accept_mutex);
1709 if (rv != APR_SUCCESS) {
1710 ap_log_error(APLOG_MARK, APLOG_EMERG, rv, s,
1711 "Couldn't set permissions on cross-process lock; "
1712 "check User and Group directives");
1713 mpm_state = AP_MPMQ_STOPPING;
1719 if (ap_run_pre_mpm(s->process->pool, SB_SHARED) != OK) {
1720 mpm_state = AP_MPMQ_STOPPING;
1723 /* fix the generation number in the global score; we just got a new,
1724 * cleared scoreboard
1726 ap_scoreboard_image->global->running_generation = ap_my_generation;
1730 /* Don't thrash... */
1731 if (max_spare_threads < min_spare_threads + ap_threads_per_child)
1732 max_spare_threads = min_spare_threads + ap_threads_per_child;
1734 /* If we're doing a graceful_restart then we're going to see a lot
1735 * of children exiting immediately when we get into the main loop
1736 * below (because we just sent them AP_SIG_GRACEFUL). This happens pretty
1737 * rapidly... and for each one that exits we'll start a new one until
1738 * we reach at least daemons_min_free. But we may be permitted to
1739 * start more than that, so we'll just keep track of how many we're
1740 * supposed to start up without the 1 second penalty between each fork.
1742 remaining_children_to_start = ap_daemons_to_start;
1743 if (remaining_children_to_start > ap_daemons_limit) {
1744 remaining_children_to_start = ap_daemons_limit;
1747 startup_children(remaining_children_to_start);
1748 remaining_children_to_start = 0;
1751 /* give the system some time to recover before kicking into
1752 * exponential mode */
1753 hold_off_on_exponential_spawning = 10;
1756 ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf,
1757 "%s configured -- resuming normal operations",
1758 ap_get_server_version());
1759 ap_log_error(APLOG_MARK, APLOG_INFO, 0, ap_server_conf,
1760 "Server built: %s", ap_get_server_built());
1761 #ifdef AP_MPM_WANT_SET_ACCEPT_LOCK_MECH
1762 ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf,
1763 "AcceptMutex: %s (default: %s)",
1764 apr_proc_mutex_name(accept_mutex),
1765 apr_proc_mutex_defname());
1767 restart_pending = shutdown_pending = 0;
1768 mpm_state = AP_MPMQ_RUNNING;
1770 server_main_loop(remaining_children_to_start);
1771 mpm_state = AP_MPMQ_STOPPING;
1773 if (shutdown_pending) {
1774 /* Time to gracefully shut down:
1775 * Kill child processes, tell them to call child_exit, etc...
1776 * (By "gracefully" we don't mean graceful in the same sense as
1777 * "apachectl graceful" where we allow old connections to finish.)
1779 ap_mpm_pod_killpg(pod, ap_daemons_limit, FALSE);
1780 ap_reclaim_child_processes(1); /* Start with SIGTERM */
1783 /* cleanup pid file on normal shutdown */
1784 const char *pidfile = NULL;
1785 pidfile = ap_server_root_relative (pconf, ap_pid_fname);
1786 if ( pidfile != NULL && unlink(pidfile) == 0)
1787 ap_log_error(APLOG_MARK, APLOG_INFO, 0,
1789 "removed PID file %s (pid=%ld)",
1790 pidfile, (long)getpid());
1792 ap_log_error(APLOG_MARK, APLOG_NOTICE, 0,
1793 ap_server_conf, "caught SIGTERM, shutting down");
1798 /* we've been told to restart */
1799 apr_signal(SIGHUP, SIG_IGN);
1802 /* not worth thinking about */
1806 /* advance to the next generation */
1807 /* XXX: we really need to make sure this new generation number isn't in
1808 * use by any of the children.
1811 ap_scoreboard_image->global->running_generation = ap_my_generation;
1814 ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf,
1815 AP_SIG_GRACEFUL_STRING " received. Doing graceful restart");
1816 /* wake up the children...time to die. But we'll have more soon */
1817 ap_mpm_pod_killpg(pod, ap_daemons_limit, TRUE);
1820 /* This is mostly for debugging... so that we know what is still
1821 * gracefully dealing with existing request.
1826 /* Kill 'em all. Since the child acts the same on the parents SIGTERM
1827 * and a SIGHUP, we may as well use the same signal, because some user
1828 * pthreads are stealing signals from us left and right.
1830 ap_mpm_pod_killpg(pod, ap_daemons_limit, FALSE);
1832 ap_reclaim_child_processes(1); /* Start with SIGTERM */
1833 ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf,
1834 "SIGHUP received. Attempting to restart");
1840 /* This really should be a post_config hook, but the error log is already
1841 * redirected by that point, so we need to do this in the open_logs phase.
1843 static int worker_open_logs(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, server_rec *s)
1851 if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) {
1852 ap_log_error(APLOG_MARK, APLOG_ALERT|APLOG_STARTUP, 0,
1853 NULL, "no listening sockets available, shutting down");
1857 #if APR_O_NONBLOCK_INHERITED
1858 for(lr = ap_listeners ; lr != NULL ; lr = lr->next) {
1859 apr_socket_opt_set(lr->sd, APR_SO_NONBLOCK, 1);
1861 #endif /* APR_O_NONBLOCK_INHERITED */
1864 if ((rv = ap_mpm_pod_open(pconf, &pod))) {
1865 ap_log_error(APLOG_MARK, APLOG_CRIT|APLOG_STARTUP, rv, NULL,
1866 "Could not open pipe-of-death.");
1873 static int worker_pre_config(apr_pool_t *pconf, apr_pool_t *plog,
1876 static int restart_num = 0;
1877 int no_detach, debug, foreground;
1878 ap_directive_t *pdir;
1879 ap_directive_t *max_clients = NULL;
1882 mpm_state = AP_MPMQ_STARTING;
1884 /* make sure that "ThreadsPerChild" gets set before "MaxClients" */
1885 for (pdir = ap_conftree; pdir != NULL; pdir = pdir->next) {
1886 if (strncasecmp(pdir->directive, "ThreadsPerChild", 15) == 0) {
1888 break; /* we're in the clear, got ThreadsPerChild first */
1891 /* now to swap the data */
1892 ap_directive_t temp;
1894 temp.directive = pdir->directive;
1895 temp.args = pdir->args;
1896 /* Make sure you don't change 'next', or you may get loops! */
1897 /* XXX: first_child, parent, and data can never be set
1898 * for these directives, right? -aaron */
1899 temp.filename = pdir->filename;
1900 temp.line_num = pdir->line_num;
1902 pdir->directive = max_clients->directive;
1903 pdir->args = max_clients->args;
1904 pdir->filename = max_clients->filename;
1905 pdir->line_num = max_clients->line_num;
1907 max_clients->directive = temp.directive;
1908 max_clients->args = temp.args;
1909 max_clients->filename = temp.filename;
1910 max_clients->line_num = temp.line_num;
1914 else if (!max_clients
1915 && strncasecmp(pdir->directive, "MaxClients", 10) == 0) {
1920 debug = ap_exists_config_define("DEBUG");
1923 foreground = one_process = 1;
1927 one_process = ap_exists_config_define("ONE_PROCESS");
1928 no_detach = ap_exists_config_define("NO_DETACH");
1929 foreground = ap_exists_config_define("FOREGROUND");
1932 /* sigh, want this only the second time around */
1933 if (restart_num++ == 1) {
1936 if (!one_process && !foreground) {
1937 rv = apr_proc_detach(no_detach ? APR_PROC_DETACH_FOREGROUND
1938 : APR_PROC_DETACH_DAEMONIZE);
1939 if (rv != APR_SUCCESS) {
1940 ap_log_error(APLOG_MARK, APLOG_CRIT, rv, NULL,
1941 "apr_proc_detach failed");
1942 return HTTP_INTERNAL_SERVER_ERROR;
1945 parent_pid = ap_my_pid = getpid();
1948 unixd_pre_config(ptemp);
1949 ap_listen_pre_config();
1950 ap_daemons_to_start = DEFAULT_START_DAEMON;
1951 min_spare_threads = DEFAULT_MIN_FREE_DAEMON * DEFAULT_THREADS_PER_CHILD;
1952 max_spare_threads = DEFAULT_MAX_FREE_DAEMON * DEFAULT_THREADS_PER_CHILD;
1953 ap_daemons_limit = server_limit;
1954 ap_threads_per_child = DEFAULT_THREADS_PER_CHILD;
1955 ap_pid_fname = DEFAULT_PIDLOG;
1956 ap_lock_fname = DEFAULT_LOCKFILE;
1957 ap_max_requests_per_child = DEFAULT_MAX_REQUESTS_PER_CHILD;
1958 ap_extended_status = 0;
1959 #ifdef AP_MPM_WANT_SET_MAX_MEM_FREE
1960 ap_max_mem_free = APR_ALLOCATOR_MAX_FREE_UNLIMITED;
1963 apr_cpystrn(ap_coredump_dir, ap_server_root, sizeof(ap_coredump_dir));
1968 static void threadpool_hooks(apr_pool_t *p)
1970 /* The worker open_logs phase must run before the core's, or stderr
1971 * will be redirected to a file, and the messages won't print to the
1974 static const char *const aszSucc[] = {"core.c", NULL};
1977 ap_hook_open_logs(worker_open_logs, NULL, aszSucc, APR_HOOK_MIDDLE);
1978 /* we need to set the MPM state before other pre-config hooks use MPM query
1979 * to retrieve it, so register as REALLY_FIRST
1981 ap_hook_pre_config(worker_pre_config, NULL, NULL, APR_HOOK_REALLY_FIRST);
1984 static const char *set_daemons_to_start(cmd_parms *cmd, void *dummy,
1987 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
1992 ap_daemons_to_start = atoi(arg);
1996 static const char *set_min_spare_threads(cmd_parms *cmd, void *dummy,
1999 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2004 min_spare_threads = atoi(arg);
2005 if (min_spare_threads <= 0) {
2006 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2007 "WARNING: detected MinSpareThreads set to non-positive.");
2008 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2009 "Resetting to 1 to avoid almost certain Apache failure.");
2010 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2011 "Please read the documentation.");
2012 min_spare_threads = 1;
2018 static const char *set_max_spare_threads(cmd_parms *cmd, void *dummy,
2021 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2026 max_spare_threads = atoi(arg);
2030 static const char *set_max_clients (cmd_parms *cmd, void *dummy,
2034 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2039 /* It is ok to use ap_threads_per_child here because we are
2040 * sure that it gets set before MaxClients in the pre_config stage. */
2041 max_clients = atoi(arg);
2042 if (max_clients < ap_threads_per_child) {
2043 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2044 "WARNING: MaxClients (%d) must be at least as large",
2046 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2047 " large as ThreadsPerChild (%d). Automatically",
2048 ap_threads_per_child);
2049 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2050 " increasing MaxClients to %d.",
2051 ap_threads_per_child);
2052 max_clients = ap_threads_per_child;
2054 ap_daemons_limit = max_clients / ap_threads_per_child;
2055 if ((max_clients > 0) && (max_clients % ap_threads_per_child)) {
2056 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2057 "WARNING: MaxClients (%d) is not an integer multiple",
2059 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2060 " of ThreadsPerChild (%d), lowering MaxClients to %d",
2061 ap_threads_per_child,
2062 ap_daemons_limit * ap_threads_per_child);
2063 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2064 " for a maximum of %d child processes,",
2066 max_clients = ap_daemons_limit * ap_threads_per_child;
2068 if (ap_daemons_limit > server_limit) {
2069 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2070 "WARNING: MaxClients of %d would require %d servers,",
2071 max_clients, ap_daemons_limit);
2072 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2073 " and would exceed the ServerLimit value of %d.",
2075 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2076 " Automatically lowering MaxClients to %d. To increase,",
2077 server_limit * ap_threads_per_child);
2078 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2079 " please see the ServerLimit directive.");
2080 ap_daemons_limit = server_limit;
2082 else if (ap_daemons_limit < 1) {
2083 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2084 "WARNING: Require MaxClients > 0, setting to 1");
2085 ap_daemons_limit = 1;
2090 static const char *set_threads_per_child (cmd_parms *cmd, void *dummy,
2093 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2098 ap_threads_per_child = atoi(arg);
2099 if (ap_threads_per_child > thread_limit) {
2100 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2101 "WARNING: ThreadsPerChild of %d exceeds ThreadLimit "
2102 "value of %d", ap_threads_per_child,
2104 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2105 "threads, lowering ThreadsPerChild to %d. To increase, please"
2106 " see the", thread_limit);
2107 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2108 " ThreadLimit directive.");
2109 ap_threads_per_child = thread_limit;
2111 else if (ap_threads_per_child < 1) {
2112 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2113 "WARNING: Require ThreadsPerChild > 0, setting to 1");
2114 ap_threads_per_child = 1;
2119 static const char *set_server_limit (cmd_parms *cmd, void *dummy, const char *arg)
2121 int tmp_server_limit;
2123 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2128 tmp_server_limit = atoi(arg);
2129 /* you cannot change ServerLimit across a restart; ignore
2132 if (first_server_limit &&
2133 tmp_server_limit != server_limit) {
2134 /* how do we log a message? the error log is a bit bucket at this
2135 * point; we'll just have to set a flag so that ap_mpm_run()
2136 * logs a warning later
2138 changed_limit_at_restart = 1;
2141 server_limit = tmp_server_limit;
2143 if (server_limit > MAX_SERVER_LIMIT) {
2144 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2145 "WARNING: ServerLimit of %d exceeds compile time limit "
2146 "of %d servers,", server_limit, MAX_SERVER_LIMIT);
2147 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2148 " lowering ServerLimit to %d.", MAX_SERVER_LIMIT);
2149 server_limit = MAX_SERVER_LIMIT;
2151 else if (server_limit < 1) {
2152 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2153 "WARNING: Require ServerLimit > 0, setting to 1");
2159 static const char *set_thread_limit (cmd_parms *cmd, void *dummy, const char *arg)
2161 int tmp_thread_limit;
2163 const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY);
2168 tmp_thread_limit = atoi(arg);
2169 /* you cannot change ThreadLimit across a restart; ignore
2172 if (first_thread_limit &&
2173 tmp_thread_limit != thread_limit) {
2174 /* how do we log a message? the error log is a bit bucket at this
2175 * point; we'll just have to set a flag so that ap_mpm_run()
2176 * logs a warning later
2178 changed_limit_at_restart = 1;
2181 thread_limit = tmp_thread_limit;
2183 if (thread_limit > MAX_THREAD_LIMIT) {
2184 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2185 "WARNING: ThreadLimit of %d exceeds compile time limit "
2186 "of %d servers,", thread_limit, MAX_THREAD_LIMIT);
2187 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2188 " lowering ThreadLimit to %d.", MAX_THREAD_LIMIT);
2189 thread_limit = MAX_THREAD_LIMIT;
2191 else if (thread_limit < 1) {
2192 ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL,
2193 "WARNING: Require ThreadLimit > 0, setting to 1");
2199 static const command_rec threadpool_cmds[] = {
2200 UNIX_DAEMON_COMMANDS,
2202 AP_INIT_TAKE1("StartServers", set_daemons_to_start, NULL, RSRC_CONF,
2203 "Number of child processes launched at server startup"),
2204 AP_INIT_TAKE1("MinSpareThreads", set_min_spare_threads, NULL, RSRC_CONF,
2205 "Minimum number of idle children, to handle request spikes"),
2206 AP_INIT_TAKE1("MaxSpareThreads", set_max_spare_threads, NULL, RSRC_CONF,
2207 "Maximum number of idle children"),
2208 AP_INIT_TAKE1("MaxClients", set_max_clients, NULL, RSRC_CONF,
2209 "Maximum number of children alive at the same time"),
2210 AP_INIT_TAKE1("ThreadsPerChild", set_threads_per_child, NULL, RSRC_CONF,
2211 "Number of threads each child creates"),
2212 AP_INIT_TAKE1("ServerLimit", set_server_limit, NULL, RSRC_CONF,
2213 "Maximum value of MaxClients for this run of Apache"),
2214 AP_INIT_TAKE1("ThreadLimit", set_thread_limit, NULL, RSRC_CONF,
2215 "Maximum worker threads in a server for this run of Apache"),
2219 module AP_MODULE_DECLARE_DATA mpm_threadpool_module = {
2221 ap_mpm_rewrite_args, /* hook to run before apache parses args */
2222 NULL, /* create per-directory config structure */
2223 NULL, /* merge per-directory config structures */
2224 NULL, /* create per-server config structure */
2225 NULL, /* merge per-server config structures */
2226 threadpool_cmds, /* command apr_table_t */
2227 threadpool_hooks /* register_hooks */