corosync  2.4.2
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2006 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
72 #include <config.h>
73 
74 #include <pthread.h>
75 #include <assert.h>
76 #include <sys/types.h>
77 #include <sys/file.h>
78 #include <sys/poll.h>
79 #include <sys/uio.h>
80 #include <sys/mman.h>
81 #include <sys/socket.h>
82 #include <sys/un.h>
83 #include <sys/time.h>
84 #include <sys/resource.h>
85 #include <sys/stat.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
88 #include <unistd.h>
89 #include <fcntl.h>
90 #include <stdlib.h>
91 #include <stdio.h>
92 #include <errno.h>
93 #include <signal.h>
94 #include <sched.h>
95 #include <time.h>
96 #include <semaphore.h>
97 #include <string.h>
98 
99 #ifdef HAVE_LIBSYSTEMD
100 #include <systemd/sd-daemon.h>
101 #endif
102 
103 #include <qb/qbdefs.h>
104 #include <qb/qblog.h>
105 #include <qb/qbloop.h>
106 #include <qb/qbutil.h>
107 #include <qb/qbipcs.h>
108 
109 #include <corosync/swab.h>
110 #include <corosync/corotypes.h>
111 #include <corosync/corodefs.h>
112 #include <corosync/list.h>
113 #include <corosync/totem/totempg.h>
114 #include <corosync/logsys.h>
115 #include <corosync/icmap.h>
116 
117 #include "quorum.h"
118 #include "totemsrp.h"
119 #include "logconfig.h"
120 #include "totemconfig.h"
121 #include "main.h"
122 #include "sync.h"
123 #include "timer.h"
124 #include "util.h"
125 #include "apidef.h"
126 #include "service.h"
127 #include "schedwrk.h"
128 
129 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
130 #define IPC_LOGSYS_SIZE 1024*64
131 #else
132 #define IPC_LOGSYS_SIZE 8192*128
133 #endif
134 
135 LOGSYS_DECLARE_SYSTEM ("corosync",
137  LOG_DAEMON,
138  LOG_INFO);
139 
140 LOGSYS_DECLARE_SUBSYS ("MAIN");
141 
142 #define SERVER_BACKLOG 5
143 
144 static int sched_priority = 0;
145 
146 static unsigned int service_count = 32;
147 
149 
150 static struct corosync_api_v1 *api = NULL;
151 
152 static int sync_in_process = 1;
153 
154 static qb_loop_t *corosync_poll_handle;
155 
156 struct sched_param global_sched_param;
157 
158 static corosync_timer_handle_t corosync_stats_timer_handle;
159 
160 static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid";
161 
162 static int ip_version = AF_INET;
163 
164 qb_loop_t *cs_poll_handle_get (void)
165 {
166  return (corosync_poll_handle);
167 }
168 
169 int cs_poll_dispatch_add (qb_loop_t * handle,
170  int fd,
171  int events,
172  void *data,
173 
174  int (*dispatch_fn) (int fd,
175  int revents,
176  void *data))
177 {
178  return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
179  dispatch_fn);
180 }
181 
182 int cs_poll_dispatch_delete(qb_loop_t * handle, int fd)
183 {
184  return qb_loop_poll_del(handle, fd);
185 }
186 
188 {
189  int i;
190 
191  for (i = 0; i < SERVICES_COUNT_MAX; i++) {
192  if (corosync_service[i] && corosync_service[i]->exec_dump_fn) {
194  }
195  }
196 }
197 
198 static void corosync_blackbox_write_to_file (void)
199 {
200  char fname[PATH_MAX];
201  char fdata_fname[PATH_MAX];
202  char time_str[PATH_MAX];
203  struct tm cur_time_tm;
204  time_t cur_time_t;
205  ssize_t res;
206 
207  cur_time_t = time(NULL);
208  localtime_r(&cur_time_t, &cur_time_tm);
209 
210  strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm);
211  snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld",
212  get_run_dir(),
213  time_str,
214  (long long int)getpid());
215 
216  if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
217  LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file");
218  }
219  snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_run_dir());
220  unlink(fdata_fname);
221  if (symlink(fname, fdata_fname) == -1) {
222  log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'",
223  fname, fdata_fname);
224  }
225 }
226 
227 static void unlink_all_completed (void)
228 {
229  api->timer_delete (corosync_stats_timer_handle);
230  qb_loop_stop (corosync_poll_handle);
231  icmap_fini();
232 }
233 
235 {
236  corosync_service_unlink_all (api, unlink_all_completed);
237 }
238 
239 static int32_t sig_diag_handler (int num, void *data)
240 {
242  return 0;
243 }
244 
245 static int32_t sig_exit_handler (int num, void *data)
246 {
247  log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal");
248  corosync_service_unlink_all (api, unlink_all_completed);
249  return 0;
250 }
251 
252 static void sigsegv_handler (int num)
253 {
254  (void)signal (SIGSEGV, SIG_DFL);
255  corosync_blackbox_write_to_file ();
256  qb_log_fini();
257  raise (SIGSEGV);
258 }
259 
260 /*
261  * QB wrapper for real signal handler
262  */
263 static int32_t sig_segv_handler (int num, void *data)
264 {
265 
266  sigsegv_handler(num);
267 
268  return 0;
269 }
270 
271 static void sigabrt_handler (int num)
272 {
273  (void)signal (SIGABRT, SIG_DFL);
274  corosync_blackbox_write_to_file ();
275  qb_log_fini();
276  raise (SIGABRT);
277 }
278 
279 /*
280  * QB wrapper for real signal handler
281  */
282 static int32_t sig_abrt_handler (int num, void *data)
283 {
284 
285  sigabrt_handler(num);
286 
287  return 0;
288 }
289 
290 #define LOCALHOST_IP inet_addr("127.0.0.1")
291 
292 static void *corosync_group_handle;
293 
294 static struct totempg_group corosync_group = {
295  .group = "a",
296  .group_len = 1
297 };
298 
299 static void serialize_lock (void)
300 {
301 }
302 
303 static void serialize_unlock (void)
304 {
305 }
306 
307 static void corosync_sync_completed (void)
308 {
310  "Completed service synchronization, ready to provide service.");
311  sync_in_process = 0;
312 
313  cs_ipcs_sync_state_changed(sync_in_process);
315  /*
316  * Inform totem to start using new message queue again
317  */
319 
320 #ifdef HAVE_LIBSYSTEMD
321  sd_notify (0, "READY=1");
322 #endif
323 }
324 
325 static int corosync_sync_callbacks_retrieve (
326  int service_id,
327  struct sync_callbacks *callbacks)
328 {
329  if (corosync_service[service_id] == NULL) {
330  return (-1);
331  }
332 
333  if (callbacks == NULL) {
334  return (0);
335  }
336 
337  callbacks->name = corosync_service[service_id]->name;
338 
339  callbacks->sync_init = corosync_service[service_id]->sync_init;
340  callbacks->sync_process = corosync_service[service_id]->sync_process;
341  callbacks->sync_activate = corosync_service[service_id]->sync_activate;
342  callbacks->sync_abort = corosync_service[service_id]->sync_abort;
343  return (0);
344 }
345 
346 static struct memb_ring_id corosync_ring_id;
347 
348 static void member_object_joined (unsigned int nodeid)
349 {
350  char member_ip[ICMAP_KEYNAME_MAXLEN];
351  char member_join_count[ICMAP_KEYNAME_MAXLEN];
352  char member_status[ICMAP_KEYNAME_MAXLEN];
353 
354  snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
355  "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
356  snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
357  "runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
358  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
359  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
360 
361  if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
362  icmap_inc(member_join_count);
363  icmap_set_string(member_status, "joined");
364  } else {
365  icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid));
366  icmap_set_uint32(member_join_count, 1);
367  icmap_set_string(member_status, "joined");
368  }
369 
371  "Member joined: %s", api->totem_ifaces_print (nodeid));
372 }
373 
374 static void member_object_left (unsigned int nodeid)
375 {
376  char member_status[ICMAP_KEYNAME_MAXLEN];
377 
378  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
379  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
380  icmap_set_string(member_status, "left");
381 
383  "Member left: %s", api->totem_ifaces_print (nodeid));
384 }
385 
386 static void confchg_fn (
387  enum totem_configuration_type configuration_type,
388  const unsigned int *member_list, size_t member_list_entries,
389  const unsigned int *left_list, size_t left_list_entries,
390  const unsigned int *joined_list, size_t joined_list_entries,
391  const struct memb_ring_id *ring_id)
392 {
393  int i;
394  int abort_activate = 0;
395 
396  if (sync_in_process == 1) {
397  abort_activate = 1;
398  }
399  sync_in_process = 1;
400  cs_ipcs_sync_state_changed(sync_in_process);
401  memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id));
402 
403  for (i = 0; i < left_list_entries; i++) {
404  member_object_left (left_list[i]);
405  }
406  for (i = 0; i < joined_list_entries; i++) {
407  member_object_joined (joined_list[i]);
408  }
409  /*
410  * Call configuration change for all services
411  */
412  for (i = 0; i < service_count; i++) {
413  if (corosync_service[i] && corosync_service[i]->confchg_fn) {
414  corosync_service[i]->confchg_fn (configuration_type,
415  member_list, member_list_entries,
416  left_list, left_list_entries,
417  joined_list, joined_list_entries, ring_id);
418  }
419  }
420 
421  if (abort_activate) {
422  sync_abort ();
423  }
424  if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) {
425  sync_save_transitional (member_list, member_list_entries, ring_id);
426  }
427  if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
428  sync_start (member_list, member_list_entries, ring_id);
429  }
430 }
431 
432 static void priv_drop (void)
433 {
434  return; /* TODO: we are still not dropping privs */
435 }
436 
437 static void corosync_tty_detach (void)
438 {
439  int devnull;
440 
441  /*
442  * Disconnect from TTY if this is not a debug run
443  */
444 
445  switch (fork ()) {
446  case -1:
448  break;
449  case 0:
450  /*
451  * child which is disconnected, run this process
452  */
453  break;
454  default:
455  exit (0);
456  break;
457  }
458 
459  /* Create new session */
460  (void)setsid();
461 
462  /*
463  * Map stdin/out/err to /dev/null.
464  */
465  devnull = open("/dev/null", O_RDWR);
466  if (devnull == -1) {
468  }
469 
470  if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
471  || dup2(devnull, 2) < 0) {
472  close(devnull);
474  }
475  close(devnull);
476 }
477 
478 static void corosync_mlockall (void)
479 {
480  int res;
481  struct rlimit rlimit;
482 
483  rlimit.rlim_cur = RLIM_INFINITY;
484  rlimit.rlim_max = RLIM_INFINITY;
485 
486 #ifndef RLIMIT_MEMLOCK
487 #define RLIMIT_MEMLOCK RLIMIT_VMEM
488 #endif
489 
490  setrlimit (RLIMIT_MEMLOCK, &rlimit);
491 
492  res = mlockall (MCL_CURRENT | MCL_FUTURE);
493  if (res == -1) {
495  "Could not lock memory of service to avoid page faults");
496  };
497 }
498 
499 
500 static void corosync_totem_stats_updater (void *data)
501 {
502  totempg_stats_t * stats;
503  uint32_t total_mtt_rx_token;
504  uint32_t total_backlog_calc;
505  uint32_t total_token_holdtime;
506  int t, prev, i;
507  int32_t token_count;
508  char key_name[ICMAP_KEYNAME_MAXLEN];
509 
510  stats = api->totem_get_stats();
511 
512  icmap_set_uint32("runtime.totem.pg.msg_reserved", stats->msg_reserved);
513  icmap_set_uint32("runtime.totem.pg.msg_queue_avail", stats->msg_queue_avail);
514  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_tx", stats->mrp->srp->orf_token_tx);
515  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_rx", stats->mrp->srp->orf_token_rx);
516  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_tx", stats->mrp->srp->memb_merge_detect_tx);
517  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_rx", stats->mrp->srp->memb_merge_detect_rx);
518  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_tx", stats->mrp->srp->memb_join_tx);
519  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_rx", stats->mrp->srp->memb_join_rx);
520  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_tx", stats->mrp->srp->mcast_tx);
521  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_retx", stats->mrp->srp->mcast_retx);
522  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_rx", stats->mrp->srp->mcast_rx);
523  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_tx", stats->mrp->srp->memb_commit_token_tx);
524  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_rx", stats->mrp->srp->memb_commit_token_rx);
525  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_tx", stats->mrp->srp->token_hold_cancel_tx);
526  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_rx", stats->mrp->srp->token_hold_cancel_rx);
527  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_entered", stats->mrp->srp->operational_entered);
528  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_token_lost", stats->mrp->srp->operational_token_lost);
529  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_entered", stats->mrp->srp->gather_entered);
530  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_token_lost", stats->mrp->srp->gather_token_lost);
531  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_entered", stats->mrp->srp->commit_entered);
532  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_token_lost", stats->mrp->srp->commit_token_lost);
533  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_entered", stats->mrp->srp->recovery_entered);
534  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_token_lost", stats->mrp->srp->recovery_token_lost);
535  icmap_set_uint64("runtime.totem.pg.mrp.srp.consensus_timeouts", stats->mrp->srp->consensus_timeouts);
536  icmap_set_uint64("runtime.totem.pg.mrp.srp.rx_msg_dropped", stats->mrp->srp->rx_msg_dropped);
537  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_gather", stats->mrp->srp->continuous_gather);
538  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_sendmsg_failures",
540 
541  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
542  stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
543 
544  if (stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ||
547  "Totem is unable to form a cluster because of an "
548  "operating system or network fault. The most common "
549  "cause of this message is that the local firewall is "
550  "configured improperly.");
551  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
552  } else {
553  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
554  }
555 
556  for (i = 0; i < stats->mrp->srp->rrp->interface_count; i++) {
557  snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.totem.pg.mrp.rrp.%u.faulty", i);
558  icmap_set_uint8(key_name, stats->mrp->srp->rrp->faulty[i]);
559  }
560  total_mtt_rx_token = 0;
561  total_token_holdtime = 0;
562  total_backlog_calc = 0;
563  token_count = 0;
564  t = stats->mrp->srp->latest_token;
565  while (1) {
566  if (t == 0)
567  prev = TOTEM_TOKEN_STATS_MAX - 1;
568  else
569  prev = t - 1;
570  if (prev == stats->mrp->srp->earliest_token)
571  break;
572  /* if tx == 0, then dropped token (not ours) */
573  if (stats->mrp->srp->token[t].tx != 0 ||
574  (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx) > 0 ) {
575  total_mtt_rx_token += (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx);
576  total_token_holdtime += (stats->mrp->srp->token[t].tx - stats->mrp->srp->token[t].rx);
577  total_backlog_calc += stats->mrp->srp->token[t].backlog_calc;
578  token_count++;
579  }
580  t = prev;
581  }
582  if (token_count) {
583  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
584  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
585  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
586  }
587 
589 
590  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
591  corosync_totem_stats_updater,
592  &corosync_stats_timer_handle);
593 }
594 
595 static void corosync_totem_stats_init (void)
596 {
597  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", 0);
598  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", 0);
599  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", 0);
600 
601  /* start stats timer */
602  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
603  corosync_totem_stats_updater,
604  &corosync_stats_timer_handle);
605 }
606 
607 static void deliver_fn (
608  unsigned int nodeid,
609  const void *msg,
610  unsigned int msg_len,
611  int endian_conversion_required)
612 {
613  const struct qb_ipc_request_header *header;
614  int32_t service;
615  int32_t fn_id;
616  uint32_t id;
617 
618  header = msg;
619  if (endian_conversion_required) {
620  id = swab32 (header->id);
621  } else {
622  id = header->id;
623  }
624 
625  /*
626  * Call the proper executive handler
627  */
628  service = id >> 16;
629  fn_id = id & 0xffff;
630 
631  if (!corosync_service[service]) {
632  return;
633  }
634  if (fn_id >= corosync_service[service]->exec_engine_count) {
635  log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)",
636  fn_id, service, corosync_service[service]->exec_engine_count);
637  return;
638  }
639 
640  icmap_fast_inc(service_stats_rx[service][fn_id]);
641 
642  if (endian_conversion_required) {
643  assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
645  ((void *)msg);
646  }
647 
649  (msg, nodeid);
650 }
651 
653  const struct iovec *iovec,
654  unsigned int iov_len,
655  unsigned int guarantee)
656 {
657  const struct qb_ipc_request_header *req = iovec->iov_base;
658  int32_t service;
659  int32_t fn_id;
660 
661  service = req->id >> 16;
662  fn_id = req->id & 0xffff;
663 
664  if (corosync_service[service]) {
665  icmap_fast_inc(service_stats_tx[service][fn_id]);
666  }
667 
668  return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
669 }
670 
671 static void corosync_ring_id_create_or_load (
672  struct memb_ring_id *memb_ring_id,
673  const struct totem_ip_address *addr)
674 {
675  int fd;
676  int res = 0;
677  char filename[PATH_MAX];
678 
679  snprintf (filename, sizeof(filename), "%s/ringid_%s",
680  get_run_dir(), totemip_print (addr));
681  fd = open (filename, O_RDONLY, 0700);
682  /*
683  * If file can be opened and read, read the ring id
684  */
685  if (fd != -1) {
686  res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
687  close (fd);
688  }
689  /*
690  * If file could not be opened or read, create a new ring id
691  */
692  if ((fd == -1) || (res != sizeof (uint64_t))) {
693  memb_ring_id->seq = 0;
694  umask(0);
695  fd = open (filename, O_CREAT|O_RDWR, 0700);
696  if (fd != -1) {
697  res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
698  close (fd);
699  if (res == -1) {
701  "Couldn't write ringid file '%s'", filename);
702 
704  }
705  } else {
707  "Couldn't create ringid file '%s'", filename);
708 
710  }
711  }
712 
713  totemip_copy(&memb_ring_id->rep, addr);
714  assert (!totemip_zero_check(&memb_ring_id->rep));
715 }
716 
717 static void corosync_ring_id_store (
718  const struct memb_ring_id *memb_ring_id,
719  const struct totem_ip_address *addr)
720 {
721  char filename[PATH_MAX];
722  int fd;
723  int res;
724 
725  snprintf (filename, sizeof(filename), "%s/ringid_%s",
726  get_run_dir(), totemip_print (addr));
727 
728  fd = open (filename, O_WRONLY, 0700);
729  if (fd == -1) {
730  fd = open (filename, O_CREAT|O_RDWR, 0700);
731  }
732  if (fd == -1) {
734  "Couldn't store new ring id %llx to stable storage",
735  memb_ring_id->seq);
736 
738  }
740  "Storing new sequence id for ring %llx", memb_ring_id->seq);
741  res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
742  close (fd);
743  if (res != sizeof(memb_ring_id->seq)) {
745  "Couldn't store new ring id %llx to stable storage",
746  memb_ring_id->seq);
747 
749  }
750 }
751 
752 static qb_loop_timer_handle recheck_the_q_level_timer;
754 {
755  totempg_check_q_level(corosync_group_handle);
757  qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC,
758  NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer);
759  }
760 }
761 
764 };
765 
766 
768  unsigned int service,
769  unsigned int id,
770  const void *msg,
771  void *sending_allowed_private_data)
772 {
774  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
775  struct iovec reserve_iovec;
776  struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
777  int sending_allowed;
778 
779  reserve_iovec.iov_base = (char *)header;
780  reserve_iovec.iov_len = header->size;
781 
783  corosync_group_handle,
784  &reserve_iovec, 1);
785  if (pd->reserved_msgs == -1) {
786  return -EINVAL;
787  }
788 
789  sending_allowed = QB_FALSE;
790  if (corosync_quorum_is_quorate() == 1 ||
791  corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) {
792  // we are quorate
793  // now check flow control
794  if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) {
795  sending_allowed = QB_TRUE;
796  } else if (pd->reserved_msgs && sync_in_process == 0) {
797  sending_allowed = QB_TRUE;
798  } else if (pd->reserved_msgs == 0) {
799  return -ENOBUFS;
800  } else /* (sync_in_process) */ {
801  return -EINPROGRESS;
802  }
803  } else {
804  return -EHOSTUNREACH;
805  }
806 
807  return (sending_allowed);
808 }
809 
810 void corosync_sending_allowed_release (void *sending_allowed_private_data)
811 {
813  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
814 
815  if (pd->reserved_msgs == -1) {
816  return;
817  }
819 }
820 
822 {
823  int ret = 0;
824 
825  assert (source != NULL);
826  if (source->nodeid == totempg_my_nodeid_get ()) {
827  ret = 1;
828  }
829  return ret;
830 }
831 
833  mar_message_source_t *source,
834  void *conn)
835 {
836  assert ((source != NULL) && (conn != NULL));
837  memset (source, 0, sizeof (mar_message_source_t));
838  source->nodeid = totempg_my_nodeid_get ();
839  source->conn = conn;
840 }
841 
844  qb_loop_timer_handle handle;
845  unsigned long long tv_prev;
846  unsigned long long max_tv_diff;
847 };
848 
849 static void timer_function_scheduler_timeout (void *data)
850 {
851  struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data;
852  unsigned long long tv_current;
853  unsigned long long tv_diff;
854 
855  tv_current = qb_util_nano_current_get ();
856 
857  if (timeout_data->tv_prev == 0) {
858  /*
859  * Initial call -> just pretent everything is ok
860  */
861  timeout_data->tv_prev = tv_current;
862  timeout_data->max_tv_diff = 0;
863  }
864 
865  tv_diff = tv_current - timeout_data->tv_prev;
866  timeout_data->tv_prev = tv_current;
867 
868  if (tv_diff > timeout_data->max_tv_diff) {
869  log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled for %0.4f ms "
870  "(threshold is %0.4f ms). Consider token timeout increase.",
871  (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC);
872  }
873 
874  /*
875  * Set next threshold, because token_timeout can change
876  */
877  timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8;
878  qb_loop_timer_add (corosync_poll_handle,
879  QB_LOOP_MED,
880  timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3,
881  timeout_data,
882  timer_function_scheduler_timeout,
883  &timeout_data->handle);
884 }
885 
886 
887 static void corosync_setscheduler (void)
888 {
889 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
890  int res;
891 
892  sched_priority = sched_get_priority_max (SCHED_RR);
893  if (sched_priority != -1) {
894  global_sched_param.sched_priority = sched_priority;
895  res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
896  if (res == -1) {
898  "Could not set SCHED_RR at priority %d",
899  global_sched_param.sched_priority);
900 
901  global_sched_param.sched_priority = 0;
902 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
903  qb_log_thread_priority_set (SCHED_OTHER, 0);
904 #endif
905  } else {
906 
907  /*
908  * Turn on SCHED_RR in logsys system
909  */
910 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
911  res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
912 #else
913  res = -1;
914 #endif
915  if (res == -1) {
917  "Could not set logsys thread priority."
918  " Can't continue because of priority inversions.");
920  }
921  }
922  } else {
924  "Could not get maximum scheduler priority");
925  sched_priority = 0;
926  }
927 #else
929  "The Platform is missing process priority setting features. Leaving at default.");
930 #endif
931 }
932 
933 
934 /* The basename man page contains scary warnings about
935  thread-safety and portability, hence this */
936 static const char *corosync_basename(const char *file_name)
937 {
938  char *base;
939  base = strrchr (file_name, '/');
940  if (base) {
941  return base + 1;
942  }
943 
944  return file_name;
945 }
946 
947 static void
948 _logsys_log_printf(int level, int subsys,
949  const char *function_name,
950  const char *file_name,
951  int file_line,
952  const char *format,
953  ...) __attribute__((format(printf, 6, 7)));
954 
955 static void
956 _logsys_log_printf(int level, int subsys,
957  const char *function_name,
958  const char *file_name,
959  int file_line,
960  const char *format, ...)
961 {
962  va_list ap;
963 
964  va_start(ap, format);
965  qb_log_from_external_source_va(function_name, corosync_basename(file_name),
966  format, level, file_line,
967  subsys, ap);
968  va_end(ap);
969 }
970 
971 static void fplay_key_change_notify_fn (
972  int32_t event,
973  const char *key_name,
974  struct icmap_notify_value new_val,
975  struct icmap_notify_value old_val,
976  void *user_data)
977 {
978  if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) {
979  fprintf(stderr,"Writetofile\n");
980  corosync_blackbox_write_to_file ();
981  }
982  if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) {
983  fprintf(stderr,"statefump\n");
985  }
986 }
987 
988 static void corosync_fplay_control_init (void)
989 {
990  icmap_track_t track = NULL;
991 
992  icmap_set_string("runtime.blackbox.dump_flight_data", "no");
993  icmap_set_string("runtime.blackbox.dump_state", "no");
994 
995  icmap_track_add("runtime.blackbox.dump_flight_data",
997  fplay_key_change_notify_fn,
998  NULL, &track);
999  icmap_track_add("runtime.blackbox.dump_state",
1001  fplay_key_change_notify_fn,
1002  NULL, &track);
1003 }
1004 
1005 /*
1006  * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
1007  * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
1008  *
1009  * Also some RO keys cannot be determined in this stage, so they are set later in
1010  * other functions (like nodelist.local_node_pos, ...)
1011  */
1012 static void set_icmap_ro_keys_flag (void)
1013 {
1014  /*
1015  * Set RO flag for all keys of internal configuration and runtime statistics
1016  */
1017  icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
1018  icmap_set_ro_access("runtime.connections.", CS_TRUE, CS_TRUE);
1019  icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
1020  icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
1021  icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE);
1022  icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE);
1023 
1024  /*
1025  * Set RO flag for constrete keys of configuration which can't be changed
1026  * during runtime
1027  */
1028  icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
1029  icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
1030  icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
1031  icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE);
1032  icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
1033  icmap_set_ro_access("totem.transport", CS_FALSE, CS_TRUE);
1034  icmap_set_ro_access("totem.cluster_name", CS_FALSE, CS_TRUE);
1035  icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);
1036  icmap_set_ro_access("totem.threads", CS_FALSE, CS_TRUE);
1037  icmap_set_ro_access("totem.version", CS_FALSE, CS_TRUE);
1038  icmap_set_ro_access("totem.nodeid", CS_FALSE, CS_TRUE);
1039  icmap_set_ro_access("totem.clear_node_high_bit", CS_FALSE, CS_TRUE);
1040  icmap_set_ro_access("qb.ipc_type", CS_FALSE, CS_TRUE);
1041  icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE);
1042  icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE);
1043 }
1044 
1045 static void main_service_ready (void)
1046 {
1047  int res;
1048 
1049  /*
1050  * This must occur after totempg is initialized because "this_ip" must be set
1051  */
1053  if (res == -1) {
1054  log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services");
1056  }
1057  cs_ipcs_init();
1058  corosync_totem_stats_init ();
1059  corosync_fplay_control_init ();
1060  sync_init (
1061  corosync_sync_callbacks_retrieve,
1062  corosync_sync_completed);
1063 }
1064 
1065 static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid)
1066 {
1067  struct flock lock;
1068  enum e_corosync_done err;
1069  char pid_s[17];
1070  int fd_flag;
1071  int lf;
1072 
1073  err = COROSYNC_DONE_EXIT;
1074 
1075  lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1076  if (lf == -1) {
1077  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file.");
1078  return (COROSYNC_DONE_ACQUIRE_LOCK);
1079  }
1080 
1081 retry_fcntl:
1082  lock.l_type = F_WRLCK;
1083  lock.l_start = 0;
1084  lock.l_whence = SEEK_SET;
1085  lock.l_len = 0;
1086  if (fcntl (lf, F_SETLK, &lock) == -1) {
1087  switch (errno) {
1088  case EINTR:
1089  goto retry_fcntl;
1090  break;
1091  case EAGAIN:
1092  case EACCES:
1093  log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running.");
1095  goto error_close;
1096  break;
1097  default:
1098  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't acquire lock. Error was %s",
1099  strerror(errno));
1101  goto error_close;
1102  break;
1103  }
1104  }
1105 
1106  if (ftruncate (lf, 0) == -1) {
1107  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s",
1108  strerror (errno));
1110  goto error_close_unlink;
1111  }
1112 
1113  memset (pid_s, 0, sizeof (pid_s));
1114  snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid);
1115 
1116 retry_write:
1117  if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1118  if (errno == EINTR) {
1119  goto retry_write;
1120  } else {
1121  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. "
1122  "Error was %s", strerror (errno));
1124  goto error_close_unlink;
1125  }
1126  }
1127 
1128  if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1129  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. "
1130  "Error was %s", strerror (errno));
1132  goto error_close_unlink;
1133  }
1134  fd_flag |= FD_CLOEXEC;
1135  if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1136  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. "
1137  "Error was %s", strerror (errno));
1139  goto error_close_unlink;
1140  }
1141 
1142  return (err);
1143 
1144 error_close_unlink:
1145  unlink (lockfile);
1146 error_close:
1147  close (lf);
1148 
1149  return (err);
1150 }
1151 
1152 int main (int argc, char **argv, char **envp)
1153 {
1154  const char *error_string;
1155  struct totem_config totem_config;
1156  int res, ch;
1157  int background, setprio, testonly;
1158  struct stat stat_out;
1159  enum e_corosync_done flock_err;
1160  uint64_t totem_config_warnings;
1161  struct scheduler_pause_timeout_data scheduler_pause_timeout_data;
1162 
1163  /* default configuration
1164  */
1165  background = 1;
1166  setprio = 1;
1167  testonly = 0;
1168 
1169  while ((ch = getopt (argc, argv, "fprtv")) != EOF) {
1170 
1171  switch (ch) {
1172  case 'f':
1173  background = 0;
1174  break;
1175  case 'p':
1176  setprio = 0;
1177  break;
1178  case 'r':
1179  setprio = 1;
1180  break;
1181  case 't':
1182  testonly = 1;
1183  break;
1184  case 'v':
1185  printf ("Corosync Cluster Engine, version '%s'\n", VERSION);
1186  printf ("Copyright (c) 2006-2009 Red Hat, Inc.\n");
1188  return EXIT_SUCCESS;
1189 
1190  break;
1191  default:
1192  fprintf(stderr, \
1193  "usage:\n"\
1194  " -f : Start application in foreground.\n"\
1195  " -p : Do not set process priority.\n"\
1196  " -t : Test configuration and exit.\n"\
1197  " -r : Set round robin realtime scheduling (default).\n"\
1198  " -v : Display version and SVN revision of Corosync and exit.\n");
1200  return EXIT_FAILURE;
1201  }
1202  }
1203 
1204  /*
1205  * Set round robin realtime scheduling with priority 99
1206  * Lock all memory to avoid page faults which may interrupt
1207  * application healthchecking
1208  */
1209  if (setprio) {
1210  corosync_setscheduler ();
1211  }
1212 
1213  corosync_mlockall ();
1214 
1215  /*
1216  * Other signals are registered later via qb_loop_signal_add
1217  */
1218  (void)signal (SIGSEGV, sigsegv_handler);
1219  (void)signal (SIGABRT, sigabrt_handler);
1220 #if MSG_NOSIGNAL != 0
1221  (void)signal (SIGPIPE, SIG_IGN);
1222 #endif
1223 
1224  if (icmap_init() != CS_OK) {
1225  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component.");
1227  }
1228  set_icmap_ro_keys_flag();
1229 
1230  /*
1231  * Initialize the corosync_api_v1 definition
1232  */
1233  api = apidef_get ();
1234 
1235  res = coroparse_configparse(icmap_get_global_map(), &error_string);
1236  if (res == -1) {
1237  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1239  }
1240 
1241  res = corosync_log_config_read (&error_string);
1242  if (res == -1) {
1243  /*
1244  * if we are here, we _must_ flush the logsys queue
1245  * and try to inform that we couldn't read the config.
1246  * this is a desperate attempt before certain death
1247  * and there is no guarantee that we can print to stderr
1248  * nor that logsys is sending the messages where we expect.
1249  */
1250  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1251  fprintf(stderr, "%s", error_string);
1252  syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1254  }
1255 
1256  if (!testonly) {
1257  log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine ('%s'): started and ready to provide service.", VERSION);
1258  log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "");
1259  }
1260 
1261  /*
1262  * Make sure required directory is present
1263  */
1264  res = stat (get_run_dir(), &stat_out);
1265  if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1266  log_printf (LOGSYS_LEVEL_ERROR, "Required directory not present %s. Please create it.", get_run_dir());
1268  }
1269 
1270  res = chdir(get_run_dir());
1271  if (res == -1) {
1272  log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to run directory %s. "
1273  "Please make sure it has correct context and rights.", get_run_dir());
1275  }
1276 
1277  res = totem_config_read (&totem_config, &error_string, &totem_config_warnings);
1278  if (res == -1) {
1279  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1281  }
1282 
1283  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) {
1284  log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored.");
1285  }
1286 
1287  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) {
1288  log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated.");
1289  }
1290 
1291  if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED) {
1292  log_printf (LOGSYS_LEVEL_WARNING, "nodeid appears both in totem section and nodelist. Nodelist one is used.");
1293  }
1294 
1295  if (totem_config_warnings != 0) {
1296  log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
1297  }
1298 
1299  res = totem_config_keyread (&totem_config, &error_string);
1300  if (res == -1) {
1301  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1303  }
1304 
1305  res = totem_config_validate (&totem_config, &error_string);
1306  if (res == -1) {
1307  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1309  }
1310 
1311  if (testonly) {
1313  }
1314 
1315  ip_version = totem_config.ip_version;
1316 
1317  totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
1318  totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
1319 
1321  totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
1322  "totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1323  "totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c");
1324 
1331  totem_config.totem_logging_configuration.log_printf = _logsys_log_printf;
1333 
1334  /*
1335  * Now we are fully initialized.
1336  */
1337  if (background) {
1338  corosync_tty_detach ();
1339  }
1340 
1341  corosync_poll_handle = qb_loop_create ();
1342 
1343  memset(&scheduler_pause_timeout_data, 0, sizeof(scheduler_pause_timeout_data));
1344  scheduler_pause_timeout_data.totem_config = &totem_config;
1345  timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1346 
1347  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1348  SIGUSR2, NULL, sig_diag_handler, NULL);
1349  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1350  SIGINT, NULL, sig_exit_handler, NULL);
1351  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1352  SIGSEGV, NULL, sig_segv_handler, NULL);
1353  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1354  SIGABRT, NULL, sig_abrt_handler, NULL);
1355  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1356  SIGQUIT, NULL, sig_exit_handler, NULL);
1357  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1358  SIGTERM, NULL, sig_exit_handler, NULL);
1359 
1360  if (logsys_thread_start() != 0) {
1361  log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread");
1363  }
1364 
1365  if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) {
1366  corosync_exit_error (flock_err);
1367  }
1368 
1369  /*
1370  * if totempg_initialize doesn't have root priveleges, it cannot
1371  * bind to a specific interface. This only matters if
1372  * there is more then one interface in a system, so
1373  * in this case, only a warning is printed
1374  */
1375  /*
1376  * Join multicast group and setup delivery
1377  * and configuration change functions
1378  */
1380  corosync_poll_handle,
1381  &totem_config);
1382 
1384  main_service_ready);
1385 
1387  &corosync_group_handle,
1388  deliver_fn,
1389  confchg_fn);
1390 
1392  corosync_group_handle,
1393  &corosync_group,
1394  1);
1395 
1396  /*
1397  * Drop root privleges to user 'corosync'
1398  * TODO: Don't really need full root capabilities;
1399  * needed capabilities are:
1400  * CAP_NET_RAW (bindtodevice)
1401  * CAP_SYS_NICE (setscheduler)
1402  * CAP_IPC_LOCK (mlockall)
1403  */
1404  priv_drop ();
1405 
1406  schedwrk_init (
1407  serialize_lock,
1408  serialize_unlock);
1409 
1410  /*
1411  * Start main processing loop
1412  */
1413  qb_loop_run (corosync_poll_handle);
1414 
1415  /*
1416  * Exit was requested
1417  */
1418  totempg_finalize ();
1419 
1420  /*
1421  * free the loop resources
1422  */
1423  qb_loop_destroy (corosync_poll_handle);
1424 
1425  /*
1426  * free up the icmap
1427  */
1428 
1429  /*
1430  * Remove pid lock file
1431  */
1432  unlink (corosync_lock_file);
1433 
1435 
1436  return EXIT_SUCCESS;
1437 }
#define CS_TRUE
Definition: corotypes.h:54
const char * name
Definition: coroapi.h:492
int32_t cs_ipcs_q_level_get(void)
Definition: ipc_glue.c:690
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
Definition: service.c:339
void(*) in log_level_security)
Definition: totem.h:82
uint64_t gather_entered
Definition: totem.h:260
const char * get_run_dir(void)
Definition: util.c:174
int corosync_quorum_is_quorate(void)
Definition: exec/quorum.c:67
void totempg_finalize(void)
Definition: totempg.c:833
Totem Single Ring Protocol.
uint64_t memb_commit_token_rx
Definition: totem.h:255
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:73
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.h:39
#define LOGSYS_LEVEL_TRACE
Definition: logsys.h:75
void totempg_trans_ack(void)
Definition: totempg.c:1530
struct totem_config * totem_config
Definition: main.c:843
#define PACKAGE_FEATURES
Definition: config.h:368
#define CS_FALSE
Definition: corotypes.h:53
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:571
uint64_t memb_join_tx
Definition: totem.h:249
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:241
qb_loop_timer_handle handle
Definition: main.c:844
void sync_abort(void)
Definition: sync.c:600
The totem_ip_address struct.
Definition: coroapi.h:111
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Definition: totem.h:274
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:214
Totem Single Ring Protocol.
int(* sync_process)(void)
Definition: coroapi.h:522
void(* exec_dump_fn)(void)
Definition: coroapi.h:502
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
Definition: sync.c:173
uint64_t mcast_rx
Definition: totem.h:253
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
Definition: coroparse.c:251
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:229
qb_loop_t * cs_poll_handle_get(void)
Definition: main.c:164
void corosync_recheck_the_q_level(void *data)
Definition: main.c:753
#define corosync_exit_error(err)
Definition: exec/util.h:70
void(* exec_endian_convert_fn)(void *msg)
Definition: coroapi.h:478
int totempg_groups_joined_release(int msg_count)
Definition: totempg.c:1324
#define VERSION
Definition: config.h:434
totem_configuration_type
The totem_configuration_type enum.
Definition: coroapi.h:132
#define LOCALSTATEDIR
Definition: config.h:352
struct message_header header
Definition: totemsrp.c:60
uint64_t memb_merge_detect_rx
Definition: totem.h:248
LOGSYS_DECLARE_SUBSYS("MAIN")
int guarantee
Definition: totemsrp.c:66
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
Definition: schedwrk.c:83
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition: icmap.c:641
void(* sync_abort)(void)
Definition: coroapi.h:524
int earliest_token
Definition: totem.h:271
uint64_t orf_token_tx
Definition: totem.h:245
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
Definition: main.c:169
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition: totempg.c:1112
uint64_t gather_token_lost
Definition: totem.h:261
#define RLIMIT_MEMLOCK
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:95
icmap_map_t icmap_get_global_map(void)
Return global icmap.
Definition: icmap.c:278
unsigned long long max_tv_diff
Definition: main.c:846
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
Definition: service.c:110
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:436
uint64_t memb_commit_token_tx
Definition: totem.h:254
#define LOGSYS_PERROR(err_num, level, fmt, args...)
The LOGSYS_PERROR macro.
Definition: logsys.h:313
cs_error_t icmap_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1057
#define log_printf(level, format, args...)
Definition: logsys.h:319
#define MAX_NO_CONT_SENDMSG_FAILURES
Definition: totem.h:62
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:477
#define TOTEM_TOKEN_STATS_MAX
Definition: totem.h:273
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
Definition: ipc_glue.c:753
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
Definition: main.c:767
void icmap_fini(void)
Finalize global icmap.
Definition: icmap.c:261
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: main.c:652
void cs_ipc_allow_connections(int32_t allow)
Definition: ipc_glue.c:155
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
Definition: service.c:394
uint64_t operational_entered
Definition: totem.h:258
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
void totempg_service_ready_register(void(*totem_service_ready)(void))
Definition: totempg.c:1499
void(* sync_activate)(void)
Definition: sync.h:46
#define ICMAP_KEYNAME_MAXLEN
Maximum length of key in icmap.
Definition: icmap.h:48
int latest_token
Definition: totem.h:272
uint64_t operational_token_lost
Definition: totem.h:259
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
Definition: totem.h:58
uint64_t consensus_timeouts
Definition: totem.h:266
int totem_config_keyread(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1450
const char * name
Definition: sync.h:48
uint64_t recovery_token_lost
Definition: totem.h:265
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:71
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
struct corosync_exec_handler * exec_engine
Definition: coroapi.h:507
unsigned int totempg_my_nodeid_get(void)
Definition: totempg.c:1490
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:611
pthread_mutex_t lock
Definition: sam.c:131
uint64_t commit_token_lost
Definition: totem.h:263
void * user_data
Definition: sam.c:127
void(* sync_abort)(void)
Definition: sync.h:47
uint64_t token_hold_cancel_rx
Definition: totem.h:257
int logsys_thread_start(void)
logsys_thread_start
Definition: logsys.c:820
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1264
#define LOGSYS_MODE_OUTPUT_SYSLOG
Definition: logsys.h:60
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
Definition: totempg.c:1280
const void * group
Definition: totempg.h:56
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:70
Linked list API.
struct totem_ip_address rep
Definition: coroapi.h:123
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:589
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition: totempg.c:1210
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:113
uint64_t mcast_retx
Definition: totem.h:252
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
Retrieve value of key key_name and store it in user preallocated value pointer.
Definition: icmap.c:739
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
Definition: logconfig.c:652
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:75
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
void cs_ipcs_init(void)
Definition: ipc_glue.c:896
totemmrp_stats_t * mrp
Definition: totem.h:288
void(* sync_activate)(void)
Definition: coroapi.h:523
The corosync_api_v1 struct.
Definition: coroapi.h:225
typedef __attribute__
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:510
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
Definition: totemconfig.h:47
uint32_t msg_queue_avail
Definition: totem.h:290
uint64_t rx_msg_dropped
Definition: totem.h:267
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
Definition: totemconfig.h:46
#define swab32(x)
The swab32 macro.
Definition: swab.h:51
uint32_t continuous_gather
Definition: totem.h:268
cs_error_t icmap_set_uint64(const char *key_name, uint64_t value)
Definition: icmap.c:623
totemrrp_stats_t * rrp
Definition: totem.h:244
cs_error_t icmap_fast_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1077
#define MILLI_2_NANO_SECONDS
Definition: coroapi.h:105
void message_source_set(mar_message_source_t *source, void *conn)
Definition: main.c:832
int(* sync_process)(void)
Definition: sync.h:45
void logsys_config_apply(void)
logsys_config_apply
Definition: logsys.c:770
uint64_t token_hold_cancel_tx
Definition: totem.h:256
int ip_version
Definition: totem.h:189
unsigned int token_timeout
Definition: totem.h:129
void totempg_check_q_level(void *instance)
Definition: totempg.c:1272
uint64_t mcast_tx
Definition: totem.h:251
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:112
The memb_ring_id struct.
Definition: coroapi.h:122
struct corosync_api_v1 * apidef_get(void)
Definition: apidef.c:147
void corosync_sending_allowed_release(void *sending_allowed_private_data)
Definition: main.c:810
void cs_ipcs_stats_update(void)
Definition: ipc_glue.c:759
uint64_t memb_merge_detect_tx
Definition: totem.h:247
uint32_t msg_reserved
Definition: totem.h:289
totemsrp_stats_t * srp
Definition: totem.h:283
#define SERVICES_COUNT_MAX
Definition: coroapi.h:463
unsigned long long tv_prev
Definition: main.c:845
uint64_t memb_join_rx
Definition: totem.h:250
void corosync_state_dump(void)
Definition: main.c:187
qb_loop_timer_handle corosync_timer_handle_t
corosync_timer_handle_t
Definition: coroapi.h:74
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
Definition: main.c:182
e_corosync_done
Definition: exec/util.h:47
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:516
int message_source_is_local(const mar_message_source_t *source)
Definition: main.c:821
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:163
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition: totempg.c:1160
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
unsigned long long seq
Definition: coroapi.h:124
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
Definition: icmap.c:587
uint64_t recovery_entered
Definition: totem.h:264
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
Set read-only access for given key (key_name) or prefix, If prefix is set.
Definition: icmap.c:1233
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
Definition: totempg.c:787
void corosync_shutdown_request(void)
Definition: main.c:234
uint32_t interface_count
Definition: totem.h:232
struct sched_param global_sched_param
Definition: main.c:156
unsigned int nodeid
Definition: coroapi.h:75
uint32_t continuous_sendmsg_failures
Definition: totem.h:269
uint8_t * faulty
Definition: totem.h:231
struct memb_ring_id ring_id
Definition: totemsrp.c:64
const char *(* totem_ifaces_print)(unsigned int nodeid)
Definition: coroapi.h:291
int main(int argc, char **argv, char **envp)
Definition: main.c:1152
cs_error_t icmap_init(void)
Initialize global icmap.
Definition: icmap.c:221
uint64_t commit_entered
Definition: totem.h:262
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
Definition: totemconfig.h:48
LOGSYS_DECLARE_SYSTEM("corosync", LOGSYS_MODE_OUTPUT_STDERR|LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_INFO)
#define LOGSYS_MODE_OUTPUT_STDERR
Definition: logsys.h:59
void *(* totem_get_stats)(void)
Definition: coroapi.h:436
uint64_t orf_token_rx
Definition: totem.h:246
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1167
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
Definition: totemconfig.c:978
void(* totem_memb_ring_id_create_or_load)(struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:191
The mar_message_source_t struct.
Definition: coroapi.h:50
void logsys_system_fini(void)
logsys_system_fini
Definition: logsys.c:285
void(* totem_memb_ring_id_store)(const struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:195