corosync  3.1.2
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2020 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
147  void *knet_context;
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
179  char *nozzle_ipaddr;
180  char *nozzle_prefix;
181  char *nozzle_macaddr;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 
199 static int totemknet_configure_compression (
200  struct totemknet_instance *instance,
201  struct totem_config *totem_config);
202 
203 static void totemknet_start_merge_detect_timeout(
204  void *knet_context);
205 
206 static void totemknet_stop_merge_detect_timeout(
207  void *knet_context);
208 
209 static void log_flush_messages (
210  void *knet_context);
211 
212 static void totemknet_instance_initialize (struct totemknet_instance *instance)
213 {
214  int res;
215 
216  memset (instance, 0, sizeof (struct totemknet_instance));
217  res = pthread_mutex_init(&instance->log_mutex, NULL);
218  /*
219  * There is not too much else what can be done.
220  */
221  assert(res == 0);
222 }
223 
224 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
225 do { \
226  (void)pthread_mutex_lock(&instance->log_mutex); \
227  instance->totemknet_log_printf ( \
228  level, subsys, function, file, line, \
229  (const char *)format, ##args); \
230  (void)pthread_mutex_unlock(&instance->log_mutex); \
231 } while (0);
232 
233 #define knet_log_printf(level, format, args...) \
234 do { \
235  knet_log_printf_lock ( \
236  level, instance->totemknet_subsys_id, \
237  __FUNCTION__, __FILE__, __LINE__, \
238  (const char *)format, ##args); \
239 } while (0);
240 
241 #define libknet_log_printf(level, format, args...) \
242 do { \
243  knet_log_printf_lock ( \
244  level, instance->knet_subsys_id, \
245  __FUNCTION__, "libknet.h", __LINE__, \
246  (const char *)format, ##args); \
247 } while (0);
248 
249 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
250 do { \
251  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
252  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
253  instance->totemknet_log_printf ( \
254  level, instance->totemknet_subsys_id, \
255  __FUNCTION__, __FILE__, __LINE__, \
256  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
257  } while(0)
258 
259 
260 #ifdef HAVE_LIBNOZZLE
261 static inline int is_ether_addr_multicast(const uint8_t *addr)
262 {
263  return (addr[0] & 0x01);
264 }
265 static inline int is_ether_addr_zero(const uint8_t *addr)
266 {
267  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
268 }
269 
270 static int ether_host_filter_fn(void *private_data,
271  const unsigned char *outdata,
272  ssize_t outdata_len,
273  uint8_t tx_rx,
274  knet_node_id_t this_host_id,
275  knet_node_id_t src_host_id,
276  int8_t *channel,
277  knet_node_id_t *dst_host_ids,
278  size_t *dst_host_ids_entries)
279 {
280  struct ether_header *eth_h = (struct ether_header *)outdata;
281  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
282  uint16_t dst_host_id;
283 
284  if (is_ether_addr_zero(dst_mac))
285  return -1;
286 
287  if (is_ether_addr_multicast(dst_mac)) {
288  return 1;
289  }
290 
291  memmove(&dst_host_id, &dst_mac[4], 2);
292 
293  dst_host_ids[0] = ntohs(dst_host_id);
294  *dst_host_ids_entries = 1;
295 
296  return 0;
297 }
298 #endif
299 
300 static int dst_host_filter_callback_fn(void *private_data,
301  const unsigned char *outdata,
302  ssize_t outdata_len,
303  uint8_t tx_rx,
304  knet_node_id_t this_host_id,
305  knet_node_id_t src_host_id,
306  int8_t *channel,
307  knet_node_id_t *dst_host_ids,
308  size_t *dst_host_ids_entries)
309 {
310  struct totem_message_header *header = (struct totem_message_header *)outdata;
311  int res;
312 
313 #ifdef HAVE_LIBNOZZLE
314  if (*channel != 0) {
315  return ether_host_filter_fn(private_data,
316  outdata, outdata_len,
317  tx_rx,
318  this_host_id, src_host_id,
319  channel,
320  dst_host_ids,
321  dst_host_ids_entries);
322  }
323 #endif
324  if (header->target_nodeid) {
325  dst_host_ids[0] = header->target_nodeid;
326  *dst_host_ids_entries = 1;
327  res = 0; /* unicast message */
328  }
329  else {
330  *dst_host_ids_entries = 0;
331  res = 1; /* multicast message */
332  }
333  return res;
334 }
335 
336 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
337 {
338  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
339 
340  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
341  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
342  knet_handle_remove_datafd(instance->knet_handle, datafd);
343  }
344 }
345 
346 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
347 {
348  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
349 
350  // TODO: what? if anything.
351  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
352 }
353 
354 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
355 {
356  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
357  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
358 
359  /* We don't need to tell corosync the actual knet MTU */
360 // instance->totemknet_mtu_changed(instance->context, data_mtu);
361 }
362 
364  void *knet_context,
365  const char *cipher_type,
366  const char *hash_type)
367 {
368  return (0);
369 }
370 
371 
372 static inline void ucast_sendmsg (
373  struct totemknet_instance *instance,
374  struct totem_ip_address *system_to,
375  const void *msg,
376  unsigned int msg_len)
377 {
378  int res = 0;
379  struct totem_message_header *header = (struct totem_message_header *)msg;
380  struct msghdr msg_ucast;
381  struct iovec iovec;
382 
383  header->target_nodeid = system_to->nodeid;
384 
385  iovec.iov_base = (void *)msg;
386  iovec.iov_len = msg_len;
387 
388  /*
389  * Build unicast message
390  */
391  memset(&msg_ucast, 0, sizeof(msg_ucast));
392  msg_ucast.msg_iov = (void *)&iovec;
393  msg_ucast.msg_iovlen = 1;
394 #ifdef HAVE_MSGHDR_CONTROL
395  msg_ucast.msg_control = 0;
396 #endif
397 #ifdef HAVE_MSGHDR_CONTROLLEN
398  msg_ucast.msg_controllen = 0;
399 #endif
400 #ifdef HAVE_MSGHDR_FLAGS
401  msg_ucast.msg_flags = 0;
402 #endif
403 #ifdef HAVE_MSGHDR_ACCRIGHTS
404  msg_ucast.msg_accrights = NULL;
405 #endif
406 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
407  msg_ucast.msg_accrightslen = 0;
408 #endif
409 
410  /*
411  * Transmit unicast message
412  * An error here is recovered by totemsrp
413  */
414 
415  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
416  if (res < 0) {
418  "sendmsg(ucast) failed (non-critical)");
419  }
420 }
421 
422 static inline void mcast_sendmsg (
423  struct totemknet_instance *instance,
424  const void *msg,
425  unsigned int msg_len,
426  int only_active)
427 {
428  int res;
429  struct totem_message_header *header = (struct totem_message_header *)msg;
430  struct msghdr msg_mcast;
431  struct iovec iovec;
432 
433  iovec.iov_base = (void *)msg;
434  iovec.iov_len = msg_len;
435 
436  header->target_nodeid = 0;
437 
438  /*
439  * Build multicast message
440  */
441  memset(&msg_mcast, 0, sizeof(msg_mcast));
442  msg_mcast.msg_iov = (void *)&iovec;
443  msg_mcast.msg_iovlen = 1;
444 #ifdef HAVE_MSGHDR_CONTROL
445  msg_mcast.msg_control = 0;
446 #endif
447 #ifdef HAVE_MSGHDR_CONTROLLEN
448  msg_mcast.msg_controllen = 0;
449 #endif
450 #ifdef HAVE_MSGHDR_FLAGS
451  msg_mcast.msg_flags = 0;
452 #endif
453 #ifdef HAVE_MSGHDR_ACCRIGHTS
454  msg_mcast.msg_accrights = NULL;
455 #endif
456 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
457  msg_mcast.msg_accrightslen = 0;
458 #endif
459 
460 
461 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
462 
463  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
464  if (res < msg_len) {
465  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
466  }
467 
468  if (!only_active || instance->send_merge_detect_message) {
469  /*
470  * Current message was sent to all nodes
471  */
473  instance->send_merge_detect_message = 0;
474  }
475 }
476 
477 static int node_compare(const void *aptr, const void *bptr)
478 {
479  uint16_t a,b;
480 
481  a = *(uint16_t *)aptr;
482  b = *(uint16_t *)bptr;
483 
484  return a > b;
485 }
486 
487 #ifndef OWN_INDEX_NONE
488 #define OWN_INDEX_NONE -1
489 #endif
490 
492  void *knet_context,
493  unsigned int nodeid,
494  struct totem_node_status *node_status)
495 {
496  int i;
497  int res = 0;
498  struct knet_link_status link_status;
499  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
500  struct knet_host_status knet_host_status;
501  uint8_t link_list[KNET_MAX_LINK];
502  size_t num_links;
503 
504  if (!instance->knet_handle) {
505  return CS_ERR_NOT_EXIST; /* Not using knet */
506  }
507 
508  if (!node_status) {
509  return CS_ERR_INVALID_PARAM;
510  }
511 
512  res = knet_host_get_status(instance->knet_handle,
513  nodeid,
514  &knet_host_status);
515  if (res) {
516  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
517  return (-1);
518  }
519  node_status->nodeid = nodeid;
520  node_status->reachable = knet_host_status.reachable;
521  node_status->remote = knet_host_status.remote;
522  node_status->external = knet_host_status.external;
523 
524 #ifdef HAVE_KNET_ONWIRE_VER
525  res = knet_handle_get_onwire_ver(instance->knet_handle,
526  nodeid,
527  &node_status->onwire_min,
528  &node_status->onwire_max,
529  &node_status->onwire_ver);
530  if (res) {
531  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
532  return (-1);
533  }
534 #endif
535  /* Get link info */
536  res = knet_link_get_link_list(instance->knet_handle,
537  nodeid, link_list, &num_links);
538  if (res) {
539  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
540  return (-1);
541  }
542 
543  for (i=0; i < num_links; i++) {
544  if (!instance->totem_config->interfaces[link_list[i]].configured) {
545  continue;
546  }
547  res = knet_link_get_status(instance->knet_handle,
548  nodeid,
549  link_list[i],
550  &link_status,
551  sizeof(link_status));
552  if (res == 0) {
553  node_status->link_status[i].enabled = link_status.enabled;
554  node_status->link_status[i].connected = link_status.connected;
555  node_status->link_status[i].dynconnected = link_status.dynconnected;
556  node_status->link_status[i].mtu = link_status.mtu;
557  memcpy(node_status->link_status[i].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
558  memcpy(node_status->link_status[i].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
559  } else {
560  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
561  }
562  }
563  return res;
564 }
565 
566 
567 
569  char ***status,
570  unsigned int *iface_count)
571 {
572  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
573  struct knet_link_status link_status;
574  knet_node_id_t host_list[KNET_MAX_HOST];
575  uint8_t link_list[KNET_MAX_LINK];
576  size_t num_hosts;
577  size_t num_links;
578  size_t link_idx;
579  int i,j;
580  char *ptr;
581  int res = 0;
582 
583  /*
584  * Don't do the whole 'link_info' bit if the caller just wants
585  * a count of interfaces.
586  */
587  if (status) {
588  int own_idx = OWN_INDEX_NONE;
589 
590  res = knet_host_get_host_list(instance->knet_handle,
591  host_list, &num_hosts);
592  if (res) {
593  return (-1);
594  }
595  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
596 
597  for (j=0; j<num_hosts; j++) {
598  if (host_list[j] == instance->our_nodeid) {
599  own_idx = j;
600  break;
601  }
602  }
603 
604  for (i=0; i<INTERFACE_MAX; i++) {
605  memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
606  if (own_idx != OWN_INDEX_NONE) {
607  instance->link_status[i][own_idx] = 'n';
608  }
609  instance->link_status[i][num_hosts] = '\0';
610  }
611 
612  /* This is all a bit "inside-out" because "status" is a set of strings per link
613  * and knet orders things by host
614  */
615  for (j=0; j<num_hosts; j++) {
616  if (own_idx != OWN_INDEX_NONE && j == own_idx) {
617  continue ;
618  }
619 
620  res = knet_link_get_link_list(instance->knet_handle,
621  host_list[j], link_list, &num_links);
622  if (res) {
623  return (-1);
624  }
625 
626  link_idx = 0;
627  for (i=0; i < num_links; i++) {
628  /*
629  * Skip over links that are unconfigured to corosync. This is basically
630  * link0 if corosync isn't using it for comms, as we will still
631  * have it set up for loopback.
632  */
633  if (!instance->totem_config->interfaces[link_list[i]].configured) {
634  continue;
635  }
636  ptr = instance->link_status[link_idx++];
637 
638  res = knet_link_get_status(instance->knet_handle,
639  host_list[j],
640  link_list[i],
641  &link_status,
642  sizeof(link_status));
643  if (res == 0) {
644  ptr[j] = '0' + (link_status.enabled |
645  link_status.connected<<1 |
646  link_status.dynconnected<<2);
647  }
648  else {
650  "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
651  ptr[j] = '?';
652  }
653  }
654  }
655  *status = instance->link_status;
656  }
657 
658  *iface_count = INTERFACE_MAX;
659 
660  return (res);
661 }
662 
664  void *knet_context)
665 {
666  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
667  int res = 0;
668  int i,j;
669  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
670  uint8_t links[KNET_MAX_LINK];
671  size_t num_nodes;
672  size_t num_links;
673 
674  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
675 
676  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
677  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
678 
679  /*
680  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
681  */
682  res = knet_handle_setfwd(instance->knet_handle, 0);
683  if (res) {
684  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
685  }
686 
687  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
688  if (res) {
689  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
690  /* Crash out anyway */
691  goto finalise_error;
692  }
693 
694  /* Tidily shut down all nodes & links. */
695  for (i=0; i<num_nodes; i++) {
696 
697  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
698  if (res) {
699  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
700  goto finalise_error;
701  }
702  for (j=0; j<num_links; j++) {
703  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
704  if (res) {
705  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
706  }
707  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
708  if (res) {
709  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
710  }
711  }
712  res = knet_host_remove(instance->knet_handle, nodes[i]);
713  if (res) {
714  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
715  }
716  }
717 
718 finalise_error:
719  res = knet_handle_free(instance->knet_handle);
720  if (res) {
721  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
722  }
723 
724  totemknet_stop_merge_detect_timeout(instance);
725 
726  log_flush_messages(instance);
727 
728  /*
729  * Error is deliberately ignored
730  */
731  (void)pthread_mutex_destroy(&instance->log_mutex);
732 
733  return (res);
734 }
735 
736 static int log_deliver_fn (
737  int fd,
738  int revents,
739  void *data)
740 {
741  struct totemknet_instance *instance = (struct totemknet_instance *)data;
742  char buffer[sizeof(struct knet_log_msg)*4];
743  char *bufptr = buffer;
744  int done = 0;
745  int len;
746 
747  len = read(fd, buffer, sizeof(buffer));
748  while (done < len) {
749  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
750  switch (msg->msglevel) {
751  case KNET_LOG_ERR:
753  knet_log_get_subsystem_name(msg->subsystem),
754  msg->msg);
755  break;
756  case KNET_LOG_WARN:
758  knet_log_get_subsystem_name(msg->subsystem),
759  msg->msg);
760  break;
761  case KNET_LOG_INFO:
763  knet_log_get_subsystem_name(msg->subsystem),
764  msg->msg);
765  break;
766  case KNET_LOG_DEBUG:
768  knet_log_get_subsystem_name(msg->subsystem),
769  msg->msg);
770  break;
771  }
772  bufptr += sizeof(struct knet_log_msg);
773  done += sizeof(struct knet_log_msg);
774  }
775  return 0;
776 }
777 
778 static int data_deliver_fn (
779  int fd,
780  int revents,
781  void *data)
782 {
783  struct totemknet_instance *instance = (struct totemknet_instance *)data;
784  struct msghdr msg_hdr;
785  struct iovec iov_recv;
786  struct sockaddr_storage system_from;
787  ssize_t msg_len;
788  int truncated_packet;
789 
790  iov_recv.iov_base = instance->iov_buffer;
791  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
792 
793  msg_hdr.msg_name = &system_from;
794  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
795  msg_hdr.msg_iov = &iov_recv;
796  msg_hdr.msg_iovlen = 1;
797 #ifdef HAVE_MSGHDR_CONTROL
798  msg_hdr.msg_control = 0;
799 #endif
800 #ifdef HAVE_MSGHDR_CONTROLLEN
801  msg_hdr.msg_controllen = 0;
802 #endif
803 #ifdef HAVE_MSGHDR_FLAGS
804  msg_hdr.msg_flags = 0;
805 #endif
806 #ifdef HAVE_MSGHDR_ACCRIGHTS
807  msg_hdr.msg_accrights = NULL;
808 #endif
809 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
810  msg_hdr.msg_accrightslen = 0;
811 #endif
812 
813  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
814  if (msg_len <= 0) {
815  return (0);
816  }
817 
818  truncated_packet = 0;
819 
820 #ifdef HAVE_MSGHDR_FLAGS
821  if (msg_hdr.msg_flags & MSG_TRUNC) {
822  truncated_packet = 1;
823  }
824 #else
825  /*
826  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
827  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
828  */
829  if (bytes_received == KNET_MAX_PACKET_SIZE) {
830  truncated_packet = 1;
831  }
832 #endif
833 
834  if (truncated_packet) {
836  "Received too big message. This may be because something bad is happening"
837  "on the network (attack?), or you tried join more nodes than corosync is"
838  "compiled with (%u) or bug in the code (bad estimation of "
839  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
840  return (0);
841  }
842 
843  /*
844  * Handle incoming message
845  */
846  instance->totemknet_deliver_fn (
847  instance->context,
848  instance->iov_buffer,
849  msg_len,
850  &system_from);
851 
852  return (0);
853 }
854 
855 static void timer_function_netif_check_timeout (
856  void *data)
857 {
858  struct totemknet_instance *instance = (struct totemknet_instance *)data;
859  int i;
860 
861  for (i=0; i < INTERFACE_MAX; i++) {
862  if (!instance->totem_config->interfaces[i].configured) {
863  continue;
864  }
865  instance->totemknet_iface_change_fn (instance->context,
866  &instance->my_ids[i],
867  i);
868  }
869 }
870 
871 static void knet_set_access_list_config(struct totemknet_instance *instance)
872 {
873 #ifdef HAVE_KNET_ACCESS_LIST
874  uint32_t value;
875  cs_error_t err;
876 
877  value = instance->totem_config->block_unlisted_ips;
878  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
879 
880  err = knet_handle_enable_access_lists(instance->knet_handle, value);
881  if (err) {
882  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
883  }
884 #endif
885 }
886 
887 
888 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
889 static void totemknet_refresh_config(
890  int32_t event,
891  const char *key_name,
892  struct icmap_notify_value new_val,
893  struct icmap_notify_value old_val,
894  void *user_data)
895 {
896  uint8_t reloading;
897  uint32_t value;
898  uint32_t link_no;
899  size_t num_nodes;
900  knet_node_id_t host_ids[KNET_MAX_HOST];
901  int i;
902  int err;
903  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
904 
905  ENTER();
906 
907  /*
908  * If a full reload is in progress then don't do anything until it's done and
909  * can reconfigure it all atomically
910  */
911  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
912  return;
913  }
914 
915  knet_set_access_list_config(instance);
916 
917  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
918 
920  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
921  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
922  if (err) {
923  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
924  }
925  }
926 
927  /* Configure link parameters for each node */
928  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
929  if (err != 0) {
930  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
931  }
932 
933  for (i=0; i<num_nodes; i++) {
934  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
935  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
936  continue;
937  }
938 
939  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
940  instance->totem_config->interfaces[link_no].knet_ping_interval,
941  instance->totem_config->interfaces[link_no].knet_ping_timeout,
942  instance->totem_config->interfaces[link_no].knet_ping_precision);
943  if (err) {
944  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
945  }
946  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
947  instance->totem_config->interfaces[link_no].knet_pong_count);
948  if (err) {
949  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
950  }
951  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
952  instance->totem_config->interfaces[link_no].knet_link_priority);
953  if (err) {
954  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
955  }
956 
957  }
958  }
959 
960  LEAVE();
961 }
962 
963 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
964 {
965  icmap_track_t icmap_track_totem = NULL;
966  icmap_track_t icmap_track_reload = NULL;
967 
968  ENTER();
969 
970  icmap_track_add("totem.",
972  totemknet_refresh_config,
973  instance,
974  &icmap_track_totem);
975 
976  icmap_track_add("config.totemconfig_reload_in_progress",
978  totemknet_refresh_config,
979  instance,
980  &icmap_track_reload);
981 
982  LEAVE();
983 }
984 
985 static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
986 {
987 
988  return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
989  strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
990 
991 }
992 
993 static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
994 {
995  struct knet_handle_crypto_cfg crypto_cfg;
996  int res;
997 
998  /* These have already been validated */
999  memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
1000  memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
1001  memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
1002  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1003  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1004 
1005 #ifdef HAVE_KNET_CRYPTO_RECONF
1006 
1007  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
1008  crypto_cfg.crypto_model,
1009  crypto_cfg.crypto_cipher_type,
1010  crypto_cfg.crypto_hash_type,
1011  instance->totem_config->crypto_index
1012  );
1013 
1014  /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
1015  if (!totemknet_is_crypto_enabled(instance)) {
1016  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1017  if (res) {
1018  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
1019  }
1020  }
1021 
1022  /* use_config will be called later when all nodes are synced */
1023  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
1024  if (res == -1) {
1025  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
1026  goto exit_error;
1027  }
1028  if (res == -2) {
1029  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
1030  goto exit_error;
1031  }
1032 #else
1033  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
1034  crypto_cfg.crypto_model,
1035  crypto_cfg.crypto_cipher_type,
1036  crypto_cfg.crypto_hash_type
1037  );
1038 
1039  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1040  if (res == -1) {
1041  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1042  goto exit_error;
1043  }
1044  if (res == -2) {
1045  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1046  goto exit_error;
1047  }
1048 #endif
1049 
1050 
1051 exit_error:
1052  return res;
1053 }
1054 
1055 /*
1056  * Create an instance
1057  */
1059  qb_loop_t *poll_handle,
1060  void **knet_context,
1061  struct totem_config *totem_config,
1062  totemsrp_stats_t *stats,
1063  void *context,
1064 
1065  void (*deliver_fn) (
1066  void *context,
1067  const void *msg,
1068  unsigned int msg_len,
1069  const struct sockaddr_storage *system_from),
1070 
1071  void (*iface_change_fn) (
1072  void *context,
1073  const struct totem_ip_address *iface_address,
1074  unsigned int link_no),
1075 
1076  void (*mtu_changed) (
1077  void *context,
1078  int net_mtu),
1079 
1080  void (*target_set_completed) (
1081  void *context))
1082 {
1083  struct totemknet_instance *instance;
1084  char *tmp_str;
1085  int8_t channel=0;
1086  int allow_knet_handle_fallback=0;
1087  int res;
1088  int i;
1089 
1090  instance = malloc (sizeof (struct totemknet_instance));
1091  if (instance == NULL) {
1092  return (-1);
1093  }
1094 
1095  totemknet_instance_initialize (instance);
1096 
1097  instance->totem_config = totem_config;
1098 
1099  /*
1100  * Configure logging
1101  */
1102  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1109 
1110  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1111 
1112  /*
1113  * Initialize local variables for totemknet
1114  */
1115 
1116  instance->our_nodeid = instance->totem_config->node_id;
1117 
1118  for (i=0; i< INTERFACE_MAX; i++) {
1119  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1120  instance->my_ids[i].nodeid = instance->our_nodeid;
1121  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1122 
1123  /* Needed for totemsrp */
1124  totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
1125  }
1126 
1127  instance->poll_handle = poll_handle;
1128 
1129  instance->context = context;
1130  instance->totemknet_deliver_fn = deliver_fn;
1131 
1132  instance->totemknet_iface_change_fn = iface_change_fn;
1133 
1134  instance->totemknet_mtu_changed = mtu_changed;
1135 
1136  instance->totemknet_target_set_completed = target_set_completed;
1137 
1138  instance->loopback_link = 0;
1139 
1140  res = pipe(instance->logpipes);
1141  if (res == -1) {
1142  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1143  goto exit_error;
1144  }
1145  if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1146  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1147  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1148  goto exit_error;
1149  }
1150 
1151  if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
1152  if (strcmp(tmp_str, "yes") == 0) {
1153  allow_knet_handle_fallback = 1;
1154  }
1155  free(tmp_str);
1156  }
1157 
1158 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1159  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1160 #else
1161  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1162 #endif
1163 
1164  if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
1165  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
1166 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1167  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1168 #else
1169  instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1170 #endif
1171  }
1172 
1173  if (!instance->knet_handle) {
1174  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1175  goto exit_error;
1176  }
1177 
1178  knet_set_access_list_config(instance);
1179 
1180  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1181  if (res) {
1182  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1183  }
1184  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1185  if (res) {
1186  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1187  }
1188  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1189  if (res) {
1190  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1191  }
1192  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1193  if (res) {
1194  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1195  }
1196  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1197  if (res) {
1198  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1199  }
1200  global_instance = instance;
1201 
1202  /* Get an fd into knet */
1203  instance->knet_fd = 0;
1204  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1205  if (res) {
1206  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1207  goto exit_error;
1208  }
1209 
1210  /* Enable crypto if requested */
1211 #ifdef HAVE_KNET_CRYPTO_RECONF
1212  if (totemknet_is_crypto_enabled(instance)) {
1213  res = totemknet_set_knet_crypto(instance);
1214  if (res == 0) {
1215  res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1216  if (res) {
1217  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1218  goto exit_error;
1219  }
1220  } else {
1221  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1222  goto exit_error;
1223  }
1224  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1225  if (res) {
1226  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1227  goto exit_error;
1228  }
1229 
1230  } else {
1231  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1232  if (res) {
1233  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1234  goto exit_error;
1235  }
1236  }
1237 #else
1238  if (totemknet_is_crypto_enabled(instance)) {
1239  res = totemknet_set_knet_crypto(instance);
1240  if (res) {
1241  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1242  goto exit_error;
1243  }
1244  }
1245 #endif
1246 
1247  /* Set up compression */
1248  if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1249  /* Not fatal, but will log */
1250  (void)totemknet_configure_compression(instance, totem_config);
1251  }
1252 
1253  knet_handle_setfwd(instance->knet_handle, 1);
1254 
1255  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1256  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1257  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1258  }
1259  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1260  instance->link_mode = KNET_LINK_POLICY_RR;
1261  }
1262 
1263  for (i=0; i<INTERFACE_MAX; i++) {
1264  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1265  if (!instance->link_status[i]) {
1266  goto exit_error;
1267  }
1268  }
1269 
1270  qb_loop_poll_add (instance->poll_handle,
1271  QB_LOOP_MED,
1272  instance->logpipes[0],
1273  POLLIN, instance, log_deliver_fn);
1274 
1275  qb_loop_poll_add (instance->poll_handle,
1276  QB_LOOP_HIGH,
1277  instance->knet_fd,
1278  POLLIN, instance, data_deliver_fn);
1279 
1280  /*
1281  * Upper layer isn't ready to receive message because it hasn't
1282  * initialized yet. Add short timer to check the interfaces.
1283  */
1284  qb_loop_timer_add (instance->poll_handle,
1285  QB_LOOP_MED,
1286  100*QB_TIME_NS_IN_MSEC,
1287  (void *)instance,
1288  timer_function_netif_check_timeout,
1289  &instance->timer_netif_check_timeout);
1290 
1291  totemknet_start_merge_detect_timeout(instance);
1292 
1293  /* Start listening for config changes */
1294  totemknet_add_config_notifications(instance);
1295 
1296  /* Add stats keys to icmap */
1298 
1299  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1300  *knet_context = instance;
1301 
1302  return (0);
1303 
1304 exit_error:
1305  log_flush_messages(instance);
1306  free(instance);
1307  return (-1);
1308 }
1309 
1311 {
1312  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1313  return malloc(KNET_MAX_PACKET_SIZE + 512);
1314 }
1315 
1317 {
1318  return free (ptr);
1319 }
1320 
1322  void *knet_context,
1323  int processor_count)
1324 {
1325  return (0);
1326 }
1327 
1328 int totemknet_recv_flush (void *knet_context)
1329 {
1330  return (0);
1331 }
1332 
1333 int totemknet_send_flush (void *knet_context)
1334 {
1335  return (0);
1336 }
1337 
1339  void *knet_context,
1340  const void *msg,
1341  unsigned int msg_len)
1342 {
1343  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1344  int res = 0;
1345 
1346  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1347 
1348  return (res);
1349 }
1351  void *knet_context,
1352  const void *msg,
1353  unsigned int msg_len)
1354 {
1355  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1356  int res = 0;
1357 
1358  mcast_sendmsg (instance, msg, msg_len, 0);
1359 
1360  return (res);
1361 }
1362 
1364  void *knet_context,
1365  const void *msg,
1366  unsigned int msg_len)
1367 {
1368  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1369  int res = 0;
1370 
1371  mcast_sendmsg (instance, msg, msg_len, 1);
1372 
1373  return (res);
1374 }
1375 
1376 
1377 extern int totemknet_iface_check (void *knet_context)
1378 {
1379  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1380  int res = 0;
1381 
1382  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1383 
1384  return (res);
1385 }
1386 
1387 extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1388 {
1389  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1390 
1391  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1392 }
1393 
1395  void *knet_context,
1396  unsigned int nodeid)
1397 {
1398  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1399  int res = 0;
1400 
1401  instance->token_target.nodeid = nodeid;
1402 
1403  instance->totemknet_target_set_completed (instance->context);
1404 
1405  return (res);
1406 }
1407 
1409  void *knet_context)
1410 {
1411  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1412  unsigned int res;
1413  struct sockaddr_storage system_from;
1414  struct msghdr msg_hdr;
1415  struct iovec iov_recv;
1416  struct pollfd ufd;
1417  int nfds;
1418  int msg_processed = 0;
1419 
1420  iov_recv.iov_base = instance->iov_buffer;
1421  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1422 
1423  msg_hdr.msg_name = &system_from;
1424  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1425  msg_hdr.msg_iov = &iov_recv;
1426  msg_hdr.msg_iovlen = 1;
1427 #ifdef HAVE_MSGHDR_CONTROL
1428  msg_hdr.msg_control = 0;
1429 #endif
1430 #ifdef HAVE_MSGHDR_CONTROLLEN
1431  msg_hdr.msg_controllen = 0;
1432 #endif
1433 #ifdef HAVE_MSGHDR_FLAGS
1434  msg_hdr.msg_flags = 0;
1435 #endif
1436 #ifdef HAVE_MSGHDR_ACCRIGHTS
1437  msg_msg_hdr.msg_accrights = NULL;
1438 #endif
1439 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1440  msg_msg_hdr.msg_accrightslen = 0;
1441 #endif
1442 
1443  do {
1444  ufd.fd = instance->knet_fd;
1445  ufd.events = POLLIN;
1446  nfds = poll (&ufd, 1, 0);
1447  if (nfds == 1 && ufd.revents & POLLIN) {
1448  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1449  if (res != -1) {
1450  msg_processed = 1;
1451  } else {
1452  msg_processed = -1;
1453  }
1454  }
1455  } while (nfds == 1);
1456 
1457  return (msg_processed);
1458 }
1459 
1460 int totemknet_iface_set (void *knet_context,
1461  const struct totem_ip_address *local_addr,
1462  unsigned short ip_port,
1463  unsigned int iface_no)
1464 {
1465  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1466 
1467  totemip_copy(&instance->my_ids[iface_no], local_addr);
1468 
1469  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1470 
1471  instance->ip_port[iface_no] = ip_port;
1472 
1473  return 0;
1474 }
1475 
1476 
1478  void *knet_context,
1479  const struct totem_ip_address *local,
1480  const struct totem_ip_address *member,
1481  int link_no)
1482 {
1483  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1484  int err;
1485  int port = instance->ip_port[link_no];
1486  struct sockaddr_storage remote_ss;
1487  struct sockaddr_storage local_ss;
1488  int addrlen;
1489  int i;
1490  int host_found = 0;
1491  knet_node_id_t host_ids[KNET_MAX_HOST];
1492  size_t num_host_ids;
1493 
1494  /* Only create 1 loopback link and use link 0 */
1495  if (member->nodeid == instance->our_nodeid) {
1496  if (!instance->loopback_link) {
1497  link_no = 0;
1498  instance->loopback_link = 1;
1499  } else {
1500  /* Already done */
1501  return 0;
1502  }
1503  }
1504 
1505  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1506  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1507 
1508 
1509  /* Only add the host if it doesn't already exist in knet */
1510  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1511  if (err) {
1512  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1513  return -1;
1514  }
1515  for (i=0; i<num_host_ids; i++) {
1516  if (host_ids[i] == member->nodeid) {
1517  host_found = 1;
1518  }
1519  }
1520 
1521  if (!host_found) {
1522  err = knet_host_add(instance->knet_handle, member->nodeid);
1523  if (err != 0 && errno != EEXIST) {
1524  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1525  return -1;
1526  }
1527  } else {
1528  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1529  }
1530 
1531 
1532  if (err == 0) {
1533  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1534  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1535  return -1;
1536  }
1537  }
1538 
1539  memset(&local_ss, 0, sizeof(local_ss));
1540  memset(&remote_ss, 0, sizeof(remote_ss));
1541  /* Casts to remove const */
1542  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1543  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1544 
1545  if (member->nodeid == instance->our_nodeid) {
1546  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1547 
1548  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1549  KNET_TRANSPORT_LOOPBACK,
1550  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1551  }
1552  else {
1553  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1554  instance->totem_config->interfaces[link_no].knet_transport,
1555  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1556  }
1557  if (err) {
1558  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1559  return -1;
1560  }
1561 
1562  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1563  instance->totem_config->interfaces[link_no].knet_link_priority);
1564 
1565  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1566  instance->totem_config->interfaces[link_no].knet_link_priority);
1567  if (err) {
1568  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1569  }
1570 
1571  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1572  get done in totemknet_refresh_config */
1573  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1574  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1575  instance->totem_config->interfaces[link_no].knet_ping_interval,
1576  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1577  instance->totem_config->interfaces[link_no].knet_ping_precision);
1578  if (err) {
1579  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1580  }
1581  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1582  instance->totem_config->interfaces[link_no].knet_pong_count);
1583  if (err) {
1584  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1585  }
1586  }
1587 
1588  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1589  if (err) {
1590  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1591  return -1;
1592  }
1593 
1594  /* register stats */
1595  stats_knet_add_member(member->nodeid, link_no);
1596  return (0);
1597 }
1598 
1600  void *knet_context,
1601  const struct totem_ip_address *token_target,
1602  int link_no)
1603 {
1604  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1605  int res;
1606  uint8_t link_list[KNET_MAX_LINK];
1607  size_t num_links;
1608 
1609  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1610 
1611  /* Don't remove the link with the loopback on it until we shut down */
1612  if (token_target->nodeid == instance->our_nodeid) {
1613  return 0;
1614  }
1615 
1616  /* Tidy stats */
1617  stats_knet_del_member(token_target->nodeid, link_no);
1618 
1619  /* Remove the link first */
1620  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1621  if (res != 0) {
1622  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1623  return res;
1624  }
1625 
1626  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1627  if (res != 0) {
1628  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1629  return res;
1630  }
1631 
1632  /* If this is the last link, then remove the node */
1633  res = knet_link_get_link_list(instance->knet_handle,
1634  token_target->nodeid, link_list, &num_links);
1635  if (res) {
1636  return (0); /* not really failure */
1637  }
1638 
1639  if (num_links == 0) {
1640  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1641  }
1642  return res;
1643 }
1644 
1646  void *knet_context)
1647 {
1648  return (0);
1649 }
1650 
1651 
1652 static int totemknet_configure_compression (
1653  struct totemknet_instance *instance,
1654  struct totem_config *totem_config)
1655 {
1656  struct knet_handle_compress_cfg compress_cfg;
1657  int res = 0;
1658 
1659  assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1660  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1661 
1662  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1663  compress_cfg.compress_level = totem_config->knet_compression_level;
1664 
1665  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1666  if (res) {
1667  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1668  }
1669  return res;
1670 }
1671 
1673  void *knet_context,
1674  struct totem_config *totem_config)
1675 {
1676  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1677  int res = 0;
1678 
1679  (void)totemknet_configure_compression(instance, totem_config);
1680 
1681 #ifdef HAVE_LIBNOZZLE
1682  /* Set up nozzle device(s). Return code is ignored, because inability
1683  * configure nozzle is not fatal problem, errors are logged and
1684  * there is not much else we can do */
1685  (void)setup_nozzle(instance);
1686 #endif
1687 
1688  if (totem_config->crypto_changed) {
1689  /* Flip crypto_index */
1690  totem_config->crypto_index = 3-totem_config->crypto_index;
1691  res = totemknet_set_knet_crypto(instance);
1692 
1693  knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1694  totem_config->crypto_model,
1695  totem_config->crypto_cipher_type,
1696  totem_config->crypto_hash_type);
1697  }
1698  return (res);
1699 }
1700 
1701 
1703  void *knet_context,
1704  struct totem_config *totem_config,
1706 {
1707 #ifdef HAVE_KNET_CRYPTO_RECONF
1708  int res;
1709  int config_to_use;
1710  int config_to_clear;
1711  struct knet_handle_crypto_cfg crypto_cfg;
1712  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1713 
1714  knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1715 
1716  switch (phase) {
1718  config_to_use = totem_config->crypto_index;
1719  if (!totemknet_is_crypto_enabled(instance)) {
1720  config_to_use = 0; /* we are clearing it */
1721  }
1722 
1723  /* Enable the new config on this node */
1724  res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1725  if (res == -1) {
1726  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1727  }
1728  break;
1729 
1731  /*
1732  * All nodes should now have the new config. clear the old one out
1733  * OR disable crypto entirely if that's what the new config insists on.
1734  */
1735  config_to_clear = 3-totem_config->crypto_index;
1736  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1737 
1738  strcpy(crypto_cfg.crypto_model, "none");
1739  strcpy(crypto_cfg.crypto_cipher_type, "none");
1740  strcpy(crypto_cfg.crypto_hash_type, "none");
1741  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1742  if (res == -1) {
1743  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1744  }
1745  if (res == -2) {
1746  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1747  }
1748 
1749  /* If crypto is enabled then disable all cleartext reception */
1750  if (totemknet_is_crypto_enabled(instance)) {
1751  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1752  if (res) {
1753  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1754  }
1755  }
1756  }
1757 #endif
1758  return 0;
1759 }
1760 
1762  void *knet_context)
1763 {
1764  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1765 
1766  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1767 }
1768 
1769 /* For the stats module */
1771  knet_node_id_t node, uint8_t link_no,
1772  struct knet_link_status *status)
1773 {
1774  int res;
1775  int ret = CS_OK;
1776 
1777  /* We are probably not using knet */
1778  if (!global_instance) {
1779  return CS_ERR_NOT_EXIST;
1780  }
1781 
1782  if (link_no >= INTERFACE_MAX) {
1783  return CS_ERR_NOT_EXIST; /* Invalid link number */
1784  }
1785 
1786  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1787  if (res) {
1788  switch (errno) {
1789  case EINVAL:
1790  ret = CS_ERR_INVALID_PARAM;
1791  break;
1792  case EBUSY:
1793  ret = CS_ERR_BUSY;
1794  break;
1795  case EDEADLK:
1796  ret = CS_ERR_TRY_AGAIN;
1797  break;
1798  default:
1799  ret = CS_ERR_LIBRARY;
1800  break;
1801  }
1802  }
1803 
1804  return (ret);
1805 }
1806 
1808  struct knet_handle_stats *stats)
1809 {
1810  int res;
1811 
1812  /* We are probably not using knet */
1813  if (!global_instance) {
1814  return CS_ERR_NOT_EXIST;
1815  }
1816 
1817  res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1818  if (res != 0) {
1819  return (qb_to_cs_error(-errno));
1820  }
1821 
1822  return CS_OK;
1823 }
1824 
1825 static void timer_function_merge_detect_timeout (
1826  void *data)
1827 {
1828  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1829 
1830  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1831  instance->send_merge_detect_message = 1;
1832  }
1833 
1835 
1836  totemknet_start_merge_detect_timeout(instance);
1837 }
1838 
1839 static void totemknet_start_merge_detect_timeout(
1840  void *knet_context)
1841 {
1842  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1843 
1844  qb_loop_timer_add(instance->poll_handle,
1845  QB_LOOP_MED,
1846  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1847  (void *)instance,
1848  timer_function_merge_detect_timeout,
1849  &instance->timer_merge_detect_timeout);
1850 
1851 }
1852 
1853 static void totemknet_stop_merge_detect_timeout(
1854  void *knet_context)
1855 {
1856  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1857 
1858  qb_loop_timer_del(instance->poll_handle,
1859  instance->timer_merge_detect_timeout);
1860 }
1861 
1862 static void log_flush_messages (void *knet_context)
1863 {
1864  struct pollfd pfd;
1865  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1866  int cont;
1867 
1868  cont = 1;
1869 
1870  while (cont) {
1871  pfd.fd = instance->logpipes[0];
1872  pfd.events = POLLIN;
1873  pfd.revents = 0;
1874 
1875  if ((poll(&pfd, 1, 0) > 0) &&
1876  (pfd.revents & POLLIN) &&
1877  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1878  cont = 1;
1879  } else {
1880  cont = 0;
1881  }
1882  }
1883 }
1884 
1885 
1886 #ifdef HAVE_LIBNOZZLE
1887 #define NOZZLE_NAME "nozzle.name"
1888 #define NOZZLE_IPADDR "nozzle.ipaddr"
1889 #define NOZZLE_PREFIX "nozzle.ipprefix"
1890 #define NOZZLE_MACADDR "nozzle.macaddr"
1891 
1892 #define NOZZLE_CHANNEL 1
1893 
1894 
1895 static char *get_nozzle_script_dir(void *knet_context)
1896 {
1897  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1898  char filename[PATH_MAX + FILENAME_MAX + 1];
1899  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1900  int res;
1901  const char *dirname_res;
1902 
1903  /*
1904  * Build script directory based on corosync.conf file location
1905  */
1906  res = snprintf(filename, sizeof(filename), "%s",
1908  if (res >= sizeof(filename)) {
1909  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1910  return NULL;
1911  }
1912 
1913  dirname_res = dirname(filename);
1914 
1915  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1916  dirname_res, "updown.d");
1917  if (res >= sizeof(updown_dirname)) {
1918  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1919  return NULL;
1920  }
1921  return updown_dirname;
1922 }
1923 
1924 /*
1925  * Deliberately doesn't return the status as caller doesn't care.
1926  * The result will be logged though
1927  */
1928 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1929 {
1930  int res;
1931  char *exec_string;
1932 
1933  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1934  if (res == -1 && errno != ENOENT) {
1935  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1936  } else if (res == -2) {
1937  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1938  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1939  }
1940 }
1941 
1942 /*
1943  * Reparse IP address to add in our node ID
1944  * IPv6 addresses must end in '::'
1945  * IPv4 addresses must just be valid
1946  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1947  *
1948  * Returns the modified IP address as a string to pass into libnozzle
1949  */
1950 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1951  const char *input_addr,
1952  const char *prefix, int nodeid,
1953  char *output_addr, size_t output_len)
1954 {
1955  char *coloncolon;
1956  int bits;
1957  int max_prefix = 64;
1958  uint32_t nodeid_mask;
1959  uint32_t addr_mask;
1960  uint32_t masked_nodeid;
1961  struct in_addr *addr;
1962  struct totem_ip_address totemip;
1963 
1964  coloncolon = strstr(input_addr, "::");
1965  if (!coloncolon) {
1966  max_prefix = 30;
1967  }
1968 
1969  bits = atoi(prefix);
1970  if (bits < 8 || bits > max_prefix) {
1971  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1972  return -1;
1973  }
1974 
1975  /* IPv6 is easy */
1976  if (coloncolon) {
1977  memcpy(output_addr, input_addr, coloncolon-input_addr);
1978  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1979  return 0;
1980  }
1981 
1982  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1983  * add in the masked_nodeid and 'print' it out again
1984  */
1985  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1986  addr_mask = UINT32_MAX ^ nodeid_mask;
1987  masked_nodeid = nodeid & nodeid_mask;
1988 
1989  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1990  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1991  return -1;
1992  }
1993  addr = (struct in_addr *)&totemip.addr;
1994  addr->s_addr &= htonl(addr_mask);
1995  addr->s_addr |= htonl(masked_nodeid);
1996 
1997  inet_ntop(AF_INET, addr, output_addr, output_len);
1998  return 0;
1999 }
2000 
2001 static int create_nozzle_device(void *knet_context, const char *name,
2002  const char *ipaddr, const char *prefix,
2003  const char *macaddr)
2004 {
2005  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2006  char device_name[IFNAMSIZ+1];
2007  size_t size = IFNAMSIZ;
2008  int8_t channel = NOZZLE_CHANNEL;
2009  nozzle_t nozzle_dev;
2010  int nozzle_fd;
2011  int res;
2012  char *updown_dir;
2013  char parsed_ipaddr[INET6_ADDRSTRLEN];
2014  char mac[19];
2015 
2016  memset(device_name, 0, size);
2017  memset(&mac, 0, sizeof(mac));
2018  strncpy(device_name, name, size);
2019 
2020  updown_dir = get_nozzle_script_dir(knet_context);
2021  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
2022 
2023  nozzle_dev = nozzle_open(device_name, size, updown_dir);
2024  if (!nozzle_dev) {
2025  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
2026  return -1;
2027  }
2028  instance->nozzle_handle = nozzle_dev;
2029 
2030  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
2031  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
2032  goto out_clean;
2033  }
2034 
2035  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
2036  /* Prints its own errors */
2037  goto out_clean;
2038  }
2039  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
2040  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
2041  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
2042  goto out_clean;
2043  }
2044 
2045  nozzle_fd = nozzle_get_fd(nozzle_dev);
2046  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
2047 
2048  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
2049  if (res != 0) {
2050  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
2051  goto out_clean;
2052  }
2053 
2054  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
2055 
2056  res = nozzle_set_up(nozzle_dev);
2057  if (res != 0) {
2058  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
2059  goto out_clean;
2060  }
2061  run_nozzle_script(instance, NOZZLE_UP, "up");
2062 
2063  return 0;
2064 
2065 out_clean:
2066  nozzle_close(nozzle_dev);
2067  return -1;
2068 }
2069 
2070 static int remove_nozzle_device(void *knet_context)
2071 {
2072  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2073  int res;
2074  int datafd;
2075 
2076  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
2077  if (res != 0) {
2078  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2079  return -1;
2080  }
2081 
2082  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
2083  if (res != 0) {
2084  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2085  return -1;
2086  }
2087 
2088  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
2089  res = nozzle_set_down(instance->nozzle_handle);
2090  if (res != 0) {
2091  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
2092  return -1;
2093  }
2094  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
2095 
2096  res = nozzle_close(instance->nozzle_handle);
2097  if (res != 0) {
2098  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
2099  return -1;
2100  }
2101  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2102  return 0;
2103 }
2104 
2105 static void free_nozzle(struct totemknet_instance *instance)
2106 {
2107  free(instance->nozzle_name);
2108  free(instance->nozzle_ipaddr);
2109  free(instance->nozzle_prefix);
2110  free(instance->nozzle_macaddr);
2111 
2112  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2113  instance->nozzle_macaddr = NULL;
2114 }
2115 
2116 static int setup_nozzle(void *knet_context)
2117 {
2118  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2119  char *ipaddr_str = NULL;
2120  char *name_str = NULL;
2121  char *prefix_str = NULL;
2122  char *macaddr_str = NULL;
2123  char mac[32];
2124  int name_res;
2125  int macaddr_res;
2126  int res = -1;
2127 
2128  /*
2129  * Return value ignored on purpose. icmap_get_string changes
2130  * ipaddr_str/prefix_str only on success.
2131  */
2132  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2133  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2134  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2135  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2136 
2137  /* Is is being removed? */
2138  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2139  remove_nozzle_device(instance);
2140  free_nozzle(instance);
2141  goto out_free;
2142  }
2143 
2144  if (!name_str) {
2145  /* no nozzle */
2146  goto out_free;
2147  }
2148 
2149  if (!ipaddr_str) {
2150  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2151  goto out_free;
2152  }
2153 
2154  if (!prefix_str) {
2155  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2156  goto out_free;
2157  }
2158 
2159  if (macaddr_str && strlen(macaddr_str) != 17) {
2160  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2161  goto out_free;
2162  }
2163  if (!macaddr_str) {
2164  macaddr_str = (char*)"54:54:01:00:00:00";
2165  }
2166 
2167  if (instance->nozzle_name &&
2168  (strcmp(name_str, instance->nozzle_name) == 0) &&
2169  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2170  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2171  (instance->nozzle_macaddr == NULL ||
2172  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2173  /* Nothing has changed */
2174  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2175  goto out_free;
2176  }
2177 
2178  /* Add nodeid into MAC address */
2179  memcpy(mac, macaddr_str, 12);
2180  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2181  instance->our_nodeid >> 8,
2182  instance->our_nodeid & 0xFF);
2183  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2184 
2185  if (name_res == CS_OK && name_str) {
2186  /* Reconfigure */
2187  if (instance->nozzle_name) {
2188  remove_nozzle_device(instance);
2189  free_nozzle(instance);
2190  }
2191 
2192  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2193  mac);
2194 
2195  instance->nozzle_name = strdup(name_str);
2196  instance->nozzle_ipaddr = strdup(ipaddr_str);
2197  instance->nozzle_prefix = strdup(prefix_str);
2198  instance->nozzle_macaddr = strdup(macaddr_str);
2199  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2200  !instance->nozzle_prefix) {
2201  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2202  /*
2203  * This 'free' will cause a complete reconfigure of the device next time we reload
2204  * but will also let the the current device keep working until then.
2205  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2206  */
2207  free_nozzle(instance);
2208  }
2209  }
2210 
2211 out_free:
2212  free(name_str);
2213  free(ipaddr_str);
2214  free(prefix_str);
2215  if (macaddr_res == CS_OK) {
2216  free(macaddr_str);
2217  }
2218 
2219  return res;
2220 }
2221 #endif // HAVE_LIBNOZZLE
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
int knet_ping_precision
Definition: totem.h:94
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:233
int totemknet_log_level_security
Definition: totemknet.c:124
cfg_message_crypto_reconfig_phase_t
Definition: totem.h:154
qb_loop_t * poll_handle
Definition: totemknet.c:96
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1599
struct knet_link_status link_status[KNET_MAX_LINK]
Definition: totem.h:271
int crypto_changed
Definition: totem.h:231
#define MSG_NOSIGNAL
Definition: totemknet.c:83
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
int knet_link_priority
Definition: totem.h:91
uint32_t value
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
struct totem_interface * interfaces
Definition: totem.h:165
void stats_knet_add_handle(void)
Definition: stats.c:736
int totemknet_log_level_error
Definition: totemknet.c:126
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:241
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
The totem_ip_address struct.
Definition: coroapi.h:111
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:256
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:114
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:233
int send_merge_detect_message
Definition: totemknet.c:169
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:663
uint32_t knet_compression_threshold
Definition: totem.h:235
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:227
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:205
unsigned int knet_pmtud_interval
Definition: totem.h:169
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1770
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1316
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1761
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:249
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:723
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
unsigned int private_key_len
Definition: totem.h:176
int totemknet_log_level_notice
Definition: totemknet.c:130
uint8_t reachable
Definition: totem.h:265
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
#define INTERFACE_MAX
Definition: coroapi.h:88
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:363
unsigned int block_unlisted_ips
Definition: totem.h:245
uint8_t onwire_ver
Definition: totem.h:270
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1350
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1377
void(*) void knet_context)
Definition: totemknet.c:145
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:868
unsigned int nodeid
Definition: totem.h:264
unsigned int node_id
Definition: totem.h:167
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
uint8_t configured
Definition: totem.h:89
int totemknet_log_level_warning
Definition: totemknet.c:128
#define OWN_INDEX_NONE
Definition: totemknet.c:488
const char * corosync_get_config_file(void)
Definition: main.c:206
void * user_data
Definition: sam.c:127
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1394
struct totem_config * totem_config
Definition: totemknet.c:161
unsigned int nodeid
Definition: coroapi.h:112
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1672
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1460
int knet_transport
Definition: totem.h:96
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1321
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1328
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1333
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:174
uint8_t onwire_min
Definition: totem.h:268
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition: totemknet.c:1702
struct totem_ip_address boundto
Definition: totem.h:84
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:892
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:101
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1807
struct totemknet_instance * global_instance
Definition: totemknet.c:187
struct totem_message_header header
Definition: totemsrp.c:260
uint16_t ip_port
Definition: totem.h:87
int knet_compression_level
Definition: totem.h:237
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:306
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
#define ENTER
Definition: logsys.h:324
unsigned int net_mtu
Definition: totem.h:209
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:225
struct totem_ip_address token_target
Definition: totemknet.c:163
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
uint8_t external
Definition: totem.h:267
int crypto_index
Definition: totem.h:229
pthread_mutex_t log_mutex
Definition: totemknet.c:176
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:223
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1477
uint8_t onwire_max
Definition: totem.h:269
int knet_pong_count
Definition: totem.h:95
struct totemknet_instance * instance
Definition: totemknet.c:192
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
int knet_ping_interval
Definition: totem.h:92
const void * msg
Definition: totemknet.c:190
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
int knet_ping_timeout
Definition: totem.h:93
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:264
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:207
typedef __attribute__
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:713
uint8_t remote
Definition: totem.h:266
struct srp_addr system_from
Definition: totemsrp.c:261
char type
Definition: totem.h:55
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1387
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1645
unsigned int merge_timeout
Definition: totem.h:197
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1363
int totemknet_log_level_debug
Definition: totemknet.c:132
unsigned int target_nodeid
Definition: totem.h:132
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1338
struct totem_ip_address bindnet
Definition: totem.h:83
unsigned int nodeid
Definition: coroapi.h:75
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1408
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
unsigned int msg_len
Definition: totemknet.c:191
#define LEAVE
Definition: logsys.h:325
int totemknet_nodestatus_get(void *knet_context, unsigned int nodeid, struct totem_node_status *node_status)
Definition: totemknet.c:491
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:1058
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:568
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem.nodeid", "totem.version", ...
Definition: icmap.h:85
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1310
knet_handle_t knet_handle
Definition: totemknet.c:98