Re: Routing socket and interrupt levels

[Matt Thomas <matt%3am-software.com@localhost>]

On Mar 25, 2008, at 3:33 PM, Andrew Doran wrote:

> Messages can be sent through the routing socket from device interrupt
> handlers at IPL_VM. The one I saw was notification of link state  
> change. The
> socket code operates at a much lower level, IPL_SOFTNET, and  
> shouldn't be
> called from a higher level. The below fixes it by enqueuing routing  
> messages
> to a soft interrupt. Any objections?
>
> Thanks,
> Andrew
>
> Index: rtsock.c
> ===================================================================
> RCS file: /cvsroot/src/sys/net/rtsock.c,v
> retrieving revision 1.98
> diff -u -r1.98 rtsock.c
> --- rtsock.c    20 Feb 2008 17:05:53 -0000      1.98
> +++ rtsock.c    25 Mar 2008 22:25:27 -0000
> @@ -75,6 +75,7 @@
> #include <sys/protosw.h>
> #include <sys/sysctl.h>
> #include <sys/kauth.h>
> +#include <sys/intr.h>
> #ifdef RTSOCK_DEBUG
> #include <netinet/in.h>
> #endif /* RTSOCK_DEBUG */
> @@ -91,6 +92,10 @@
> struct  sockaddr route_src = { .sa_len = 2, .sa_family = PF_ROUTE, };
> struct  sockproto route_proto = { .sp_family = PF_ROUTE, };
>
> +int    route_maxqlen = IFQ_MAXLEN;
> +static struct  ifqueue route_intrq;
> +static void    *route_sih;
> +
> struct walkarg {
>         int     w_op;
>         int     w_arg;
> @@ -111,6 +116,7 @@
> static int sysctl_iflist(int, struct walkarg *, int);
> static int sysctl_rtable(SYSCTLFN_PROTO);
> static inline void rt_adjustcount(int, int);
> +static void route_enqueue(struct mbuf *);

static void route_enqueue(struct mbuf *, int);

> /* Sleazy use of local variables throughout file, warning!!!! */
> #define dst     info.rti_info[RTAX_DST]
> @@ -460,7 +466,7 @@
>         if (family)
>                 route_proto.sp_protocol = family;
>         if (m)
> -               raw_input(m, &route_proto, &route_src, &route_dst);
> +               route_enqueue(m);
>         if (rp)
>                 rp->rcb_proto.sp_family = PF_ROUTE;
>     }
> @@ -712,7 +718,7 @@
>                 return;
>         mtod(m, struct rt_msghdr *)->rtm_addrs = rtinfo->rti_addrs;
>         route_proto.sp_protocol = sa ? sa->sa_family : 0;

> -       raw_input(m, &route_proto, &route_src, &route_dst);
> +       route_enqueue(m);

route_enqueue(m, sa ? sa->sa_family : 0);

> }
>
> /*
> @@ -741,7 +747,7 @@
>         if (m == NULL)
>                 return;
>         route_proto.sp_protocol = 0;
> -       raw_input(m, &route_proto, &route_src, &route_dst);
> +       route_enqueue(m);

route_enqueue(m, 0);

> #ifdef COMPAT_14
>         memset(&info, 0, sizeof(info));
>         memset(&oifm, 0, sizeof(oifm));
> @@ -770,7 +776,7 @@
>         if (m == NULL)
>                 return;
>         route_proto.sp_protocol = 0;
> -       raw_input(m, &route_proto, &route_src, &route_dst);
> +       route_enqueue(m);

route_enqueue(m, 0);

> #endif
> }
>
> @@ -833,7 +839,7 @@
>                         mtod(m, struct rt_msghdr *)->rtm_addrs = info.rti_addrs;
>                 }
>                 route_proto.sp_protocol = sa ? sa->sa_family : 0;
>
> -               raw_input(m, &route_proto, &route_src, &route_dst);
> +               route_enqueue(m);

route_enqueue(m, sa ? sa->sa_family : 0);

>         }
> }
>
> @@ -867,7 +873,7 @@
>         if (m == NULL)
>                 return;
>         route_proto.sp_protocol = 0;
> -       raw_input(m, &route_proto, &route_src, &route_dst);
> +       route_enqueue(m);

route_enqueue(m, 0);

> }
>
> /*
> @@ -910,7 +916,7 @@
>                 m->m_pkthdr.len += data_len;
>         mtod(m, struct if_announcemsghdr *)->ifan_msglen += data_len;
>         route_proto.sp_protocol = 0;
> -       raw_input(m, &route_proto, &route_src, &route_dst);
> +       route_enqueue(m);

route_enqueue(m, 0);
> }
>
> /*
> @@ -1164,6 +1170,54 @@
> }
>
> /*
> + * Routing message software interrupt routine
> + */
> +static void
> +route_intr(void *cookie)
> +{
> +       int s;
> +       struct mbuf *m;
> +
> +       while (!IF_IS_EMPTY(&route_intrq)) {
> +               s = splnet();
> +               IF_DEQUEUE(&route_intrq, m);
> +               splx(s);
> +               if (m == NULL)
> +                       break;

Make route_proto a local;

                route_proto.sp_family = (uintptr_t) M_GETCTX(m);

> +               raw_input(m, &route_proto, &route_src, &route_dst);
> +       }
> +}
> +
> +/*
> + * Enqueue a message to the software interrupt routine.
> + */
> +static void
> +route_enqueue(struct mbuf *m)
, int family
> +{
> +       int s, wasempty;
> +
> +       s = splnet();
> +       if (IF_QFULL(&route_intrq)) {
> +               IF_DROP(&route_intrq);
> +               m_freem(m);
> +       } else {
> +               wasempty = IF_IS_EMPTY(&route_intrq);
                 M_SETCTX(m, family);
> +               IF_ENQUEUE(&route_intrq, m);
> +               if (wasempty)
> +                       softint_schedule(route_sih);
> +       }
> +       splx(s);
> +}
> +
> +void
> +rt_init(void)
> +{
> +
> +       route_intrq.ifq_maxlen = route_maxqlen;
> +       route_sih = softint_establish(SOFTINT_NET, route_intr, NULL);
> +}
> +
> +/*
>  * Definitions of protocols supported in the ROUTE domain.
>  */