Fix a few bugs, tentative tun multiqueue support

pass-1/hide-eid.c

@@ -5,8 +5,11 @@
 #include <unistd.h>
 #include <errno.h>
 
+#include <sys/types.h>
+#include <sys/wait.h>
 #include <sys/uio.h>
 
+#define MAX_SESSIONS 32
 
 int process_icmpv4_rloc_update( struct rlocs *reg, struct packet *packet  )
 {
@@ -77,35 +80,30 @@ int process_packet( struct rlocs *reg, struct packet *pkt, struct rsp_data *frag
     return wrap_packet( reg, pkt, frag1, frag2 );
 }
 
-/*
+
- * Entry point. Expects an invocation like:
+#define ARG_NUM_SESSIONS  1
- * hide-eid <filename of rloc database> <listen_ifname> <output_ifname>
+#define ARG_RLOC_DB       2
- * 
+#define ARG_IFNAME        3
- * hide-eid wraps packets that come to it unwrapped, and unwraps packets that
+#define ARG_LAST          4
- * come to it wrapped. This makes it handy for all-in-one boxes
+
- */ 
+int do_session( int multi, int argc, char **argv )
-int main(int argc, char** argv)
 {
-    struct session session;
     struct packet recv_pkt;
+    struct session session;
     
     struct rsp_data frag1;
     struct rsp_data frag2;
     ssize_t count;
 
-    if ( argc < 4 ) {
-        warn( "Usage: %s <rloc database> <listen_ifname> <output_ifname> [ <rloc> <keyfile> ]n", argv[0] );
-        return 1;
-    }
-    
     rlocs_init();
-    if ( !session_setup( &session, argv[1], argv[2], argv[3] ) ) {
+
+    if ( !session_setup( &session, argv[ARG_RLOC_DB], argv[ARG_IFNAME], multi ) ) {
         warn( "Failed to set up session, exiting" );
         return 1;
     }
 
-    if  (argc > 4 ) {
+    if  (argc > ARG_LAST ) {
-        if ( !session_upgrade_rlocs( &session, argc - 4, argv + 4 ) ) {
+        if ( !session_upgrade_rlocs( &session, argc - ARG_LAST, argv + ARG_LAST ) ) {
             warn( "Failed to upgrade rlocs for session, exiting" );
             session_teardown( &session );
             return 1;
@@ -122,7 +120,7 @@ int main(int argc, char** argv)
     while( 1 ) {
         int rsp_count;
         
-        if ( ( count = read( session.listen_if, &recv_pkt, sizeof( struct packet ) ) ) < 0 ) {
+        if ( ( count = read( session.fd, &recv_pkt, sizeof( struct packet ) ) ) < 0 ) {
             warn( "Failed to get a packet (%s)", strerror( errno ) );
             break;
         }
@@ -138,13 +136,13 @@ int main(int argc, char** argv)
         }
         
         if ( rsp_count > 0 ) {
-            if ( ( count = writev( session.output_if, frag1.iovs, frag1.count ) ) < 0 ) {
+            if ( ( count = writev( session.fd, frag1.iovs, frag1.count ) ) < 0 ) {
                 debug( "Error writing processed packet to output: %s", strerror(errno) );
             }
         }
         
         if ( rsp_count == 2 ) {
-            if ( ( count = writev( session.output_if, frag2.iovs, frag2.count ) ) < 0 ) {
+            if ( ( count = writev( session.fd, frag2.iovs, frag2.count ) ) < 0 ) {
                 debug( "Error writing second processed packet to output: %s", strerror(errno) );
             }
         }
@@ -156,3 +154,81 @@ int main(int argc, char** argv)
 
     return 0;
 }
+
+/*
+ * Entry point. Expects an invocation like:
+ * hide-eid <filename of rloc database> <listen_ifname> <output_ifname>
+ * 
+ * hide-eid wraps packets that come to it unwrapped, and unwraps packets that
+ * come to it wrapped. This makes it handy for all-in-one boxes
+ */ 
+int main(int argc, char **argv)
+{
+    pid_t pids[MAX_SESSIONS];
+    int num_sessions, i;
+    int num_returned = 0, ret = 0;
+
+    if ( argc < ARG_LAST ) {
+        warn( "Usage: %s <number of sessions> <rloc database> <ifname> [ <rloc> <keyfile> ]n", argv[0] );
+        return 1;
+    }
+    
+    memset( &pids, 0, sizeof( pid_t ) * MAX_SESSIONS );
+
+    num_sessions = atoi( argv[ARG_NUM_SESSIONS] );
+
+    if ( num_sessions <= 0 || num_sessions > MAX_SESSIONS ) {
+        warn( "Bad value: %s for number of sessions", argv[ARG_NUM_SESSIONS] );
+    }
+
+    // Don't spawn a child process if there's only 1 of us
+    if ( num_sessions == 1 ) {
+        return do_session( 0, argc, argv );
+    }
+
+    for( i = 0 ;  i < num_sessions ; i++ ) {
+        pids[i] = fork();
+        if ( pids[i] == -1 ) {
+            warn( "fork() failure!" );
+            exit(1);
+        }
+
+        if ( pids[i] == 0 ) {
+            int ret = do_session( 1, argc, argv );
+            info( "Child process %i exiting with status %i", getpid(), ret );
+            exit( ret );
+        } else {
+            info( "Child session %i started", pids[i] );
+        }
+        
+    }
+
+    // wait on all sessions returning
+    // todo: pass sigterm, etc to children
+    while ( num_returned < num_sessions ) {
+        int status;
+        pid_t pid;
+
+        pid = waitpid( -1, &status, 0 );
+        if ( pid == -1 ) {
+            warn( "waitpid() failure: %s", strerror(errno) );
+            ret = 1;
+            break;
+        }
+
+        if ( WIFEXITED( status ) ) {
+            for ( i = 0 ; i < num_sessions ; i++ ) {
+                if ( pids[i] == pid ) {
+                    info( "Child pid %i exited", pid );
+                    pids[i] = -1;
+                    num_returned += 1;
+                    // TODO: if exit status was bad, set ret to 1
+                    break;
+                }
+            }
+        }
+    }
+    
+    return ret;
+
+}

pass-1/rlocs.c

@@ -293,7 +293,7 @@ struct rlocs* rlocs_new( char* filename )
     json_object* config;
     ssize_t have_read = 0;
     char* json_text;
-    int fd;
+    int fd, ret;
     
     fd = open( filename, O_RDONLY );
     if ( errno < 0 ) {
@@ -301,8 +301,8 @@ struct rlocs* rlocs_new( char* filename )
         return NULL;
     }
     
-    fstat( fd, &filedata );
+    ret = fstat( fd, &filedata );
-    if ( errno < 0 ) {
+    if ( ret < 0 ) {
         warn( "Error %s (%i) getting size of %s", strerror(errno), errno, filename );
         return NULL;        
     }
@@ -477,36 +477,6 @@ int rlocs_find_two_ipv4(
 }
 
 
-/* Replaces the public key in the rloc struct with a private key so we can
- * unwrap, as well as wrap, packets.
- */
-int rloc_add_private_key( struct rloc *rloc, char *filename )
-{
-    BIO *key_data = BIO_new_file( filename, "r" );    
-    EC_KEY* key = PEM_read_bio_ECPrivateKey( key_data, NULL, NULL, NULL );
-    
-    if ( key == NULL ) {
-        warn( "Failed to add private key %s", filename );
-        goto fail;
-    }
-    
-    if ( !EVP_PKEY_assign_EC_KEY( rloc->key, key ) ) {
-        warn( "Failed to assign private key in %s to rloc", filename );
-        goto fail;
-    }
-    
-    return 1;
-
-fail:
-    show_ssl_errors();
-
-    if ( key != NULL ) {
-        EC_KEY_free( key );
-    }
-
-    return 0;    
-}
-
 int rlocs_update_peer_context(struct rlocs *reg, struct rloc *x, struct rloc *y)
 {
     struct peer_context *entry = &reg->peer_contexts[x->context_id][y->context_id];
@@ -576,6 +546,43 @@ fail:
     return 0;
 }
 
+/* Replaces the public key in the rloc struct with a private key so we can
+ * unwrap, as well as wrap, packets.
+ */
+int rlocs_add_private_key( struct rlocs *reg, struct rloc *rloc, char *filename )
+{
+    int i;
+    BIO *key_data = BIO_new_file( filename, "r" );    
+    EC_KEY* key = PEM_read_bio_ECPrivateKey( key_data, NULL, NULL, NULL );
+    
+    if ( key == NULL ) {
+        warn( "Failed to add private key %s", filename );
+        goto fail;
+    }
+    
+    if ( !EVP_PKEY_assign_EC_KEY( rloc->key, key ) ) {
+        warn( "Failed to assign private key in %s to rloc", filename );
+        goto fail;
+    }
+
+    // We need to update all the peer contexts touching this rloc now
+    for ( i = 0 ; i < reg->num_entries ; i++ ) {
+        rlocs_update_peer_context( reg, rloc, &reg->entries[i] );
+        rlocs_update_peer_context( reg, &reg->entries[i], rloc );
+    }
+    
+    return 1;
+
+fail:
+    show_ssl_errors();
+
+    if ( key != NULL ) {
+        EC_KEY_free( key );
+    }
+
+    return 0;    
+}
+
 
 struct peer_context *rlocs_get_peer_ctx( struct rlocs *reg, struct rloc *x, struct rloc *y )
 {

pass-1/rlocs.h

@@ -31,6 +31,7 @@ struct peer_context {
 typedef struct peer_context PeerCtx;
 
 struct rloc {
+    int in_use;
     short family;
     union {
         struct in_addr  ip4;
@@ -98,7 +99,7 @@ int rlocs_find_two_ipv4(
 
 struct peer_context *rlocs_get_peer_ctx( struct rlocs *reg, struct rloc *x, struct rloc *y );
 
-int rloc_add_private_key( struct rloc *rloc, char *filename );
+int rlocs_add_private_key( struct rlocs *reg, struct rloc *rloc, char *filename );
 
 void rlocs_debug_output( struct rlocs *reg );
 

pass-1/util.c

@@ -14,12 +14,16 @@
 #include <arpa/inet.h>
 #include <netinet/ip.h>
 
+#ifndef IFF_MULTI_QUEUE
+#define IFF_MULTI_QUEUE 0x0100
+#endif
+
 void* xmalloc( size_t bytes )
 {
     void* result = malloc( bytes );
     
     if ( bytes > 0 && result == NULL ) {
-        warn( "Couldn't allocate memory, exiting!" );
+        warn( "Couldn't allocate %zu bytes, exiting!", bytes );
         exit(2);
         
     }
@@ -28,25 +32,52 @@ void* xmalloc( size_t bytes )
     return result;
 }
 
-int create_tun( const char* name )
+int tun_has_multiqueue( int tun_fd )
+{
+    int features;
+
+    if ( ioctl( tun_fd, TUNGETFEATURES, &features ) < 0) {
+        warn("Kernel doesn't support TUNGETFEATURES, assuming no multiqueue");
+        features = 0;
+    }
+
+    return features & IFF_MULTI_QUEUE;
+}
+
+int create_tun( const char *name, int multi )
 {
     int fd, err;
     struct ifreq ifr;
 
-   if ( ( fd = open( "/dev/net/tun", O_RDWR ) ) < 0 ) {
+    if ( ( fd = open( "/dev/net/tun", O_RDWR ) ) < 0 ) {
-        warn( "Error %s opening tun to create %s", strerror(errno), name );
+        warn( "Error %s (%i) opening tun to create %s", strerror(errno), errno, name );
         return -1;
     }
     
     memset( &ifr, 0, sizeof( struct ifreq ) );
     
-    ifr.ifr_flags = IFF_TUN | IFF_NO_PI | IFF_UP;
+    ifr.ifr_flags = IFF_TUN | IFF_NO_PI;
+    
+    if ( multi ) {
+        if ( !tun_has_multiqueue( fd ) ) {
+            warn( "multiqueue requested but kernel doesn't support it" );
+            close( fd );
+            return -1;
+        }
+
+        debug( "Creating multi-queue device" );
+        ifr.ifr_flags |= IFF_MULTI_QUEUE;
+    }
+
+
     strncpy( ifr.ifr_name, name, IFNAMSIZ );
     
-    if ( (err = ioctl( fd, TUNSETIFF, (void*) &ifr ) ) < 0 ) {
+    while ( (err = ioctl( fd, TUNSETIFF, (void*) &ifr ) ) < 0 ) {
-        warn( "Error creating tun device %s: %s", name, strerror(errno) );
+        if ( errno != EBUSY ) {
-        close( fd );
+            warn( "Error creating tun device %s: %s (%i)", name, strerror(errno), errno );
-        return -1;
+            close( fd );
+            return -1;
+        }
     }
     
     return fd;
@@ -82,11 +113,9 @@ int link_set_up( char *link_name, int state )
 }
 
 
-int session_setup( struct session *session, char *config_file, char *listen_if, char *output_if  )
+int session_setup( struct session *session, char *config_file, char *ifname, int multi  )
 {
     memset( session, 0, sizeof( struct session ) );
-    session->listen_if = -1;
-    session->output_if = -1;
 
     session->rlocs = rlocs_new( config_file );
 
@@ -97,34 +126,13 @@ int session_setup( struct session *session, char *config_file, char *listen_if,
 
     rlocs_debug_output( session->rlocs );
     
-    // TODO: We can scale the tun architecture by using multiqueue and having
+    session->fd = create_tun( ifname, multi );
-    // a bunch of workers, rather than this noddy scheme. If we don't jump 
+    if ( session->fd == -1 ) {
-    // directly to something saner, anyway...
+        warn( "Error opening %s for listening", ifname );
-
-    session->listen_if = create_tun( listen_if );
-    if ( session->listen_if == -1 ) {
-        warn( "Error opening %s for listening", listen_if );
         rlocs_free( session->rlocs );
         return 0;
     }
-    link_set_up( listen_if, 1 );
+    link_set_up( ifname, 1 );
-    
-    
-    if ( strcmp( listen_if, output_if ) == 0 ) {
-        session->same_if = 1;
-        session->output_if = session->listen_if;
-    } else {
-        session->same_if = 0;
-        session->output_if = create_tun( output_if );
-
-        if ( session->output_if == -1 ) {
-            warn( "Error opening %s for outputting", output_if );
-            rlocs_free( session->rlocs );
-            close( session->listen_if );
-            return 0;
-        }
-        link_set_up( output_if, 1 );
-    }
 
     return 1;
 }
@@ -164,7 +172,7 @@ int session_upgrade_rlocs( struct session *session, int argc, char** args )
             return 0;
         }
         
-        if ( !rloc_add_private_key( rloc, filename ) ) {
+        if ( !rlocs_add_private_key( session->rlocs, rloc, filename ) ) {
             warn( "Couldn't upgrade rloc %s with %s", rloc_str, filename );
             return 0;
         }
@@ -179,12 +187,8 @@ int session_upgrade_rlocs( struct session *session, int argc, char** args )
 void session_teardown( struct session *session )
 {
     rlocs_free( session->rlocs );
-    if ( session->listen_if >= 0 ) {
+    if ( session->fd >= 0 ) {
-        close( session->listen_if );
+        close( session->fd );
-    }
-
-    if ( session->output_if >= 0 && !session->same_if ) {
-        close( session->output_if );
     }
 }
 

pass-1/util.h

@@ -14,20 +14,20 @@
 #define warn(msg, ...) { fprintf( stderr, msg, ##__VA_ARGS__ ) ; fprintf( stderr, "\n" ); }
 
 
+
+
 void* xmalloc( size_t bytes );
-int create_tun( const char* name );
+int create_tun( const char* name, int multi );
 
 int link_set_up( char *link_name, int state );
 
 /* Our programs use this common struct to take advantage of common init code */
 struct session {
     struct rlocs *rlocs;
-    int listen_if;
+    int fd;
-    int output_if;
-    int same_if;
 };
 
-int session_setup( struct session *session, char *config_file, char *listen_if, char *output_if );
+int session_setup( struct session *session, char *config_file, char *ifname, int multi );
 
 
 /* We take an array of 2n rlocs to upgrade. First element of each pair is
@@ -35,7 +35,6 @@ int session_setup( struct session *session, char *config_file, char *listen_if,
  */
 int session_upgrade_rlocs( struct session *session, int argc, char** args );
 void session_teardown( struct session *session );
-
 int sha256sum( unsigned char *src, size_t src_len, unsigned char dst[SHA256_DIGEST_LENGTH] );