Skip to content

Commit

Permalink
QUIC: separate UDP framework for QUIC.
Browse files Browse the repository at this point in the history 
Previously, QUIC used the existing UDP framework, which was created for UDP in
Stream.  However the way QUIC connections are created and looked up is different
from the way UDP connections in Stream are created and looked up.  Now these
two implementations are decoupled.
  • Loading branch information
@arut
arut committed Apr 20, 2022
1 parent c24c27b commit 9d81ef7
Show file tree
Hide file tree
Showing 13 changed files with 585 additions and 142 deletions.
1 change: 1 addition & 0 deletions auto/modules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1344,6 +1344,7 @@ if [ $USE_OPENSSL_QUIC = YES ]; then
src/event/quic/ngx_event_quic_output.h \
src/event/quic/ngx_event_quic_socket.h"
ngx_module_srcs="src/event/quic/ngx_event_quic.c \
src/event/quic/ngx_event_quic_udp.c \
src/event/quic/ngx_event_quic_transport.c \
src/event/quic/ngx_event_quic_protection.c \
src/event/quic/ngx_event_quic_frames.c \
Expand Down
4 changes: 0 additions & 4 deletions src/core/ngx_connection.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,6 @@ ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,

ngx_memcpy(ls->addr_text.data, text, len);

#if !(NGX_WIN32)
ngx_rbtree_init(&ls->rbtree, &ls->sentinel, ngx_udp_rbtree_insert_value);
#endif

ls->fd = (ngx_socket_t) -1;
ls->type = SOCK_STREAM;

Expand Down
12 changes: 10 additions & 2 deletions src/event/ngx_event.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -886,8 +886,16 @@ ngx_event_process_init(ngx_cycle_t *cycle)

#else

rev->handler = (c->type == SOCK_STREAM) ? ngx_event_accept
: ngx_event_recvmsg;
if (c->type == SOCK_STREAM) {
rev->handler = ngx_event_accept;

#if (NGX_QUIC)
} else if (ls[i].quic) {
rev->handler = ngx_quic_recvmsg;
#endif
} else {
rev->handler = ngx_event_recvmsg;
}

#if (NGX_HAVE_REUSEPORT)

Expand Down
173 changes: 59 additions & 114 deletions src/event/ngx_event_udp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,25 @@
static void ngx_close_accepted_udp_connection(ngx_connection_t *c);
static ssize_t ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf,
size_t size);
static ngx_int_t ngx_create_udp_connection(ngx_connection_t *c);
static ngx_int_t ngx_insert_udp_connection(ngx_connection_t *c);
static ngx_connection_t *ngx_lookup_udp_connection(ngx_listening_t *ls,
ngx_str_t *key, struct sockaddr *local_sockaddr, socklen_t local_socklen);
struct sockaddr *sockaddr, socklen_t socklen,
struct sockaddr *local_sockaddr, socklen_t local_socklen);


void
ngx_event_recvmsg(ngx_event_t *ev)
{
size_t len;
ssize_t n;
ngx_str_t key;
ngx_buf_t buf;
ngx_log_t *log;
ngx_err_t err;
socklen_t local_socklen;
socklen_t socklen, local_socklen;
ngx_event_t *rev, *wev;
struct iovec iov[1];
struct msghdr msg;
ngx_sockaddr_t sa, lsa;
ngx_udp_dgram_t dgram;
struct sockaddr *local_sockaddr;
struct sockaddr *sockaddr, *local_sockaddr;
ngx_listening_t *ls;
ngx_event_conf_t *ecf;
ngx_connection_t *c, *lc;
Expand Down Expand Up @@ -110,21 +108,21 @@ ngx_event_recvmsg(ngx_event_t *ev)
}
#endif

dgram.sockaddr = msg.msg_name;
dgram.socklen = msg.msg_namelen;
sockaddr = msg.msg_name;
socklen = msg.msg_namelen;

if (dgram.socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
dgram.socklen = sizeof(ngx_sockaddr_t);
if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
socklen = sizeof(ngx_sockaddr_t);
}

if (dgram.socklen == 0) {
if (socklen == 0) {

/*
* on Linux recvmsg() returns zero msg_namelen
* when receiving packets from unbound AF_UNIX sockets
*/

dgram.socklen = sizeof(struct sockaddr);
socklen = sizeof(struct sockaddr);
ngx_memzero(&sa, sizeof(struct sockaddr));
sa.sockaddr.sa_family = ls->sockaddr->sa_family;
}
Expand Down Expand Up @@ -152,35 +150,8 @@ ngx_event_recvmsg(ngx_event_t *ev)

#endif

key.data = (u_char *) dgram.sockaddr;
key.len = dgram.socklen;

#if (NGX_HAVE_UNIX_DOMAIN)

if (dgram.sockaddr->sa_family == AF_UNIX) {
struct sockaddr_un *saun = (struct sockaddr_un *) dgram.sockaddr;

if (dgram.socklen <= (socklen_t) offsetof(struct sockaddr_un,
sun_path)
|| saun->sun_path[0] == '\0')
{
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0,
"unbound unix socket");
key.len = 0;
}
}

#endif

#if (NGX_QUIC)
if (ls->quic) {
if (ngx_quic_get_packet_dcid(ev->log, buffer, n, &key) != NGX_OK) {
goto next;
}
}
#endif

c = ngx_lookup_udp_connection(ls, &key, local_sockaddr, local_socklen);
c = ngx_lookup_udp_connection(ls, sockaddr, socklen, local_sockaddr,
local_socklen);

if (c) {

Expand All @@ -202,22 +173,18 @@ ngx_event_recvmsg(ngx_event_t *ev)

buf.pos = buffer;
buf.last = buffer + n;
buf.start = buf.pos;
buf.end = buffer + sizeof(buffer);

rev = c->read;

dgram.buffer = &buf;

c->udp->dgram = &dgram;
c->udp->buffer = &buf;

rev->ready = 1;
rev->active = 0;

rev->handler(rev);

if (c->udp) {
c->udp->dgram = NULL;
c->udp->buffer = NULL;
}

rev->ready = 0;
Expand All @@ -240,7 +207,7 @@ ngx_event_recvmsg(ngx_event_t *ev)

c->shared = 1;
c->type = SOCK_DGRAM;
c->socklen = dgram.socklen;
c->socklen = socklen;

#if (NGX_STAT_STUB)
(void) ngx_atomic_fetch_add(ngx_stat_active, 1);
Expand All @@ -252,21 +219,13 @@ ngx_event_recvmsg(ngx_event_t *ev)
return;
}

len = dgram.socklen;

#if (NGX_QUIC)
if (ls->quic) {
len = NGX_SOCKADDRLEN;
}
#endif

c->sockaddr = ngx_palloc(c->pool, len);
c->sockaddr = ngx_palloc(c->pool, socklen);
if (c->sockaddr == NULL) {
ngx_close_accepted_udp_connection(c);
return;
}

ngx_memcpy(c->sockaddr, dgram.sockaddr, dgram.socklen);
ngx_memcpy(c->sockaddr, sockaddr, socklen);

log = ngx_palloc(c->pool, sizeof(ngx_log_t));
if (log == NULL) {
Expand Down Expand Up @@ -369,7 +328,7 @@ ngx_event_recvmsg(ngx_event_t *ev)
}
#endif

if (ngx_create_udp_connection(c) != NGX_OK) {
if (ngx_insert_udp_connection(c) != NGX_OK) {
ngx_close_accepted_udp_connection(c);
return;
}
Expand Down Expand Up @@ -412,17 +371,17 @@ ngx_udp_shared_recv(ngx_connection_t *c, u_char *buf, size_t size)
ssize_t n;
ngx_buf_t *b;

if (c->udp == NULL || c->udp->dgram == NULL) {
if (c->udp == NULL || c->udp->buffer == NULL) {
return NGX_AGAIN;
}

b = c->udp->dgram->buffer;
b = c->udp->buffer;

n = ngx_min(b->last - b->pos, (ssize_t) size);

ngx_memcpy(buf, b->pos, n);

c->udp->dgram = NULL;
c->udp->buffer = NULL;

c->read->ready = 0;
c->read->active = 1;
Expand Down Expand Up @@ -458,8 +417,8 @@ ngx_udp_rbtree_insert_value(ngx_rbtree_node_t *temp,
udpt = (ngx_udp_connection_t *) temp;
ct = udpt->connection;

rc = ngx_memn2cmp(udp->key.data, udpt->key.data,
udp->key.len, udpt->key.len);
rc = ngx_cmp_sockaddr(c->sockaddr, c->socklen,
ct->sockaddr, ct->socklen, 1);

if (rc == 0 && c->listening->wildcard) {
rc = ngx_cmp_sockaddr(c->local_sockaddr, c->local_socklen,
Expand All @@ -485,18 +444,12 @@ ngx_udp_rbtree_insert_value(ngx_rbtree_node_t *temp,


static ngx_int_t
ngx_create_udp_connection(ngx_connection_t *c)
ngx_insert_udp_connection(ngx_connection_t *c)
{
ngx_str_t key;
uint32_t hash;
ngx_pool_cleanup_t *cln;
ngx_udp_connection_t *udp;

#if (NGX_QUIC)
if (c->listening->quic) {
return NGX_OK;
}
#endif

if (c->udp) {
return NGX_OK;
}
Expand All @@ -506,46 +459,32 @@ ngx_create_udp_connection(ngx_connection_t *c)
return NGX_ERROR;
}

cln = ngx_pool_cleanup_add(c->pool, 0);
if (cln == NULL) {
return NGX_ERROR;
}

cln->data = c;
cln->handler = ngx_delete_udp_connection;

key.data = (u_char *) c->sockaddr;
key.len = c->socklen;

ngx_insert_udp_connection(c, udp, &key);

c->udp = udp;

return NGX_OK;
}


void
ngx_insert_udp_connection(ngx_connection_t *c, ngx_udp_connection_t *udp,
ngx_str_t *key)
{
uint32_t hash;
udp->connection = c;

ngx_crc32_init(hash);

ngx_crc32_update(&hash, key->data, key->len);
ngx_crc32_update(&hash, (u_char *) c->sockaddr, c->socklen);

if (c->listening->wildcard) {
ngx_crc32_update(&hash, (u_char *) c->local_sockaddr, c->local_socklen);
}

ngx_crc32_final(hash);

udp->connection = c;
udp->key = *key;
udp->node.key = hash;

cln = ngx_pool_cleanup_add(c->pool, 0);
if (cln == NULL) {
return NGX_ERROR;
}

cln->data = c;
cln->handler = ngx_delete_udp_connection;

ngx_rbtree_insert(&c->listening->rbtree, &udp->node);

c->udp = udp;

return NGX_OK;
}


Expand All @@ -565,24 +504,36 @@ ngx_delete_udp_connection(void *data)


static ngx_connection_t *
ngx_lookup_udp_connection(ngx_listening_t *ls, ngx_str_t *key,
struct sockaddr *local_sockaddr, socklen_t local_socklen)
ngx_lookup_udp_connection(ngx_listening_t *ls, struct sockaddr *sockaddr,
socklen_t socklen, struct sockaddr *local_sockaddr, socklen_t local_socklen)
{
uint32_t hash;
ngx_int_t rc;
ngx_connection_t *c;
ngx_rbtree_node_t *node, *sentinel;
ngx_udp_connection_t *udp;

if (key->len == 0) {
return NULL;
#if (NGX_HAVE_UNIX_DOMAIN)

if (sockaddr->sa_family == AF_UNIX) {
struct sockaddr_un *saun = (struct sockaddr_un *) sockaddr;

if (socklen <= (socklen_t) offsetof(struct sockaddr_un, sun_path)
|| saun->sun_path[0] == '\0')
{
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ngx_cycle->log, 0,
"unbound unix socket");
return NULL;
}
}

#endif

node = ls->rbtree.root;
sentinel = ls->rbtree.sentinel;

ngx_crc32_init(hash);
ngx_crc32_update(&hash, key->data, key->len);
ngx_crc32_update(&hash, (u_char *) sockaddr, socklen);

if (ls->wildcard) {
ngx_crc32_update(&hash, (u_char *) local_sockaddr, local_socklen);
Expand All @@ -608,21 +559,15 @@ ngx_lookup_udp_connection(ngx_listening_t *ls, ngx_str_t *key,

c = udp->connection;

rc = ngx_memn2cmp(key->data, udp->key.data, key->len, udp->key.len);
rc = ngx_cmp_sockaddr(sockaddr, socklen,
c->sockaddr, c->socklen, 1);

if (rc == 0 && ls->wildcard) {
rc = ngx_cmp_sockaddr(local_sockaddr, local_socklen,
c->local_sockaddr, c->local_socklen, 1);
}

if (rc == 0) {

#if (NGX_QUIC)
if (ls->quic && c->udp != udp) {
c->udp = udp;
}
#endif

return c;
}

Expand Down
Loading

0 comments on commit 9d81ef7

Please sign in to comment.