Skip to content

Commit

Permalink
Upstream hash: speedup consistent hash init.
Browse files Browse the repository at this point in the history
Repeatedly calling ngx_http_upstream_add_chash_point() to create
the points array in sorted order, is O(n^2) to the total weight.
This can cause nginx startup and reconfigure to be substantially
delayed.  For example, when total weight is 1000, startup takes
5s on a modern laptop.

Replace this with a linear insertion followed by QuickSort and
duplicates removal.  Startup for total weight of 1000 reduces to 40ms.

Based on a patch by Wai Keen Woon.
  • Loading branch information
arut committed Mar 2, 2015
1 parent de3adad commit bf7d76b
Showing 1 changed file with 31 additions and 21 deletions.
52 changes: 31 additions & 21 deletions src/http/modules/ngx_http_upstream_hash_module.c
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ static ngx_int_t ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc,

static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us);
static void ngx_http_upstream_add_chash_point(
ngx_http_upstream_chash_points_t *points, uint32_t hash, ngx_str_t *server);
static int ngx_libc_cdecl
ngx_http_upstream_chash_cmp_points(const void *one, const void *two);
static ngx_uint_t ngx_http_upstream_find_chash_point(
ngx_http_upstream_chash_points_t *points, uint32_t hash);
static ngx_int_t ngx_http_upstream_init_chash_peer(ngx_http_request_t *r,
Expand Down Expand Up @@ -360,41 +360,51 @@ ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
ngx_crc32_update(&hash, (u_char *) &prev_hash, sizeof(uint32_t));
ngx_crc32_final(hash);

ngx_http_upstream_add_chash_point(points, hash, &peer->server);
points->point[points->number].hash = hash;
points->point[points->number].server = server;
points->number++;

prev_hash = hash;
}
}

ngx_qsort(points->point,
points->number,
sizeof(ngx_http_upstream_chash_point_t),
ngx_http_upstream_chash_cmp_points);

for (i = 0, j = 1; j < points->number; j++) {
if (points->point[i].hash != points->point[j].hash) {
points->point[++i] = points->point[j];
}
}

points->number = i + 1;

hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module);
hcf->points = points;

return NGX_OK;
}


static void
ngx_http_upstream_add_chash_point(ngx_http_upstream_chash_points_t *points,
uint32_t hash, ngx_str_t *server)
static int ngx_libc_cdecl
ngx_http_upstream_chash_cmp_points(const void *one, const void *two)
{
size_t size;
ngx_uint_t i;
ngx_http_upstream_chash_point_t *point;
ngx_http_upstream_chash_point_t *first =
(ngx_http_upstream_chash_point_t *) one;
ngx_http_upstream_chash_point_t *second =
(ngx_http_upstream_chash_point_t *) two;

i = ngx_http_upstream_find_chash_point(points, hash);
point = &points->point[i];
if (first->hash < second->hash) {
return -1;

if (point->hash == hash) {
return;
}

size = (points->number - i) * sizeof(ngx_http_upstream_chash_point_t);

ngx_memmove(point + 1, point, size);
} else if (first->hash > second->hash) {
return 1;

points->number++;
point->hash = hash;
point->server = server;
} else {
return 0;
}
}


Expand Down

0 comments on commit bf7d76b

Please sign in to comment.