Skip to content

Commit

Permalink
Merge branch 'jt/avoid-prefetch-when-able-in-diff'
Browse files Browse the repository at this point in the history 
"git diff" in a partial clone learned to avoid lazy loading blob
objects in more casese when they are not needed.

* jt/avoid-prefetch-when-able-in-diff:
  diff: restrict when prefetching occurs
  diff: refactor object read
  diff: make diff_populate_filespec_options struct
  promisor-remote: accept 0 as oid_nr in function
  • Loading branch information
@gitster
gitster committed Apr 28, 2020
2 parents 772d785 + 95acf11 commit 8f5dc5a
Show file tree
Hide file tree
Showing 10 changed files with 267 additions and 71 deletions.
5 changes: 2 additions & 3 deletions builtin/index-pack.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1368,9 +1368,8 @@ static void fix_unresolved_deltas(struct hashfile *f)
continue;
oid_array_append(&to_fetch, &d->oid);
}
if (to_fetch.nr)
promisor_remote_get_direct(the_repository,
to_fetch.oid, to_fetch.nr);
promisor_remote_get_direct(the_repository,
to_fetch.oid, to_fetch.nr);
oid_array_clear(&to_fetch);
}

Expand Down
157 changes: 107 additions & 50 deletions diff.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,7 @@ static int fill_mmfile(struct repository *r, mmfile_t *mf,
mf->size = 0;
return 0;
}
else if (diff_populate_filespec(r, one, 0))
else if (diff_populate_filespec(r, one, NULL))
return -1;

mf->ptr = one->data;
Expand All @@ -585,9 +585,13 @@ static int fill_mmfile(struct repository *r, mmfile_t *mf,
static unsigned long diff_filespec_size(struct repository *r,
struct diff_filespec *one)
{
struct diff_populate_filespec_options dpf_options = {
.check_size_only = 1,
};

if (!DIFF_FILE_VALID(one))
return 0;
diff_populate_filespec(r, one, CHECK_SIZE_ONLY);
diff_populate_filespec(r, one, &dpf_options);
return one->size;
}

Expand Down Expand Up @@ -3020,6 +3024,9 @@ static void show_dirstat(struct diff_options *options)
struct diff_filepair *p = q->queue[i];
const char *name;
unsigned long copied, added, damage;
struct diff_populate_filespec_options dpf_options = {
.check_size_only = 1,
};

name = p->two->path ? p->two->path : p->one->path;

Expand Down Expand Up @@ -3047,19 +3054,19 @@ static void show_dirstat(struct diff_options *options)
}

if (DIFF_FILE_VALID(p->one) && DIFF_FILE_VALID(p->two)) {
diff_populate_filespec(options->repo, p->one, 0);
diff_populate_filespec(options->repo, p->two, 0);
diff_populate_filespec(options->repo, p->one, NULL);
diff_populate_filespec(options->repo, p->two, NULL);
diffcore_count_changes(options->repo,
p->one, p->two, NULL, NULL,
&copied, &added);
diff_free_filespec_data(p->one);
diff_free_filespec_data(p->two);
} else if (DIFF_FILE_VALID(p->one)) {
diff_populate_filespec(options->repo, p->one, CHECK_SIZE_ONLY);
diff_populate_filespec(options->repo, p->one, &dpf_options);
copied = added = 0;
diff_free_filespec_data(p->one);
} else if (DIFF_FILE_VALID(p->two)) {
diff_populate_filespec(options->repo, p->two, CHECK_SIZE_ONLY);
diff_populate_filespec(options->repo, p->two, &dpf_options);
copied = 0;
added = p->two->size;
diff_free_filespec_data(p->two);
Expand Down Expand Up @@ -3339,13 +3346,17 @@ static void emit_binary_diff(struct diff_options *o,
int diff_filespec_is_binary(struct repository *r,
struct diff_filespec *one)
{
struct diff_populate_filespec_options dpf_options = {
.check_binary = 1,
};

if (one->is_binary == -1) {
diff_filespec_load_driver(one, r->index);
if (one->driver->binary != -1)
one->is_binary = one->driver->binary;
else {
if (!one->data && DIFF_FILE_VALID(one))
diff_populate_filespec(r, one, CHECK_BINARY);
diff_populate_filespec(r, one, &dpf_options);
if (one->is_binary == -1 && one->data)
one->is_binary = buffer_is_binary(one->data,
one->size);
Expand Down Expand Up @@ -3677,8 +3688,8 @@ static void builtin_diffstat(const char *name_a, const char *name_b,
}

else if (complete_rewrite) {
diff_populate_filespec(o->repo, one, 0);
diff_populate_filespec(o->repo, two, 0);
diff_populate_filespec(o->repo, one, NULL);
diff_populate_filespec(o->repo, two, NULL);
data->deleted = count_lines(one->data, one->size);
data->added = count_lines(two->data, two->size);
}
Expand Down Expand Up @@ -3914,9 +3925,10 @@ static int diff_populate_gitlink(struct diff_filespec *s, int size_only)
*/
int diff_populate_filespec(struct repository *r,
struct diff_filespec *s,
unsigned int flags)
const struct diff_populate_filespec_options *options)
{
int size_only = flags & CHECK_SIZE_ONLY;
int size_only = options ? options->check_size_only : 0;
int check_binary = options ? options->check_binary : 0;
int err = 0;
int conv_flags = global_conv_flags_eol;
/*
Expand Down Expand Up @@ -3986,7 +3998,7 @@ int diff_populate_filespec(struct repository *r,
* opening the file and inspecting the contents, this
* is probably fine.
*/
if ((flags & CHECK_BINARY) &&
if (check_binary &&
s->size > big_file_threshold && s->is_binary == -1) {
s->is_binary = 1;
return 0;
Expand All @@ -4011,22 +4023,43 @@ int diff_populate_filespec(struct repository *r,
}
}
else {
enum object_type type;
if (size_only || (flags & CHECK_BINARY)) {
type = oid_object_info(r, &s->oid, &s->size);
if (type < 0)
die("unable to read %s",
oid_to_hex(&s->oid));
struct object_info info = {
.sizep = &s->size
};

if (!(size_only || check_binary))
/*
* Set contentp, since there is no chance that merely
* the size is sufficient.
*/
info.contentp = &s->data;

if (options && options->missing_object_cb) {
if (!oid_object_info_extended(r, &s->oid, &info,
OBJECT_INFO_LOOKUP_REPLACE |
OBJECT_INFO_SKIP_FETCH_OBJECT))
goto object_read;
options->missing_object_cb(options->missing_object_data);
}
if (oid_object_info_extended(r, &s->oid, &info,
OBJECT_INFO_LOOKUP_REPLACE))
die("unable to read %s", oid_to_hex(&s->oid));

object_read:
if (size_only || check_binary) {
if (size_only)
return 0;
if (s->size > big_file_threshold && s->is_binary == -1) {
s->is_binary = 1;
return 0;
}
}
s->data = repo_read_object_file(r, &s->oid, &type, &s->size);
if (!s->data)
die("unable to read %s", oid_to_hex(&s->oid));
if (!info.contentp) {
info.contentp = &s->data;
if (oid_object_info_extended(r, &s->oid, &info,
OBJECT_INFO_LOOKUP_REPLACE))
die("unable to read %s", oid_to_hex(&s->oid));
}
s->should_free = 1;
}
return 0;
Expand Down Expand Up @@ -4144,7 +4177,7 @@ static struct diff_tempfile *prepare_temp_file(struct repository *r,
return temp;
}
else {
if (diff_populate_filespec(r, one, 0))
if (diff_populate_filespec(r, one, NULL))
die("cannot read data blob for %s", one->path);
prep_temp_blob(r->index, name, temp,
one->data, one->size,
Expand Down Expand Up @@ -6410,16 +6443,22 @@ static int diff_filespec_is_identical(struct repository *r,
{
if (S_ISGITLINK(one->mode))
return 0;
if (diff_populate_filespec(r, one, 0))
if (diff_populate_filespec(r, one, NULL))
return 0;
if (diff_populate_filespec(r, two, 0))
if (diff_populate_filespec(r, two, NULL))
return 0;
return !memcmp(one->data, two->data, one->size);
}

static int diff_filespec_check_stat_unmatch(struct repository *r,
struct diff_filepair *p)
{
struct diff_populate_filespec_options dpf_options = {
.check_size_only = 1,
.missing_object_cb = diff_queued_diff_prefetch,
.missing_object_data = r,
};

if (p->done_skip_stat_unmatch)
return p->skip_stat_unmatch_result;

Expand All @@ -6442,8 +6481,8 @@ static int diff_filespec_check_stat_unmatch(struct repository *r,
!DIFF_FILE_VALID(p->two) ||
(p->one->oid_valid && p->two->oid_valid) ||
(p->one->mode != p->two->mode) ||
diff_populate_filespec(r, p->one, CHECK_SIZE_ONLY) ||
diff_populate_filespec(r, p->two, CHECK_SIZE_ONLY) ||
diff_populate_filespec(r, p->one, &dpf_options) ||
diff_populate_filespec(r, p->two, &dpf_options) ||
(p->one->size != p->two->size) ||
!diff_filespec_is_identical(r, p->one, p->two)) /* (2) */
p->skip_stat_unmatch_result = 1;
Expand Down Expand Up @@ -6494,9 +6533,9 @@ void diffcore_fix_diff_index(void)
QSORT(q->queue, q->nr, diffnamecmp);
}

static void add_if_missing(struct repository *r,
struct oid_array *to_fetch,
const struct diff_filespec *filespec)
void diff_add_if_missing(struct repository *r,
struct oid_array *to_fetch,
const struct diff_filespec *filespec)
{
if (filespec && filespec->oid_valid &&
!S_ISGITLINK(filespec->mode) &&
Expand All @@ -6505,30 +6544,48 @@ static void add_if_missing(struct repository *r,
oid_array_append(to_fetch, &filespec->oid);
}

void diffcore_std(struct diff_options *options)
void diff_queued_diff_prefetch(void *repository)
{
if (options->repo == the_repository && has_promisor_remote()) {
/*
* Prefetch the diff pairs that are about to be flushed.
*/
int i;
struct diff_queue_struct *q = &diff_queued_diff;
struct oid_array to_fetch = OID_ARRAY_INIT;
struct repository *repo = repository;
int i;
struct diff_queue_struct *q = &diff_queued_diff;
struct oid_array to_fetch = OID_ARRAY_INIT;

for (i = 0; i < q->nr; i++) {
struct diff_filepair *p = q->queue[i];
add_if_missing(options->repo, &to_fetch, p->one);
add_if_missing(options->repo, &to_fetch, p->two);
}
if (to_fetch.nr)
/*
* NEEDSWORK: Consider deduplicating the OIDs sent.
*/
promisor_remote_get_direct(options->repo,
to_fetch.oid, to_fetch.nr);
oid_array_clear(&to_fetch);
for (i = 0; i < q->nr; i++) {
struct diff_filepair *p = q->queue[i];
diff_add_if_missing(repo, &to_fetch, p->one);
diff_add_if_missing(repo, &to_fetch, p->two);
}

/*
* NEEDSWORK: Consider deduplicating the OIDs sent.
*/
promisor_remote_get_direct(repo, to_fetch.oid, to_fetch.nr);

oid_array_clear(&to_fetch);
}

void diffcore_std(struct diff_options *options)
{
int output_formats_to_prefetch = DIFF_FORMAT_DIFFSTAT |
DIFF_FORMAT_NUMSTAT |
DIFF_FORMAT_PATCH |
DIFF_FORMAT_SHORTSTAT |
DIFF_FORMAT_DIRSTAT;

/*
* Check if the user requested a blob-data-requiring diff output and/or
* break-rewrite detection (which requires blob data). If yes, prefetch
* the diff pairs.
*
* If no prefetching occurs, diffcore_rename() will prefetch if it
* decides that it needs inexact rename detection.
*/
if (options->repo == the_repository && has_promisor_remote() &&
(options->output_format & output_formats_to_prefetch ||
options->pickaxe_opts & DIFF_PICKAXE_KINDS_MASK))
diff_queued_diff_prefetch(options->repo);

/* NOTE please keep the following in sync with diff_tree_combined() */
if (options->skip_stat_unmatch)
diffcore_skip_stat_unmatch(options);
Expand Down Expand Up @@ -6774,7 +6831,7 @@ size_t fill_textconv(struct repository *r,
*outbuf = "";
return 0;
}
if (diff_populate_filespec(r, df, 0))
if (diff_populate_filespec(r, df, NULL))
die("unable to read files to diff");
*outbuf = df->data;
return df->size;
Expand Down
12 changes: 10 additions & 2 deletions diffcore-break.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "cache.h"
#include "diff.h"
#include "diffcore.h"
#include "promisor-remote.h"

static int should_break(struct repository *r,
struct diff_filespec *src,
Expand Down Expand Up @@ -49,6 +50,8 @@ static int should_break(struct repository *r,
unsigned long delta_size, max_size;
unsigned long src_copied, literal_added, src_removed;

struct diff_populate_filespec_options options = { 0 };

*merge_score_p = 0; /* assume no deletion --- "do not break"
* is the default.
*/
Expand All @@ -62,8 +65,13 @@ static int should_break(struct repository *r,
oideq(&src->oid, &dst->oid))
return 0; /* they are the same */

if (diff_populate_filespec(r, src, 0) ||
diff_populate_filespec(r, dst, 0))
if (r == the_repository && has_promisor_remote()) {
options.missing_object_cb = diff_queued_diff_prefetch;
options.missing_object_data = r;
}

if (diff_populate_filespec(r, src, &options) ||
diff_populate_filespec(r, dst, &options))
return 0; /* error but caught downstream */

max_size = ((src->size > dst->size) ? src->size : dst->size);
Expand Down
Loading

0 comments on commit 8f5dc5a

Please sign in to comment.