diff --git a/Documentation/config/remote.adoc b/Documentation/config/remote.adoc index 91e46f66f5dd1c..d91c0c45d68b3e 100644 --- a/Documentation/config/remote.adoc +++ b/Documentation/config/remote.adoc @@ -107,6 +107,29 @@ priority configuration file (e.g. `.git/config` in a repository) to clear the values inherited from a lower priority configuration files (e.g. `$HOME/.gitconfig`). +remote..haveRefs:: + When negotiating with this remote during `git fetch` and `git push`, + the client advertises a list of commits that exist locally. In + repos with many references, this list of "haves" can be truncated. + Depending on data shape, dropping certain references may be + expensive. This multi-valued config option specifies ref patterns + whose tips should always be sent as "have" commits during fetch + negotiation with this remote. ++ +Each value is either an exact ref name (e.g. `refs/heads/release`) or a +glob pattern (e.g. `refs/heads/release/*`). The pattern syntax is the same +as for `--negotiation-tip`. ++ +If `--negotiation-tip` is used, then the have set is first restricted by +that option and then increased to include the tips specified by the +`remote..haveRefs` config values. ++ +This option is additive with the normal negotiation process: the +negotiation algorithm still runs and advertises its own selected commits, +but the refs matching `remote..haveRefs` are sent unconditionally on +top of those heuristically selected commits. This option is also used +during push negotiation when `push.negotiate` is enabled. + remote..followRemoteHEAD:: How linkgit:git-fetch[1] should handle updates to `remotes//HEAD` when fetching using the configured refspecs of a remote. diff --git a/Documentation/fetch-options.adoc b/Documentation/fetch-options.adoc index 81a9d7f9bbc11d..da25394a7f9bfe 100644 --- a/Documentation/fetch-options.adoc +++ b/Documentation/fetch-options.adoc @@ -67,7 +67,8 @@ this option multiple times, one for each matching ref name. + See also the `fetch.negotiationAlgorithm` and `push.negotiate` configuration variables documented in linkgit:git-config[1], and the -`--negotiate-only` option below. +`--negotiate-only` option below. Also, these restrictions do not apply +to any `remote..haveRefs` config values. `--negotiate-only`:: Do not fetch anything from the server, and instead print the diff --git a/fetch-pack.c b/fetch-pack.c index 6ecd468ef766a8..9fb822c6370c52 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -25,6 +25,7 @@ #include "oidset.h" #include "packfile.h" #include "odb.h" +#include "object-name.h" #include "path.h" #include "connected.h" #include "fetch-negotiator.h" @@ -332,6 +333,40 @@ static void send_filter(struct fetch_pack_args *args, } } +static int add_oid_to_oidset(const struct reference *ref, void *cb_data) +{ + struct oidset *set = cb_data; + oidset_insert(set, ref->oid); + return 0; +} + +static void resolve_have_refs(const struct string_list *have_refs, + struct oidset *result) +{ + struct string_list_item *item; + + if (!have_refs || !have_refs->nr) + return; + + for_each_string_list_item(item, have_refs) { + if (!has_glob_specials(item->string)) { + struct object_id oid; + if (repo_get_oid(the_repository, item->string, &oid)) + continue; + if (!odb_has_object(the_repository->objects, &oid, 0)) + continue; + oidset_insert(result, &oid); + } else { + struct refs_for_each_ref_options opts = { + .pattern = item->string, + }; + refs_for_each_ref_ext( + get_main_ref_store(the_repository), + add_oid_to_oidset, result, &opts); + } + } +} + static int find_common(struct fetch_negotiator *negotiator, struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, @@ -347,6 +382,7 @@ static int find_common(struct fetch_negotiator *negotiator, struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; struct packet_reader reader; + struct oidset have_refs_oids = OIDSET_INIT; if (args->stateless_rpc && multi_ack == 1) die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed"); @@ -474,7 +510,24 @@ static int find_common(struct fetch_negotiator *negotiator, trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository); flushes = 0; retval = -1; + + /* Send unconditional haves from remote..haveRefs */ + resolve_have_refs(args->have_refs, &have_refs_oids); + if (oidset_size(&have_refs_oids)) { + struct oidset_iter iter; + oidset_iter_init(&have_refs_oids, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(&req_buf, "have %s\n", + oid_to_hex(oid)); + print_verbose(args, "have %s", oid_to_hex(oid)); + } + } + while ((oid = negotiator->next(negotiator))) { + /* avoid duplicate oids from remote..haveRefs */ + if (oidset_contains(&have_refs_oids, oid)) + continue; packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); print_verbose(args, "have %s", oid_to_hex(oid)); in_vain++; @@ -584,6 +637,7 @@ static int find_common(struct fetch_negotiator *negotiator, flushes++; } strbuf_release(&req_buf); + oidset_clear(&have_refs_oids); if (!got_ready || !no_done) consume_shallow_list(args, &reader); @@ -1305,12 +1359,25 @@ static void add_common(struct strbuf *req_buf, struct oidset *common) static int add_haves(struct fetch_negotiator *negotiator, struct strbuf *req_buf, - int *haves_to_send) + int *haves_to_send, + struct oidset *have_refs_oids) { int haves_added = 0; const struct object_id *oid; + /* Send unconditional haves from remote..haveRefs */ + if (have_refs_oids) { + struct oidset_iter iter; + oidset_iter_init(have_refs_oids, &iter); + + while ((oid = oidset_iter_next(&iter))) + packet_buf_write(req_buf, "have %s\n", + oid_to_hex(oid)); + } + while ((oid = negotiator->next(negotiator))) { + if (have_refs_oids && oidset_contains(have_refs_oids, oid)) + continue; packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); if (++haves_added >= *haves_to_send) break; @@ -1358,7 +1425,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, - int sideband_all, int seen_ack) + int sideband_all, int seen_ack, + struct oidset *have_refs_oids) { int haves_added; int done_sent = 0; @@ -1413,7 +1481,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, /* Add all of the common commits we've found in previous rounds */ add_common(&req_buf, common); - haves_added = add_haves(negotiator, &req_buf, haves_to_send); + haves_added = add_haves(negotiator, &req_buf, haves_to_send, + have_refs_oids); *in_vain += haves_added; trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added); trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain); @@ -1657,6 +1726,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, struct ref *ref = copy_ref_list(orig_ref); enum fetch_state state = FETCH_CHECK_LOCAL; struct oidset common = OIDSET_INIT; + struct oidset have_refs_oids = OIDSET_INIT; struct packet_reader reader; int in_vain = 0, negotiation_started = 0; int negotiation_round = 0; @@ -1708,6 +1778,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, reader.me = "fetch-pack"; } + resolve_have_refs(args->have_refs, &have_refs_oids); + while (state != FETCH_DONE) { switch (state) { case FETCH_CHECK_LOCAL: @@ -1747,7 +1819,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, &common, &haves_to_send, &in_vain, reader.use_sideband, - seen_ack)) { + seen_ack, + &have_refs_oids)) { trace2_region_leave_printf("negotiation_v2", "round", the_repository, "%d", negotiation_round); @@ -1883,6 +1956,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, negotiator->release(negotiator); oidset_clear(&common); + oidset_clear(&have_refs_oids); return ref; } @@ -2181,12 +2255,14 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, const struct string_list *server_options, int stateless_rpc, int fd[], - struct oidset *acked_commits) + struct oidset *acked_commits, + const struct string_list *have_refs) { struct fetch_negotiator negotiator; struct packet_reader reader; struct object_array nt_object_array = OBJECT_ARRAY_INIT; struct strbuf req_buf = STRBUF_INIT; + struct oidset have_refs_oids = OIDSET_INIT; int haves_to_send = INITIAL_FLUSH; int in_vain = 0; int seen_ack = 0; @@ -2205,6 +2281,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, add_to_object_array, &nt_object_array); + resolve_have_refs(have_refs, &have_refs_oids); + trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository); while (!last_iteration) { int haves_added; @@ -2221,7 +2299,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, packet_buf_write(&req_buf, "wait-for-done"); - haves_added = add_haves(&negotiator, &req_buf, &haves_to_send); + haves_added = add_haves(&negotiator, &req_buf, &haves_to_send, + &have_refs_oids); in_vain += haves_added; if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN)) last_iteration = 1; @@ -2273,6 +2352,7 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, clear_common_flag(acked_commits); object_array_clear(&nt_object_array); + oidset_clear(&have_refs_oids); negotiator.release(&negotiator); strbuf_release(&req_buf); } diff --git a/fetch-pack.h b/fetch-pack.h index 9d3470366f85ec..59f292eb92702a 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -23,6 +23,13 @@ struct fetch_pack_args { */ const struct oid_array *negotiation_tips; + /* + * If non-empty, ref patterns whose tips should always be sent + * as "have" lines during negotiation, regardless of what the + * negotiation algorithm selects. + */ + const struct string_list *have_refs; + unsigned deepen_relative:1; unsigned quiet:1; unsigned keep_pack:1; @@ -93,7 +100,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, const struct string_list *server_options, int stateless_rpc, int fd[], - struct oidset *acked_commits); + struct oidset *acked_commits, + const struct string_list *have_refs); /* * Print an appropriate error message for each sought ref that wasn't diff --git a/remote.c b/remote.c index 7ca2a6501b4920..7c06c8df1b98c9 100644 --- a/remote.c +++ b/remote.c @@ -152,6 +152,7 @@ static struct remote *make_remote(struct remote_state *remote_state, refspec_init_push(&ret->push); refspec_init_fetch(&ret->fetch); string_list_init_dup(&ret->server_options); + string_list_init_dup(&ret->have_refs); ALLOC_GROW(remote_state->remotes, remote_state->remotes_nr + 1, remote_state->remotes_alloc); @@ -179,6 +180,7 @@ static void remote_clear(struct remote *remote) FREE_AND_NULL(remote->http_proxy); FREE_AND_NULL(remote->http_proxy_authmethod); string_list_clear(&remote->server_options, 0); + string_list_clear(&remote->have_refs, 0); } static void add_merge(struct branch *branch, const char *name) @@ -562,6 +564,10 @@ static int handle_config(const char *key, const char *value, } else if (!strcmp(subkey, "serveroption")) { return parse_transport_option(key, value, &remote->server_options); + } else if (!strcmp(subkey, "haverefs")) { + if (!value) + return config_error_nonbool(key); + string_list_append(&remote->have_refs, value); } else if (!strcmp(subkey, "followremotehead")) { const char *no_warn_branch; if (!strcmp(value, "never")) diff --git a/remote.h b/remote.h index fc052945ee451d..4cd90530f1c84a 100644 --- a/remote.h +++ b/remote.h @@ -117,6 +117,7 @@ struct remote { char *http_proxy_authmethod; struct string_list server_options; + struct string_list have_refs; enum follow_remote_head_settings follow_remote_head; const char *no_warn_branch; diff --git a/send-pack.c b/send-pack.c index 67d6987b1ccd7e..33ebe0717d32a9 100644 --- a/send-pack.c +++ b/send-pack.c @@ -433,6 +433,7 @@ static void reject_invalid_nonce(const char *nonce, int len) static void get_commons_through_negotiation(struct repository *r, const char *url, + const char *remote_name, const struct ref *remote_refs, struct oid_array *commons) { @@ -452,7 +453,12 @@ static void get_commons_through_negotiation(struct repository *r, nr_negotiation_tip++; } } - strvec_push(&child.args, url); + /* + * Use the remote name so the subprocess can find + * remote..haveRefs config. Fall back to the URL if no + * remote name is available. + */ + strvec_push(&child.args, remote_name ? remote_name : url); if (!nr_negotiation_tip) { child_process_clear(&child); @@ -528,7 +534,8 @@ int send_pack(struct repository *r, repo_config_get_bool(r, "push.negotiate", &push_negotiate); if (push_negotiate) { trace2_region_enter("send_pack", "push_negotiate", r); - get_commons_through_negotiation(r, args->url, remote_refs, &commons); + get_commons_through_negotiation(r, args->url, args->remote_name, + remote_refs, &commons); trace2_region_leave("send_pack", "push_negotiate", r); } diff --git a/send-pack.h b/send-pack.h index c5ded2d2006f13..71d7685e5da4eb 100644 --- a/send-pack.h +++ b/send-pack.h @@ -18,6 +18,7 @@ struct repository; struct send_pack_args { const char *url; + const char *remote_name; unsigned verbose:1, quiet:1, porcelain:1, diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 5dcb4b51a47d88..a4fb1706e715e1 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -1728,6 +1728,79 @@ test_expect_success REFFILES "HEAD is updated even with conflicts" ' ) ' +test_expect_success 'remote..haveRefs includes configured refs as haves' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # With --negotiation-tip restricting tips, only alpha_1 is + # normally sent. Configure remote.origin.haveRefs to also include beta_1. + git -C client config --add remote.origin.haveRefs refs/tags/beta_1 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s && + + ALPHA_1=$(git -C client rev-parse alpha_1) && + test_grep "fetch> have $ALPHA_1" trace && + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace +' + +test_expect_success 'remote..haveRefs works with glob patterns' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + git -C client config --add remote.origin.haveRefs "refs/tags/beta_*" && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace && + BETA_2=$(git -C client rev-parse beta_2) && + test_grep "fetch> have $BETA_2" trace +' + +test_expect_success 'remote..haveRefs is additive with negotiation' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # Without --negotiation-tip, all local refs are used as tips. + # remote.origin.haveRefs should add its refs unconditionally on top. + git -C client config --add remote.origin.haveRefs refs/tags/beta_1 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace +' + +test_expect_success 'remote..haveRefs ignores non-existent refs silently' ' + setup_negotiation_tip server server 0 && + + git -C client config --add remote.origin.haveRefs refs/tags/nonexistent && + git -C client fetch --quiet \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s 2>err && + test_must_be_empty err +' + +test_expect_success 'remote..haveRefs avoids duplicates with negotiator' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # Configure a ref that will also be a negotiation tip. + # fetch should still complete successfully. + ALPHA_1=$(git -C client rev-parse alpha_1) && + git -C client config --add remote.origin.haveRefs refs/tags/alpha_1 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s && + + # alpha_1 should appear as a have + test_grep "fetch> have $ALPHA_1" trace >matches && + test_line_count = 1 matches +' + . "$TEST_DIRECTORY"/lib-httpd.sh start_httpd diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 29e2f176081561..35380c3a9f512b 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -254,6 +254,21 @@ test_expect_success 'push with negotiation does not attempt to fetch submodules' ! grep "Fetching submodule" err ' +test_expect_success 'push with negotiation and remote..haveRefs' ' + test_when_finished rm -rf haverefs && + mk_empty haverefs && + git push haverefs $the_first_commit:refs/remotes/origin/first_commit && + test_commit -C haverefs unrelated_commit && + git -C haverefs config receive.hideRefs refs/remotes/origin/first_commit && + test_when_finished "rm event" && + GIT_TRACE2_EVENT="$(pwd)/event" \ + git -c protocol.version=2 -c push.negotiate=1 \ + -c remote.haverefs.haveRefs=refs/heads/main \ + push haverefs refs/heads/main:refs/remotes/origin/main && + test_grep \"key\":\"total_rounds\" event && + grep_wrote 2 event # 1 commit, 1 tree +' + test_expect_success 'push without wildcard' ' mk_empty testrepo && @@ -1349,7 +1364,7 @@ test_expect_success 'fetch follows tags by default' ' git for-each-ref >tmp1 && sed -n "p; s|refs/heads/main$|refs/remotes/origin/main|p" tmp1 | sed -n "p; s|refs/heads/main$|refs/remotes/origin/HEAD|p" | - sort -k 4 >../expect + sort -k 3 >../expect ) && test_when_finished "rm -rf dst" && git init dst && diff --git a/transport.c b/transport.c index 107f4fa5dce96a..8ca648003eb612 100644 --- a/transport.c +++ b/transport.c @@ -464,6 +464,7 @@ static int fetch_refs_via_pack(struct transport *transport, args.stateless_rpc = transport->stateless_rpc; args.server_options = transport->server_options; args.negotiation_tips = data->options.negotiation_tips; + args.have_refs = &transport->remote->have_refs; args.reject_shallow_remote = transport->smart_options->reject_shallow; if (!data->finished_handshake) { @@ -495,7 +496,8 @@ static int fetch_refs_via_pack(struct transport *transport, transport->server_options, transport->stateless_rpc, data->fd, - data->options.acked_commits); + data->options.acked_commits, + &transport->remote->have_refs); ret = 0; } goto cleanup; @@ -919,6 +921,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re args.atomic = !!(flags & TRANSPORT_PUSH_ATOMIC); args.push_options = transport->push_options; args.url = transport->url; + args.remote_name = transport->remote->name; if (flags & TRANSPORT_PUSH_CERT_ALWAYS) args.push_cert = SEND_PACK_PUSH_CERT_ALWAYS;