Skip to content

odb: add write_packfile, for_each_unique_abbrev, convert_object_id #2074

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MayCXC wants to merge 1 commit into
base: master
Choose a base branch
from
Open

odb: add write_packfile, for_each_unique_abbrev, convert_object_id #2074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions builtin/cat-file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "object-file.h"
#include "object-name.h"
#include "odb.h"
#include "odb/source.h"
#include "odb/streaming.h"
#include "replace-object.h"
#include "promisor-remote.h"
Expand Down Expand Up @@ -859,8 +860,17 @@ static void batch_each_object(struct batch_options *opt,
*/
odb_prepare_alternates(the_repository->objects);
for (source = the_repository->objects->sources; source; source = source->next) {
int ret = odb_source_loose_for_each_object(source, NULL, batch_one_object_oi,
&payload, flags);
int ret;
if (!source->packed) {
/*
* Non-files source: dispatch through vtable.
*/
ret = odb_source_for_each_object(source, NULL,
batch_one_object_oi, &payload, flags);
} else {
ret = odb_source_loose_for_each_object(source, NULL,
batch_one_object_oi, &payload, flags);
}
if (ret)
break;
}
Expand All @@ -882,11 +892,14 @@ static void batch_each_object(struct batch_options *opt,
struct object_info oi = { 0 };

for (source = the_repository->objects->sources; source; source = source->next) {
struct odb_source_files *files = odb_source_files_downcast(source);
int ret = packfile_store_for_each_object(files->packed, &oi,
batch_one_object_oi, &payload, flags);
if (ret)
break;
if (!source->packed)
continue;
{
int ret = packfile_store_for_each_object(source->packed, &oi,
batch_one_object_oi, &payload, flags);
if (ret)
break;
}
}
}

Expand Down
71 changes: 61 additions & 10 deletions builtin/fsck.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "object-file.h"
#include "object-name.h"
#include "odb.h"
#include "odb/source.h"
#include "odb/streaming.h"
#include "path.h"
#include "read-cache-ll.h"
Expand Down Expand Up @@ -769,24 +770,74 @@ static int fsck_subdir(unsigned int nr, const char *path UNUSED, void *data)
return 0;
}

static int fsck_vtable_object(const struct object_id *oid,
struct object_info *oi UNUSED,
void *data UNUSED)
{
enum object_type type;
unsigned long size;
void *contents;
struct object *obj;
int eaten;

contents = odb_read_object(the_repository->objects, oid,
&type, &size);
if (!contents) {
errors_found |= ERROR_OBJECT;
error(_("%s: object missing"),
describe_object(oid));
return 0;
}

obj = parse_object_buffer(the_repository, oid, type,
size, contents, &eaten);
if (!obj) {
errors_found |= ERROR_OBJECT;
error(_("%s: object corrupt or missing"),
describe_object(oid));
if (!eaten)
free(contents);
return 0;
}
obj->flags |= HAS_OBJ;
if (fsck_obj(obj, contents, size))
errors_found |= ERROR_OBJECT;
if (!eaten)
free(contents);
return 0;
}

static void fsck_source(struct odb_source *source)
{
struct progress *progress = NULL;
struct for_each_loose_cb cb_data = {
.progress = progress,
};

if (verbose)
fprintf_ln(stderr, _("Checking object directory"));

if (show_progress)
progress = start_progress(the_repository,
_("Checking object directories"), 256);
if (!source->packed) {
/*
* Non-files source: iterate objects through the vtable
* and fsck each one.
*/
odb_source_for_each_object(source, NULL,
fsck_vtable_object, NULL, 0);
return;
}

for_each_loose_file_in_source(source, fsck_loose,
fsck_cruft, fsck_subdir, &cb_data);
display_progress(progress, 256);
stop_progress(&progress);
{
struct for_each_loose_cb cb_data = {
.progress = progress,
};

if (show_progress)
progress = start_progress(the_repository,
_("Checking object directories"), 256);

for_each_loose_file_in_source(source, fsck_loose,
fsck_cruft, fsck_subdir, &cb_data);
display_progress(progress, 256);
stop_progress(&progress);
}
}

static int fsck_cache_tree(struct cache_tree *it, const char *index_path)
Expand Down
108 changes: 28 additions & 80 deletions builtin/pack-objects.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "packfile.h"
#include "object-file.h"
#include "odb.h"
#include "odb/source.h"
#include "odb/streaming.h"
#include "replace-object.h"
#include "dir.h"
Expand Down Expand Up @@ -1541,57 +1542,15 @@ static int have_duplicate_entry(const struct object_id *oid,

static int want_cruft_object_mtime(struct repository *r,
const struct object_id *oid,
unsigned flags, uint32_t mtime)
unsigned flags UNUSED, uint32_t mtime UNUSED)
{
struct odb_source *source;

for (source = r->objects->sources; source; source = source->next) {
struct odb_source_files *files = odb_source_files_downcast(source);
struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags);

for (; *cache; cache++) {
struct packed_git *p = *cache;
off_t ofs;
uint32_t candidate_mtime;

ofs = find_pack_entry_one(oid, p);
if (!ofs)
continue;

/*
* We have a copy of the object 'oid' in a non-cruft
* pack. We can avoid packing an additional copy
* regardless of what the existing copy's mtime is since
* it is outside of a cruft pack.
*/
if (!p->is_cruft)
return 0;

/*
* If we have a copy of the object 'oid' in a cruft
* pack, then either read the cruft pack's mtime for
* that object, or, if that can't be loaded, assume the
* pack's mtime itself.
*/
if (!load_pack_mtimes(p)) {
uint32_t pos;
if (offset_to_pack_pos(p, ofs, &pos) < 0)
continue;
candidate_mtime = nth_packed_mtime(p, pos);
} else {
candidate_mtime = p->mtime;
}

/*
* We have a surviving copy of the object in a cruft
* pack whose mtime is greater than or equal to the one
* we are considering. We can thus avoid packing an
* additional copy of that object.
*/
if (mtime <= candidate_mtime)
return 0;
}
}
/*
* Check if the object exists in a kept source. Dispatches through
* the vtable: files backends check kept packs, non-files backends
* check their own kept tracking via OBJECT_INFO_KEPT_ONLY.
*/
if (odb_has_object_kept(r->objects, oid))
return 0;

return -1;
}
Expand Down Expand Up @@ -1657,7 +1616,7 @@ static int want_found_object(const struct object_id *oid, int exclude,
return 0;
if (ignore_packed_keep_in_core && p->pack_keep_in_core)
return 0;
if (has_object_kept_pack(p->repo, oid, flags))
if (odb_has_object_kept(p->repo->objects, oid))
return 0;
} else {
/*
Expand Down Expand Up @@ -1726,8 +1685,6 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
uint32_t found_mtime)
{
int want;
struct packfile_list_entry *e;
struct odb_source *source;

if (!exclude && local) {
/*
Expand Down Expand Up @@ -1757,25 +1714,18 @@ static int want_object_in_pack_mtime(const struct object_id *oid,

odb_prepare_alternates(the_repository->objects);

for (source = the_repository->objects->sources; source; source = source->next) {
struct multi_pack_index *m = get_multi_pack_index(source);
struct pack_entry e;

if (m && fill_midx_entry(m, oid, &e)) {
want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime);
if (want != -1)
return want;
}
}

for (source = the_repository->objects->sources; source; source = source->next) {
struct odb_source_files *files = odb_source_files_downcast(source);
{
struct object_info oi = OBJECT_INFO_INIT;

for (e = files->packed->packs.head; e; e = e->next) {
struct packed_git *p = e->pack;
want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
if (!exclude && want > 0)
packfile_list_prepend(&files->packed->packs, p);
if (!odb_read_object_info_extended(the_repository->objects,
oid, &oi,
OBJECT_INFO_QUICK) &&
oi.whence == OI_PACKED) {
struct packed_git *p = oi.u.packed.pack;
want = want_object_in_pack_one(p, oid, exclude,
found_pack,
found_offset,
found_mtime);
if (want != -1)
return want;
}
Expand Down Expand Up @@ -4065,7 +4015,7 @@ static void show_cruft_commit(struct commit *commit, void *data)

static int cruft_include_check_obj(struct object *obj, void *data UNUSED)
{
return !has_object_kept_pack(to_pack.repo, &obj->oid, KEPT_PACK_IN_CORE);
return !odb_has_object_kept(to_pack.repo->objects, &obj->oid);
}

static int cruft_include_check(struct commit *commit, void *data)
Expand Down Expand Up @@ -4365,17 +4315,15 @@ static void add_objects_in_unpacked_packs(void)

odb_prepare_alternates(to_pack.repo->objects);
for (source = to_pack.repo->objects->sources; source; source = source->next) {
struct odb_source_files *files = odb_source_files_downcast(source);

if (!source->local)
continue;

if (packfile_store_for_each_object(files->packed, &oi,
add_object_in_unpacked_pack, NULL,
ODB_FOR_EACH_OBJECT_PACK_ORDER |
ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
if (odb_source_for_each_object(source, &oi,
add_object_in_unpacked_pack, NULL,
ODB_FOR_EACH_OBJECT_PACK_ORDER |
ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
die(_("cannot open pack index"));
}
}
Expand Down
14 changes: 10 additions & 4 deletions loose.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,17 @@ int repo_read_loose_object_map(struct repository *repo)

int repo_write_loose_object_map(struct repository *repo)
{
struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources);
kh_oid_map_t *map = files->loose->map->to_compat;
struct odb_source *source = repo->objects->sources;
kh_oid_map_t *map;
struct lock_file lock;
int fd;
khiter_t iter;
struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;

if (!source->loose || !source->loose->map)
return 0;
map = source->loose->map->to_compat;

if (!should_use_loose_object_map(repo))
return 0;

Expand Down Expand Up @@ -235,8 +239,10 @@ int repo_loose_object_map_oid(struct repository *repo,
khiter_t pos;

for (source = repo->objects->sources; source; source = source->next) {
struct odb_source_files *files = odb_source_files_downcast(source);
struct loose_object_map *loose_map = files->loose->map;
struct loose_object_map *loose_map;
if (!source->loose)
continue;
loose_map = source->loose->map;
if (!loose_map)
continue;
map = (to == repo->compat_hash_algo) ?
Expand Down
7 changes: 4 additions & 3 deletions midx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,

struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
{
struct odb_source_files *files = odb_source_files_downcast(source);
packfile_store_prepare(files->packed);
return files->packed->midx;
if (!source->packed)
return NULL;
packfile_store_prepare(source->packed);
return source->packed->midx;
}

static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
Expand Down
Loading
Loading