diff --git a/contentcuration/contentcuration/tests/test_exportchannel.py b/contentcuration/contentcuration/tests/test_exportchannel.py index 2817ed840e..fdd1d51269 100644 --- a/contentcuration/contentcuration/tests/test_exportchannel.py +++ b/contentcuration/contentcuration/tests/test_exportchannel.py @@ -16,6 +16,7 @@ from kolibri_content.router import cleanup_content_database_connection from kolibri_content.router import get_active_content_database from kolibri_content.router import set_active_content_database +from le_utils.constants import content_kinds from le_utils.constants import exercises from le_utils.constants import format_presets from le_utils.constants import modalities @@ -57,6 +58,10 @@ pytestmark = pytest.mark.django_db +# Larger than the signed 32-bit maximum (2_147_483_647); ~3 GB. +LARGE_FILE_SIZE = 3 * 1024 ** 3 + + def description(): return "".join(random.sample(string.printable, 20)) @@ -387,6 +392,36 @@ def setUp(self): lesson_topic.extra_fields = {"options": {"modality": modalities.LESSON}} lesson_topic.save() + document_kind, _ = cc.ContentKind.objects.get_or_create( + kind=content_kinds.DOCUMENT + ) + large_file_node = cc.ContentNode( + kind=document_kind, + parent=self.content_channel.main_tree, + title="Large file node", + node_id=uuid.uuid4(), + content_id=uuid.uuid4(), + sort_order=1, + complete=True, + ) + large_file_node.save() + + large_db_file = create_studio_file( + b"large file body", preset="document", ext="pdf" + )["db_file"] + # A >2.1 GB file cannot fit the legacy 32-bit File.file_size column; its + # true size lives in the studio#5974 file_size_bigint shadow, with the + # legacy file_size left NULL. + large_db_file.file_size = None + large_db_file.contentnode = large_file_node + large_db_file.save() + # Set the shadow directly; the mirror trigger leaves it alone because + # file_size is unchanged (NULL). + cc.File.objects.filter(pk=large_db_file.pk).update( + file_size_bigint=LARGE_FILE_SIZE + ) + self.large_file_checksum = large_db_file.checksum + set_channel_icon_encoding(self.content_channel) self.tempdb = create_content_database( self.content_channel, True, self.admin_user.id, True @@ -506,6 +541,22 @@ def test_contentnode_file_size_data(self): for file in files.prefetch_related("local_file"): self.assertEqual(file.file_size, file.local_file.file_size) + def test_localfile_file_size_bigint_matches_small_files(self): + # Files that fit in 32 bits write the same value to both columns. + local_files = kolibri_models.LocalFile.objects.exclude( + pk=self.large_file_checksum + ) + assert local_files.count() > 0 + for local_file in local_files: + self.assertEqual(local_file.file_size_bigint, local_file.file_size) + + def test_localfile_large_file_size_bigint(self): + # A >2.1 GB file keeps its real size in file_size_bigint and NULLs the + # legacy 32-bit file_size. + local_file = kolibri_models.LocalFile.objects.get(pk=self.large_file_checksum) + self.assertEqual(local_file.file_size_bigint, LARGE_FILE_SIZE) + self.assertIsNone(local_file.file_size) + def test_file_included_presets_renderable(self): # Every non-supplementary (renderable) exported file carries its own preset bit. files = kolibri_models.File.objects.filter(supplementary=False) diff --git a/contentcuration/contentcuration/utils/publish.py b/contentcuration/contentcuration/utils/publish.py index 9b4cfe2c3b..cca7cc272d 100644 --- a/contentcuration/contentcuration/utils/publish.py +++ b/contentcuration/contentcuration/utils/publish.py @@ -65,6 +65,8 @@ PERSEUS_IMG_DIR = exercises.IMG_PLACEHOLDER + "/images" THUMBNAIL_DIMENSION = 128 MIN_SCHEMA_VERSION = "1" +# Largest value the legacy 32-bit LocalFile.file_size / File.file_size columns hold. +INT_32BIT_MAX = 2 ** 31 - 1 PUBLISHING_UPDATE_THRESHOLD = 3600 @@ -659,11 +661,23 @@ def create_associated_file_objects(kolibrinode, ccnode): create_associated_thumbnail(ccnode, ccfilemodel) or ccfilemodel ) + # The true size lives in the studio#5974 file_size_bigint shadow (the + # legacy 32-bit file_size cannot hold >2.1 GB); fall back to file_size + # for rows the shadow has not been backfilled onto yet. + real_size = ccfilemodel.file_size_bigint + if real_size is None: + real_size = ccfilemodel.file_size + if real_size is not None and real_size > INT_32BIT_MAX: + legacy_size = None + else: + legacy_size = real_size + kolibrilocalfilemodel, new = kolibrimodels.LocalFile.objects.get_or_create( pk=ccfilemodel.checksum, defaults={ "extension": fformat.extension, - "file_size": ccfilemodel.file_size, + "file_size": legacy_size, + "file_size_bigint": real_size, }, ) @@ -687,7 +701,7 @@ def create_associated_file_objects(kolibrinode, ccnode): checksum=ccfilemodel.checksum, extension=fformat.extension, available=True, # TODO: Set this to False, once we have availability stamping implemented in Kolibri - file_size=ccfilemodel.file_size, + file_size=legacy_size, contentnode=kolibrinode, preset=preset.pk, supplementary=preset.supplementary, diff --git a/contentcuration/kolibri_content/base_models.py b/contentcuration/kolibri_content/base_models.py index b5ed17e1bc..c2d5417fca 100644 --- a/contentcuration/kolibri_content/base_models.py +++ b/contentcuration/kolibri_content/base_models.py @@ -152,6 +152,7 @@ class LocalFile(models.Model): ) available = models.BooleanField(default=False) file_size = models.IntegerField(blank=True, null=True) + file_size_bigint = models.BigIntegerField(blank=True, null=True) class Meta: abstract = True diff --git a/contentcuration/kolibri_content/migrations/0025_localfile_file_size_bigint.py b/contentcuration/kolibri_content/migrations/0025_localfile_file_size_bigint.py new file mode 100644 index 0000000000..02956bb005 --- /dev/null +++ b/contentcuration/kolibri_content/migrations/0025_localfile_file_size_bigint.py @@ -0,0 +1,17 @@ +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("content", "0024_file_included_presets"), + ] + + operations = [ + migrations.AddField( + model_name="localfile", + name="file_size_bigint", + field=models.BigIntegerField(blank=True, null=True), + ), + ] diff --git a/contentcuration/kolibri_public/migrations/0010_localfile_file_size_bigint.py b/contentcuration/kolibri_public/migrations/0010_localfile_file_size_bigint.py new file mode 100644 index 0000000000..aaaae04079 --- /dev/null +++ b/contentcuration/kolibri_public/migrations/0010_localfile_file_size_bigint.py @@ -0,0 +1,17 @@ +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("kolibri_public", "0009_file_included_presets"), + ] + + operations = [ + migrations.AddField( + model_name="localfile", + name="file_size_bigint", + field=models.BigIntegerField(blank=True, null=True), + ), + ]