diff --git a/.github/workflows/check-code-and-docs-validation.yml b/.github/workflows/check-code-and-docs-validation.yml index c61f398a..20221f40 100644 --- a/.github/workflows/check-code-and-docs-validation.yml +++ b/.github/workflows/check-code-and-docs-validation.yml @@ -1,4 +1,4 @@ -name: Run unit tests +name: Check code and docs validation on: workflow_dispatch: diff --git a/.github/workflows/run-unit-tests-docker.yml b/.github/workflows/run-unit-tests-docker.yml index 53de574b..c17e555e 100644 --- a/.github/workflows/run-unit-tests-docker.yml +++ b/.github/workflows/run-unit-tests-docker.yml @@ -22,5 +22,10 @@ jobs: - name: Generate the .env file and the SECRET_KEY run: make envfile + - name: Build Docker image + run: docker compose -f compose.yml -f compose.build.yml build + - name: Run tests - run: docker compose run web python ./manage.py test --verbosity=2 --noinput + run: | + docker compose -f compose.yml -f compose.build.yml run web \ + python ./manage.py test --verbosity=2 --noinput --parallel auto diff --git a/Makefile b/Makefile index cf78903b..f5572f1e 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ superuser: ${MANAGE} createsuperuser ######################################################################################## -# Utilities +# Linters / docs ######################################################################################## DOCS_LOCATION=./docs @@ -76,36 +76,12 @@ docs: uvx --from sphinx==9.1.0 --with furo==2025.12.19 sphinx-build -b html ${DOCS_LOCATION} ${DOCS_LOCATION}/_build/html/ ######################################################################################## - -VENV_LOCATION=.venv -ACTIVATE?=. ${VENV_LOCATION}/bin/activate; -#MANAGE=${VENV_LOCATION}/bin/python manage.py -# Do not depend on Python to generate the SECRET_KEY -GET_SECRET_KEY=`head -c50 /dev/urandom | base64 | head -c50` -# Customize with `$ make envfile ENV_FILE=/etc/dejacode/.env` -ENV_FILE=.env -DOCKER_COMPOSE=docker compose -f docker-compose.yml -DOCKER_EXEC=${DOCKER_COMPOSE} exec -DB_NAME=dejacode_db -DB_USERNAME=dejacode -DB_PASSWORD=dejacode -DB_CONTAINER_NAME=db -DB_INIT_FILE=./data/postgresql/initdb.sql.gz -POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 -TIMESTAMP=$(shell date +"%Y-%m-%d_%H%M") - -conf: virtualenv - @echo "-> Install dependencies" - uv sync --frozen - @echo "-> Create the var/ directory" - @mkdir -p var - -dev: virtualenv - @echo "-> Configure and install development dependencies" - uv sync --frozen --extra dev +# Utilities +######################################################################################## outdated: @echo "-> Check for outdated packages (with 7 days cooldown)" + uv sync --frozen --quiet uv pip list --outdated \ --no-config \ --index-url https://pypi.org/simple \ @@ -119,25 +95,61 @@ upgrade: exit 1; \ fi @echo "-> Download $(PACKAGE) wheels for Linux x86_64" - pip download $(PACKAGE) \ + uvx pip download $(PACKAGE) \ --only-binary=:all: \ --platform manylinux_2_28_x86_64 \ --platform manylinux_2_17_x86_64 \ --python-version 3.14 \ --dest ./thirdparty/dist/ @echo "-> Download $(PACKAGE) wheels for macOS ARM64" - pip download $(PACKAGE) \ + uvx pip download $(PACKAGE) \ --only-binary=:all: \ --platform macosx_11_0_arm64 \ --python-version 3.14 \ --dest ./thirdparty/dist/ @echo "-> Update pyproject.toml and uv.lock" - uv add $(PACKAGE) + uvx uv add $(PACKAGE) lock: @echo "-> Regenerate uv.lock from local wheels" uv lock +clean: + @echo "-> Clean the Python env" + rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/ + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + +######################################################################################## +# Local venv commands (legacy) +######################################################################################## + +VENV_LOCATION=.venv +ACTIVATE?=. ${VENV_LOCATION}/bin/activate; +#MANAGE=${VENV_LOCATION}/bin/python manage.py +# Do not depend on Python to generate the SECRET_KEY +GET_SECRET_KEY=`head -c50 /dev/urandom | base64 | head -c50` +# Customize with `$ make envfile ENV_FILE=/etc/dejacode/.env` +ENV_FILE=.env +DOCKER_COMPOSE=docker compose -f docker-compose.yml +DOCKER_EXEC=${DOCKER_COMPOSE} exec +DB_NAME=dejacode_db +DB_USERNAME=dejacode +DB_PASSWORD=dejacode +DB_CONTAINER_NAME=db +DB_INIT_FILE=./data/postgresql/initdb.sql.gz +POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 +TIMESTAMP=$(shell date +"%Y-%m-%d_%H%M") + +conf: virtualenv + @echo "-> Install dependencies" + uv sync --frozen + @echo "-> Create the var/ directory" + @mkdir -p var + +dev: virtualenv + @echo "-> Configure and install development dependencies" + uv sync --frozen --extra dev + envfile: @echo "-> Create the .env file and generate a secret key" @if test -f ${ENV_FILE}; then echo "${ENV_FILE} file exists already"; exit 1; fi @@ -148,15 +160,6 @@ envfile_dev: envfile @echo "-> Update the .env file for development" @echo DATABASE_PASSWORD=\"dejacode\" >> ${ENV_FILE} -check-deploy: - @echo "-> Check Django deployment settings" - ${MANAGE} check --deploy - -clean: - @echo "-> Clean the Python env" - rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/ - find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete - initdb: @echo "-> Stop Docker services that access the database" ${DOCKER_COMPOSE} stop web worker @@ -181,4 +184,4 @@ psql: log: ${DOCKER_COMPOSE} logs --tail="100" ${SERVICE} -.PHONY: virtualenv conf dev lock upgrade envfile envfile_dev check outdated doc8 valid check-deploy clean initdb postgresdb postgresdb_clean migrate run test docs build psql bash shell log superuser +.PHONY: virtualenv conf dev lock upgrade envfile envfile_dev check outdated doc8 valid clean initdb postgresdb postgresdb_clean migrate run test docs build psql bash shell log superuser diff --git a/component_catalog/tests/test_importers.py b/component_catalog/tests/test_importers.py index eceb65d1..bb05c372 100644 --- a/component_catalog/tests/test_importers.py +++ b/component_catalog/tests/test_importers.py @@ -601,10 +601,10 @@ def test_importers_view_num_queries_view(self): with self.assertMaxQueries(9): self.client.get(reverse("admin:component_catalog_package_import")) - with self.assertNumQueries(4): + with self.assertMaxQueries(5): self.client.get(reverse("admin:organization_owner_import")) - with self.assertMaxQueries(10): + with self.assertMaxQueries(11): self.client.get(reverse("admin:component_catalog_component_import")) def test_component_import_keywords(self): diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index 5efafcb1..c776d85d 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -52,6 +52,7 @@ from dje.models import ExternalReference from dje.models import ExternalSource from dje.models import History +from dje.tests import MaxQueryMixin from dje.tests import add_perm from dje.tests import add_perms from dje.tests import create_superuser @@ -77,7 +78,7 @@ User = get_user_model() -class ComponentUserViewsTestCase(TestCase): +class ComponentUserViewsTestCase(MaxQueryMixin, TestCase): def setUp(self): self.nexb_dataspace = Dataspace.objects.create(name="nexB") self.nexb_user = User.objects.create_superuser( @@ -980,7 +981,7 @@ def test_component_catalog_details_view_num_queries(self): History.log_change(self.basic_user, self.component1, "Changed version.") History.log_change(self.nexb_user, self.component1, "Changed notes.") - with self.assertNumQueries(32): + with self.assertMaxQueries(33): self.client.get(url) def test_component_catalog_details_view_package_tab_fields_visibility(self): @@ -1095,7 +1096,7 @@ def test_component_catalog_component_create_ajax_view(self): self.assertContains(response, expected, html=True) -class PackageUserViewsTestCase(TestCase): +class PackageUserViewsTestCase(MaxQueryMixin, TestCase): testfiles_location = join(dirname(__file__), "testfiles") def setUp(self): @@ -1133,7 +1134,7 @@ def setUp(self): def test_package_list_view_num_queries(self): self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(16): + with self.assertMaxQueries(17): self.client.get(reverse("component_catalog:package_list")) def test_package_list_view_pagination(self): @@ -1271,7 +1272,7 @@ def test_package_details_view_num_queries(self): ) self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(30): + with self.assertMaxQueries(31): self.client.get(self.package1.get_absolute_url()) def test_package_details_view_content(self): @@ -3797,7 +3798,7 @@ def test_component_catalog_package_update_view_save_as_with_collect_data( self.assertEqual(1, len(mock_collect_data.mock_calls)) -class ComponentListViewTestCase(TestCase): +class ComponentListViewTestCase(MaxQueryMixin, TestCase): def setUp(self): self.dataspace = Dataspace.objects.create( name="nexB", @@ -3887,7 +3888,7 @@ def setUp(self): def test_component_catalog_list_view_num_queries(self): self.client.login(username="nexb_user", password="t3st") - with self.assertNumQueries(17): + with self.assertMaxQueries(18): self.client.get(reverse("component_catalog:component_list")) def test_component_catalog_list_view_default(self): diff --git a/license_library/tests/test_views.py b/license_library/tests/test_views.py index cb053ef3..fe39564c 100644 --- a/license_library/tests/test_views.py +++ b/license_library/tests/test_views.py @@ -31,7 +31,7 @@ from organization.models import Subowner -class LicenseListViewTestCase(TestCase): +class LicenseListViewTestCase(MaxQueryMixin, TestCase): def setUp(self): self.nexb_dataspace = Dataspace.objects.create( name="nexB", @@ -286,7 +286,7 @@ def test_license_library_list_previous_next_license_link(self): def test_license_library_list_view_num_queries(self): self.client.login(username="nexb_user", password="t3st") - with self.assertNumQueries(16): + with self.assertMaxQueries(17): self.client.get(reverse("license_library:license_list")) def test_license_profile_column_availability_in_license_list_view(self): diff --git a/organization/tests/test_views.py b/organization/tests/test_views.py index 4ee02880..5ca61966 100644 --- a/organization/tests/test_views.py +++ b/organization/tests/test_views.py @@ -13,6 +13,7 @@ from django.urls import reverse from dje.models import Dataspace +from dje.tests import MaxQueryMixin from dje.tests import add_perm from dje.tests import create_superuser from dje.tests import create_user @@ -24,7 +25,7 @@ Component = apps.get_model("component_catalog", "Component") -class OwnerUserViewsTestCase(TestCase): +class OwnerUserViewsTestCase(MaxQueryMixin, TestCase): def setUp(self): self.dataspace = Dataspace.objects.create(name="Dataspace") self.super_user = create_superuser("super_user", self.dataspace) @@ -94,12 +95,12 @@ def test_object_details_view_tab_owner(self): def test_owner_list_view_num_queries(self): self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(13): + with self.assertMaxQueries(14): self.client.get(reverse("organization:owner_list")) def test_owner_details_view_num_queries(self): self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(18): + with self.assertMaxQueries(19): self.client.get(self.owner1.get_absolute_url()) def test_owner_list_view_search_unicode_utf8_name_support(self): diff --git a/product_portfolio/tests/test_views.py b/product_portfolio/tests/test_views.py index c4eb4ba2..d2463648 100644 --- a/product_portfolio/tests/test_views.py +++ b/product_portfolio/tests/test_views.py @@ -136,7 +136,7 @@ def test_product_portfolio_detail_view_tab_inventory_and_hierarchy_availability( ProductComponent.objects.create( product=self.product1, component=self.component1, dataspace=self.dataspace ) - with self.assertNumQueries(27): + with self.assertMaxQueries(28): response = self.client.get(url) self.assertContains(response, expected1) self.assertContains(response, expected2) @@ -162,7 +162,7 @@ def test_product_portfolio_detail_view_tab_inventory_availability(self): ProductPackage.objects.create( product=self.product1, package=self.package1, dataspace=self.dataspace ) - with self.assertNumQueries(25): + with self.assertMaxQueries(26): response = self.client.get(url) self.assertContains(response, expected) @@ -267,7 +267,7 @@ def test_product_portfolio_detail_view_tab_dependency_view(self): resolved_to_package=package2, ) - with self.assertMaxQueries(9): + with self.assertMaxQueries(10): response = self.client.get(url) self.assertContains(response, "4 results") @@ -289,7 +289,7 @@ def test_product_portfolio_detail_view_tab_vulnerability_queryset(self): self.assertEqual(4, product1.packages.vulnerable().count()) url = product1.get_url("tab_vulnerabilities") - with self.assertMaxQueries(11): + with self.assertMaxQueries(12): response = self.client.get(url) self.assertContains(response, "4 results") @@ -357,7 +357,7 @@ def test_product_portfolio_tab_vulnerability_view_queries(self): make_vulnerability_analysis(product_package2, vulnerability2) url = product1.get_url("tab_vulnerabilities") - with self.assertNumQueries(11): + with self.assertMaxQueries(12): self.client.get(url) def test_product_portfolio_tab_vulnerability_risk_threshold(self): diff --git a/pyproject.toml b/pyproject.toml index 97bf784e..bb8931a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,8 @@ dependencies = [ "django-guardian==3.3.1", "django-environ==0.13.0", "django-debug-toolbar==6.3.0", + # Parallel testing + "tblib==3.2.2", # CAPTCHA "altcha==1.0.0", "django_altcha==0.10.0", @@ -96,7 +98,7 @@ dependencies = [ "XlsxWriter==3.2.9", # Markdown "markdown==3.10.2", - "bleach==6.3.0", + "bleach==6.4.0", "bleach_allowlist==1.0.3", "webencodings==0.5.1", # Authentication @@ -160,8 +162,6 @@ dependencies = [ dev = [ # Linter and Validation "ruff==0.15.14", - # Parallel testing - "tblib==3.2.2" ] [project.urls] diff --git a/reporting/tests/test_views.py b/reporting/tests/test_views.py index ae34dcfa..ed6b40bb 100644 --- a/reporting/tests/test_views.py +++ b/reporting/tests/test_views.py @@ -19,6 +19,7 @@ from component_catalog.models import Component from dje.copier import copy_object from dje.models import Dataspace +from dje.tests import MaxQueryMixin from license_library.models import License from license_library.models import LicenseCategory from organization.models import Owner @@ -32,7 +33,7 @@ from reporting.models import Report -class ReportDetailsViewTestCase(TestCase): +class ReportDetailsViewTestCase(MaxQueryMixin, TestCase): def setUp(self): self.dataspace = Dataspace.objects.create(name="nexB") self.owner = Owner.objects.create(dataspace=self.dataspace, name="My Fancy Owner Name") @@ -1166,7 +1167,7 @@ def test_report_list_view_num_queries(self): # Needed to clear the queries from the License batch creation in setUp self.client.get(url) - with self.assertNumQueries(9): + with self.assertMaxQueries(10): self.client.get(url) def test_run_report_view_query_using_related_fields(self): diff --git a/thirdparty/dist/bleach-6.3.0-py3-none-any.whl.ABOUT b/thirdparty/dist/bleach-6.3.0-py3-none-any.whl.ABOUT deleted file mode 100644 index 7814d78e..00000000 --- a/thirdparty/dist/bleach-6.3.0-py3-none-any.whl.ABOUT +++ /dev/null @@ -1,18 +0,0 @@ -about_resource: bleach-6.3.0-py3-none-any.whl -name: bleach -version: 6.3.0 -download_url: https://files.pythonhosted.org/packages/cd/3a/577b549de0cc09d95f11087ee63c739bba856cd3952697eec4c4bb91350a/bleach-6.3.0-py3-none-any.whl -package_url: pkg:pypi/bleach@6.3.0 -license_expression: apache-2.0 AND mit -copyright: Copyright bleach project contributors -attribute: yes -track_changes: yes -checksum_md5: 582f05dac01de36bf93c8e05b9cba11b -checksum_sha1: 74792becf1c32fb1edd3e04594acaee490969170 -licenses: - - key: apache-2.0 - name: Apache License 2.0 - file: apache-2.0.LICENSE - - key: mit - name: MIT License - file: mit.LICENSE diff --git a/thirdparty/dist/bleach-6.3.0-py3-none-any.whl b/thirdparty/dist/bleach-6.4.0-py3-none-any.whl similarity index 72% rename from thirdparty/dist/bleach-6.3.0-py3-none-any.whl rename to thirdparty/dist/bleach-6.4.0-py3-none-any.whl index 6dd8ebad..f57d1f2d 100644 Binary files a/thirdparty/dist/bleach-6.3.0-py3-none-any.whl and b/thirdparty/dist/bleach-6.4.0-py3-none-any.whl differ diff --git a/uv.lock b/uv.lock index dc5af854..e43038b4 100644 --- a/uv.lock +++ b/uv.lock @@ -11,7 +11,7 @@ supported-markers = [ ] [options] -exclude-newer = "2026-06-01T14:55:56.000049Z" +exclude-newer = "2026-06-11T13:07:42.745317Z" exclude-newer-span = "P7D" [[package]] @@ -68,13 +68,13 @@ wheels = [ [[package]] name = "bleach" -version = "6.3.0" +version = "6.4.0" source = { registry = "thirdparty/dist" } dependencies = [ { name = "webencodings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] wheels = [ - { path = "bleach-6.3.0-py3-none-any.whl" }, + { path = "bleach-6.4.0-py3-none-any.whl" }, ] [[package]] @@ -282,6 +282,7 @@ dependencies = [ { name = "sortedcontainers", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "sqlparse", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "swapper", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "tblib", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-inspection", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uritemplate", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -295,7 +296,6 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "tblib", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.metadata] @@ -305,7 +305,7 @@ requires-dist = [ { name = "annotated-types", specifier = "==0.7.0" }, { name = "asgiref", specifier = "==3.11.1" }, { name = "attrs", specifier = "==25.4.0" }, - { name = "bleach", specifier = "==6.3.0" }, + { name = "bleach", specifier = "==6.4.0" }, { name = "bleach-allowlist", specifier = "==1.0.3" }, { name = "boolean-py", specifier = "==5.0" }, { name = "certifi", specifier = "==2026.5.20" }, @@ -389,7 +389,7 @@ requires-dist = [ { name = "sortedcontainers", specifier = "==2.4.0" }, { name = "sqlparse", specifier = "==0.5.5" }, { name = "swapper", specifier = "==1.4.0" }, - { name = "tblib", marker = "extra == 'dev'", specifier = "==3.2.2" }, + { name = "tblib", specifier = "==3.2.2" }, { name = "typing-extensions", specifier = "==4.15.0" }, { name = "typing-inspection", specifier = "==0.4.2" }, { name = "uritemplate", specifier = "==4.2.0" }, diff --git a/vulnerabilities/tests/test_views.py b/vulnerabilities/tests/test_views.py index fd8fbb80..b0604396 100644 --- a/vulnerabilities/tests/test_views.py +++ b/vulnerabilities/tests/test_views.py @@ -12,12 +12,13 @@ from component_catalog.tests import make_component from component_catalog.tests import make_package from dje.models import Dataspace +from dje.tests import MaxQueryMixin from dje.tests import create_superuser from vulnerabilities.models import Vulnerability from vulnerabilities.tests import make_vulnerability -class VulnerabilityViewsTestCase(TestCase): +class VulnerabilityViewsTestCase(MaxQueryMixin, TestCase): def setUp(self): self.dataspace = Dataspace.objects.create( name="Dataspace", @@ -35,7 +36,7 @@ def setUp(self): def test_vulnerability_list_view_num_queries(self): self.client.login(username=self.super_user.username, password="secret") - with self.assertNumQueries(7): + with self.assertMaxQueries(8): response = self.client.get(reverse("vulnerabilities:vulnerability_list")) vulnerability_count = Vulnerability.objects.count() diff --git a/workflow/tests/test_views.py b/workflow/tests/test_views.py index 9157f829..e822b0f0 100644 --- a/workflow/tests/test_views.py +++ b/workflow/tests/test_views.py @@ -26,6 +26,7 @@ from component_catalog.models import Subcomponent from dje.models import Dataspace from dje.models import History +from dje.tests import MaxQueryMixin from dje.tests import add_perm from dje.tests import create_superuser from dje.tests import create_user @@ -1643,7 +1644,7 @@ def test_workflow_notification_request_comment_slack_payload(self): self.assertEqual(expected, payload) -class RequestInComponentCatalogTestCase(TestCase): +class RequestInComponentCatalogTestCase(MaxQueryMixin, TestCase): def setUp(self): self.nexb_dataspace = Dataspace.objects.create(name="nexB") self.user = create_superuser("nexb_user", self.nexb_dataspace) @@ -1810,7 +1811,7 @@ def test_component_catalog_details_view_with_requests_num_queries(self): self.assertEqual(3, self.component1.get_requests(self.user).count()) - with self.assertNumQueries(28): + with self.assertMaxQueries(29): self.client.get(url) @override_settings(ANONYMOUS_USERS_DATASPACE="nexB")