From 129fe2c6188bae5c019cb1b057370e32558da562 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 12:45:25 +0400 Subject: [PATCH 1/7] add CREATE_DEPENDENCIES_DEFAULT setting Signed-off-by: tdruez --- dejacode/settings.py | 6 ++++++ product_portfolio/forms.py | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dejacode/settings.py b/dejacode/settings.py index fb026ceb..952e011a 100644 --- a/dejacode/settings.py +++ b/dejacode/settings.py @@ -586,6 +586,11 @@ def get_fake_redis_connection(config, use_strict_redis): "propagate": False, "level": DEJACODE_LOG_LEVEL, }, + "product_portfolio": { + "handlers": ["null"] if IS_TESTS else ["console"], + "propagate": False, + "level": "DEBUG" if DEBUG else DEJACODE_LOG_LEVEL, + }, }, } @@ -717,6 +722,7 @@ def get_fake_redis_connection(config, use_strict_redis): # Default to 5 seconds. DEJACODE_INTEGRATION_REQUESTS_TIMEOUT = env.int("DEJACODE_INTEGRATION_REQUESTS_TIMEOUT", default=5) +CREATE_DEPENDENCIES_DEFAULT = env.bool("CREATE_DEPENDENCIES_DEFAULT", default=True) if IS_TESTS: # Silent the django-axes logging during tests diff --git a/product_portfolio/forms.py b/product_portfolio/forms.py index 7119230d..6c7482b3 100644 --- a/product_portfolio/forms.py +++ b/product_portfolio/forms.py @@ -9,6 +9,7 @@ import json from django import forms +from django.conf import settings from django.core.exceptions import ValidationError from django.db import transaction from django.forms import BaseModelFormSet @@ -557,7 +558,7 @@ class ImportFromScanForm(forms.Form): create_dependencies = forms.BooleanField( label=_("Create Dependencies"), required=False, - initial=False, + initial=settings.CREATE_DEPENDENCIES_DEFAULT, help_text=_( "When checked, dependency relationships between packages discovered in the " "import will be created on the Product." @@ -664,7 +665,7 @@ class BaseProductImportFormView(forms.Form): create_dependencies = forms.BooleanField( label=_("Create Dependencies"), required=False, - initial=False, + initial=settings.CREATE_DEPENDENCIES_DEFAULT, help_text=_( "When checked, dependency relationships between packages discovered in the " "import will be created on the Product." @@ -1002,7 +1003,7 @@ class PullProjectDataForm(forms.Form): create_dependencies = forms.BooleanField( label=_("Create Dependencies"), required=False, - initial=False, + initial=settings.CREATE_DEPENDENCIES_DEFAULT, help_text=_( "When checked, dependency relationships between packages discovered in the " "import will be created on the Product." From 09349a402c710bbef42566f9813e6acbd3e7cc8c Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 12:45:35 +0400 Subject: [PATCH 2/7] mount local .env file Signed-off-by: tdruez --- compose.dev.yml | 3 +-- product_portfolio/importers.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 14dcc333..564c0db4 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,8 +22,6 @@ services: web: build: . image: dejacode:dev - env_file: - - .env environment: - DATABASE_HOST=db - DATABASE_PASSWORD=dejacode @@ -37,6 +35,7 @@ services: ports: - "8000:8000" volumes: + - ./.env:/opt/dejacode/.env - ./component_catalog:/opt/dejacode/component_catalog - ./dejacode:/opt/dejacode/dejacode - ./dejacode_toolkit:/opt/dejacode/dejacode_toolkit diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 28d5768e..a14752b5 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -693,11 +693,21 @@ def __init__( scancodeio = ScanCodeIO(user.dataspace) self.packages = scancodeio.fetch_project_packages(self.project_uuid) self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) + if not self.packages and not self.dependencies: raise Exception("Packages could not be fetched from ScanCode.io") + dataspace = product.dataspace + self.default_review_status = ProductRelationStatus.objects.get_default_on_addition_qs( + dataspace + ).first() + self.default_purpose = ProductItemPurpose.objects.get_default_on_addition_qs( + dataspace + ).first() + def save(self): self.import_packages() + if self.create_dependencies: self.import_dependencies() @@ -780,8 +790,11 @@ def import_package(self, package_data): "license_expression": package.license_expression, "notes": "Imported from ScanCode.io", "created_by": self.user, + "review_status": self.default_review_status, + "purpose": self.default_purpose, }, ) + package_uid = package_data.get("package_uid") or package.uuid self.package_uid_mapping[package_uid] = package @@ -846,7 +859,7 @@ def look_for_existing_package(self, package_data): return package # 2. If the package data does not include a download_url value: - # Attemp to find an existing package using purl-only match. + # Attempt to find an existing package using purl-only match. if not package_data.get("download_url"): purl_lookups = {field: package_data.get(field, "") for field in PACKAGE_URL_FIELDS} same_purl_packages = package_qs.filter(**purl_lookups) From c31906ee3e6809fada49e85a2cc4677f3f7da166 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 12:49:48 +0400 Subject: [PATCH 3/7] conditionally fetch dependencies Signed-off-by: tdruez --- product_portfolio/importers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index a14752b5..7bbd960b 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -690,13 +690,7 @@ def __init__( self.infer_download_urls = infer_download_urls self.create_dependencies = create_dependencies - scancodeio = ScanCodeIO(user.dataspace) - self.packages = scancodeio.fetch_project_packages(self.project_uuid) - self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) - - if not self.packages and not self.dependencies: - raise Exception("Packages could not be fetched from ScanCode.io") - + # Pre-fetch once to avoid repeated DB lookups in DefaultOnAdditionMixin.save(). dataspace = product.dataspace self.default_review_status = ProductRelationStatus.objects.get_default_on_addition_qs( dataspace @@ -705,6 +699,14 @@ def __init__( dataspace ).first() + scancodeio = ScanCodeIO(user.dataspace) + self.packages = scancodeio.fetch_project_packages(self.project_uuid) + if not self.packages: + raise Exception("Packages could not be fetched from ScanCode.io") + + if self.create_dependencies: + self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) + def save(self): self.import_packages() From 80095db2efec2980d5c6c5512ed363b068f14d64 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 13:21:19 +0400 Subject: [PATCH 4/7] add logging Signed-off-by: tdruez --- product_portfolio/importers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 7bbd960b..f54379e1 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -7,6 +7,8 @@ # import json +import logging +import time from collections import defaultdict from contextlib import suppress @@ -48,6 +50,8 @@ from product_portfolio.models import ProductRelationStatus from product_portfolio.models import ScanCodeProject +logger = logging.getLogger(__name__) + class CleanProductMixin(ComponentRelatedFieldImportMixin): def clean_product(self): @@ -708,16 +712,27 @@ def __init__( self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) def save(self): + save_start = time.perf_counter() + + step_start = time.perf_counter() self.import_packages() + logger.info(f"import_packages: {time.perf_counter() - step_start:.1f}s") if self.create_dependencies: + step_start = time.perf_counter() self.import_dependencies() + logger.info(f"import_dependencies: {time.perf_counter() - step_start:.1f}s") if self.scan_all_packages: transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user)) + logger.info("scan_all_packages: scheduled") if self.user.dataspace.enable_vulnerablecodedb_access: + step_start = time.perf_counter() self.product.fetch_vulnerabilities() + logger.info(f"fetch_vulnerabilities: {time.perf_counter() - step_start:.1f}s") + + logger.info(f"save total: {time.perf_counter() - save_start:.1f}s") return dict(self.created), dict(self.existing), dict(self.errors) From f48d426a4f73d1128f1b28d26902c7156641d4e7 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 13:25:49 +0400 Subject: [PATCH 5/7] refine save Signed-off-by: tdruez --- product_portfolio/importers.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index f54379e1..226c49cd 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -10,6 +10,7 @@ import logging import time from collections import defaultdict +from contextlib import contextmanager from contextlib import suppress from django import forms @@ -687,6 +688,7 @@ def __init__( self.package_uid_mapping = {} self.user = user + self.dataspace = product.dataspace self.project_uuid = project_uuid self.product = product self.update_existing = update_existing @@ -694,25 +696,22 @@ def __init__( self.infer_download_urls = infer_download_urls self.create_dependencies = create_dependencies - # Pre-fetch once to avoid repeated DB lookups in DefaultOnAdditionMixin.save(). - dataspace = product.dataspace - self.default_review_status = ProductRelationStatus.objects.get_default_on_addition_qs( - dataspace - ).first() - self.default_purpose = ProductItemPurpose.objects.get_default_on_addition_qs( - dataspace - ).first() + def save(self): + save_start = time.perf_counter() + + scancodeio = ScanCodeIO(self.dataspace) - scancodeio = ScanCodeIO(user.dataspace) + step_start = time.perf_counter() self.packages = scancodeio.fetch_project_packages(self.project_uuid) + logger.info(f"fetch_project_packages: {time.perf_counter() - step_start:.1f}s") + if not self.packages: raise Exception("Packages could not be fetched from ScanCode.io") if self.create_dependencies: + step_start = time.perf_counter() self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) - - def save(self): - save_start = time.perf_counter() + logger.info(f"fetch_project_dependencies: {time.perf_counter() - step_start:.1f}s") step_start = time.perf_counter() self.import_packages() @@ -737,6 +736,14 @@ def save(self): return dict(self.created), dict(self.existing), dict(self.errors) def import_packages(self): + # Pre-fetch once to avoid repeated DB lookups in DefaultOnAdditionMixin.save(). + self.default_review_status = ProductRelationStatus.objects.get_default_on_addition_qs( + self.dataspace + ).first() + self.default_purpose = ProductItemPurpose.objects.get_default_on_addition_qs( + self.dataspace + ).first() + for package_data in self.packages: self.import_package(package_data) From 55a36796755320b0d20773952a3e8ef77e7842eb Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 13:27:39 +0400 Subject: [PATCH 6/7] refining logging Signed-off-by: tdruez --- product_portfolio/importers.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 226c49cd..021ebaf3 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -54,6 +54,14 @@ logger = logging.getLogger(__name__) +@contextmanager +def log_elapsed(label): + """Context manager that logs the elapsed time of the wrapped block.""" + start = time.perf_counter() + yield + logger.info(f"{label}: {time.perf_counter() - start:.1f}s") + + class CleanProductMixin(ComponentRelatedFieldImportMixin): def clean_product(self): queryset = Product.objects.get_queryset(self.user) @@ -701,35 +709,30 @@ def save(self): scancodeio = ScanCodeIO(self.dataspace) - step_start = time.perf_counter() - self.packages = scancodeio.fetch_project_packages(self.project_uuid) - logger.info(f"fetch_project_packages: {time.perf_counter() - step_start:.1f}s") + with log_elapsed("fetch_project_packages"): + self.packages = scancodeio.fetch_project_packages(self.project_uuid) if not self.packages: raise Exception("Packages could not be fetched from ScanCode.io") if self.create_dependencies: - step_start = time.perf_counter() - self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) - logger.info(f"fetch_project_dependencies: {time.perf_counter() - step_start:.1f}s") + with log_elapsed("fetch_project_dependencies"): + self.dependencies = scancodeio.fetch_project_dependencies(self.project_uuid) - step_start = time.perf_counter() - self.import_packages() - logger.info(f"import_packages: {time.perf_counter() - step_start:.1f}s") + with log_elapsed("import_packages"): + self.import_packages() if self.create_dependencies: - step_start = time.perf_counter() - self.import_dependencies() - logger.info(f"import_dependencies: {time.perf_counter() - step_start:.1f}s") + with log_elapsed("import_dependencies"): + self.import_dependencies() if self.scan_all_packages: transaction.on_commit(lambda: self.product.scan_all_packages_task(self.user)) logger.info("scan_all_packages: scheduled") if self.user.dataspace.enable_vulnerablecodedb_access: - step_start = time.perf_counter() - self.product.fetch_vulnerabilities() - logger.info(f"fetch_vulnerabilities: {time.perf_counter() - step_start:.1f}s") + with log_elapsed("fetch_vulnerabilities"): + self.product.fetch_vulnerabilities() logger.info(f"save total: {time.perf_counter() - save_start:.1f}s") From da9734b3e4691feeaa9c0f79d4222d65294aa56a Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 18 Jun 2026 13:59:18 +0400 Subject: [PATCH 7/7] improve error logging Signed-off-by: tdruez --- product_portfolio/importers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/product_portfolio/importers.py b/product_portfolio/importers.py index 021ebaf3..a790a42c 100644 --- a/product_portfolio/importers.py +++ b/product_portfolio/importers.py @@ -804,8 +804,9 @@ def import_package(self, package_data): if not package: try: package = Package.create_from_data(self.user, package_data, validate=True) - except ValidationError as errors: - self.errors["package"].append(str(errors)) + except ValidationError as error: + logger.error(f"Failed to create package: {error}\n{package_data}") + self.errors["package"].append(str(error)) return self.created["package"].append(str(package))