refactor: aquanavi data update and documents renaming

.gitignore

@@ -38,6 +38,7 @@ local_dev/paper_preview
 # php files
 /server/classes/headstart/vendor
 /server/classes/headstart/var/*
+**.phpunit.cache
 
 # R files
 server/preprocessing/other-scripts/.Rhistory
@@ -46,6 +47,9 @@ server/preprocessing/other-scripts/renv
 .Rprofile
 .Rproj*
 /*.Rproj
+*/**/*.Rproj
+*/**/*.RData
+server/workers/*/renv
 .Rproj.user
 
 # python files
@@ -66,6 +70,9 @@ server/preprocessing/other-scripts/renv
 # temporal
 local_dev/config_local_headstart.ini
 local_dev/config_local_searchflow.ini
+local_dev/tools/**/*.csv
+local_dev/tools/**/*.json
+local_dev/tools/**/*.log
 
 # mac os
 .DS_Store

examples/public/main_integration/base_sg.html

@@ -20,7 +20,7 @@
     <meta name="keywords" content="knowldege visualization, open knowledge, open science">
 
     <meta name="description"
-        content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+        content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/public/main_integration/openaire.html

@@ -20,7 +20,7 @@
     <meta name="keywords" content="knowldege visualization, open knowledge, open science">
 
     <meta name="description"
-          content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+          content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/templates/base.html

@@ -22,7 +22,7 @@
     <link rel="icon" type="image/png" sizes="16x16" href="./main_integration/img/favicon-16x16.png?v=xQz6nej7eR">
 
     <meta name="description"
-        content="Get an overview of digital education, find relevant papers, and identify important concepts.">
+        content="Get an overview of digital education, find relevant research outputs, and identify important concepts.">
 
     <!-- FAVICONS -->
     <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=xQz6nej7eR">

examples/templates/pubmed.html

@@ -22,7 +22,7 @@
     <link rel="icon" type="image/png" sizes="16x16" href="./main_integration/img/favicon-16x16.png?v=xQz6nej7eR">
 
     <meta name="description"
-        content="Get an overview of &quot;climate change&quot;, find relevant papers, and identify important concepts.">
+        content="Get an overview of &quot;climate change&quot;, find relevant research outputs, and identify important concepts.">
 
     <link type="text/css" rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css">
     <link href='https://fonts.googleapis.com/css?family=Open+Sans:300,400,600,700,800' rel='stylesheet' type='text/css'>

examples/templates/triple.html

@@ -7,7 +7,7 @@
     <title>Overview of research on psychology</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="description"
-        content="Get an overview of psychology, find relevant papers, and identify important concepts.">
+        content="Get an overview of psychology, find relevant research outputs, and identify important concepts.">
 
     <script type="text/javascript" src="./triple/data-config-km.js"></script>
     <script type="text/javascript" src="./triple/data-config-sg.js"></script>

local_dev/config_local_headstart.ini

@@ -35,6 +35,8 @@ snapshot_php = "headstart/server/services/snapshot/headstart_snapshot.php"
 snapshot_local_protocol = "http://"
 # Size of thumbnail width
 snapshot_width = "1200px"
+# chrome executable path optional, only needed if puppeteer cannot find chrome automatically
+chrome_executable_path = "/path/to/chrome"
 
 [connection]
 # PostgreSQL database

server/preprocessing/conf/config.ini

@@ -32,7 +32,7 @@ storage_path = "/path/to/storage/"
 snapshot_php = "server/services/snapshot/headstart_snapshot.php"
 # snapshot_local_protocol fallback for non-server environments
 snapshot_local_protocol = "http://"
-# chrome executable path (optional, only needed if puppeteer cannot find chrome automatically)
+# chrome executable path optional, only needed if puppeteer cannot find chrome automatically
 chrome_executable_path = "/path/to/chrome"
 
 

server/preprocessing/other-scripts/openaire.R

@@ -52,11 +52,6 @@ get_papers <- function(query, params) {
                          funder = funder,
                          format = 'xml')
     pubs_metadata <- parse_response(response)
-    # FYI: The deactivation of the fill_dois() function is a hotfix
-    # to enable creation of OpenAIRE project maps
-    # TODO: root cause analysis of failure mode
-    # TODO: refactor/replace/remove the function, decision pending
-    #pubs_metadata <- fill_dois(pubs_metadata)
   },
   error = function(err){
     olog$warn(paste0("vis_id:", .GlobalEnv$VIS_ID, "publications: ", err))
@@ -193,34 +188,6 @@ parse_response <- function(response) {
   }
 }
 
-fill_dois <- function(df) {
-  missing_doi_indices <- which(is.na(df$doi))
-  titles <- df[missing_doi_indices,]$title
-  if (exists("DEBUG") && DEBUG) {
-    olog$debug(paste("Missing DOIs:", length(titles)))
-    olog$debug("Time for filling missing DOIs")
-    olog$debug(system.time(cr_works(query=queries(titles), async=TRUE)))
-  }
-  tryCatch({
-    if (length(titles) > 1) {
-      response <- cr_works(query=queries(titles))
-      candidates <- lapply(response, get_doi_candidates)
-      dois <- mapply(check_distance, titles, candidates, USE.NAMES=FALSE)
-    } else if (length(titles) == 1) {
-      response <- cr_works(flq=c('query.title'=titles))$data
-      candidate_response = response[1,]
-      dois <- check_distance(titles, candidate_response)
-    } else {
-      dois <- ""
-    }
-    df$doi[c(missing_doi_indices)] <- dois
-    }, error=function(err){
-      olog$error(paste("vis_id:", .GlobalEnv$VIS_ID, "DOI enrichment failed:", paste(err)))
-    }
-  )
-  return (df)
-}
-
 get_doi_candidates <- function(response){
   if (nrow(response) >= 1) {
     candidate = response[1,c('doi', 'title')]

server/preprocessing/other-scripts/update_contentproviders.R

@@ -0,0 +1,50 @@
+rm(list = ls())
+
+args <- commandArgs(TRUE)
+wd <- args[1]
+setwd(wd)
+
+renv::activate()
+renv::restore(lockfile = '../renv.lock')
+Sys.setlocale(category = "LC_ALL", locale = "en_US.UTF-8")
+
+library(jsonlite)
+library(rbace)
+library(logging)
+
+source('utils.R')
+if (Sys.getenv("LOGLEVEL") == "DEBUG") {
+  DEBUG <- FALSE
+} else {
+  DEBUG <- TRUE
+}
+
+if (DEBUG == TRUE) {
+  setup_logging('DEBUG')
+} else {
+  setup_logging('INFO')
+}
+
+log <- getLogger('update_contentproviders')
+
+output_path <- if (length(args) >= 2) args[2] else {
+  file.path(wd, "../../workers/common/common/contentproviders.json")
+}
+output_path <- normalizePath(output_path, mustWork = FALSE)
+
+tryCatch({
+  contentproviders <- bs_repositories("")
+  if (is.null(contentproviders) || nrow(contentproviders) == 0) {
+    stop("No content providers retrieved.")
+  }
+  triple <- list(name = "GoTriple", internal_name = "fttriple")
+  contentproviders <- rbind(contentproviders, triple)
+
+  write(toJSON(contentproviders, pretty = TRUE), file = output_path)
+  log$info(paste("contentproviders.json updated:", output_path,
+                 "—", nrow(contentproviders), "entries"))
+}, error = function(err) {
+  log$error(paste("Content provider update failed", "base",
+                  "update_contentproviders", "", err, sep = "||"))
+  quit(status = 1)
+})

server/workers/build_docker_images.sh

@@ -3,6 +3,26 @@
 # Defines the script directory
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
+CONTENTPROVIDERS_FILE="$SCRIPT_DIR/common/common/contentproviders.json"
+
+# Update contentproviders.json cache from the running dev-base-1 container
+echo ""
+echo "Updating contentproviders.json cache..."
+echo ""
+docker exec dev-base-1 Rscript /headstart/other-scripts/update_contentproviders.R \
+  /headstart/other-scripts \
+  /common/contentproviders.json
+docker cp dev-base-1:/common/contentproviders.json "$CONTENTPROVIDERS_FILE"
+
+# Commit if the file changed
+cd "$SCRIPT_DIR/../.." && git diff --quiet "$CONTENTPROVIDERS_FILE"
+if [ $? -ne 0 ]; then
+  echo "contentproviders.json changed, committing..."
+  git add "$CONTENTPROVIDERS_FILE"
+  git commit -m "update of contentprovider.json cache"
+fi
+cd "$SCRIPT_DIR"
+
 # Define the list of services
 services=("api" "persistence" "dataprocessing" "base" "pubmed" "openaire" "orcid" "metrics")
 

server/workers/common/common/aquanavi/constants/constants.py

@@ -0,0 +1,15 @@
+COLUMNS = {
+    "url": "url",
+    "name": "Name",
+    "country": "Country",
+    "continent": "Continent",
+    "equipment": "Equipment",
+    "research_topics": "Research Topics",
+    "specialist_areas": "Specialist areas",
+    "grand_challenges": "Primary interests",
+    "controlled_parameters": "Controlled Parameters",
+    "description": "Description of Facility",
+    "location": "Facility location(s) split",
+    "years_of_experiments": "Years of Mesocosm Experiments",
+    "photos_of_experiments": "Photos of experiments/installations images",
+}

server/workers/common/common/aquanavi/helpers/coordinates.py

@@ -0,0 +1,95 @@
+import ast
+import math
+
+from ..constants.constants import COLUMNS
+
+# Cap jitter so markers stay near the original location.
+MAX_OFFSET_METERS = 30.0
+# Controls how quickly the offset grows for the 2nd, 3rd, ... duplicates.
+GROWTH_FACTOR = 6.0
+# Golden angle for an even spiral distribution.
+GOLDEN_ANGLE_DEGREES = 137.50776405003785
+# Approx conversion: meters per 1 degree of latitude.
+METERS_PER_DEGREE_LAT = 111_111.0
+# Avoid division by ~0 for longitude conversion near the poles.
+COS_EPSILON = 1e-6
+
+def get_row_coordinates_with_collision_offset(row, seen_coordinates):
+    """
+    Returns row coordinates with a small deterministic offset applied for duplicates.
+
+    Args:
+        row (pandas.Series): DataFrame row.
+        seen_coordinates (dict[tuple[float, float], int]): Occurrence counter by exact (lat, lon).
+
+    Returns:
+        tuple[float|None, float|None]: (latitude, longitude), possibly offset for duplicates.
+    """
+    latitude, longitude = get_latitude_longitude(row)
+
+    duplicate_index = 0
+    if latitude is not None and longitude is not None:
+        key = (latitude, longitude)
+        duplicate_index = seen_coordinates.get(key, 0)
+        seen_coordinates[key] = duplicate_index + 1
+
+    return offset_duplicate_coordinates(latitude, longitude, duplicate_index)
+
+def get_latitude_longitude(row):
+    """
+    The function returns a list with latitude and longitude.
+
+    Args:
+        row (str): String DataFrame.
+
+    Returns:
+        list: A list with latitude and longitude (or None).
+    """
+    coordinates_string = str(row[COLUMNS['location']]).strip()
+
+    latitude, longitude = None, None
+
+    if "," in coordinates_string:
+        try:
+            coords = ast.literal_eval(coordinates_string)
+            if isinstance(coords, (list, tuple)) and len(coords) == 2:
+                latitude = float(str(coords[0]).strip())
+                longitude = float(str(coords[1]).strip())
+        except Exception:
+            latitude, longitude = None, None
+
+    return [latitude, longitude]
+
+def offset_duplicate_coordinates(latitude, longitude, duplicate_index):
+    """
+    Applies a tiny deterministic offset (meters) to avoid marker collision for identical coordinates.
+
+    The offset is intentionally limited to keep the point nearby while making collisions visible.
+
+    Args:
+        latitude (float|None): Latitude.
+        longitude (float|None): Longitude.
+        duplicate_index (int): 0 for the first occurrence (no offset), 1..N for duplicates.
+
+    Returns:
+        tuple[float|None, float|None]: (latitude, longitude) with offset applied.
+    """
+    if latitude is None or longitude is None:
+        return latitude, longitude
+    if duplicate_index <= 0:
+        return latitude, longitude
+
+    golden_angle_rad = math.radians(GOLDEN_ANGLE_DEGREES)
+    angle = duplicate_index * golden_angle_rad
+
+    radius_m = min(GROWTH_FACTOR * math.sqrt(duplicate_index), MAX_OFFSET_METERS)
+    east_m = radius_m * math.cos(angle)
+    north_m = radius_m * math.sin(angle)
+
+    lat_rad = math.radians(latitude)
+    meters_per_degree_lon = METERS_PER_DEGREE_LAT * max(math.cos(lat_rad), COS_EPSILON)
+
+    dlat = north_m / METERS_PER_DEGREE_LAT
+    dlon = east_m / meters_per_degree_lon
+
+    return latitude + dlat, longitude + dlon

server/workers/common/common/aquanavi/helpers/text_content_validation.py

@@ -0,0 +1,12 @@
+import pandas as pd  # type: ignore[reportMissingImports]
+
+def has_meaningful_value(row, column_name: str) -> bool:
+    """
+    Treat pandas NaN / None / '' / whitespace-only (e.g. '\\n\\n') as empty.
+    """
+    value = row[column_name]
+
+    if pd.isna(value):
+        return False
+
+    return str(value).strip() != ""