Allow pushing user-allocation membership to Keycloak

.github/workflows/test-functional-keycloak.yaml

@@ -0,0 +1,35 @@
+name: test-functional-keycloak
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.12
+
+      - name: Upgrade and install packages
+        run: |
+          bash ./ci/setup-ubuntu.sh
+
+      - name: Install Keycloak
+        run: |
+          bash ./ci/setup-keycloak.sh
+
+      - name: Install ColdFront and plugin
+        run: |
+          ./ci/setup.sh
+
+      - name: Run functional tests
+        run: |
+          ./ci/run_functional_tests_keycloak.sh

ci/run_functional_tests_keycloak.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -xe
+
+if [[ ! "${CI}" == "true" ]]; then
+    source /tmp/coldfront_venv/bin/activate
+fi
+
+export DJANGO_SETTINGS_MODULE="local_settings"
+export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+
+export KEYCLOAK_BASE_URL="http://localhost:8080"
+export KEYCLOAK_REALM="master"
+export KEYCLOAK_CLIENT_ID="coldfront"
+export KEYCLOAK_CLIENT_SECRET="nomoresecret"
+
+coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.keycloak
+coverage report

ci/setup-keycloak.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -xe
+
+sudo docker rm -f keycloak
+
+sudo docker run -d --name keycloak \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \
+    -p 8080:8080 \
+    -p 8443:8443 \
+    quay.io/keycloak/keycloak:25.0 start-dev
+
+# wait for keycloak to be ready
+until curl -s http://localhost:8080/auth/realms/master; do
+    echo "Waiting for Keycloak to be ready..."
+    sleep 5
+done
+
+# Create client and add admin role to client's service account
+ACCESS_TOKEN=$(curl -X POST "http://localhost:8080/realms/master/protocol/openid-connect/token" \
+     -d "username=admin" \
+     -d "password=nomoresecret" \
+     -d "grant_type=password" \
+     -d "client_id=admin-cli" \
+     -d "scope=openid" \
+| jq -r '.access_token')
+
+
+curl -X POST "http://localhost:8080/admin/realms/master/clients" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "clientId": "coldfront",
+        "secret": "nomoresecret",
+        "redirectUris": ["http://localhost:8080/*"],
+        "serviceAccountsEnabled": true
+    }'
+
+COLDFRONT_CLIENT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients?clientId=coldfront" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.[0].id')
+
+
+COLDFRONT_SERVICE_ACCOUNT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients/$COLDFRONT_CLIENT_ID/service-account-user" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+| jq -r '.id')
+
+ADMIN_ROLE_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/roles/admin" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.id')
+
+# Add admin role to the service account user
+curl -X POST "http://localhost:8080/admin/realms/master/users/$COLDFRONT_SERVICE_ACCOUNT_ID/role-mappings/realm" \
+    -H "Authorization: Bearer $ACCESS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '[
+            {
+                "id": "'$ADMIN_ROLE_ID'",
+                "name": "admin"
+            }
+        ]'

requirements.txt

@@ -12,3 +12,4 @@ python-novaclient
 python-neutronclient
 python-swiftclient
 pytz
+requests

src/coldfront_plugin_cloud/attributes.py

@@ -24,7 +24,6 @@ class CloudAllocationAttribute:
 RESOURCE_API_URL = "OpenShift API Endpoint URL"
 RESOURCE_IDENTITY_NAME = "OpenShift Identity Provider Name"
 RESOURCE_ROLE = "Role for User in Project"
-RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
 
 RESOURCE_FEDERATION_PROTOCOL = "OpenStack Federation Protocol"
 RESOURCE_IDP = "OpenStack Identity Provider"
@@ -35,6 +34,8 @@ class CloudAllocationAttribute:
 
 RESOURCE_EULA_URL = "EULA URL"
 RESOURCE_CLUSTER_NAME = "Internal Cluster Name"
+RESOURCE_QUOTA_RESOURCES = "Available Quota Resources"
+RESOURCE_KEYCLOAK_GROUP_TEMPLATE = "Template String for Keycloak Group Names"
 
 RESOURCE_ATTRIBUTES = [
     CloudResourceAttribute(name=RESOURCE_AUTH_URL),
@@ -45,6 +46,7 @@ class CloudAllocationAttribute:
     CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_ROLE),
     CloudResourceAttribute(name=RESOURCE_QUOTA_RESOURCES),
+    CloudResourceAttribute(name=RESOURCE_KEYCLOAK_GROUP_TEMPLATE),
     CloudResourceAttribute(name=RESOURCE_USER_DOMAIN),
     CloudResourceAttribute(name=RESOURCE_EULA_URL),
     CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK),

src/coldfront_plugin_cloud/kc_client.py

@@ -0,0 +1,77 @@
+import os
+
+import requests
+
+
+class KeyCloakAPIClient:
+    def __init__(self):
+        self.base_url = os.getenv("KEYCLOAK_BASE_URL")
+        self.realm = os.getenv("KEYCLOAK_REALM")
+        self.client_id = os.getenv("KEYCLOAK_CLIENT_ID")
+        self.client_secret = os.getenv("KEYCLOAK_CLIENT_SECRET")
+
+        self.token_url = (
+            f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token"
+        )
+
+    @property
+    def api_client(self):
+        params = {
+            "grant_type": "client_credentials",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+        }
+        r = requests.post(self.token_url, data=params)
+        r.raise_for_status()
+        headers = {
+            "Authorization": ("Bearer %s" % r.json()["access_token"]),
+            "Content-Type": "application/json",
+        }
+        session = requests.session()
+        session.headers.update(headers)
+        return session
+
+    def create_group(self, group_name):
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups"
+        payload = {"name": group_name}
+        response = self.api_client.post(url, json=payload)
+
+        # If group already exists, ignore and move on
+        if response.status_code not in (201, 409):
+            response.raise_for_status()
+
+    def get_group_id(self, group_name) -> str | None:
+        """Return None if group not found"""
+        query = f"search={group_name}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/groups?{query}"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        r_json = r.json()
+        return r_json[0]["id"] if r_json else None
+
+    def get_user_id(self, cf_username) -> str | None:
+        """Return None if user not found"""
+        # (Quan) Coldfront usernames map to Keycloak usernames
+        # https://github.com/nerc-project/coldfront-plugin-cloud/pull/249#discussion_r2953393852
+        query = f"username={cf_username}&exact=true"
+        url = f"{self.base_url}/admin/realms/{self.realm}/users?{query}"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        r_json = r.json()
+        return r_json[0]["id"] if r_json else None
+
+    def add_user_to_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.put(url)
+        r.raise_for_status()
+
+    def remove_user_from_group(self, user_id, group_id):
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}"
+        r = self.api_client.delete(url)
+        r.raise_for_status()
+
+    def get_user_groups(self, user_id) -> list[str]:
+        url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups"
+        r = self.api_client.get(url)
+        r.raise_for_status()
+        return [group["name"] for group in r.json()]

src/coldfront_plugin_cloud/signals.py

@@ -6,8 +6,10 @@
 from coldfront_plugin_cloud.tasks import (
     activate_allocation,
     add_user_to_allocation,
+    add_user_to_keycloak,
     disable_allocation,
     remove_user_from_allocation,
+    remove_user_from_keycloak,
 )
 from coldfront.core.allocation.signals import (
     allocation_activate,
@@ -25,6 +27,10 @@ def is_async():
     return os.getenv("REDIS_HOST")
 
 
+def is_keycloak_enabled():
+    return os.getenv("KEYCLOAK_BASE_URL")
+
+
 @receiver(allocation_activate)
 @receiver(allocation_change_approved)
 def activate_allocation_receiver(sender, **kwargs):
@@ -48,11 +54,18 @@ def activate_allocation_user_receiver(sender, **kwargs):
     allocation_user_pk = kwargs.get("allocation_user_pk")
     if is_async():
         async_task(add_user_to_allocation, allocation_user_pk)
+        if is_keycloak_enabled():
+            async_task(add_user_to_keycloak, allocation_user_pk)
     else:
         add_user_to_allocation(allocation_user_pk)
+        if is_keycloak_enabled():
+            add_user_to_keycloak(allocation_user_pk)
 
 
 @receiver(allocation_remove_user)
 def allocation_remove_user_receiver(sender, **kwargs):
     allocation_user_pk = kwargs.get("allocation_user_pk")
     remove_user_from_allocation(allocation_user_pk)
+
+    if is_keycloak_enabled():
+        remove_user_from_keycloak(allocation_user_pk)

src/coldfront_plugin_cloud/tasks.py

@@ -1,8 +1,14 @@
 import datetime
 import logging
 import time
+import functools
+from string import Template
 
-from coldfront.core.allocation.models import Allocation, AllocationUser
+from coldfront.core.allocation.models import (
+    Allocation,
+    AllocationUser,
+    AllocationAttribute,
+)
 
 from coldfront_plugin_cloud import (
     attributes,
@@ -12,11 +18,17 @@
     esi,
     openshift_vm,
     utils,
+    kc_client,
 )
 
 logger = logging.getLogger(__name__)
 
 
+@functools.lru_cache()
+def get_kc_client():
+    return kc_client.KeyCloakAPIClient()
+
+
 def find_allocator(allocation) -> base.ResourceAllocator:
     allocators = {
         "openstack": openstack.OpenStackResourceAllocator,
@@ -128,3 +140,85 @@ def remove_user_from_allocation(allocation_user_pk):
             allocator.remove_role_from_user(username, project_id)
         else:
             logger.warning("No project has been created. Nothing to disable.")
+
+
+def _clean_template_string(template_string: str) -> str:
+    return template_string.replace(" ", "_").lower()
+
+
+def _get_keycloak_group_name(allocation: Allocation, template_string: str) -> str:
+    """
+    Acceptable variables for the group name template string is:
+    - $resource_name: the name of the resource (e.g. "OpenShift")
+    - Any allocation attribute defined for the allocation, with spaces replaced by underscores and
+    all lowercase (e.g. for `Project Name`, the variable would be `$project_name`)
+    """
+    resource_name = allocation.resources.first().name
+    allocation_attrs_list: list[AllocationAttribute] = (
+        allocation.allocationattribute_set.all()
+    )
+
+    template_sub_dict = {"resource_name": resource_name}
+    for attr in allocation_attrs_list:
+        template_sub_dict[
+            _clean_template_string(attr.allocation_attribute_type.name)
+        ] = attr.value
+
+    return Template(template_string).substitute(**template_sub_dict)
+
+
+def add_user_to_keycloak(allocation_user_pk):
+    allocation_user = AllocationUser.objects.get(pk=allocation_user_pk)
+    allocation = allocation_user.allocation
+
+    kc_admin_client = get_kc_client()
+    username = allocation_user.user.username
+
+    group_name_template = allocation.resources.first().get_attribute(
+        attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE
+    )
+    if group_name_template is None:
+        logger.info(
+            f"Keycloak enabled but no group name template specified for resource {allocation.resources.first().name}. Skipping addition to Keycloak group"
+        )
+        return
+
+    if (user_id := kc_admin_client.get_user_id(username)) is None:
+        logger.warning(f"User {username} not found in Keycloak, cannot add to group.")
+        return
+
+    group_name = _get_keycloak_group_name(allocation, group_name_template)
+    kc_admin_client.create_group(group_name)
+    group_id = kc_admin_client.get_group_id(group_name)
+    kc_admin_client.add_user_to_group(user_id, group_id)
+
+
+def remove_user_from_keycloak(allocation_user_pk):
+    allocation_user = AllocationUser.objects.get(pk=allocation_user_pk)
+    allocation = allocation_user.allocation
+
+    kc_admin_client = get_kc_client()
+    username = allocation_user.user.username
+
+    group_name_template = allocation.resources.first().get_attribute(
+        attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE
+    )
+    if group_name_template is None:
+        logger.info(
+            f"Keycloak enabled but no group name template specified for resource {allocation.resources.first().name}. Skipping removal from Keycloak group"
+        )
+        return
+
+    if (user_id := kc_admin_client.get_user_id(username)) is None:
+        logger.warning(
+            f"User {username} not found in Keycloak, cannot remove from group."
+        )
+        return
+
+    group_name = _get_keycloak_group_name(allocation, group_name_template)
+    if (group_id := kc_admin_client.get_group_id(group_name)) is None:
+        logger.warning(
+            f"Group {group_name} not found in Keycloak, skipping removal for user {username}."
+        )
+        return
+    kc_admin_client.remove_user_from_group(user_id, group_id)