diff --git a/.github/workflows/test-functional-keycloak.yaml b/.github/workflows/test-functional-keycloak.yaml new file mode 100644 index 00000000..3ffb60be --- /dev/null +++ b/.github/workflows/test-functional-keycloak.yaml @@ -0,0 +1,35 @@ +name: test-functional-keycloak + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + + - name: Upgrade and install packages + run: | + bash ./ci/setup-ubuntu.sh + + - name: Install Keycloak + run: | + bash ./ci/setup-keycloak.sh + + - name: Install ColdFront and plugin + run: | + ./ci/setup.sh + + - name: Run functional tests + run: | + ./ci/run_functional_tests_keycloak.sh diff --git a/ci/run_functional_tests_keycloak.sh b/ci/run_functional_tests_keycloak.sh new file mode 100755 index 00000000..1eb9df10 --- /dev/null +++ b/ci/run_functional_tests_keycloak.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -xe + +if [[ ! "${CI}" == "true" ]]; then + source /tmp/coldfront_venv/bin/activate +fi + +export DJANGO_SETTINGS_MODULE="local_settings" +export PYTHONWARNINGS="ignore:Unverified HTTPS request" + +export KEYCLOAK_BASE_URL="http://localhost:8080" +export KEYCLOAK_REALM="master" +export KEYCLOAK_CLIENT_ID="coldfront" +export KEYCLOAK_CLIENT_SECRET="nomoresecret" + +coverage run --source="." -m django test coldfront_plugin_cloud.tests.functional.keycloak +coverage report diff --git a/ci/setup-keycloak.sh b/ci/setup-keycloak.sh new file mode 100755 index 00000000..70394387 --- /dev/null +++ b/ci/setup-keycloak.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -xe + +sudo docker rm -f keycloak + +sudo docker run -d --name keycloak \ + -e KEYCLOAK_ADMIN=admin \ + -e KEYCLOAK_ADMIN_PASSWORD=nomoresecret \ + -p 8080:8080 \ + -p 8443:8443 \ + quay.io/keycloak/keycloak:25.0 start-dev + +# wait for keycloak to be ready +until curl -s http://localhost:8080/auth/realms/master; do + echo "Waiting for Keycloak to be ready..." + sleep 5 +done + +# Create client and add admin role to client's service account +ACCESS_TOKEN=$(curl -X POST "http://localhost:8080/realms/master/protocol/openid-connect/token" \ + -d "username=admin" \ + -d "password=nomoresecret" \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "scope=openid" \ +| jq -r '.access_token') + + +curl -X POST "http://localhost:8080/admin/realms/master/clients" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "clientId": "coldfront", + "secret": "nomoresecret", + "redirectUris": ["http://localhost:8080/*"], + "serviceAccountsEnabled": true + }' + +COLDFRONT_CLIENT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients?clientId=coldfront" \ + -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.[0].id') + + +COLDFRONT_SERVICE_ACCOUNT_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/clients/$COLDFRONT_CLIENT_ID/service-account-user" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ +| jq -r '.id') + +ADMIN_ROLE_ID=$(curl -X GET "http://localhost:8080/admin/realms/master/roles/admin" \ + -H "Authorization: Bearer $ACCESS_TOKEN" | jq -r '.id') + +# Add admin role to the service account user +curl -X POST "http://localhost:8080/admin/realms/master/users/$COLDFRONT_SERVICE_ACCOUNT_ID/role-mappings/realm" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -H "Content-Type: application/json" \ + -d '[ + { + "id": "'$ADMIN_ROLE_ID'", + "name": "admin" + } + ]' diff --git a/requirements.txt b/requirements.txt index b1e6c65b..3c9959d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ python-novaclient python-neutronclient python-swiftclient pytz +requests diff --git a/src/coldfront_plugin_cloud/attributes.py b/src/coldfront_plugin_cloud/attributes.py index 1f09e878..725710a5 100644 --- a/src/coldfront_plugin_cloud/attributes.py +++ b/src/coldfront_plugin_cloud/attributes.py @@ -24,7 +24,6 @@ class CloudAllocationAttribute: RESOURCE_API_URL = "OpenShift API Endpoint URL" RESOURCE_IDENTITY_NAME = "OpenShift Identity Provider Name" RESOURCE_ROLE = "Role for User in Project" -RESOURCE_QUOTA_RESOURCES = "Available Quota Resources" RESOURCE_FEDERATION_PROTOCOL = "OpenStack Federation Protocol" RESOURCE_IDP = "OpenStack Identity Provider" @@ -35,6 +34,8 @@ class CloudAllocationAttribute: RESOURCE_EULA_URL = "EULA URL" RESOURCE_CLUSTER_NAME = "Internal Cluster Name" +RESOURCE_QUOTA_RESOURCES = "Available Quota Resources" +RESOURCE_KEYCLOAK_GROUP_TEMPLATE = "Template String for Keycloak Group Names" RESOURCE_ATTRIBUTES = [ CloudResourceAttribute(name=RESOURCE_AUTH_URL), @@ -45,6 +46,7 @@ class CloudAllocationAttribute: CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN), CloudResourceAttribute(name=RESOURCE_ROLE), CloudResourceAttribute(name=RESOURCE_QUOTA_RESOURCES), + CloudResourceAttribute(name=RESOURCE_KEYCLOAK_GROUP_TEMPLATE), CloudResourceAttribute(name=RESOURCE_USER_DOMAIN), CloudResourceAttribute(name=RESOURCE_EULA_URL), CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK), diff --git a/src/coldfront_plugin_cloud/kc_client.py b/src/coldfront_plugin_cloud/kc_client.py new file mode 100644 index 00000000..295529d1 --- /dev/null +++ b/src/coldfront_plugin_cloud/kc_client.py @@ -0,0 +1,77 @@ +import os + +import requests + + +class KeyCloakAPIClient: + def __init__(self): + self.base_url = os.getenv("KEYCLOAK_BASE_URL") + self.realm = os.getenv("KEYCLOAK_REALM") + self.client_id = os.getenv("KEYCLOAK_CLIENT_ID") + self.client_secret = os.getenv("KEYCLOAK_CLIENT_SECRET") + + self.token_url = ( + f"{self.base_url}/realms/{self.realm}/protocol/openid-connect/token" + ) + + @property + def api_client(self): + params = { + "grant_type": "client_credentials", + "client_id": self.client_id, + "client_secret": self.client_secret, + } + r = requests.post(self.token_url, data=params) + r.raise_for_status() + headers = { + "Authorization": ("Bearer %s" % r.json()["access_token"]), + "Content-Type": "application/json", + } + session = requests.session() + session.headers.update(headers) + return session + + def create_group(self, group_name): + url = f"{self.base_url}/admin/realms/{self.realm}/groups" + payload = {"name": group_name} + response = self.api_client.post(url, json=payload) + + # If group already exists, ignore and move on + if response.status_code not in (201, 409): + response.raise_for_status() + + def get_group_id(self, group_name) -> str | None: + """Return None if group not found""" + query = f"search={group_name}&exact=true" + url = f"{self.base_url}/admin/realms/{self.realm}/groups?{query}" + r = self.api_client.get(url) + r.raise_for_status() + r_json = r.json() + return r_json[0]["id"] if r_json else None + + def get_user_id(self, cf_username) -> str | None: + """Return None if user not found""" + # (Quan) Coldfront usernames map to Keycloak usernames + # https://github.com/nerc-project/coldfront-plugin-cloud/pull/249#discussion_r2953393852 + query = f"username={cf_username}&exact=true" + url = f"{self.base_url}/admin/realms/{self.realm}/users?{query}" + r = self.api_client.get(url) + r.raise_for_status() + r_json = r.json() + return r_json[0]["id"] if r_json else None + + def add_user_to_group(self, user_id, group_id): + url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}" + r = self.api_client.put(url) + r.raise_for_status() + + def remove_user_from_group(self, user_id, group_id): + url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups/{group_id}" + r = self.api_client.delete(url) + r.raise_for_status() + + def get_user_groups(self, user_id) -> list[str]: + url = f"{self.base_url}/admin/realms/{self.realm}/users/{user_id}/groups" + r = self.api_client.get(url) + r.raise_for_status() + return [group["name"] for group in r.json()] diff --git a/src/coldfront_plugin_cloud/signals.py b/src/coldfront_plugin_cloud/signals.py index cdcec841..fa29407e 100644 --- a/src/coldfront_plugin_cloud/signals.py +++ b/src/coldfront_plugin_cloud/signals.py @@ -6,8 +6,10 @@ from coldfront_plugin_cloud.tasks import ( activate_allocation, add_user_to_allocation, + add_user_to_keycloak, disable_allocation, remove_user_from_allocation, + remove_user_from_keycloak, ) from coldfront.core.allocation.signals import ( allocation_activate, @@ -25,6 +27,10 @@ def is_async(): return os.getenv("REDIS_HOST") +def is_keycloak_enabled(): + return os.getenv("KEYCLOAK_BASE_URL") + + @receiver(allocation_activate) @receiver(allocation_change_approved) def activate_allocation_receiver(sender, **kwargs): @@ -48,11 +54,18 @@ def activate_allocation_user_receiver(sender, **kwargs): allocation_user_pk = kwargs.get("allocation_user_pk") if is_async(): async_task(add_user_to_allocation, allocation_user_pk) + if is_keycloak_enabled(): + async_task(add_user_to_keycloak, allocation_user_pk) else: add_user_to_allocation(allocation_user_pk) + if is_keycloak_enabled(): + add_user_to_keycloak(allocation_user_pk) @receiver(allocation_remove_user) def allocation_remove_user_receiver(sender, **kwargs): allocation_user_pk = kwargs.get("allocation_user_pk") remove_user_from_allocation(allocation_user_pk) + + if is_keycloak_enabled(): + remove_user_from_keycloak(allocation_user_pk) diff --git a/src/coldfront_plugin_cloud/tasks.py b/src/coldfront_plugin_cloud/tasks.py index 6adf8c71..b55dfbd1 100644 --- a/src/coldfront_plugin_cloud/tasks.py +++ b/src/coldfront_plugin_cloud/tasks.py @@ -1,8 +1,14 @@ import datetime import logging import time +import functools +from string import Template -from coldfront.core.allocation.models import Allocation, AllocationUser +from coldfront.core.allocation.models import ( + Allocation, + AllocationUser, + AllocationAttribute, +) from coldfront_plugin_cloud import ( attributes, @@ -12,11 +18,17 @@ esi, openshift_vm, utils, + kc_client, ) logger = logging.getLogger(__name__) +@functools.lru_cache() +def get_kc_client(): + return kc_client.KeyCloakAPIClient() + + def find_allocator(allocation) -> base.ResourceAllocator: allocators = { "openstack": openstack.OpenStackResourceAllocator, @@ -128,3 +140,85 @@ def remove_user_from_allocation(allocation_user_pk): allocator.remove_role_from_user(username, project_id) else: logger.warning("No project has been created. Nothing to disable.") + + +def _clean_template_string(template_string: str) -> str: + return template_string.replace(" ", "_").lower() + + +def _get_keycloak_group_name(allocation: Allocation, template_string: str) -> str: + """ + Acceptable variables for the group name template string is: + - $resource_name: the name of the resource (e.g. "OpenShift") + - Any allocation attribute defined for the allocation, with spaces replaced by underscores and + all lowercase (e.g. for `Project Name`, the variable would be `$project_name`) + """ + resource_name = allocation.resources.first().name + allocation_attrs_list: list[AllocationAttribute] = ( + allocation.allocationattribute_set.all() + ) + + template_sub_dict = {"resource_name": resource_name} + for attr in allocation_attrs_list: + template_sub_dict[ + _clean_template_string(attr.allocation_attribute_type.name) + ] = attr.value + + return Template(template_string).substitute(**template_sub_dict) + + +def add_user_to_keycloak(allocation_user_pk): + allocation_user = AllocationUser.objects.get(pk=allocation_user_pk) + allocation = allocation_user.allocation + + kc_admin_client = get_kc_client() + username = allocation_user.user.username + + group_name_template = allocation.resources.first().get_attribute( + attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE + ) + if group_name_template is None: + logger.info( + f"Keycloak enabled but no group name template specified for resource {allocation.resources.first().name}. Skipping addition to Keycloak group" + ) + return + + if (user_id := kc_admin_client.get_user_id(username)) is None: + logger.warning(f"User {username} not found in Keycloak, cannot add to group.") + return + + group_name = _get_keycloak_group_name(allocation, group_name_template) + kc_admin_client.create_group(group_name) + group_id = kc_admin_client.get_group_id(group_name) + kc_admin_client.add_user_to_group(user_id, group_id) + + +def remove_user_from_keycloak(allocation_user_pk): + allocation_user = AllocationUser.objects.get(pk=allocation_user_pk) + allocation = allocation_user.allocation + + kc_admin_client = get_kc_client() + username = allocation_user.user.username + + group_name_template = allocation.resources.first().get_attribute( + attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE + ) + if group_name_template is None: + logger.info( + f"Keycloak enabled but no group name template specified for resource {allocation.resources.first().name}. Skipping removal from Keycloak group" + ) + return + + if (user_id := kc_admin_client.get_user_id(username)) is None: + logger.warning( + f"User {username} not found in Keycloak, cannot remove from group." + ) + return + + group_name = _get_keycloak_group_name(allocation, group_name_template) + if (group_id := kc_admin_client.get_group_id(group_name)) is None: + logger.warning( + f"Group {group_name} not found in Keycloak, skipping removal for user {username}." + ) + return + kc_admin_client.remove_user_from_group(user_id, group_id) diff --git a/src/coldfront_plugin_cloud/tests/functional/keycloak/__init__.py b/src/coldfront_plugin_cloud/tests/functional/keycloak/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/coldfront_plugin_cloud/tests/functional/keycloak/test_keycloak.py b/src/coldfront_plugin_cloud/tests/functional/keycloak/test_keycloak.py new file mode 100644 index 00000000..e515d82a --- /dev/null +++ b/src/coldfront_plugin_cloud/tests/functional/keycloak/test_keycloak.py @@ -0,0 +1,231 @@ +from django.contrib.auth.models import User +from coldfront.core.resource.models import ResourceAttribute, ResourceAttributeType + +from coldfront_plugin_cloud import tasks, kc_client, attributes, utils +from coldfront_plugin_cloud.tests import base + + +class TestKeyCloakUserManagement(base.TestBase): + @classmethod + def setUpTestData(cls) -> None: + super().setUpTestData() + cls.kc_admin_client = kc_client.KeyCloakAPIClient() + cls.resource = cls.new_openshift_resource( + name="Test Resource", + ) + ResourceAttribute.objects.get_or_create( + resource_attribute_type=ResourceAttributeType.objects.get( + name=attributes.RESOURCE_KEYCLOAK_GROUP_TEMPLATE + ), + resource=cls.resource, + value="$resource_name/$allocated_project_id", + ) + + def new_keycloak_user(self, cf_username): + url = f"{self.kc_admin_client.base_url}/admin/realms/{self.kc_admin_client.realm}/users" + payload = { + "username": cf_username, + "enabled": True, + "email": cf_username, + } + r = self.kc_admin_client.api_client.post(url, json=payload) + r.raise_for_status() + + def new_user(self, username=None, add_to_keycloak=True) -> User: + user = super().new_user(username) + if add_to_keycloak: + self.new_keycloak_user(user.username) + return user + + def new_allocation( + self, project, resource, quantity, status="Active", attr_value="Test Value" + ): + allocation = super().new_allocation(project, resource, quantity, status) + utils.set_attribute_on_allocation( + allocation, attributes.ALLOCATION_PROJECT_ID, attr_value + ) + return allocation + + def test_user_added_to_allocation(self): + """Test that when a user is added to an allocation, they exist in Keycloak and are in the project group.""" + user = self.new_user() + project = self.new_project(pi=user) + allocation = self.new_allocation(project, self.resource, 1) + allocation_user = self.new_allocation_user(allocation, user) + + # Simulate triggering the allocation activate signal + tasks.add_user_to_keycloak(allocation_user.pk) + + # Check that the user exists in Keycloak + user_id = self.kc_admin_client.get_user_id(user.username) + self.assertIsNotNone(user_id) + + # Check that the user is in the project group + # Group name determined by the RESOURCE_KEYCLOAK_GROUP_TEMPLATE attribute, set to "$resource_name/$allocated_project_id" in tests + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertIn(f"{self.resource.name}/Test Value", user_groups) + + def test_user_removed_from_allocation(self): + """Test that when a user is removed from an allocation, they are removed from the project group.""" + user = self.new_user() + project = self.new_project(pi=user) + allocation = self.new_allocation(project, self.resource, 1) + allocation_user = self.new_allocation_user(allocation, user) + + tasks.add_user_to_keycloak(allocation_user.pk) + + user_id = self.kc_admin_client.get_user_id(user.username) + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertIn(f"{self.resource.name}/Test Value", user_groups) + + tasks.remove_user_from_keycloak(allocation_user.pk) + + # Check that the user is no longer in the group + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertNotIn(f"{self.resource.name}/Test Value", user_groups) + + def test_user_not_in_keycloak_added_to_allocation(self): + """Test that when a user not in Keycloak is added to an allocation, they are not added to the group.""" + user = self.new_user(add_to_keycloak=False) + project = self.new_project(pi=user) + allocation = self.new_allocation( + project, self.resource, 1, attr_value="Test Not Created" + ) + allocation_user = self.new_allocation_user(allocation, user) + + # Should not raise error + tasks.add_user_to_keycloak(allocation_user.pk) + + user_id = self.kc_admin_client.get_user_id(user.username) + self.assertIsNone(user_id) + + # Verify the group was not created at all + group_id = self.kc_admin_client.get_group_id( + f"{self.resource.name}/Test Not Created" + ) + self.assertIsNone(group_id) + + def test_user_not_in_keycloak_removed_from_allocation(self): + """Test that when a user not in Keycloak is removed from an allocation, no error occurs.""" + user = self.new_user(add_to_keycloak=False) + project = self.new_project(pi=user) + allocation = self.new_allocation(project, self.resource, 1) + allocation_user = self.new_allocation_user(allocation, user) + + # Verify the user doesn't exist in Keycloak + user_id = self.kc_admin_client.get_user_id(user.username) + self.assertIsNone(user_id) + + # Try to remove the user from the allocation (should not raise an error) + tasks.remove_user_from_keycloak(allocation_user.pk) + + def test_multiple_users_in_same_allocation(self): + """Test that multiple users can be added to the same allocation and are all in the group.""" + pi = self.new_user() + project = self.new_project(pi=pi) + allocation = self.new_allocation(project, self.resource, 2) + + # Add multiple users to the allocation + users = [self.new_user() for _ in range(3)] + allocation_users = [ + self.new_allocation_user(allocation, user) for user in users + ] + + for allocation_user in allocation_users: + tasks.add_user_to_keycloak(allocation_user.pk) + + # Verify all users are in the group + for user in users: + user_id = self.kc_admin_client.get_user_id(user.username) + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertIn(f"{self.resource.name}/Test Value", user_groups) + + def test_remove_one_user_keeps_others_in_group(self): + """Test that removing one user from an allocation doesn't affect other users in the group.""" + pi = self.new_user() + project = self.new_project(pi=pi) + allocation = self.new_allocation(project, self.resource, 2) + + users = [self.new_user() for _ in range(2)] + allocation_users = [ + self.new_allocation_user(allocation, user) for user in users + ] + + for allocation_user in allocation_users: + tasks.add_user_to_keycloak(allocation_user.pk) + + tasks.remove_user_from_keycloak(allocation_users[0].pk) + + # Verify all users except the removed one are still in the group + user1_id = self.kc_admin_client.get_user_id(users[0].username) + user1_groups = self.kc_admin_client.get_user_groups(user1_id) + self.assertNotIn(f"{self.resource.name}/Test Value", user1_groups) + + user2_id = self.kc_admin_client.get_user_id(users[1].username) + user2_groups = self.kc_admin_client.get_user_groups(user2_id) + self.assertIn(f"{self.resource.name}/Test Value", user2_groups) + + def test_user_in_multiple_allocations_groups(self): + """Test that a user can be in multiple project groups from different allocations.""" + user = self.new_user() + + project1 = self.new_project(pi=user) + allocation1 = self.new_allocation( + project1, self.resource, 1, attr_value="Test Value 1" + ) + + project2 = self.new_project(pi=user) + allocation2 = self.new_allocation( + project2, self.resource, 1, attr_value="Test Value 2" + ) + + # Add user to both allocations + allocation_user1 = self.new_allocation_user(allocation1, user) + allocation_user2 = self.new_allocation_user(allocation2, user) + + tasks.add_user_to_keycloak(allocation_user1.pk) + tasks.add_user_to_keycloak(allocation_user2.pk) + + # Verify user is in both groups + user_id = self.kc_admin_client.get_user_id(user.username) + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertIn(f"{self.resource.name}/Test Value 1", user_groups) + self.assertIn(f"{self.resource.name}/Test Value 2", user_groups) + + # Remove user from first allocation + tasks.remove_user_from_keycloak(allocation_user1.pk) + + # Verify user is now only in second group + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertNotIn(f"{self.resource.name}/Test Value 1", user_groups) + self.assertIn(f"{self.resource.name}/Test Value 2", user_groups) + + def test_user_added_without_keycloak_group_template(self): + """Test that when the Keycloak group template attribute is not present on the resource, the user is not added to group and a log message is captured.""" + # Create a resource without the Keycloak group template attribute + resource_no_template = self.new_openshift_resource(name="Resource No Template") + + user = self.new_user() + project = self.new_project(pi=user) + allocation = self.new_allocation( + project, resource_no_template, 1, attr_value="Test No Template" + ) + allocation_user = self.new_allocation_user(allocation, user) + + # Capture the log message + with self.assertLogs("coldfront_plugin_cloud.tasks", level="INFO") as log: + tasks.add_user_to_keycloak(allocation_user.pk) + + # Verify the warning was logged + self.assertEqual(len(log.records), 1) + self.assertIn( + "Keycloak enabled but no group name template specified for resource Resource No Template", + log.records[0].getMessage(), + ) + self.assertIn(resource_no_template.name, log.records[0].getMessage()) + + # Verify the user exists in Keycloak but is not in any groups + user_id = self.kc_admin_client.get_user_id(user.username) + self.assertIsNotNone(user_id) + user_groups = self.kc_admin_client.get_user_groups(user_id) + self.assertEqual(user_groups, [])