diff --git a/scripts/deploy/__init__.py b/scripts/deploy/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/scripts/deploy/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/deploy/pixi.py b/scripts/deploy/pixi.py new file mode 100644 index 000000000..e22623ad4 --- /dev/null +++ b/scripts/deploy/pixi.py @@ -0,0 +1,76 @@ +import concurrent.futures +from collections.abc import Callable + +from fabric import Group, GroupResult, Result + +from deploy.misc import Connection, hide_output, print_debug, print_info + +_INTERNET_STATUS_ATTR = "_bitbots_deploy_internet_available" +_PIXI_REPOSITORY_URL = "https://data.bit-bots.de/conda-misc/output/" +_PIXI_REPOSITORY_CHECK_COMMAND = f"curl -sSfLI --max-time 5 {_PIXI_REPOSITORY_URL}" + + +def robot_has_internet(connection: Connection) -> bool: + cached_status: bool | None = getattr(connection, _INTERNET_STATUS_ATTR, None) + if cached_status is not None: + return cached_status + + print_debug( + f"Checking for internet connection from {connection.original_host} to Pixi repository: {_PIXI_REPOSITORY_URL}" + ) + + cmd = _PIXI_REPOSITORY_CHECK_COMMAND + print_debug(f"Calling '{cmd}' on: {connection.host}") + result = connection.run(cmd, hide=hide_output(), warn=True) + available = result.ok + + setattr(connection, _INTERNET_STATUS_ATTR, available) + print_debug( + f"Internet connection to Pixi repository on {connection.original_host} " + f"is{' ' if available else ' NOT '}available." + ) + if not available: + print_info( + f"Internet connection is not available on {connection.original_host}. " + "Remote pixi run commands will use --as-is." + ) + return available + + +def clear_robot_internet_status(connection: Connection) -> None: + if hasattr(connection, _INTERNET_STATUS_ATTR): + delattr(connection, _INTERNET_STATUS_ATTR) + + +def pixi_run_command(connection: Connection, remote_workspace: str, pixi_command: str) -> str: + as_is_argument = "" if robot_has_internet(connection) else " --as-is" + return f"cd {remote_workspace} && pixi run{as_is_argument} --environment robot {pixi_command}" + + +def pixi_install_if_online(connection: Connection, remote_workspace: str) -> Result: + if not robot_has_internet(connection): + command = "# Skipping pixi install because the robot has no internet connection." + stdout = "Skipped pixi install because the robot has no internet connection." + print_debug(stdout) + return Result(connection=connection, command=command, stdout=stdout, exited=0) + + cmd = f"cd {remote_workspace} && pixi install --environment robot" + print_debug("Installing dependencies on remote host to verify synchronization and architecture match.") + print_debug(f"Calling '{cmd}' on: {connection.host}") + return connection.run(cmd, hide=hide_output(), warn=True) + + +def run_for_connections(connections: Group, command_factory: Callable[[Connection], str]) -> GroupResult: + def _run_single(connection: Connection) -> Result: + cmd = command_factory(connection) + print_debug(f"Calling '{cmd}' on: {connection.host}") + return connection.run(cmd, hide=hide_output(), warn=True) + + results = GroupResult() + with concurrent.futures.ThreadPoolExecutor(max_workers=len(connections)) as executor: + futures = [executor.submit(_run_single, connection) for connection in connections] + + for future in futures: + result = future.result() + results[result.connection] = result + return results diff --git a/scripts/deploy/tasks/build.py b/scripts/deploy/tasks/build.py index b6468d284..6ec648251 100644 --- a/scripts/deploy/tasks/build.py +++ b/scripts/deploy/tasks/build.py @@ -1,7 +1,8 @@ -from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error -from deploy.tasks.abstract_task import AbstractTask from fabric import Group, GroupResult -from fabric.exceptions import GroupException + +from deploy.misc import get_connections_from_succeeded, print_debug, print_error +from deploy.pixi import pixi_run_command, run_for_connections +from deploy.tasks.abstract_task import AbstractTask class Build(AbstractTask): @@ -42,16 +43,15 @@ def _clean(self, connections: Group) -> GroupResult: :return: The results of the task. """ print_debug(f"Cleaning the following packages before building: {self._package}") - cmd_clean = f"cd {self._remote_workspace} && pixi run --environment robot clean {self._package}" - - print_debug(f"Calling '{cmd_clean}'") - try: - results = connections.run(cmd_clean, hide=hide_output()) + results = run_for_connections( + connections, + lambda connection: pixi_run_command(connection, self._remote_workspace, f"clean {self._package}"), + ) + if results.succeeded: print_debug(f"Clean succeeded on the following hosts: {self._succeeded_hosts(results)}") - except GroupException as e: - for connection, result in e.result.failed.items(): + if results.failed: + for connection, result in results.failed.items(): print_error(f"Clean on {connection.host} failed with the following errors: {result.stderr}") - return e.result return results def _build(self, connections: Group) -> GroupResult: @@ -63,14 +63,13 @@ def _build(self, connections: Group) -> GroupResult: """ print_debug("Building packages") - cmd = f"cd {self._remote_workspace} && pixi run --environment robot build {self._package}" - - print_debug(f"Calling '{cmd}'") - try: - results = connections.run(cmd, hide=hide_output()) + results = run_for_connections( + connections, + lambda connection: pixi_run_command(connection, self._remote_workspace, f"build {self._package}"), + ) + if results.succeeded: print_debug(f"Build succeeded on the following hosts: {self._succeeded_hosts(results)}") - except GroupException as e: - for connection in e.result.failed.keys(): + if results.failed: + for connection in results.failed.keys(): print_error(f"Build on {connection.host} failed!") - return e.result return results diff --git a/scripts/deploy/tasks/configure.py b/scripts/deploy/tasks/configure.py index 7be90b84a..ca9b9c541 100644 --- a/scripts/deploy/tasks/configure.py +++ b/scripts/deploy/tasks/configure.py @@ -1,5 +1,8 @@ import concurrent.futures +from fabric import Group, GroupResult, Result +from rich.prompt import Prompt + from deploy.misc import ( CONSOLE, Connection, @@ -9,9 +12,8 @@ print_error, print_info, ) +from deploy.pixi import clear_robot_internet_status, pixi_run_command from deploy.tasks.abstract_task import AbstractTaskWhichRequiresSudo -from fabric import Group, GroupResult, Result -from rich.prompt import Prompt class Configure(AbstractTaskWhichRequiresSudo): @@ -59,7 +61,11 @@ def _configure_single(connection: Connection) -> Result: :return: The result of the task. """ print_info(f"Configuring game settings on {connection.host}...") - cmd = f"cd {self._remote_workspace} && pixi run --environment robot python3 {self._remote_workspace}/src/bitbots_misc/bitbots_parameter_blackboard/bitbots_parameter_blackboard/game_settings.py" + cmd = pixi_run_command( + connection, + self._remote_workspace, + f"python3 {self._remote_workspace}/src/bitbots_misc/bitbots_parameter_blackboard/bitbots_parameter_blackboard/game_settings.py", + ) print_debug(f"Calling '{cmd}'") results = connection.run( @@ -183,6 +189,8 @@ def _configure_single(connection: Connection, answered_connection_id: str) -> Re print_error( f"Could not set priority of connection {answered_connection_id} to 100 on {connection.host}" ) + else: + clear_robot_internet_status(connection) return set_priority_result # Get wifi UUID to enable from user diff --git a/scripts/deploy/tasks/launch.py b/scripts/deploy/tasks/launch.py index a4334d376..12df5dee9 100644 --- a/scripts/deploy/tasks/launch.py +++ b/scripts/deploy/tasks/launch.py @@ -1,10 +1,12 @@ import re -from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error, print_success -from deploy.tasks.abstract_task import AbstractTask from fabric import Group, GroupResult, Result from fabric.exceptions import GroupException +from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error, print_success +from deploy.pixi import pixi_run_command, run_for_connections +from deploy.tasks.abstract_task import AbstractTask + class Launch(AbstractTask): def __init__(self, remote_workspace: str, tmux_session_name: str) -> None: @@ -55,18 +57,16 @@ def _check_nodes_already_running(self, connections: Group) -> GroupResult: :return: Results, with success if ROS 2 nodes are not already running """ print_debug("Checking if ROS 2 nodes are already running") - cmd = f"cd {self._remote_workspace} && pixi run --environment robot ros2 node list -c" - - print_debug(f"Calling '{cmd}'") - try: - node_list_results = connections.run(cmd, hide=hide_output()) - print_debug(f"Calling '{cmd}' succeeded on: {self._succeeded_hosts(node_list_results)}") - except GroupException as e: - print_error(f"Calling '{cmd}' failed on: {self._failed_hosts(e.result)}") - if not e.result.succeeded: # TODO: Does run immediately fail if one host fails? Do we even get here? - return e.result - else: - node_list_results = e.result + node_list_results = run_for_connections( + connections, + lambda connection: pixi_run_command(connection, self._remote_workspace, "ros2 node list -c"), + ) + if node_list_results.succeeded: + print_debug(f"Checking ROS 2 nodes succeeded on: {self._succeeded_hosts(node_list_results)}") + if node_list_results.failed: + print_error(f"Checking ROS 2 nodes failed on: {self._failed_hosts(node_list_results)}") + if not node_list_results.succeeded: + return node_list_results # Check if output was ^0$ (zero for no nodes running) for all remote machines # Create new group result with success if no nodes are running @@ -151,12 +151,8 @@ def _check_tmux_session_already_running(self, connections: Group) -> GroupResult def _launch_teamplayer(self, connections: Group) -> GroupResult: print_debug("Launching teamplayer") - # Create tmux session - cmd = f"tmux new-session -d -s {self._tmux_session_name} && tmux send-keys -t {self._tmux_session_name} 'cd {self._remote_workspace} && pixi run --environment robot ros2 launch bitbots_bringup teamplayer.launch record:=true tts:=false' Enter" - - print_debug(f"Calling '{cmd}'") - try: - results = connections.run(cmd, hide=hide_output()) + results = run_for_connections(connections, self._launch_command) + if results.succeeded: # Print commands to connect to teamplayer tmux session help_cmds = "" for connection in results.succeeded: @@ -164,9 +160,19 @@ def _launch_teamplayer(self, connections: Group) -> GroupResult: print_success( f"Teamplayer launched successfully on {self._succeeded_hosts(results)}!\nTo attach to the tmux session, run:\n\n{help_cmds}" ) - except GroupException as e: + if results.failed: print_error( - f"Creating tmux session called {self._tmux_session_name} failed OR launching teamplayer failed on the following hosts: {self._failed_hosts(e.result)}" + f"Creating tmux session called {self._tmux_session_name} failed OR launching teamplayer failed on the following hosts: {self._failed_hosts(results)}" ) - return e.result return results + + def _launch_command(self, connection) -> str: + pixi_cmd = pixi_run_command( + connection, + self._remote_workspace, + "ros2 launch bitbots_bringup teamplayer.launch record:=true tts:=false", + ) + return ( + f"tmux new-session -d -s {self._tmux_session_name} && " + f"tmux send-keys -t {self._tmux_session_name} '{pixi_cmd}' Enter" + ) diff --git a/scripts/deploy/tasks/sync.py b/scripts/deploy/tasks/sync.py index 92e9cb0f0..bbfe3e33b 100644 --- a/scripts/deploy/tasks/sync.py +++ b/scripts/deploy/tasks/sync.py @@ -1,9 +1,11 @@ import concurrent.futures import os +from fabric import Group, GroupResult, Result + from deploy.misc import Connection, be_quiet, hide_output, print_debug, print_error +from deploy.pixi import pixi_install_if_online from deploy.tasks.abstract_task import AbstractTask -from fabric import Group, GroupResult, Result class Sync(AbstractTask): @@ -67,11 +69,7 @@ def _sync_single(connection: Connection) -> Result | None: print_error(f"Rsync command failed to execute on host {connection.host}") return - # Verify syncing succeeded and system architectures matches by running "pixi install --environment robot" - cmd = f"cd {self._remote_workspace} && pixi install --environment robot" - print_debug("Installing dependencies on remote host to verify synchronization and architecture match.") - print_debug(f"Calling '{cmd}' on: {connection.host}") - verify_result = connection.run(cmd, hide=hide_output()) + verify_result = pixi_install_if_online(connection, self._remote_workspace) return verify_result diff --git a/scripts/deploy/tests/test_pixi.py b/scripts/deploy/tests/test_pixi.py new file mode 100644 index 000000000..cfc44bc81 --- /dev/null +++ b/scripts/deploy/tests/test_pixi.py @@ -0,0 +1,81 @@ +import sys +from pathlib import Path + +_SCRIPTS_PATH = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_SCRIPTS_PATH)) + +from deploy import pixi # noqa: E402 + +_PIXI_REPOSITORY_CHECK_COMMAND = "curl -sSfLI --max-time 2 https://data.bit-bots.de/conda-misc/output/" + + +def test_pixi_run_command_uses_as_is_when_robot_is_offline(): + connection = _FakeConnection(internet_available=False) + + command = pixi.pixi_run_command(connection, "~/bitbots/bitbots_main", "build bitbots_utils") + + assert command == "cd ~/bitbots/bitbots_main && pixi run --as-is --environment robot build bitbots_utils" + + +def test_pixi_run_command_omits_as_is_when_robot_is_online(): + connection = _FakeConnection(internet_available=True) + + command = pixi.pixi_run_command(connection, "~/bitbots/bitbots_main", "build bitbots_utils") + + assert command == "cd ~/bitbots/bitbots_main && pixi run --environment robot build bitbots_utils" + + +def test_robot_has_internet_caches_result(): + connection = _FakeConnection(internet_available=False) + + assert pixi.robot_has_internet(connection) is False + assert pixi.robot_has_internet(connection) is False + + assert connection.commands == [_PIXI_REPOSITORY_CHECK_COMMAND] + + +def test_pixi_install_is_skipped_when_robot_is_offline(): + connection = _FakeConnection(internet_available=False) + + result = pixi.pixi_install_if_online(connection, "~/bitbots/bitbots_main") + + assert result.ok + assert result.command == "# Skipping pixi install because the robot has no internet connection." + assert connection.commands == [_PIXI_REPOSITORY_CHECK_COMMAND] + + +def test_pixi_install_runs_when_robot_is_online(): + connection = _FakeConnection(internet_available=True) + + result = pixi.pixi_install_if_online(connection, "~/bitbots/bitbots_main") + + assert result.ok + assert connection.commands == [ + _PIXI_REPOSITORY_CHECK_COMMAND, + "cd ~/bitbots/bitbots_main && pixi install --environment robot", + ] + + +class _FakeConnection: + def __init__(self, internet_available: bool): + self.host = "10.0.0.1" + self.original_host = "nuc" + self.internet_available = internet_available + self.commands = [] + + def run(self, command, hide=None, warn=False): + self.commands.append(command) + if command == _PIXI_REPOSITORY_CHECK_COMMAND: + return _FakeResult(self, command, ok=self.internet_available) + return _FakeResult(self, command, ok=True) + + +class _FakeResult: + def __init__(self, connection: _FakeConnection, command: str, ok: bool): + self.connection = connection + self.command = command + self.ok = ok + self.failed = not ok + self.stdout = "" + self.stderr = "" + self.exited = int(not ok)