Deploy: Implement internet connectivity checks

scripts/deploy/__init__.py

@@ -0,0 +1 @@
+

scripts/deploy/pixi.py

@@ -0,0 +1,76 @@
+import concurrent.futures
+from collections.abc import Callable
+
+from fabric import Group, GroupResult, Result
+
+from deploy.misc import Connection, hide_output, print_debug, print_info
+
+_INTERNET_STATUS_ATTR = "_bitbots_deploy_internet_available"
+_PIXI_REPOSITORY_URL = "https://data.bit-bots.de/conda-misc/output/"
+_PIXI_REPOSITORY_CHECK_COMMAND = f"curl -sSfLI --max-time 5 {_PIXI_REPOSITORY_URL}"
+
+
+def robot_has_internet(connection: Connection) -> bool:
+    cached_status: bool | None = getattr(connection, _INTERNET_STATUS_ATTR, None)
+    if cached_status is not None:
+        return cached_status
+
+    print_debug(
+        f"Checking for internet connection from {connection.original_host} to Pixi repository: {_PIXI_REPOSITORY_URL}"
+    )
+
+    cmd = _PIXI_REPOSITORY_CHECK_COMMAND
+    print_debug(f"Calling '{cmd}' on: {connection.host}")
+    result = connection.run(cmd, hide=hide_output(), warn=True)
+    available = result.ok
+
+    setattr(connection, _INTERNET_STATUS_ATTR, available)
+    print_debug(
+        f"Internet connection to Pixi repository on {connection.original_host} "
+        f"is{' ' if available else ' NOT '}available."
+    )
+    if not available:
+        print_info(
+            f"Internet connection is not available on {connection.original_host}. "
+            "Remote pixi run commands will use --as-is."
+        )
+    return available
+
+
+def clear_robot_internet_status(connection: Connection) -> None:
+    if hasattr(connection, _INTERNET_STATUS_ATTR):
+        delattr(connection, _INTERNET_STATUS_ATTR)
+
+
+def pixi_run_command(connection: Connection, remote_workspace: str, pixi_command: str) -> str:
+    as_is_argument = "" if robot_has_internet(connection) else " --as-is"
+    return f"cd {remote_workspace} && pixi run{as_is_argument} --environment robot {pixi_command}"
+
+
+def pixi_install_if_online(connection: Connection, remote_workspace: str) -> Result:
+    if not robot_has_internet(connection):
+        command = "# Skipping pixi install because the robot has no internet connection."
+        stdout = "Skipped pixi install because the robot has no internet connection."
+        print_debug(stdout)
+        return Result(connection=connection, command=command, stdout=stdout, exited=0)
+
+    cmd = f"cd {remote_workspace} && pixi install --environment robot"
+    print_debug("Installing dependencies on remote host to verify synchronization and architecture match.")
+    print_debug(f"Calling '{cmd}' on: {connection.host}")
+    return connection.run(cmd, hide=hide_output(), warn=True)
+
+
+def run_for_connections(connections: Group, command_factory: Callable[[Connection], str]) -> GroupResult:
+    def _run_single(connection: Connection) -> Result:
+        cmd = command_factory(connection)
+        print_debug(f"Calling '{cmd}' on: {connection.host}")
+        return connection.run(cmd, hide=hide_output(), warn=True)
+
+    results = GroupResult()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=len(connections)) as executor:
+        futures = [executor.submit(_run_single, connection) for connection in connections]
+
+    for future in futures:
+        result = future.result()
+        results[result.connection] = result
+    return results

scripts/deploy/tasks/build.py

@@ -1,7 +1,8 @@
-from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error
-from deploy.tasks.abstract_task import AbstractTask
 from fabric import Group, GroupResult
-from fabric.exceptions import GroupException
+
+from deploy.misc import get_connections_from_succeeded, print_debug, print_error
+from deploy.pixi import pixi_run_command, run_for_connections
+from deploy.tasks.abstract_task import AbstractTask
 
 
 class Build(AbstractTask):
@@ -42,16 +43,15 @@ def _clean(self, connections: Group) -> GroupResult:
         :return: The results of the task.
         """
         print_debug(f"Cleaning the following packages before building: {self._package}")
-        cmd_clean = f"cd {self._remote_workspace} && pixi run --environment robot clean {self._package}"
-
-        print_debug(f"Calling '{cmd_clean}'")
-        try:
-            results = connections.run(cmd_clean, hide=hide_output())
+        results = run_for_connections(
+            connections,
+            lambda connection: pixi_run_command(connection, self._remote_workspace, f"clean {self._package}"),
+        )
+        if results.succeeded:
             print_debug(f"Clean succeeded on the following hosts: {self._succeeded_hosts(results)}")
-        except GroupException as e:
-            for connection, result in e.result.failed.items():
+        if results.failed:
+            for connection, result in results.failed.items():
                 print_error(f"Clean on {connection.host} failed with the following errors: {result.stderr}")
-            return e.result
         return results
 
     def _build(self, connections: Group) -> GroupResult:
@@ -63,14 +63,13 @@ def _build(self, connections: Group) -> GroupResult:
         """
         print_debug("Building packages")
 
-        cmd = f"cd {self._remote_workspace} && pixi run --environment robot build {self._package}"
-
-        print_debug(f"Calling '{cmd}'")
-        try:
-            results = connections.run(cmd, hide=hide_output())
+        results = run_for_connections(
+            connections,
+            lambda connection: pixi_run_command(connection, self._remote_workspace, f"build {self._package}"),
+        )
+        if results.succeeded:
             print_debug(f"Build succeeded on the following hosts: {self._succeeded_hosts(results)}")
-        except GroupException as e:
-            for connection in e.result.failed.keys():
+        if results.failed:
+            for connection in results.failed.keys():
                 print_error(f"Build on {connection.host} failed!")
-            return e.result
         return results

scripts/deploy/tasks/configure.py

@@ -1,5 +1,8 @@
 import concurrent.futures
 
+from fabric import Group, GroupResult, Result
+from rich.prompt import Prompt
+
 from deploy.misc import (
     CONSOLE,
     Connection,
@@ -9,9 +12,8 @@
     print_error,
     print_info,
 )
+from deploy.pixi import clear_robot_internet_status, pixi_run_command
 from deploy.tasks.abstract_task import AbstractTaskWhichRequiresSudo
-from fabric import Group, GroupResult, Result
-from rich.prompt import Prompt
 
 
 class Configure(AbstractTaskWhichRequiresSudo):
@@ -59,7 +61,11 @@ def _configure_single(connection: Connection) -> Result:
             :return: The result of the task.
             """
             print_info(f"Configuring game settings on {connection.host}...")
-            cmd = f"cd {self._remote_workspace} && pixi run --environment robot python3 {self._remote_workspace}/src/bitbots_misc/bitbots_parameter_blackboard/bitbots_parameter_blackboard/game_settings.py"
+            cmd = pixi_run_command(
+                connection,
+                self._remote_workspace,
+                f"python3 {self._remote_workspace}/src/bitbots_misc/bitbots_parameter_blackboard/bitbots_parameter_blackboard/game_settings.py",
+            )
 
             print_debug(f"Calling '{cmd}'")
             results = connection.run(
@@ -183,6 +189,8 @@ def _configure_single(connection: Connection, answered_connection_id: str) -> Re
                     print_error(
                         f"Could not set priority of connection {answered_connection_id} to 100 on {connection.host}"
                     )
+                else:
+                    clear_robot_internet_status(connection)
                 return set_priority_result
 
         # Get wifi UUID to enable from user

scripts/deploy/tasks/launch.py

@@ -1,10 +1,12 @@
 import re
 
-from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error, print_success
-from deploy.tasks.abstract_task import AbstractTask
 from fabric import Group, GroupResult, Result
 from fabric.exceptions import GroupException
 
+from deploy.misc import get_connections_from_succeeded, hide_output, print_debug, print_error, print_success
+from deploy.pixi import pixi_run_command, run_for_connections
+from deploy.tasks.abstract_task import AbstractTask
+
 
 class Launch(AbstractTask):
     def __init__(self, remote_workspace: str, tmux_session_name: str) -> None:
@@ -55,18 +57,16 @@ def _check_nodes_already_running(self, connections: Group) -> GroupResult:
         :return: Results, with success if ROS 2 nodes are not already running
         """
         print_debug("Checking if ROS 2 nodes are already running")
-        cmd = f"cd {self._remote_workspace} && pixi run --environment robot ros2 node list -c"
-
-        print_debug(f"Calling '{cmd}'")
-        try:
-            node_list_results = connections.run(cmd, hide=hide_output())
-            print_debug(f"Calling '{cmd}' succeeded on: {self._succeeded_hosts(node_list_results)}")
-        except GroupException as e:
-            print_error(f"Calling '{cmd}' failed on: {self._failed_hosts(e.result)}")
-            if not e.result.succeeded:  # TODO: Does run immediately fail if one host fails? Do we even get here?
-                return e.result
-            else:
-                node_list_results = e.result
+        node_list_results = run_for_connections(
+            connections,
+            lambda connection: pixi_run_command(connection, self._remote_workspace, "ros2 node list -c"),
+        )
+        if node_list_results.succeeded:
+            print_debug(f"Checking ROS 2 nodes succeeded on: {self._succeeded_hosts(node_list_results)}")
+        if node_list_results.failed:
+            print_error(f"Checking ROS 2 nodes failed on: {self._failed_hosts(node_list_results)}")
+            if not node_list_results.succeeded:
+                return node_list_results
 
         # Check if output was ^0$ (zero for no nodes running) for all remote machines
         # Create new group result with success if no nodes are running
@@ -151,22 +151,28 @@ def _check_tmux_session_already_running(self, connections: Group) -> GroupResult
 
     def _launch_teamplayer(self, connections: Group) -> GroupResult:
         print_debug("Launching teamplayer")
-        # Create tmux session
-        cmd = f"tmux new-session -d -s {self._tmux_session_name} && tmux send-keys -t {self._tmux_session_name} 'cd {self._remote_workspace} && pixi run --environment robot ros2 launch bitbots_bringup teamplayer.launch record:=true tts:=false' Enter"
-
-        print_debug(f"Calling '{cmd}'")
-        try:
-            results = connections.run(cmd, hide=hide_output())
+        results = run_for_connections(connections, self._launch_command)
+        if results.succeeded:
             # Print commands to connect to teamplayer tmux session
             help_cmds = ""
             for connection in results.succeeded:
                 help_cmds += f"{connection.original_host} : [bold]ssh {connection.user}@{connection.host} -t 'tmux attach-session -t {self._tmux_session_name}'[/bold]\n"
             print_success(
                 f"Teamplayer launched successfully on {self._succeeded_hosts(results)}!\nTo attach to the tmux session, run:\n\n{help_cmds}"
             )
-        except GroupException as e:
+        if results.failed:
             print_error(
-                f"Creating tmux session called {self._tmux_session_name} failed OR launching teamplayer failed on the following hosts: {self._failed_hosts(e.result)}"
+                f"Creating tmux session called {self._tmux_session_name} failed OR launching teamplayer failed on the following hosts: {self._failed_hosts(results)}"
             )
-            return e.result
         return results
+
+    def _launch_command(self, connection) -> str:
+        pixi_cmd = pixi_run_command(
+            connection,
+            self._remote_workspace,
+            "ros2 launch bitbots_bringup teamplayer.launch record:=true tts:=false",
+        )
+        return (
+            f"tmux new-session -d -s {self._tmux_session_name} && "
+            f"tmux send-keys -t {self._tmux_session_name} '{pixi_cmd}' Enter"
+        )

scripts/deploy/tasks/sync.py

@@ -1,9 +1,11 @@
 import concurrent.futures
 import os
 
+from fabric import Group, GroupResult, Result
+
 from deploy.misc import Connection, be_quiet, hide_output, print_debug, print_error
+from deploy.pixi import pixi_install_if_online
 from deploy.tasks.abstract_task import AbstractTask
-from fabric import Group, GroupResult, Result
 
 
 class Sync(AbstractTask):
@@ -67,11 +69,7 @@ def _sync_single(connection: Connection) -> Result | None:
                 print_error(f"Rsync command failed to execute on host {connection.host}")
                 return
 
-            # Verify syncing succeeded and system architectures matches by running "pixi install --environment robot"
-            cmd = f"cd {self._remote_workspace} && pixi install --environment robot"
-            print_debug("Installing dependencies on remote host to verify synchronization and architecture match.")
-            print_debug(f"Calling '{cmd}' on: {connection.host}")
-            verify_result = connection.run(cmd, hide=hide_output())
+            verify_result = pixi_install_if_online(connection, self._remote_workspace)
 
             return verify_result
 

scripts/deploy/tests/test_pixi.py

@@ -0,0 +1,81 @@
+import sys
+from pathlib import Path
+
+_SCRIPTS_PATH = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(_SCRIPTS_PATH))
+
+from deploy import pixi  # noqa: E402
+
+_PIXI_REPOSITORY_CHECK_COMMAND = "curl -sSfLI --max-time 2 https://data.bit-bots.de/conda-misc/output/"
+
+
+def test_pixi_run_command_uses_as_is_when_robot_is_offline():
+    connection = _FakeConnection(internet_available=False)
+
+    command = pixi.pixi_run_command(connection, "~/bitbots/bitbots_main", "build bitbots_utils")
+
+    assert command == "cd ~/bitbots/bitbots_main && pixi run --as-is --environment robot build bitbots_utils"
+
+
+def test_pixi_run_command_omits_as_is_when_robot_is_online():
+    connection = _FakeConnection(internet_available=True)
+
+    command = pixi.pixi_run_command(connection, "~/bitbots/bitbots_main", "build bitbots_utils")
+
+    assert command == "cd ~/bitbots/bitbots_main && pixi run --environment robot build bitbots_utils"
+
+
+def test_robot_has_internet_caches_result():
+    connection = _FakeConnection(internet_available=False)
+
+    assert pixi.robot_has_internet(connection) is False
+    assert pixi.robot_has_internet(connection) is False
+
+    assert connection.commands == [_PIXI_REPOSITORY_CHECK_COMMAND]
+
+
+def test_pixi_install_is_skipped_when_robot_is_offline():
+    connection = _FakeConnection(internet_available=False)
+
+    result = pixi.pixi_install_if_online(connection, "~/bitbots/bitbots_main")
+
+    assert result.ok
+    assert result.command == "# Skipping pixi install because the robot has no internet connection."
+    assert connection.commands == [_PIXI_REPOSITORY_CHECK_COMMAND]
+
+
+def test_pixi_install_runs_when_robot_is_online():
+    connection = _FakeConnection(internet_available=True)
+
+    result = pixi.pixi_install_if_online(connection, "~/bitbots/bitbots_main")
+
+    assert result.ok
+    assert connection.commands == [
+        _PIXI_REPOSITORY_CHECK_COMMAND,
+        "cd ~/bitbots/bitbots_main && pixi install --environment robot",
+    ]
+
+
+class _FakeConnection:
+    def __init__(self, internet_available: bool):
+        self.host = "10.0.0.1"
+        self.original_host = "nuc"
+        self.internet_available = internet_available
+        self.commands = []
+
+    def run(self, command, hide=None, warn=False):
+        self.commands.append(command)
+        if command == _PIXI_REPOSITORY_CHECK_COMMAND:
+            return _FakeResult(self, command, ok=self.internet_available)
+        return _FakeResult(self, command, ok=True)
+
+
+class _FakeResult:
+    def __init__(self, connection: _FakeConnection, command: str, ok: bool):
+        self.connection = connection
+        self.command = command
+        self.ok = ok
+        self.failed = not ok
+        self.stdout = ""
+        self.stderr = ""
+        self.exited = int(not ok)