diff --git a/CHANGELOG.md b/CHANGELOG.md index 043afc6..bbada9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.52.0] +## [0.53.0] - 2026-06-05 + +### Added + +- `tilebox-workflows`: Added worker runtime mode for executing registered tasks over the worker RPC service, backed by a + shared task executor facade used by both worker and polling runners. + +## [0.52.0] - 2026-05-08 ### Added @@ -368,7 +375,8 @@ the first client that does not cache data (since it's already on the local file - Released under the [MIT](https://opensource.org/license/mit) license. - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc` -[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.52.0...HEAD +[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.53.0...HEAD +[0.53.0]: https://github.com/tilebox/tilebox-python/compare/v0.52.0...v0.53.0 [0.52.0]: https://github.com/tilebox/tilebox-python/compare/v0.51.0...v0.52.0 [0.51.0]: https://github.com/tilebox/tilebox-python/compare/v0.50.1...v0.51.0 [0.50.1]: https://github.com/tilebox/tilebox-python/compare/v0.50.0...v0.50.1 diff --git a/tilebox-datasets/tilebox/datasets/datasets/v1/collections_pb2_grpc.py b/tilebox-datasets/tilebox/datasets/datasets/v1/collections_pb2_grpc.py index 91428b7..e385b8b 100644 --- a/tilebox-datasets/tilebox/datasets/datasets/v1/collections_pb2_grpc.py +++ b/tilebox-datasets/tilebox/datasets/datasets/v1/collections_pb2_grpc.py @@ -6,7 +6,7 @@ from tilebox.datasets.datasets.v1 import core_pb2 as datasets_dot_v1_dot_core__pb2 -class CollectionServiceStub(object): +class CollectionServiceStub: """CollectionService is the service definition for the Tilebox datasets service, which provides access to datasets """ @@ -38,7 +38,7 @@ def __init__(self, channel): _registered_method=True) -class CollectionServiceServicer(object): +class CollectionServiceServicer: """CollectionService is the service definition for the Tilebox datasets service, which provides access to datasets """ @@ -97,7 +97,7 @@ def add_CollectionServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class CollectionService(object): +class CollectionService: """CollectionService is the service definition for the Tilebox datasets service, which provides access to datasets """ diff --git a/tilebox-datasets/tilebox/datasets/datasets/v1/data_access_pb2_grpc.py b/tilebox-datasets/tilebox/datasets/datasets/v1/data_access_pb2_grpc.py index 6159b22..016cd82 100644 --- a/tilebox-datasets/tilebox/datasets/datasets/v1/data_access_pb2_grpc.py +++ b/tilebox-datasets/tilebox/datasets/datasets/v1/data_access_pb2_grpc.py @@ -6,7 +6,7 @@ from tilebox.datasets.datasets.v1 import data_access_pb2 as datasets_dot_v1_dot_data__access__pb2 -class DataAccessServiceStub(object): +class DataAccessServiceStub: """DataAccessService provides data access and querying capabilities for Tilebox datasets. """ @@ -28,7 +28,7 @@ def __init__(self, channel): _registered_method=True) -class DataAccessServiceServicer(object): +class DataAccessServiceServicer: """DataAccessService provides data access and querying capabilities for Tilebox datasets. """ @@ -67,7 +67,7 @@ def add_DataAccessServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class DataAccessService(object): +class DataAccessService: """DataAccessService provides data access and querying capabilities for Tilebox datasets. """ diff --git a/tilebox-datasets/tilebox/datasets/datasets/v1/data_ingestion_pb2_grpc.py b/tilebox-datasets/tilebox/datasets/datasets/v1/data_ingestion_pb2_grpc.py index 2e8a9c1..2bec596 100644 --- a/tilebox-datasets/tilebox/datasets/datasets/v1/data_ingestion_pb2_grpc.py +++ b/tilebox-datasets/tilebox/datasets/datasets/v1/data_ingestion_pb2_grpc.py @@ -5,7 +5,7 @@ from tilebox.datasets.datasets.v1 import data_ingestion_pb2 as datasets_dot_v1_dot_data__ingestion__pb2 -class DataIngestionServiceStub(object): +class DataIngestionServiceStub: """DataIngestionService provides data ingestion and deletion capabilities for Tilebox datasets. """ @@ -27,7 +27,7 @@ def __init__(self, channel): _registered_method=True) -class DataIngestionServiceServicer(object): +class DataIngestionServiceServicer: """DataIngestionService provides data ingestion and deletion capabilities for Tilebox datasets. """ @@ -64,7 +64,7 @@ def add_DataIngestionServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class DataIngestionService(object): +class DataIngestionService: """DataIngestionService provides data ingestion and deletion capabilities for Tilebox datasets. """ diff --git a/tilebox-datasets/tilebox/datasets/datasets/v1/datasets_pb2_grpc.py b/tilebox-datasets/tilebox/datasets/datasets/v1/datasets_pb2_grpc.py index 3d0a4c6..791b945 100644 --- a/tilebox-datasets/tilebox/datasets/datasets/v1/datasets_pb2_grpc.py +++ b/tilebox-datasets/tilebox/datasets/datasets/v1/datasets_pb2_grpc.py @@ -6,7 +6,7 @@ from tilebox.datasets.datasets.v1 import datasets_pb2 as datasets_dot_v1_dot_datasets__pb2 -class DatasetServiceStub(object): +class DatasetServiceStub: """DatasetsService is the CRUD service for Tilebox datasets. """ @@ -43,7 +43,7 @@ def __init__(self, channel): _registered_method=True) -class DatasetServiceServicer(object): +class DatasetServiceServicer: """DatasetsService is the CRUD service for Tilebox datasets. """ @@ -113,7 +113,7 @@ def add_DatasetServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class DatasetService(object): +class DatasetService: """DatasetsService is the CRUD service for Tilebox datasets. """ diff --git a/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.py b/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.py index 438d902..1089882 100644 --- a/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.py +++ b/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.py @@ -27,7 +27,7 @@ from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16tilebox/v1/query.proto\x12\ntilebox.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xac\x01\n\nIDInterval\x12)\n\x08start_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x07startId\x12%\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"n\n\nPagination\x12\"\n\x05limit\x18\x01 \x01(\x03\x42\x0c\xaa\x01\x02\x08\x01\xbaH\x04\"\x02(\x01R\x05limit\x12<\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x05\xaa\x01\x02\x08\x01R\rstartingAfterBj\n\x0e\x63om.tilebox.v1B\nQueryProtoP\x01\xa2\x02\x03TXX\xaa\x02\nTilebox.V1\xca\x02\nTilebox\\V1\xe2\x02\x16Tilebox\\V1\\GPBMetadata\xea\x02\x0bTilebox::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16tilebox/v1/query.proto\x12\ntilebox.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xac\x01\n\nIDInterval\x12)\n\x08start_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x07startId\x12%\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"n\n\nPagination\x12\"\n\x05limit\x18\x01 \x01(\x03\x42\x0c\xaa\x01\x02\x08\x01\xbaH\x04\"\x02(\x01R\x05limit\x12<\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x05\xaa\x01\x02\x08\x01R\rstartingAfter*l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02\x42j\n\x0e\x63om.tilebox.v1B\nQueryProtoP\x01\xa2\x02\x03TXX\xaa\x02\nTilebox.V1\xca\x02\nTilebox\\V1\xe2\x02\x16Tilebox\\V1\\GPBMetadata\xea\x02\x0bTilebox::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -39,6 +39,8 @@ _globals['_PAGINATION'].fields_by_name['limit']._serialized_options = b'\252\001\002\010\001\272H\004\"\002(\001' _globals['_PAGINATION'].fields_by_name['starting_after']._loaded_options = None _globals['_PAGINATION'].fields_by_name['starting_after']._serialized_options = b'\252\001\002\010\001' + _globals['_SORTDIRECTION']._serialized_start=617 + _globals['_SORTDIRECTION']._serialized_end=725 _globals['_TIMEINTERVAL']._serialized_start=122 _globals['_TIMEINTERVAL']._serialized_end=328 _globals['_IDINTERVAL']._serialized_start=331 diff --git a/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.pyi b/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.pyi index 2e43519..1f6a92f 100644 --- a/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.pyi +++ b/tilebox-datasets/tilebox/datasets/tilebox/v1/query_pb2.pyi @@ -1,6 +1,7 @@ from tilebox.datasets.buf.validate import validate_pb2 as _validate_pb2 from google.protobuf import timestamp_pb2 as _timestamp_pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from collections.abc import Mapping as _Mapping @@ -8,6 +9,15 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor +class SortDirection(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + SORT_DIRECTION_UNSPECIFIED: _ClassVar[SortDirection] + SORT_DIRECTION_ASCENDING: _ClassVar[SortDirection] + SORT_DIRECTION_DESCENDING: _ClassVar[SortDirection] +SORT_DIRECTION_UNSPECIFIED: SortDirection +SORT_DIRECTION_ASCENDING: SortDirection +SORT_DIRECTION_DESCENDING: SortDirection + class TimeInterval(_message.Message): __slots__ = ("start_time", "end_time", "start_exclusive", "end_inclusive") START_TIME_FIELD_NUMBER: _ClassVar[int] diff --git a/tilebox-workflows/tests/clusters/test_client.py b/tilebox-workflows/tests/clusters/test_client.py index 4b88b34..2a2f8b1 100644 --- a/tilebox-workflows/tests/clusters/test_client.py +++ b/tilebox-workflows/tests/clusters/test_client.py @@ -10,7 +10,9 @@ from tilebox.workflows.data import ( Cluster, ) -from tilebox.workflows.workflows.v1.core_pb2 import Cluster as ClusterMessage +from tilebox.workflows.workflows.v1.workflows_pb2 import ( + Cluster as ClusterMessage, +) from tilebox.workflows.workflows.v1.workflows_pb2 import ( CreateClusterRequest, DeleteClusterRequest, diff --git a/tilebox-workflows/tests/runner/testdata/recordings/fibonacci_workflow.rpcs.bin b/tilebox-workflows/tests/runner/testdata/recordings/fibonacci_workflow.rpcs.bin index 46486e4..b8e7674 100644 --- a/tilebox-workflows/tests/runner/testdata/recordings/fibonacci_workflow.rpcs.bin +++ b/tilebox-workflows/tests/runner/testdata/recordings/fibonacci_workflow.rpcs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24ff20b8b66a30cf11e5998477066b500830280160eaa5dd1e1c5895ea920a47 -size 9032 +oid sha256:ede65a3f196290e2aee71cddc45f67c9e9b2c6eb1374bf6beb0b8406f1f4d8c4 +size 8923 diff --git a/tilebox-workflows/tests/runner/testdata/recordings/flaky_task.rpcs.bin b/tilebox-workflows/tests/runner/testdata/recordings/flaky_task.rpcs.bin index fbe63a6..eff2a13 100644 --- a/tilebox-workflows/tests/runner/testdata/recordings/flaky_task.rpcs.bin +++ b/tilebox-workflows/tests/runner/testdata/recordings/flaky_task.rpcs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b8081ec6c43e20ee4ef13ff7ae89a24f695cd85e10c0346341d77a9fec5a285f -size 6061 +oid sha256:54b3862b1e4c863f27e9f2e29958e3122ea8d197459fdb87e7b3d81ff0373f59 +size 5932 diff --git a/tilebox-workflows/tests/runner/testdata/recordings/optional_subbranch.rpcs.bin b/tilebox-workflows/tests/runner/testdata/recordings/optional_subbranch.rpcs.bin index f52efd2..ed9e4bc 100644 --- a/tilebox-workflows/tests/runner/testdata/recordings/optional_subbranch.rpcs.bin +++ b/tilebox-workflows/tests/runner/testdata/recordings/optional_subbranch.rpcs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cac0cd1cd31ec5c62949704159a113d2f69a249c35997eac113a2ac695d196d5 -size 4430 +oid sha256:432d42a185a620843815a821a9c04617459b351dc873a2dc31101657639d13e0 +size 4325 diff --git a/tilebox-workflows/tests/runner/testdata/recordings/optional_subtask.rpcs.bin b/tilebox-workflows/tests/runner/testdata/recordings/optional_subtask.rpcs.bin index 51df487..1f0cf07 100644 --- a/tilebox-workflows/tests/runner/testdata/recordings/optional_subtask.rpcs.bin +++ b/tilebox-workflows/tests/runner/testdata/recordings/optional_subtask.rpcs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3eee76fab8b6018fbba6e0332a8a5a193a1095ceab3e4a0b2e2b586a5d520f9b -size 3526 +oid sha256:1260ba9f7798ef3b53a76beac7943ee4857f7c5035c10425702ff121dc64966d +size 3413 diff --git a/tilebox-workflows/tests/runner/testdata/recordings/progress.rpcs.bin b/tilebox-workflows/tests/runner/testdata/recordings/progress.rpcs.bin index 994e726..8d3f01a 100644 --- a/tilebox-workflows/tests/runner/testdata/recordings/progress.rpcs.bin +++ b/tilebox-workflows/tests/runner/testdata/recordings/progress.rpcs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:432fe539191e3c122d88dfe9a9ad7ff4c797df7ab0390c1ab69f454ed8d80858 -size 4359 +oid sha256:e30083e0e53ed859997e01bf4abdd1c06d873ab3ac96f84497778e7d267d8481 +size 4258 diff --git a/tilebox-workflows/tilebox/workflows/__init__.py b/tilebox-workflows/tilebox/workflows/__init__.py index 13c7df2..722288c 100644 --- a/tilebox-workflows/tilebox/workflows/__init__.py +++ b/tilebox-workflows/tilebox/workflows/__init__.py @@ -5,9 +5,10 @@ from tilebox.workflows.client import Client from tilebox.workflows.data import Job +from tilebox.workflows.runner.runner import Runner from tilebox.workflows.task import ExecutionContext, Task -__all__ = ["Client", "ExecutionContext", "Job", "Task"] +__all__ = ["Client", "ExecutionContext", "Job", "Runner", "Task"] def _init_logging(level: str = "INFO") -> None: diff --git a/tilebox-workflows/tilebox/workflows/client.py b/tilebox-workflows/tilebox/workflows/client.py index 32f3ce3..f22199f 100644 --- a/tilebox-workflows/tilebox/workflows/client.py +++ b/tilebox-workflows/tilebox/workflows/client.py @@ -1,10 +1,9 @@ import logging import os import warnings -from uuid import uuid4 +from uuid import UUID, uuid4 from _tilebox.grpc.channel import open_channel, parse_channel_info -from tilebox.datasets.sync.client import Client as DatasetsClient from tilebox.workflows.automations.client import AutomationClient, AutomationService from tilebox.workflows.cache import JobCache, NoCache from tilebox.workflows.clusters.client import ClusterClient, ClusterSlugLike, to_cluster_slug @@ -22,13 +21,21 @@ _create_tilebox_logger_provider, ) from tilebox.workflows.observability.tracing import WorkflowTracer +from tilebox.workflows.runner.executor import LazyStorageLocations +from tilebox.workflows.runner.runner import Runner from tilebox.workflows.runner.task_runner import TaskRunner, _LeaseRenewer from tilebox.workflows.runner.task_service import TaskService +from tilebox.workflows.task import Task class Client: def __init__( - self, *, url: str = "https://api.tilebox.com", token: str | None = None, name: str | None = None + self, + *, + url: str = "https://api.tilebox.com", + token: str | None = None, + name: str | None = None, + client_id: UUID | None = None, ) -> None: """ Create a Tilebox workflows client. @@ -39,13 +46,14 @@ def __init__( name: An optional name of the client, used as service.name for telemetry. If not set, defaults to the service name provided by `tilebox.workflows.observability.tracing.configure_otel_tracing`, or "tilebox-python" if no external tracer is configured. + client_id: An optional stable id used to scope internal loggers. Defaults to a random id. """ token = _token_from_env(url, token) self._auth: dict[str, str] = {"token": token, "url": url} self._channel = open_channel(url, token) # configure logging and tracing - self._client_id = uuid4() # a random uuid to scope loggers to this client instance + self._client_id = client_id or uuid4() # a random uuid to scope loggers to this client instance self._logger_provider = _create_tilebox_logger_provider(service=name, url=url, token=token) # task logger is the logger available for users to emit logs from within a Task.execute method, via @@ -107,9 +115,10 @@ def jobs(self) -> JobClient: def runner( self, cluster: ClusterSlugLike | None = None, - tasks: list[type] | None = None, + tasks: list[type[Task]] | None = None, cache: JobCache | None = None, context: type[RunnerContext] | None = None, + runner: Runner | None = None, ) -> TaskRunner: """Initialize a task runner. @@ -118,30 +127,25 @@ def runner( tasks: A list of task the runner is able to execute. cache: The cache to share between tasks. context: The type of the runner context to use. Defaults to RunnerContext. + runner: A runner definition containing tasks, cache and context configuration. Returns: A task runner. """ + if runner is not None and (tasks is not None or cache is not None or context is not None): + raise ValueError("Pass either runner or tasks/cache/context, not both.") + + runner_definition = runner or Runner(tasks=tasks, cache=cache, context=context) if cache is None: - cache = NoCache() # a no-op cache that will raise an error if it's used + cache = runner_definition.cache or NoCache() # a no-op cache that will raise an error if it's used found_cluster = self.clusters().find(to_cluster_slug(cluster or "")) - try: - storage_locations = self.automations().storage_locations() - except: # noqa: E722 - # if fetching storage locations fails, we just disable this feature, and don't crash all runners - # lets refactor this to a lazy loading mechanism in the future - storage_locations = [] - - runner_context_type = context or RunnerContext - runner_context = runner_context_type( - self._tracer, - datasets_client=DatasetsClient(**self._auth), # ty: ignore[invalid-argument-type] - storage_locations=storage_locations, - ) + runner_context_type = runner_definition.context or RunnerContext + runner_context = runner_context_type(self._tracer) + runner_context.storage_locations = LazyStorageLocations(self, runner_context) - runner = TaskRunner( + task_runner = TaskRunner( TaskService(self._channel), found_cluster.slug, cache, @@ -152,11 +156,10 @@ def runner( runner_logger=StructuredLogger(self._runner_logger, {}), ) - if tasks is not None: - for task in tasks: - runner.register(task) + for task in runner_definition.tasks_by_identifier.values(): + task_runner.register(task) - return runner + return task_runner def clusters(self) -> ClusterClient: """ diff --git a/tilebox-workflows/tilebox/workflows/data.py b/tilebox-workflows/tilebox/workflows/data.py index 2a1999a..482ba8a 100644 --- a/tilebox-workflows/tilebox/workflows/data.py +++ b/tilebox-workflows/tilebox/workflows/data.py @@ -36,10 +36,9 @@ except ModuleNotFoundError: from typing import Any as S3Client -from tilebox.datasets.sync.client import Client as DatasetsClient from tilebox.workflows.observability.tracing import NoopWorkflowTracer, WorkflowTracer from tilebox.workflows.workflows.v1 import automation_pb2 as automation_pb -from tilebox.workflows.workflows.v1 import core_pb2, job_pb2, task_pb2 +from tilebox.workflows.workflows.v1 import core_pb2, job_pb2, task_pb2, workflows_pb2 _VERSION_PATTERN = re.compile(r"^v(\d+)\.(\d+)$") # matches a version string in the format "v3.2" @@ -323,19 +322,35 @@ class Cluster: deletable: bool @classmethod # lets use typing.Self once we require python >= 3.11 - def from_message(cls, cluster: core_pb2.Cluster) -> "Cluster": + def from_message(cls, cluster: workflows_pb2.Cluster) -> "Cluster": """Convert a Cluster protobuf message to a Cluster object.""" return cls(slug=cluster.slug, display_name=cluster.display_name, deletable=cluster.deletable) - def to_message(self) -> core_pb2.Cluster: + def to_message(self) -> workflows_pb2.Cluster: """Convert a Cluster object to a Cluster protobuf message.""" - return core_pb2.Cluster(slug=self.slug, display_name=self.display_name, deletable=self.deletable) + return workflows_pb2.Cluster(slug=self.slug, display_name=self.display_name, deletable=self.deletable) + + +@dataclass(order=True, frozen=True) +class Workflow: + slug: str + name: str + description: str + + @classmethod + def from_message(cls, workflow: workflows_pb2.Workflow) -> "Workflow": + """Convert a Workflow protobuf message to a Workflow object.""" + return cls(slug=workflow.slug, name=workflow.name, description=workflow.description) + + def to_message(self) -> workflows_pb2.Workflow: + """Convert a Workflow object to a Workflow protobuf message.""" + return workflows_pb2.Workflow(slug=self.slug, name=self.name, description=self.description) @dataclass class NextTaskToRun: cluster_slug: str - identifiers: dict[TaskIdentifier, type] + identifiers: dict[TaskIdentifier, type[Any]] # from message not needed, as we never return this from the server @@ -474,6 +489,51 @@ def to_message(self) -> task_pb2.ComputedTask: ) +@dataclass +class FailedTask: + task_id: UUID + display: str | None + was_workflow_error: bool + progress_updates: list[ProgressIndicator] + + @classmethod + def from_message(cls, failed_task: task_pb2.TaskFailedRequest) -> "FailedTask": + """Convert a TaskFailedRequest protobuf message to a FailedTask object.""" + return cls( + task_id=uuid_message_to_uuid(failed_task.task_id), + display=failed_task.display, + was_workflow_error=failed_task.was_workflow_error, + progress_updates=[ProgressIndicator.from_message(progress) for progress in failed_task.progress_updates], + ) + + @classmethod + def from_task_error( + cls, + task: Task, + error: Exception, + was_workflow_error: bool, + progress_updates: list[ProgressIndicator], + ) -> "FailedTask": + # job output is limited to 1KB, so truncate the error message if necessary + error_message = repr(error)[: (1024 - len(task.display or "None") - 1)] + display = f"{task.display}" if error_message == "" else f"{task.display}\n{error_message}" + return cls( + task_id=task.id, + display=display, + was_workflow_error=was_workflow_error, + progress_updates=progress_updates, + ) + + def to_message(self) -> task_pb2.TaskFailedRequest: + """Convert a FailedTask object to a TaskFailedRequest protobuf message.""" + return task_pb2.TaskFailedRequest( + task_id=uuid_to_uuid_message(self.task_id), + display=self.display, + was_workflow_error=self.was_workflow_error, + progress_updates=[progress.to_message() for progress in self.progress_updates], + ) + + def _parse_version(version: str) -> tuple[int, int]: """ Parse the major and minor version from a string in the format "vMajor.Minor" and returns them as tuple of ints. @@ -924,13 +984,11 @@ class RunnerContext: def __init__( self, tracer: WorkflowTracer | None = None, - datasets_client: DatasetsClient | None = None, storage_locations: list[StorageLocation] | None = None, ) -> None: if tracer is None: tracer = NoopWorkflowTracer() self.tracer = tracer - self.datasets_client = datasets_client self.storage_locations = { sl.id: sl._with_runner_context(self) # noqa: SLF001 for sl in storage_locations or [] diff --git a/tilebox-workflows/tilebox/workflows/interceptors.py b/tilebox-workflows/tilebox/workflows/interceptors.py deleted file mode 100644 index 225e615..0000000 --- a/tilebox-workflows/tilebox/workflows/interceptors.py +++ /dev/null @@ -1,62 +0,0 @@ -import functools -from collections.abc import Callable -from typing import Any, Protocol, TypeAlias, cast - -from tilebox.workflows.task import ExecutionContext, Task, _task_meta - -ForwardExecution: TypeAlias = Callable[[ExecutionContext], Any] - - -class Interceptor(Protocol): - def __call__(self, task: Task, call_next: ForwardExecution, context: ExecutionContext) -> None: ... - - -class InterceptorType(Protocol): - __original_interceptor_func__: Interceptor - - def __call__(self, task_cls: type) -> type: ... - - -def execution_interceptor(func: Interceptor) -> InterceptorType: - """Decorator to convert a function into an execution interceptor. - - Example: - >>> @execution_interceptor - >>> def my_interceptor(task: Task, next: Interceptor, context: ExecutionContext) -> None: - >>> print("Before") - >>> result = next(context) - >>> print("After") - >>> return result - - Afterwards, my_interceptor can be used as an interceptor in a task definition. - >>> @my_interceptor - >>> @task - >>> class MyTask: - >>> ... - - - Args: - func: The function to convert into an interceptor. - - Returns: - The interceptor function. - """ - - @functools.wraps(func) - def wrap(task_cls: type) -> type: - meta = _task_meta(task_cls) - meta.interceptors.insert(0, func) - return task_cls - - wrapped: InterceptorType = cast(InterceptorType, wrap) - # needed internally for task runner interceptors to get the original function - wrapped.__original_interceptor_func__ = func - - return wrapped - - -@execution_interceptor -def print_executions(task: Task, call_next: ForwardExecution, context: ExecutionContext) -> None: - """Print executing tasks.""" - print(f"Executing {task}") # noqa: T201 - return call_next(context) diff --git a/tilebox-workflows/tilebox/workflows/runner/__init__.py b/tilebox-workflows/tilebox/workflows/runner/__init__.py index e69de29..bd7cb57 100644 --- a/tilebox-workflows/tilebox/workflows/runner/__init__.py +++ b/tilebox-workflows/tilebox/workflows/runner/__init__.py @@ -0,0 +1,4 @@ +from tilebox.workflows.runner.runner import Runner +from tilebox.workflows.runner.task_runner import TaskRunner + +__all__ = ["Runner", "TaskRunner"] diff --git a/tilebox-workflows/tilebox/workflows/runner/__main__.py b/tilebox-workflows/tilebox/workflows/runner/__main__.py new file mode 100644 index 0000000..b81a609 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/runner/__main__.py @@ -0,0 +1,55 @@ +import argparse +import importlib +from collections.abc import Sequence +from typing import Any + +from tilebox.workflows.runner.runner import Runner +from tilebox.workflows.runner.worker_server import serve_runner + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m tilebox.workflows.runner", + description="Start a Tilebox workflow worker runtime.", + ) + parser.add_argument("runner", help="Runner object import path, for example 'my_workflow.runner:runner'.") + args = parser.parse_args(argv) + + serve_runner(_import_runner(args.runner)) + return 0 + + +def _import_runner(import_path: str) -> Runner: + module_name, separator, object_path = import_path.partition(":") + if not module_name or not separator or not object_path: + raise SystemExit( + "Expected runner import path in the format ':', for example 'my_workflow.runner:runner'." + ) + + try: + module = importlib.import_module(module_name) + except Exception as error: + raise SystemExit(f"Failed to import module {module_name!r}: {error}") from error + + try: + obj = _get_attribute(module, object_path) + except AttributeError as error: + raise SystemExit(f"Module {module_name!r} has no runner object {object_path!r}.") from error + + if not isinstance(obj, Runner): + raise SystemExit( + f"Expected {import_path!r} to resolve to tilebox.workflows.Runner, " + f"got {type(obj).__module__}.{type(obj).__qualname__}." + ) + + return obj + + +def _get_attribute(obj: Any, dotted_path: str) -> Any: + for part in dotted_path.split("."): + obj = getattr(obj, part) + return obj + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tilebox-workflows/tilebox/workflows/runner/executor.py b/tilebox-workflows/tilebox/workflows/runner/executor.py new file mode 100644 index 0000000..163f74d --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/runner/executor.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import json +import logging +from base64 import b64encode +from collections.abc import Callable, Iterator, MutableMapping, Sequence +from contextlib import AbstractContextManager, contextmanager +from typing import TYPE_CHECKING +from uuid import UUID +from warnings import warn + +from opentelemetry.trace.status import StatusCode + +from tilebox.workflows.cache import JobCache +from tilebox.workflows.data import ( + ComputedTask, + FailedTask, + ProgressIndicator, + StorageLocation, + Task, +) +from tilebox.workflows.observability.logging import StructuredLogger +from tilebox.workflows.observability.tracing import NoopWorkflowTracer, WorkflowTracer, start_job_span +from tilebox.workflows.runner.runner import Runner +from tilebox.workflows.task import ExecutionContext as ExecutionContextBase +from tilebox.workflows.task import FutureTask, ProgressUpdate, RunnerContext, merge_future_tasks_to_submissions +from tilebox.workflows.task import Task as TaskInstance + +if TYPE_CHECKING: + from tilebox.workflows.client import Client + +_MAX_TASK_PROGRESS_INDICATORS = 1000 + + +class TaskExecutor: + def __init__( # noqa: PLR0913 + self, + runner: Runner, + cache: JobCache, + tracer: WorkflowTracer, + task_logger: StructuredLogger, + runner_context: RunnerContext, + fallback_cluster: str, + ) -> None: + self.runner = runner + self.cache = cache + self.tracer = tracer + self.task_logger = task_logger + self.fallback_cluster = fallback_cluster + self.runner_context = runner_context + + def execute_task( + self, + task: Task, + wrap_execute_context_manager: Callable[[Task, ExecutionContext], AbstractContextManager[None]] | None = None, + ) -> ComputedTask | FailedTask: + context: ExecutionContext | None = None + try: + if task.job is None: + self.task_logger.error(f"Task {task.id} has no job associated with it.", task_id=str(task.id)) + return FailedTask(task.id, task.display, was_workflow_error=False, progress_updates=[]) + + try: + task_class = self.runner.tasks_by_identifier[task.identifier] + except KeyError: + self.task_logger.error( + f"Task {task.id} has unknown identifier {task.identifier}.", + task_id=str(task.id), + identifier=str(task.identifier), + ) + return FailedTask(task.id, task.display, was_workflow_error=False, progress_updates=[]) + + context = ExecutionContext(self, task, self.cache.group(str(task.job.id))) + if wrap_execute_context_manager is None: + wrap_execute_context_manager = _noop_context_manager + + with start_job_span(self.tracer, task.job, f"task/{task.id}") as span: + task_repr = task_class.__name__ + span.update_name(f"task/{task_repr}") + span.set_attribute("task_id", str(task.id)) + span.set_attribute("identifier.name", task.identifier.name) + span.set_attribute("identifier.version", task.identifier.version) + + try: + task_instance = task_class._deserialize(task.input, self.runner_context) # noqa: SLF001 + _set_task_input_span_attribute(span, task.input) + with wrap_execute_context_manager(task, context): + _execute(task_instance, context) + + return ComputedTask( + id=task.id, + display=task.display, + sub_tasks=merge_future_tasks_to_submissions( + context._sub_tasks, # noqa: SLF001 + self.fallback_cluster, + ), + progress_updates=_finalize_mutable_progress_trackers(context._progress_indicators), # noqa: SLF001 + ) + except json.JSONDecodeError: + workflow_error = ValueError(f"Failed to deserialize input for task execution {task.id}") + span.record_exception(workflow_error) + span.set_status(StatusCode.ERROR, "Task failed with exception") + return FailedTask.from_task_error( + task, + workflow_error, + was_workflow_error=True, + progress_updates=_finalize_mutable_progress_trackers(context._progress_indicators), # noqa: SLF001 + ) + except Exception as error: # noqa: BLE001 + span.record_exception(error) + span.set_status(StatusCode.ERROR, "Task failed with exception") + return FailedTask.from_task_error( + task, + error, + was_workflow_error=True, + progress_updates=_finalize_mutable_progress_trackers(context._progress_indicators), # noqa: SLF001 + ) + except Exception as error: # noqa: BLE001 + progress_updates = [] + if context is not None: + progress_updates = _finalize_mutable_progress_trackers(context._progress_indicators) # noqa: SLF001 + return FailedTask.from_task_error( + task, + error, + was_workflow_error=False, + progress_updates=progress_updates, + ) + + +class ExecutionContext(ExecutionContextBase): + def __init__(self, executor: TaskExecutor, task: Task, job_cache: JobCache) -> None: + self._executor = executor + self.current_task = task + self.job_cache = job_cache + self._sub_tasks: list[FutureTask] = [] + self._progress_indicators: dict[str | None, ProgressUpdate] = {} + if executor is None or task is None: + # Some tests instantiate an execution context only to exercise local subtask merging helpers. + self._logger = StructuredLogger(logging.getLogger("tilebox.workflows.noop")) + else: + self._logger = executor.task_logger.bind(task_id=str(task.id)) + + def submit_subtask( + self, + task: TaskInstance, + depends_on: FutureTask | list[FutureTask] | None = None, + cluster: str | None = None, + max_retries: int = 0, + optional: bool = False, + ) -> FutureTask: + dependencies: list[int] = [] + + if depends_on is None: + depends_on = [] + elif isinstance(depends_on, FutureTask): + depends_on = [depends_on] + elif not isinstance(depends_on, list): + raise TypeError(f"Invalid dependency. Expected FutureTask or list[FutureTask], got {type(depends_on)}") + + for dep in depends_on: + if not isinstance(dep, FutureTask): + raise TypeError(f"Invalid dependency. Expected FutureTask, got {type(dep)}") + if dep.index >= len(self._sub_tasks): + raise ValueError(f"Dependent task {dep.index} does not exist") + dependencies.append(dep.index) + subtask = FutureTask( + index=len(self._sub_tasks), + task=task, + # cyclic dependencies are not allowed, they are detected by the server and will result in an error + depends_on=dependencies, + cluster=cluster, + max_retries=max_retries, + optional=optional, + ) + self._sub_tasks.append(subtask) + return subtask + + def submit_subtasks( + self, + tasks: Sequence[TaskInstance], + depends_on: FutureTask | list[FutureTask] | None = None, + cluster: str | None = None, + max_retries: int = 0, + optional: bool = False, + ) -> list[FutureTask]: + return [ + self.submit_subtask( + task, cluster=cluster, max_retries=max_retries, depends_on=depends_on, optional=optional + ) + for task in tasks + ] + + def submit_batch( + self, tasks: Sequence[TaskInstance], cluster: str | None = None, max_retries: int = 0 + ) -> list[FutureTask]: + warn( + "submit_batch is deprecated, use submit_subtasks instead", + DeprecationWarning, + stacklevel=2, + ) + return self.submit_subtasks(tasks, cluster=cluster, max_retries=max_retries) + + def progress(self, label: str | None = None) -> ProgressUpdate: + if label == "": + label = None + + if label in self._progress_indicators: + return self._progress_indicators[label] + + # this is our server side limit to prevent mistakes / abuse, so let's not allow to go beyond that already + # client side + if len(self._progress_indicators) > _MAX_TASK_PROGRESS_INDICATORS: + raise ValueError(f"Cannot create more than {_MAX_TASK_PROGRESS_INDICATORS} progress indicators per task.") + + progress_bar = ProgressUpdate(label) + self._progress_indicators[label] = progress_bar + return progress_bar + + @property + def runner_context(self) -> RunnerContext: + if self._executor is None: + return RunnerContext() + return self._executor.runner_context + + @property + def logger(self) -> StructuredLogger: + return self._logger + + @property + def tracer(self) -> WorkflowTracer: + if self._executor is None: + return NoopWorkflowTracer() + return self._executor.tracer + + +class LazyStorageLocations(MutableMapping[UUID, StorageLocation]): + def __init__(self, client: Client, runner_context: RunnerContext) -> None: + self._client = client + self._runner_context = runner_context + self._locations: dict[UUID, StorageLocation] = {} + self._loaded = False + + def _load(self) -> None: + if self._loaded: + return + self._locations = { + location.id: location._with_runner_context(self._runner_context) # noqa: SLF001 + for location in self._client.automations().storage_locations() + } + self._loaded = True + + def __getitem__(self, key: UUID) -> StorageLocation: + self._load() + return self._locations[key] + + def __setitem__(self, key: UUID, value: StorageLocation) -> None: + self._load() + self._locations[key] = value + + def __delitem__(self, key: UUID) -> None: + self._load() + del self._locations[key] + + def __iter__(self) -> Iterator[UUID]: + self._load() + return iter(self._locations) + + def __len__(self) -> int: + self._load() + return len(self._locations) + + +def _finalize_mutable_progress_trackers( + progress_bars: dict[str | None, ProgressUpdate], +) -> list[ProgressIndicator]: + return [ProgressIndicator(label, bar._total, bar._done) for label, bar in progress_bars.items()] # noqa: SLF001 + + +def _set_task_input_span_attribute(span: object, task_input: bytes | None) -> None: + task_input_span_attr = "" + if task_input is not None: + try: + task_input_span_attr = task_input.decode("utf-8") + except UnicodeDecodeError: + task_input_span_attr = b64encode(task_input).decode("ascii") + span.set_attribute("input", task_input_span_attr) # ty: ignore[unresolved-attribute] + + +def _execute(task: TaskInstance, context: ExecutionContext) -> None: + return task.execute(context) + + +@contextmanager +def _noop_context_manager(task: Task, context: ExecutionContext) -> Iterator[None]: # noqa: ARG001 + yield diff --git a/tilebox-workflows/tilebox/workflows/runner/runner.py b/tilebox-workflows/tilebox/workflows/runner/runner.py new file mode 100644 index 0000000..6edcce9 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/runner/runner.py @@ -0,0 +1,45 @@ +from tilebox.workflows.cache import JobCache +from tilebox.workflows.data import TaskIdentifier +from tilebox.workflows.task import RunnerContext, Task, TaskMeta + + +class Runner: + def __init__( + self, + *, + tasks: list[type[Task]] | None = None, + cache: JobCache | None = None, + context: type[RunnerContext] | None = None, + ) -> None: + self.cache = cache + self.context = context + self._tasks_by_identifier: dict[TaskIdentifier, type[Task]] = {} + + for task in tasks or []: + self.register(task) + + def register(self, task: type[Task]) -> None: + """Register a task that can be executed by this runner.""" + meta = TaskMeta.for_task(task) # ensures that this is a valid task + if not meta.executable: + task_repr = task.__name__ + if meta.identifier.name != task.__name__: + task_repr += f" ({meta.identifier.name})" + raise ValueError( + f"Task {task_repr} is not executable. It must have an execute method in order to " + f"register it with a task runner." + ) + if meta.identifier in self._tasks_by_identifier: + raise ValueError( + f"Duplicate task identifier: A task '{meta.identifier.name}' with version '{meta.identifier.version}' " + f"is already registered." + ) + self._tasks_by_identifier[meta.identifier] = task + + @property + def task_identifiers(self) -> list[TaskIdentifier]: + return list(self._tasks_by_identifier) + + @property + def tasks_by_identifier(self) -> dict[TaskIdentifier, type[Task]]: + return self._tasks_by_identifier diff --git a/tilebox-workflows/tilebox/workflows/runner/task_runner.py b/tilebox-workflows/tilebox/workflows/runner/task_runner.py index 2903a9a..1bcfb9d 100644 --- a/tilebox-workflows/tilebox/workflows/runner/task_runner.py +++ b/tilebox-workflows/tilebox/workflows/runner/task_runner.py @@ -1,14 +1,9 @@ -import contextlib -import json -import logging import random import signal import threading -from base64 import b64encode -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Iterator from contextlib import contextmanager from datetime import timedelta -from functools import partial from multiprocessing import get_context from multiprocessing.context import SpawnProcess from queue import Empty, Queue @@ -17,7 +12,6 @@ from types import FrameType, TracebackType from typing import Any, TypeAlias, TypeVar from uuid import UUID -from warnings import warn try: from typing import Self # ty: ignore[unresolved-import] @@ -25,29 +19,23 @@ from typing_extensions import Self from loguru import logger -from opentelemetry.trace.status import StatusCode from tenacity import retry, retry_if_exception_type, stop_when_event_set, wait_random_exponential from tenacity.stop import stop_base from _tilebox.grpc.channel import open_channel from _tilebox.grpc.error import InternalServerError -from tilebox.datasets.sync.dataset import DatasetClient from tilebox.workflows.cache import JobCache -from tilebox.workflows.data import ComputedTask, Idling, NextTaskToRun, ProgressIndicator, Task, TaskLease -from tilebox.workflows.interceptors import Interceptor, InterceptorType +from tilebox.workflows.data import ComputedTask, FailedTask, Idling, NextTaskToRun, Task, TaskLease from tilebox.workflows.observability.logging import StructuredLogger -from tilebox.workflows.observability.tracing import WorkflowTracer, start_job_span +from tilebox.workflows.observability.tracing import WorkflowTracer +from tilebox.workflows.runner.executor import ExecutionContext, TaskExecutor +from tilebox.workflows.runner.runner import Runner from tilebox.workflows.runner.task_service import TaskService -from tilebox.workflows.task import ExecutionContext as ExecutionContextBase -from tilebox.workflows.task import ( - FutureTask, - ProgressUpdate, - RunnerContext, - TaskMeta, - merge_future_tasks_to_submissions, -) +from tilebox.workflows.task import RunnerContext from tilebox.workflows.task import Task as TaskInstance +__all__ = ["ExecutionContext", "PollingTaskRunner", "TaskRunner"] + # The time we give a task to finish it's execution when a runner shutdown is requested before we forcefully stop it _SHUTDOWN_GRACE_PERIOD = timedelta(seconds=2) @@ -190,51 +178,25 @@ class RunnerShutdown(Exception): # noqa: N818 class _GracefulShutdown: - def __init__(self, grace_period: timedelta, service: TaskService) -> None: + def __init__(self, grace_period: timedelta, polling_runner: "PollingTaskRunner") -> None: """ - Graceful shutdown is a context manager that can be used to delay SIGTERM and SIGINT signals for a grace period. - That way work can finish cleanly before the process is terminated. - - Workers can check if they should shut down by calling the is_shutting_down method. - - After the grace period has passed, the same signal will be re-raised. - - It has special support for marking a task as failed immediately on interrupt, which is done if we are - currently executing a tasks execute function, since this is user code and we have no control over how long it - takes to execute. + Graceful shutdown policy owned by the user-facing task runner facade. - Args: - grace_period: Timedelta to delay the signal by in order to enable a graceful shutdown. - service: A reference to the task service, so that we can mark a task as failed on interrupt if necessary. + The polling runner owns task claiming, lease management, and failure reporting. This context manager only + translates process interrupts into polling-runner lifecycle calls. """ self._interrupted = Event() self._grace_period = grace_period - self._service = service + self._polling_runner = polling_runner self._original_sigterm: _HANDLER = None self._original_sigint: _HANDLER = None - # special handling for marking a task as failed on interrupt - self._task_mutex = threading.Lock() - self._task: Task | None = None - self._context: ExecutionContext | None = None - def _external_interrupt_handler(self, signum: int, frame: FrameType | None) -> None: """Signal handler for SIGTERM and SIGINT.""" self._interrupted.set() - - with self._task_mutex: - if self._task is not None: - progress = [] - if self._context is not None: - progress = _finalize_mutable_progress_trackers(self._context._progress_indicators) # noqa: SLF001 - self._service.task_failed( - self._task, - RunnerShutdown("Task was interrupted"), - was_workflow_error=False, - progress_updates=progress, - ) + self._polling_runner.stop_requesting_new_tasks() # fetch the handler we want to call after the grace period original_handler = self._original_sigterm if signum == signal.SIGTERM else self._original_sigint @@ -245,10 +207,9 @@ def _external_interrupt_handler(self, signum: int, frame: FrameType | None) -> N if not callable(original_handler): original_handler = signal.default_int_handler - with self._task_mutex: - if self._task is not None: - # if a task is currently running let's delay for the grace period, and then call the original handler - sleep(self._grace_period.total_seconds()) + if self._polling_runner.active_task is not None: + sleep(self._grace_period.total_seconds()) + self._polling_runner.interrupt_active_task() # this will stop the process: original_handler(signum, frame) @@ -287,375 +248,212 @@ def _revert_to_original_signal_handlers(self) -> None: self._original_sigint = None self._original_sigterm = None - @contextmanager - def mark_task_as_failed_on_interrupt(self, task: Task, context: "ExecutionContext") -> Iterator[None]: - """ - A context manager to enable marking a task as failed immediately on interrupt. - Only while the context manager is active the task will be marked as failed on interrupt. This is useful to - wrap the execution of a task, since we have no way of knowing how long it will take to execute, its user code. - """ - with self._task_mutex: - self._task = task - self._context = context - try: - yield - finally: - with self._task_mutex: - self._task = None - self._context = None - - -class TaskRunner: - def __init__( # noqa: PLR0913 +class PollingTaskRunner: + def __init__( self, service: TaskService, cluster: str, - cache: JobCache, - tracer: WorkflowTracer, + executor: TaskExecutor, lease_renewer: _LeaseRenewer, - context: RunnerContext, - task_logger: StructuredLogger, runner_logger: StructuredLogger, ) -> None: self._service = service - self.tasks_to_run = NextTaskToRun(cluster_slug=cluster, identifiers={}) - self.cache = cache - self.tracer = tracer - self.task_logger = task_logger self.runner_logger = runner_logger - self._interceptors: list[Interceptor] = [] + self._cluster = cluster + self._executor = executor self._lease_renewer = lease_renewer self._lease_renewer.start() # Start the lease extension process in the background - self._context = context + self._request_new_tasks = True + self._active_mutex = threading.Lock() + self._active_task: Task | None = None - def register(self, task: type) -> None: - """Register a task that can be executed by the task runner. + @property + def active_task(self) -> Task | None: + with self._active_mutex: + return self._active_task - Args: - task: The task to register. - """ - meta = TaskMeta.for_task(task) # ensures that this is a valid task - if not meta.executable: - task_repr = task.__name__ - if meta.identifier.name != task.__name__: - task_repr += f" ({meta.identifier.name})" - raise ValueError( - f"Task {task_repr} is not executable. It must have an execute method in order to " - f"register it with a task runner." - ) - if meta.identifier in self.tasks_to_run.identifiers: - raise ValueError( - f"Duplicate task identifier: A task '{meta.identifier.name}' with version '{meta.identifier.version}' " - f"is already registered." - ) - self.tasks_to_run.identifiers[meta.identifier] = task - - def add_interceptor(self, interceptor: InterceptorType) -> None: - """Add an interceptor to the task runner. + @property + def tasks_to_run(self) -> NextTaskToRun: + return NextTaskToRun(cluster_slug=self._cluster, identifiers=self._executor.runner.tasks_by_identifier) - Args: - interceptor: The interceptor to add. - """ - if not hasattr(interceptor, "__original_interceptor_func__"): - raise ValueError("Interceptor must be created with @execution_interceptor decorator.") - self._interceptors.append(interceptor.__original_interceptor_func__) + def stop_requesting_new_tasks(self) -> None: + self._request_new_tasks = False - def run_forever(self) -> None: + def is_requesting_tasks(self) -> bool: + return self._request_new_tasks + + def interrupt_active_task(self) -> None: + task = self.active_task + if task is None: + return + self._service.task_failed( + task, + RunnerShutdown("Task was interrupted"), + was_workflow_error=False, + progress_updates=[], + ) + + def run_forever(self, shutdown_context: _GracefulShutdown) -> None: """ - Run the task runner forever. This will poll for new tasks and execute them as they come in. + Run the polling runner forever. This will poll for new tasks and execute them as they come in. If no tasks are available, it will sleep for a short time and then try again. """ - self._run(stop_when_idling=False) + self._run(shutdown_context, stop_when_idling=False) - def run_all(self) -> None: + def run_all(self, shutdown_context: _GracefulShutdown) -> None: """ - Run the task runner and execute all tasks, until there are no more tasks available. + Run the polling runner and execute all tasks, until there are no more tasks available. """ - self._run(stop_when_idling=True) + self._run(shutdown_context, stop_when_idling=True) - def _run(self, stop_when_idling: bool = True) -> None: # noqa: C901 + def _run(self, shutdown_context: _GracefulShutdown, stop_when_idling: bool = True) -> None: # noqa: C901 """ - Run the task runner forever. This will poll for new tasks and execute them as they come in. + Run the polling runner forever. This will poll for new tasks and execute them as they come in. If no tasks are available, it will sleep for a short time and then try again. """ work: Task | Idling | None = None - # capture interrupt signals and delay them by a grace period in order to shut down gracefully - with _GracefulShutdown(_SHUTDOWN_GRACE_PERIOD, self._service) as shutdown_context: - while True: - if not isinstance(work, Task): # if we don't have a task right now, let's try to work-steal one - if shutdown_context.is_shutting_down(): # unless we received an interrupt, then we shut down - return - try: - work = self._service.next_task(task_to_run=self.tasks_to_run, computed_task=None) - except InternalServerError as e: - # We do not need to retry here, since the task runner will sleep for a while and then anyways request this again. - self.runner_logger.error(f"Failed to get next task with error {e}") - - if isinstance(work, Task): # we received a task to execute - task = work - if task.retry_count > 0: - self.runner_logger.debug(f"Retrying task {task.id} that failed {task.retry_count} times") - work = self._execute(task, shutdown_context) # submitting the task gives us the next work item - elif isinstance(work, Idling): # we received an idling response, so let's sleep for a bit - self.runner_logger.debug("No task to run, idling") - if stop_when_idling: # if stop_when_idling is set, we can just return - return - - # now sleep for a bit and then try again, unless we receive an interrupt - idling_duration = work.suggested_idling_duration - idling_duration = min(idling_duration, _MAX_IDLING_DURATION) - idling_duration = max(idling_duration, _MIN_IDLING_DURATION) - shutdown_context.sleep(idling_duration.total_seconds()) - if shutdown_context.is_shutting_down(): - return - else: # work is None - # we didn't receive an idling response, but also not a task. This only happens if we didn't request - # a task to run, indicating that we are shutting down. - if shutdown_context.is_shutting_down(): - return - - fallback_interval = _FALLBACK_POLL_INTERVAL.total_seconds() + random.uniform( # noqa: S311 - 0, _FALLBACK_JITTER_INTERVAL.total_seconds() - ) - self.runner_logger.debug( - f"Didn't receive a task to run, nor an idling response, but runner is not shutting down. " - f"Falling back to a default idling period of {fallback_interval:.2f}s" - ) + while True: + if not isinstance(work, Task): # if we don't have a task right now, let's try to work-steal one + if shutdown_context.is_shutting_down(): # unless we received an interrupt, then we shut down + return + try: + task_to_run = self.tasks_to_run if self._request_new_tasks else None + work = self._service.next_task(task_to_run=task_to_run, computed_task=None) + except InternalServerError as e: + # We do not need to retry here, since the task runner will sleep for a while and then anyways request this again. + self.runner_logger.error(f"Failed to get next task with error {e}") + + if isinstance(work, Task): # we received a task to execute + task = work + if task.retry_count > 0: + self.runner_logger.debug(f"Retrying task {task.id} that failed {task.retry_count} times") + work = self._execute(task, shutdown_context) # submitting the task gives us the next work item + elif isinstance(work, Idling): # we received an idling response, so let's sleep for a bit + self.runner_logger.debug("No task to run, idling") + if stop_when_idling: # if stop_when_idling is set, we can just return + return + + # now sleep for a bit and then try again, unless we receive an interrupt + idling_duration = work.suggested_idling_duration + idling_duration = min(idling_duration, _MAX_IDLING_DURATION) + idling_duration = max(idling_duration, _MIN_IDLING_DURATION) + shutdown_context.sleep(idling_duration.total_seconds()) + if shutdown_context.is_shutting_down(): + return + else: # work is None + # we didn't receive an idling response, but also not a task. This only happens if we didn't request + # a task to run, indicating that we are shutting down. + if shutdown_context.is_shutting_down(): + return + + fallback_interval = _FALLBACK_POLL_INTERVAL.total_seconds() + random.uniform( # noqa: S311 + 0, _FALLBACK_JITTER_INTERVAL.total_seconds() + ) + self.runner_logger.debug( + f"Didn't receive a task to run, nor an idling response, but runner is not shutting down. " + f"Falling back to a default idling period of {fallback_interval:.2f}s" + ) - shutdown_context.sleep(fallback_interval) + shutdown_context.sleep(fallback_interval) def _execute(self, task: Task, shutdown_context: _GracefulShutdown) -> Task | Idling | None: if task.job is None: raise ValueError(f"Task {task.id} has no job associated with it.") - context = ExecutionContext(self, task, self.cache.group(str(task.job.id))) - - try: - return self._try_execute(task, context, shutdown_context) - except Exception as e: - task_repr = str(task.id) - # let's try to get the name of the task class for a better error message: - # otherwise, if it's not possible, we just log the task id - with contextlib.suppress(KeyError): - task_repr = self.tasks_to_run.identifiers[task.identifier].__name__ - self.runner_logger.exception(f"Task {task_repr} failed!") - - task_failed_retry = _retry_backoff(self._service.task_failed, stop=shutdown_context.stop_if_shutting_down()) - was_workflow_error = True - progress_updates: list[ProgressIndicator] = _finalize_mutable_progress_trackers( - context._progress_indicators # noqa: SLF001 - ) - task_failed_retry(task, e, was_workflow_error, progress_updates) - - return None - - def _try_execute( - self, task: Task, context: "ExecutionContext", shutdown_context: _GracefulShutdown - ) -> Task | Idling | None: - if task.job is None: - raise ValueError(f"Task {task.id} has no job associated with it.") - - if task.lease is None: - raise ValueError(f"Task {task.id} has no lease associated with it.") - task_repr = str(task.id) - - with ( - self._lease_renewer.lease_extension(task.id, task.lease), - start_job_span(self.tracer, task.job, f"task/{task.id}") as span, - ): - self.runner_logger.debug("Executing task", task=task_repr, input=task.input) + self.runner_logger.debug("Executing task", task=task_repr, input=task.input) + with self._active(task): try: - try: - task_class = self.tasks_to_run.identifiers[task.identifier] - task_repr = task_class.__name__ - except KeyError: - self.runner_logger.error(f"Task {task.id} has unknown task identifier {task.identifier}") - raise ValueError(f"Task {task.id} has unknown task identifier {task.identifier}") from None - - # now that we've successfully looked up the task class, we can update the span name to replace the - # task execution id with the task class name - span.update_name(f"task/{task_repr}") - span.set_attribute("task_id", str(task.id)) - span.set_attribute("identifier.name", task.identifier.name) - span.set_attribute("identifier.version", task.identifier.version) - - try: - task_instance = task_class._deserialize(task.input, self._context) # ty: ignore[possibly-missing-attribute] # noqa: SLF001 - except json.JSONDecodeError: - self.runner_logger.exception(f"Failed to deserialize input for task execution {task.id}") - raise ValueError(f"Failed to deserialize input for task execution {task.id}") from None - - # record the task input as a span attribute, but for this we need to convert it to a string - # in case of binary data, we base64 encode it (e.g. protobuf message content) - task_input_span_attr = "" - if task.input is not None: - try: - task_input_span_attr = task.input.decode("utf-8") - except UnicodeDecodeError: - task_input_span_attr = b64encode(task.input).decode("ascii") - span.set_attribute("input", task_input_span_attr) - - # if we receive an interrupt exactly when running the user defined execute function, it is quite - # likely that we don't finish in time. So we mark the task as failed in that case immediately. - # If for some reason it does finish in time, the failed status will be overwritten by the computed - # status done later on in this function. - with shutdown_context.mark_task_as_failed_on_interrupt(task, context): - _execute(task_instance, context, self._interceptors) - - # if we received a stop signal, we should not request a next task - request_new_task = not shutdown_context.is_shutting_down() - task_to_run = self.tasks_to_run if request_new_task else None - computed_task = ComputedTask( - id=task.id, - display=task.display, - sub_tasks=merge_future_tasks_to_submissions( - context._sub_tasks, # noqa: SLF001 - # if not otherwise specified, we use the cluster of the runner for all subtasks, which is also - # the cluster of the parent task - self.tasks_to_run.cluster_slug, - ), - progress_updates=_finalize_mutable_progress_trackers(context._progress_indicators), # noqa: SLF001 - ) - - next_task_retry = _retry_backoff(self._service.next_task, stop=shutdown_context.stop_if_shutting_down()) - # mark the task as computed and get the next one - return next_task_retry(task_to_run=task_to_run, computed_task=computed_task) - - except Exception as e: - # catch all exceptions and re-raise them, since we just want to mark spans as failed - span.record_exception(e) - span.set_status(StatusCode.ERROR, "Task failed with exception") - - self.runner_logger.exception(f"Task {task_repr} failed!") + if task.lease is None: + raise ValueError(f"Task {task.id} has no lease associated with it.") # noqa: TRY301 + with self._lease_renewer.lease_extension(task.id, task.lease): + result = self._executor.execute_task(task) + except Exception as error: # noqa: BLE001 + result = FailedTask.from_task_error(task, error, was_workflow_error=False, progress_updates=[]) + + task_class = self.tasks_to_run.identifiers.get(task.identifier) + if task_class is not None: + task_repr = task_class.__name__ + + if isinstance(result, ComputedTask): + request_new_task = not shutdown_context.is_shutting_down() + task_to_run = self.tasks_to_run if request_new_task else None + next_task_retry = _retry_backoff(self._service.next_task, stop=shutdown_context.stop_if_shutting_down()) + return next_task_retry(task_to_run=task_to_run, computed_task=result) + + if isinstance(result, FailedTask): + self.runner_logger.error(f"Task {task_repr} failed!", display=result.display) + task_failed_retry = _retry_backoff(self._service.task_failed, stop=shutdown_context.stop_if_shutting_down()) + task_failed_retry(result) + return None - raise e from None # reraise, since we just wanted to mark it as failed in the span + raise TypeError(f"Unexpected task execution result: {type(result)}") + @contextmanager + def _active(self, task: Task) -> Iterator[None]: + with self._active_mutex: + self._active_task = task + try: + yield + finally: + with self._active_mutex: + self._active_task = None -class ExecutionContext(ExecutionContextBase): - def __init__(self, runner: TaskRunner, task: Task, job_cache: JobCache) -> None: - self._runner = runner - self.current_task = task - self.job_cache = job_cache - self._sub_tasks: list[FutureTask] = [] - self._progress_indicators: dict[str | None, ProgressUpdate] = {} - if runner is None or task is None: - # Some tests instantiate an execution context only to exercise local subtask merging helpers. - self._logger = StructuredLogger(logging.getLogger("tilebox.workflows.noop")) - else: - self._logger = runner.task_logger.bind(task_id=str(task.id)) - def submit_subtask( +class TaskRunner: + def __init__( # noqa: PLR0913 self, - task: TaskInstance, - depends_on: FutureTask | list[FutureTask] | None = None, - cluster: str | None = None, - max_retries: int = 0, - optional: bool = False, - ) -> FutureTask: - dependencies: list[int] = [] - - if depends_on is None: - depends_on = [] - elif isinstance(depends_on, FutureTask): - depends_on = [depends_on] - elif not isinstance(depends_on, list): - raise TypeError(f"Invalid dependency. Expected FutureTask or list[FutureTask], got {type(depends_on)}") - - for dep in depends_on: - if not isinstance(dep, FutureTask): - raise TypeError(f"Invalid dependency. Expected FutureTask, got {type(dep)}") - if dep.index >= len(self._sub_tasks): - raise ValueError(f"Dependent task {dep.index} does not exist") - dependencies.append(dep.index) - subtask = FutureTask( - index=len(self._sub_tasks), - task=task, - # cyclic dependencies are not allowed, they are detected by the server and will result in an error - depends_on=dependencies, - cluster=cluster, - max_retries=max_retries, - optional=optional, + service: TaskService, + cluster: str, + cache: JobCache, + tracer: WorkflowTracer, + lease_renewer: _LeaseRenewer, + context: RunnerContext, + task_logger: StructuredLogger, + runner_logger: StructuredLogger, + ) -> None: + self._runner = Runner(cache=cache) + self._executor = TaskExecutor( + self._runner, + cache, + tracer, + task_logger, + context, + cluster, ) - self._sub_tasks.append(subtask) - return subtask - - def submit_subtasks( - self, - tasks: Sequence[TaskInstance], - depends_on: FutureTask | list[FutureTask] | None = None, - cluster: str | None = None, - max_retries: int = 0, - optional: bool = False, - ) -> list[FutureTask]: - return [ - self.submit_subtask( - task, cluster=cluster, max_retries=max_retries, depends_on=depends_on, optional=optional - ) - for task in tasks - ] - - def submit_batch( - self, tasks: Sequence[TaskInstance], cluster: str | None = None, max_retries: int = 0 - ) -> list[FutureTask]: - warn( - "submit_batch is deprecated, use submit_subtasks instead", - DeprecationWarning, - stacklevel=2, + self._polling_runner = PollingTaskRunner( + service, + cluster, + self._executor, + lease_renewer, + runner_logger, ) - return self.submit_subtasks(tasks, cluster=cluster, max_retries=max_retries) - - def progress(self, label: str | None = None) -> ProgressUpdate: - if label == "": - label = None - - if label in self._progress_indicators: - return self._progress_indicators[label] - - # this is our server side limit to prevent mistakes / abuse, so let's not allow to go beyond that already - # client side - if len(self._progress_indicators) > _MAX_TASK_PROGRESS_INDICATORS: - raise ValueError(f"Cannot create more than {_MAX_TASK_PROGRESS_INDICATORS} progress indicators per task.") - - progress_bar = ProgressUpdate(label) - self._progress_indicators[label] = progress_bar - return progress_bar - - @property - def runner_context(self) -> RunnerContext: - return self._runner._context # noqa: SLF001 @property - def logger(self) -> StructuredLogger: - return self._logger - - @property - def tracer(self) -> WorkflowTracer: - return self._runner.tracer - - def _dataset(self, dataset_id: str) -> DatasetClient: - """Needed by the timeseries integration, to resolve a dataset id to a RemoteTimeseriesDataset.""" - client = self._runner._context.datasets_client # noqa: SLF001 - if client is None: - raise ValueError("No datasets client configured.") - - return client.dataset(dataset_id) - - -def _finalize_mutable_progress_trackers( - progress_bars: dict[str | None, ProgressUpdate], -) -> list[ProgressIndicator]: - return [ProgressIndicator(label, bar._total, bar._done) for label, bar in progress_bars.items()] # noqa: SLF001 + def tasks_to_run(self) -> NextTaskToRun: + return self._polling_runner.tasks_to_run + def register(self, task: type[TaskInstance]) -> None: + """Register a task that can be executed by the task runner. -def _execute(task: TaskInstance, context: ExecutionContext, additional_interceptors: list[Interceptor]) -> None: - interceptors: list[Interceptor] = additional_interceptors + TaskMeta.for_task(task).interceptors + Args: + task: The task to register. + """ + self._runner.register(task) - # chain interceptors in reverse order before eventually calling the actual task.execute function: - next_func = task.execute - for interceptor in interceptors[::-1]: - next_func = partial(interceptor, task, next_func) + def run_forever(self) -> None: + """ + Run the task runner forever. This will poll for new tasks and execute them as they come in. + If no tasks are available, it will sleep for a short time and then try again. + """ + with _GracefulShutdown(_SHUTDOWN_GRACE_PERIOD, self._polling_runner) as shutdown_context: + self._polling_runner.run_forever(shutdown_context) - return next_func(context) # call the first func in the chain, it will call the next one and so on + def run_all(self) -> None: + """ + Run the task runner and execute all tasks, until there are no more tasks available. + """ + with _GracefulShutdown(_SHUTDOWN_GRACE_PERIOD, self._polling_runner) as shutdown_context: + self._polling_runner.run_all(shutdown_context) diff --git a/tilebox-workflows/tilebox/workflows/runner/task_service.py b/tilebox-workflows/tilebox/workflows/runner/task_service.py index e9c8643..e65bdb3 100644 --- a/tilebox-workflows/tilebox/workflows/runner/task_service.py +++ b/tilebox-workflows/tilebox/workflows/runner/task_service.py @@ -6,6 +6,7 @@ from _tilebox.grpc.error import with_pythonic_errors from tilebox.workflows.data import ( ComputedTask, + FailedTask, Idling, NextTaskToRun, ProgressIndicator, @@ -17,7 +18,6 @@ from tilebox.workflows.workflows.v1.task_pb2 import ( NextTaskRequest, NextTaskResponse, - TaskFailedRequest, TaskLeaseRequest, ) from tilebox.workflows.workflows.v1.task_pb2_grpc import TaskServiceStub @@ -49,18 +49,18 @@ def next_task(self, task_to_run: NextTaskToRun | None, computed_task: ComputedTa return None def task_failed( - self, task: Task, error: Exception, was_workflow_error: bool, progress_updates: list[ProgressIndicator] + self, + task: Task | FailedTask, + error: Exception | None = None, + was_workflow_error: bool | None = None, + progress_updates: list[ProgressIndicator] | None = None, ) -> None: - # job ouptut is limited to 1KB, so truncate the error message if necessary - error_message = repr(error)[: (1024 - len(task.display or "None") - 1)] - display = f"{task.display}" if error_message == "" else f"{task.display}\n{error_message}" - - request = TaskFailedRequest( - task_id=uuid_to_uuid_message(task.id), - was_workflow_error=was_workflow_error, - display=display, - progress_updates=[progress.to_message() for progress in progress_updates], - ) + if isinstance(task, FailedTask): + request = task.to_message() + else: + if error is None or was_workflow_error is None or progress_updates is None: + raise ValueError("error, was_workflow_error, and progress_updates are required when passing a Task") + request = FailedTask.from_task_error(task, error, was_workflow_error, progress_updates).to_message() self.service.TaskFailed(request) def extend_task_lease(self, task_id: UUID, requested_lease: int) -> TaskLease: diff --git a/tilebox-workflows/tilebox/workflows/runner/worker_server.py b/tilebox-workflows/tilebox/workflows/runner/worker_server.py new file mode 100644 index 0000000..6818465 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/runner/worker_server.py @@ -0,0 +1,57 @@ +import os +import threading +from concurrent import futures +from pathlib import Path + +import grpc + +from tilebox.workflows.runner.runner import Runner +from tilebox.workflows.runner.worker_service import WorkerServiceServicer +from tilebox.workflows.workflows.v1 import worker_pb2_grpc + +WORKER_ADDRESS_ENV = "TILEBOX_WORKER_ADDRESS" + + +def serve_runner(runner: Runner, address: str | None = None) -> None: + address = address or os.environ.get(WORKER_ADDRESS_ENV) + if not address: + raise RuntimeError( + f"{WORKER_ADDRESS_ENV} is not set. Set it to a local gRPC address, for example " + f"'unix:///tmp/tilebox-worker.sock'." + ) + + bind_address = _normalize_grpc_address(address) + _unlink_stale_unix_socket(bind_address) + + server = grpc.server(futures.ThreadPoolExecutor()) + + def shutdown() -> None: + # server.stop() is blocking, so we run it in a separate thread + # server.stop(5) means we stop accepting new requests immediately, but we give existing requests up to 5 + # seconds to finish before we forcefully terminate them + threading.Thread(target=server.stop, args=(5,), daemon=True).start() + + worker_pb2_grpc.add_WorkerServiceServicer_to_server(WorkerServiceServicer(runner, shutdown), server) + port = server.add_insecure_port(bind_address) + if port == 0: + raise RuntimeError(f"Failed to bind worker server to {address!r}") + + server.start() + server.wait_for_termination() + + +def _normalize_grpc_address(address: str) -> str: + if address.startswith("unix://"): + return "unix:" + address.removeprefix("unix://") + return address + + +def _unlink_stale_unix_socket(address: str) -> None: + if not address.startswith("unix:"): + return + path = address.removeprefix("unix:") + if not path: + return + socket_path = Path(path) + if socket_path.exists(): + socket_path.unlink() diff --git a/tilebox-workflows/tilebox/workflows/runner/worker_service.py b/tilebox-workflows/tilebox/workflows/runner/worker_service.py new file mode 100644 index 0000000..7424ee3 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/runner/worker_service.py @@ -0,0 +1,86 @@ +from collections.abc import Callable + +import grpc +from google.protobuf.empty_pb2 import Empty + +from tilebox.datasets.uuid import uuid_message_to_uuid +from tilebox.workflows.cache import NoCache +from tilebox.workflows.client import Client +from tilebox.workflows.data import Cluster, ComputedTask, FailedTask, Task +from tilebox.workflows.observability.logging import StructuredLogger +from tilebox.workflows.runner.executor import LazyStorageLocations, TaskExecutor +from tilebox.workflows.runner.runner import Runner +from tilebox.workflows.task import RunnerContext +from tilebox.workflows.workflows.v1 import core_pb2, worker_pb2, worker_pb2_grpc + + +class WorkerServiceServicer(worker_pb2_grpc.WorkerServiceServicer): + def __init__( + self, + runner: Runner, + shutdown: Callable[[], None], + ) -> None: + self._runner = runner + self._shutdown = shutdown + self._executor: TaskExecutor | None = None + + def ListRegisteredTasks(self, request: Empty, context: grpc.ServicerContext) -> core_pb2.TaskIdentifiers: # noqa: ARG002, N802 + return core_pb2.TaskIdentifiers( + identifiers=[identifier.to_message() for identifier in self._runner.task_identifiers] + ) + + def InitializeWorker( # noqa: N802 + self, + request: worker_pb2.InitializeRunnerRequest, + context: grpc.ServicerContext, # noqa: ARG002 + ) -> worker_pb2.InitializeRunnerResponse: + runner_id = uuid_message_to_uuid(request.runner_id) + cluster = Cluster.from_message(request.cluster) if request.HasField("cluster") else None + + api_connection = request.api_connection if request.HasField("api_connection") else None + api_url = api_connection.url if api_connection and api_connection.url else "https://api.tilebox.com" + api_token = api_connection.token if api_connection and api_connection.token else None + + client = Client(url=api_url, token=api_token, client_id=runner_id) + tracer = client._tracer # noqa: SLF001 + task_logger = StructuredLogger(client._task_logger, {}) # noqa: SLF001 + + context_type = self._runner.context or RunnerContext + runner_context = context_type(tracer) + runner_context.storage_locations = LazyStorageLocations(client, runner_context) + + self._executor = TaskExecutor( + self._runner, + self._runner.cache or NoCache(), + tracer, + task_logger, + runner_context, + cluster.slug if cluster is not None else "", + ) + return worker_pb2.InitializeRunnerResponse() + + def ExecuteTask( # noqa: N802 + self, + request: core_pb2.Task, + context: grpc.ServicerContext, # noqa: ARG002 + ) -> worker_pb2.ExecuteTaskResponse: + task = Task.from_message(request) + if self._executor is None: + failed_task = FailedTask.from_task_error( + task, + RuntimeError("Worker is not initialized"), + was_workflow_error=False, + progress_updates=[], + ) + return worker_pb2.ExecuteTaskResponse(failed_task=failed_task.to_message()) + + result = self._executor.execute_task(task) + if isinstance(result, ComputedTask): + return worker_pb2.ExecuteTaskResponse(computed_task=result.to_message()) + if isinstance(result, FailedTask): + return worker_pb2.ExecuteTaskResponse(failed_task=result.to_message()) + raise TypeError(f"Unexpected task execution result: {type(result)}") + + def ShutdownWorker(self, request: Empty, context: grpc.ServicerContext) -> Empty: # noqa: ARG002, N802 + self._shutdown() + return Empty() diff --git a/tilebox-workflows/tilebox/workflows/task.py b/tilebox-workflows/tilebox/workflows/task.py index 691ccfc..9c73e14 100644 --- a/tilebox-workflows/tilebox/workflows/task.py +++ b/tilebox-workflows/tilebox/workflows/task.py @@ -1,4 +1,3 @@ -import contextlib import inspect import json import typing @@ -6,7 +5,7 @@ from base64 import b64decode, b64encode from collections import defaultdict from collections.abc import Sequence -from dataclasses import dataclass, field, fields, is_dataclass +from dataclasses import dataclass, fields, is_dataclass from types import NoneType, UnionType from typing import Any, Generic, TypeVar, cast, get_args, get_origin @@ -75,12 +74,7 @@ def __new__(cls, name: str, bases: tuple[type], attrs: dict[str, Any]) -> type: # raise as TypeError instead of a ValueError, because this runs at class creation time raise TypeError(str(err)) from None - interceptors = [] - with contextlib.suppress(TypeError): - # if possible we copy the already existing interceptors from the base class - interceptors = list(_task_meta(bases[-1]).interceptors) - - setattr(task_class, META_ATTR, TaskMeta(identifier, is_executable, interceptors)) + setattr(task_class, META_ATTR, TaskMeta(identifier, is_executable)) return task_class @@ -152,7 +146,6 @@ def _validate_execute_method( class TaskMeta: identifier: TaskIdentifier executable: bool - interceptors: list[Any] = field(default_factory=list) @staticmethod def for_task(task: type | Task) -> "TaskMeta": diff --git a/tilebox-workflows/tilebox/workflows/timeseries.py b/tilebox-workflows/tilebox/workflows/timeseries.py deleted file mode 100644 index f09f748..0000000 --- a/tilebox-workflows/tilebox/workflows/timeseries.py +++ /dev/null @@ -1,203 +0,0 @@ -import math -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from itertools import pairwise - -import xarray as xr - -# from python 3.11 onwards: typing.dataclass_transform -# from python 3.12 onwards: typing.override -from typing_extensions import dataclass_transform, override - -from tilebox.datasets.data.collection import Collection, CollectionInfo -from tilebox.datasets.data.timeseries import TimeChunk, TimeseriesDatasetChunk -from tilebox.datasets.query.id_interval import IDInterval -from tilebox.datasets.query.time_interval import TimeInterval, TimeIntervalLike -from tilebox.datasets.sync.dataset import CollectionClient -from tilebox.workflows.interceptors import ForwardExecution, execution_interceptor -from tilebox.workflows.task import ExecutionContext, Task - -_365_DAYS = timedelta(days=365) -_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) - - -@execution_interceptor -def _timeseries_dataset_chunk(task: Task, call_next: ForwardExecution, context: ExecutionContext) -> None: # noqa: C901 - if not isinstance(task, TimeseriesTask): - raise TypeError("Task is not a timeseries task. Inherit from TimeseriesTask to mark it as such.") - - chunk: TimeseriesDatasetChunk = task.timeseries_data - - # let's get a collection client - datasets_client = context.runner_context.datasets_client - dataset = datasets_client._dataset_by_id(str(chunk.dataset_id)) # ty: ignore[possibly-missing-attribute] # noqa: SLF001 - # we already know the collection id, so we can skip the lookup (we don't know the name, but don't need it) - collection_info = CollectionInfo(Collection(chunk.collection_id, "unknown"), None, None) - collection = CollectionClient(dataset, collection_info) - - # leaf case: we are already executing a specific batch of datapoints fitting in the chunk size, so let's load them - if chunk.datapoint_interval: - datapoint_interval = (chunk.datapoint_interval.start_id, chunk.datapoint_interval.end_id) - # we already are a leaf task executing for a specific datapoint interval: - datapoints = collection._find_interval( # noqa: SLF001 - datapoint_interval, - end_inclusive=chunk.datapoint_interval.end_inclusive, - skip_data=False, - show_progress=False, - ) - if not datapoints: - return # no datapoints in the interval -> we are done - - for i in range(datapoints.sizes["time"]): - datapoint = datapoints.isel(time=i) - call_next(context, datapoint) # ty: ignore[too-many-positional-arguments] - - return # we are done - - if not chunk.time_interval: - raise ValueError("Missing time_interval and data_point interval, one of them is required") - - interval = chunk.time_interval - interval_size = interval.end - interval.start - estimated_datapoints = int(chunk.datapoints_per_365_days * (interval_size / _365_DAYS)) - sub_chunks = [] - - # if we are only a little larger than the chunk size, let's submit leaf tasks next: - if estimated_datapoints < chunk.chunk_size * (chunk.branch_factor - 0.5): - for page in collection._iter_pages( # noqa: SLF001 - interval, skip_data=True, show_progress=False, page_size=chunk.chunk_size - ): - # pages here only contain datapoint ids, no metadata or data - # pages are limited to a maximum of chunk_size datapoints, but additionally also server side - # in case the chunk size is larger than the server side maximum limit - # in that case we just use an effective chunk size of the server side limit - - if page.n_datapoints > 0: - interval_chunk = replace( - chunk, - time_interval=None, - datapoint_interval=IDInterval( - start_id=page.min_id, end_id=page.max_id, start_exclusive=False, end_inclusive=True - ), - ) - sub_chunks.append(interval_chunk) - - # otherwise let's split our time range evenly into sub chunks: - else: - chunk_interval = interval_size / chunk.branch_factor - chunks = [chunk.time_interval.start + chunk_interval * i for i in range(chunk.branch_factor)] + [interval.end] - - for sub_chunk_start, sub_chunk_end in pairwise(chunks): - sub_chunks.append(replace(chunk, time_interval=TimeInterval(sub_chunk_start, sub_chunk_end))) - - subtasks = [replace(task, timeseries_data=sub_chunk) for sub_chunk in sub_chunks] - if len(subtasks) > 0: - context.submit_subtasks(subtasks) - - return - - -@_timeseries_dataset_chunk -@dataclass_transform() -class TimeseriesTask(Task): - timeseries_data: TimeseriesDatasetChunk - - @override - def execute(self, context: ExecutionContext, datapoint: xr.Dataset) -> None: # ty: ignore[invalid-method-override] - pass - - -def batch_process_timeseries_dataset( - collection: CollectionClient, interval: TimeIntervalLike, chunk_size: int -) -> TimeseriesDatasetChunk: - info = collection.info(availability=True, count=True) - - assert info.availability is not None - assert info.count is not None - - interval = TimeInterval.parse(interval).to_half_open() # our time splitting assumes half open intervals - - # estimate the number of datapoints per 365 days, to roughly know when to stop splitting into branches - datapoints_per_365_days = math.ceil(info.count / ((info.availability.end - info.availability.start) / _365_DAYS)) - - return TimeseriesDatasetChunk( - dataset_id=collection._dataset._dataset.id, # noqa: SLF001 - collection_id=info.collection.id, - time_interval=interval, - datapoint_interval=None, - branch_factor=2, - chunk_size=chunk_size, - datapoints_per_365_days=datapoints_per_365_days, - ) - - -@execution_interceptor -def _time_interval_chunk(task: Task, call_next: ForwardExecution, context: ExecutionContext) -> None: - if not isinstance(task, TimeIntervalTask): - raise TypeError("Task is not a time interval task. Inherit from TimeIntervalTask to mark it as such.") - - chunk: TimeChunk = task.interval - - start = _make_multiple(chunk.time_interval.start, chunk.chunk_size, before=True) - end = _make_multiple(chunk.time_interval.end, chunk.chunk_size, before=False) - - n = (end - start) // chunk.chunk_size - if n <= 1: # we are already a leaf task - return call_next(context, TimeInterval(start, end)) # ty: ignore[too-many-positional-arguments] - - chunks: list[datetime] = [] - if n < 4: # we are a branch task with less than 4 sub chunks, so a further split is not worth it - chunks = [start + chunk.chunk_size * i for i in range(n)] + [end] - else: # we have a large number of sub chunks, so let's split into 2 branches - # in case we can't perfectly divide in the middle due to an uneven - # number make the left half slightly larger than the right half - middle = _make_multiple(start + (end - start) / 2, chunk.chunk_size, before=False) - chunks = [start, middle, end] - - time_chunks = [ - TimeChunk(TimeInterval(chunk_start, chunk_end), chunk.chunk_size) for chunk_start, chunk_end in pairwise(chunks) - ] - - context.submit_subtasks([replace(task, interval=time_chunk) for time_chunk in time_chunks]) - return None - - -@_time_interval_chunk -@dataclass_transform() -class TimeIntervalTask(Task): - interval: TimeChunk - - @override - def execute(self, context: ExecutionContext, time_interval: TimeInterval) -> None: # ty: ignore[invalid-method-override] - pass - - -def batch_process_time_interval(interval: TimeIntervalLike, chunk_size: timedelta) -> TimeChunk: - return TimeChunk(time_interval=TimeInterval.parse(interval).to_half_open(), chunk_size=chunk_size) - - -def _make_multiple(time: datetime, duration: timedelta, start: datetime = _EPOCH, before: bool = True) -> datetime: - """ - Calculate the nearest multiple of a duration for a given datetime relative to the given start datetime. - - Depending on the before argument, this will calculate the nearest multiple immediately before or after the given - time. - - >>> _make_multiple(datetime(2021, 4, 13, 13, 12, 10), timedelta(days=1)) - datetime.datetime(2021, 4, 13, 0, 0) - - >>> _make_multiple(datetime(2021, 4, 13, 13, 12, 10), timedelta(days=1), before=False) - datetime.datetime(2021, 4, 14, 0, 0) - - Args: - time: The datetime to make a multiple. - duration: The duration to make a multiple of. - start: The start datetime to make a multiple relative to. Defaults to _EPOCH. - before: Whether to calculate the nearest multiple immediately before or after the given time. Defaults to True. - - Returns: - A datetime that is a multiple of the given duration relative to the given start datetime. - """ - n = (time - start) / duration - n = math.floor(n) if before else math.ceil(n) - return start + n * duration diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py index d3afc4c..0ab210d 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2_grpc.py @@ -7,7 +7,7 @@ from tilebox.workflows.workflows.v1 import automation_pb2 as workflows_dot_v1_dot_automation__pb2 -class AutomationServiceStub(object): +class AutomationServiceStub: """AutomationService is a service for managing automations. Currently, we support two types of triggers for automations: - Bucket triggers, which triggers tasks when an object is uploaded to a storage bucket that matches a glob pattern - Cron triggers, which triggers tasks on a schedule @@ -66,7 +66,7 @@ def __init__(self, channel): _registered_method=True) -class AutomationServiceServicer(object): +class AutomationServiceServicer: """AutomationService is a service for managing automations. Currently, we support two types of triggers for automations: - Bucket triggers, which triggers tasks when an object is uploaded to a storage bucket that matches a glob pattern - Cron triggers, which triggers tasks on a schedule @@ -191,7 +191,7 @@ def add_AutomationServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class AutomationService(object): +class AutomationService: """AutomationService is a service for managing automations. Currently, we support two types of triggers for automations: - Bucket triggers, which triggers tasks when an object is uploaded to a storage bucket that matches a glob pattern - Cron triggers, which triggers tasks on a schedule diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.py index 910bbe4..3cba2bb 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.py @@ -28,7 +28,7 @@ from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17workflows/v1/core.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\"d\n\x07\x43luster\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12!\n\x0c\x64isplay_name\x18\x03 \x01(\tR\x0b\x64isplayName\x12\x1c\n\tdeletable\x18\x04 \x01(\x08R\tdeletableJ\x04\x08\x01\x10\x02\"\xe5\x04\n\x03Job\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12!\n\x0ctrace_parent\x18\x03 \x01(\tR\x0btraceParent\x12\x1e\n\x08\x63\x61nceled\x18\x05 \x01(\x08\x42\x02\x18\x01R\x08\x63\x61nceled\x12\x43\n\x0clegacy_state\x18\x06 \x01(\x0e\x32\x1c.workflows.v1.LegacyJobStateB\x02\x18\x01R\x0blegacyState\x12=\n\x0csubmitted_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0bsubmittedAt\x12=\n\nstarted_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\tstartedAt\x12@\n\x0etask_summaries\x18\t \x03(\x0b\x32\x19.workflows.v1.TaskSummaryR\rtaskSummaries\x12\x33\n\rautomation_id\x18\n \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x32\n\x08progress\x18\x0b \x03(\x0b\x32\x16.workflows.v1.ProgressR\x08progress\x12,\n\x05state\x18\x0c \x01(\x0e\x32\x16.workflows.v1.JobStateR\x05state\x12\x45\n\x0f\x65xecution_stats\x18\r \x01(\x0b\x32\x1c.workflows.v1.ExecutionStatsR\x0e\x65xecutionStatsJ\x04\x08\x04\x10\x05\"\xaf\x03\n\x0e\x45xecutionStats\x12M\n\x15\x66irst_task_started_at\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x12\x66irstTaskStartedAt\x12K\n\x14last_task_stopped_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x11lastTaskStoppedAt\x12<\n\x0c\x63ompute_time\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0b\x63omputeTime\x12<\n\x0c\x65lapsed_time\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0b\x65lapsedTime\x12 \n\x0bparallelism\x18\x05 \x01(\x01R\x0bparallelism\x12\x1f\n\x0btotal_tasks\x18\x06 \x01(\x04R\ntotalTasks\x12\x42\n\x0etasks_by_state\x18\x07 \x03(\x0b\x32\x1c.workflows.v1.TaskStateCountR\x0ctasksByState\"U\n\x0eTaskStateCount\x12-\n\x05state\x18\x01 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12\x14\n\x05\x63ount\x18\x02 \x01(\x04R\x05\x63ount\"\x9f\x02\n\x0bTaskSummary\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x18\n\x07\x64isplay\x18\x02 \x01(\tR\x07\x64isplay\x12-\n\x05state\x18\x03 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12+\n\tparent_id\x18\x04 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x08parentId\x12\x39\n\nstarted_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x39\n\nstopped_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstoppedAtJ\x04\x08\x05\x10\x06\"S\n\x08Progress\x12\x1d\n\x05label\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x18\x64R\x05label\x12\x14\n\x05total\x18\x02 \x01(\x04R\x05total\x12\x12\n\x04\x64one\x18\x03 \x01(\x04R\x04\x64one\"\xa2\x03\n\x04Task\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12<\n\nidentifier\x18\x02 \x01(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\nidentifier\x12-\n\x05state\x18\x03 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12\x1b\n\x05input\x18\x04 \x01(\x0c\x42\x05\xaa\x01\x02\x08\x01R\x05input\x12\x1f\n\x07\x64isplay\x18\x05 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x07\x64isplay\x12#\n\x03job\x18\x06 \x01(\x0b\x32\x11.workflows.v1.JobR\x03job\x12+\n\tparent_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x08parentId\x12-\n\ndepends_on\x18\x08 \x03(\x0b\x32\x0e.tilebox.v1.IDR\tdependsOn\x12-\n\x05lease\x18\t \x01(\x0b\x32\x17.workflows.v1.TaskLeaseR\x05lease\x12\x1f\n\x0bretry_count\x18\n \x01(\x03R\nretryCount\"d\n\x0eTaskIdentifier\x12\x1e\n\x04name\x18\x01 \x01(\tB\n\xbaH\x07r\x05 \x01(\x80\x02R\x04name\x12\x32\n\x07version\x18\x02 \x01(\tB\x18\xbaH\x15r\x13 \x01\x32\x0f^v(\\d+)\\.(\\d+)$R\x07version\"1\n\x05Tasks\x12(\n\x05tasks\x18\x01 \x03(\x0b\x32\x12.workflows.v1.TaskR\x05tasks\"\x98\x02\n\x14SingleTaskSubmission\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\x12<\n\nidentifier\x18\x02 \x01(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\nidentifier\x12!\n\x07\x64isplay\x18\x04 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07\x64isplay\x12\x32\n\x0c\x64\x65pendencies\x18\x05 \x03(\x03\x42\x0e\xbaH\x0b\x92\x01\x08\"\x06\"\x04\x18?(\x00R\x0c\x64\x65pendencies\x12(\n\x0bmax_retries\x18\x06 \x01(\x03\x42\x07\xbaH\x04\"\x02(\x00R\nmaxRetries\x12\x1e\n\x05input\x18\x03 \x01(\x0c\x42\x08\xbaH\x05z\x03\x18\x80\x10R\x05input\"\x91\x02\n\x0fTaskSubmissions\x12N\n\x0btask_groups\x18\x01 \x03(\x0b\x32!.workflows.v1.TaskSubmissionGroupB\n\xbaH\x07\x92\x01\x04\x08\x01\x10@R\ntaskGroups\x12.\n\x13\x63luster_slug_lookup\x18\x02 \x03(\tR\x11\x63lusterSlugLookup\x12I\n\x11identifier_lookup\x18\x03 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\x10identifierLookup\x12\x33\n\x0e\x64isplay_lookup\x18\x04 \x03(\tB\x0c\xbaH\t\x92\x01\x06\"\x04r\x02\x10\x01R\rdisplayLookup\"\xd2\t\n\x13TaskSubmissionGroup\x12?\n\x1c\x64\x65pendencies_on_other_groups\x18\x01 \x03(\rR\x19\x64\x65pendenciesOnOtherGroups\x12\'\n\x06inputs\x18\x02 \x03(\x0c\x42\x0f\xbaH\x0c\x92\x01\t\x08\x01\"\x05z\x03\x18\x80\x10R\x06inputs\x12/\n\x13identifier_pointers\x18\x03 \x03(\x04R\x12identifierPointers\x12\x32\n\x15\x63luster_slug_pointers\x18\x04 \x03(\x04R\x13\x63lusterSlugPointers\x12)\n\x10\x64isplay_pointers\x18\x05 \x03(\x04R\x0f\x64isplayPointers\x12,\n\x12max_retries_values\x18\x06 \x03(\x03R\x10maxRetriesValues\x12\'\n\x0foptional_values\x18\x07 \x03(\x08R\x0eoptionalValues:\xe9\x06\xbaH\xe5\x06\x1a\xa6\x01\n,task_submission_group.identifiers_size_match\x12?The number of inputs must match the number of task identifiers.\x1a\x35this.inputs.size() == this.identifier_pointers.size()\x1a\xa7\x01\n.task_submission_group.cluster_slugs_size_match\x12The number of optional values must match the number of inputs.\x1aUthis.optional_values.size() == 0 || this.inputs.size() == this.optional_values.size()\"\xa9\x01\n\tTaskLease\x12/\n\x05lease\x18\x01 \x01(\x0b\x32\x19.google.protobuf.DurationR\x05lease\x12k\n%recommended_wait_until_next_extension\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR!recommendedWaitUntilNextExtension*\x8d\x01\n\x0eLegacyJobState\x12 \n\x1cLEGACY_JOB_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17LEGACY_JOB_STATE_QUEUED\x10\x01\x12\x1c\n\x18LEGACY_JOB_STATE_STARTED\x10\x02\x12\x1e\n\x1aLEGACY_JOB_STATE_COMPLETED\x10\x03*\xb3\x01\n\x08JobState\x12\x19\n\x15JOB_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13JOB_STATE_SUBMITTED\x10\x01\x12\x15\n\x11JOB_STATE_RUNNING\x10\x02\x12\x15\n\x11JOB_STATE_STARTED\x10\x03\x12\x17\n\x13JOB_STATE_COMPLETED\x10\x04\x12\x14\n\x10JOB_STATE_FAILED\x10\x05\x12\x16\n\x12JOB_STATE_CANCELED\x10\x06*\xbe\x01\n\tTaskState\x12\x1a\n\x16TASK_STATE_UNSPECIFIED\x10\x00\x12\x15\n\x11TASK_STATE_QUEUED\x10\x01\x12\x16\n\x12TASK_STATE_RUNNING\x10\x02\x12\x17\n\x13TASK_STATE_COMPUTED\x10\x03\x12\x15\n\x11TASK_STATE_FAILED\x10\x04\x12\x16\n\x12TASK_STATE_SKIPPED\x10\x05\x12\x1e\n\x1aTASK_STATE_FAILED_OPTIONAL\x10\x06\x42s\n\x10\x63om.workflows.v1B\tCoreProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17workflows/v1/core.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\"\xe5\x04\n\x03Job\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12!\n\x0ctrace_parent\x18\x03 \x01(\tR\x0btraceParent\x12\x1e\n\x08\x63\x61nceled\x18\x05 \x01(\x08\x42\x02\x18\x01R\x08\x63\x61nceled\x12\x43\n\x0clegacy_state\x18\x06 \x01(\x0e\x32\x1c.workflows.v1.LegacyJobStateB\x02\x18\x01R\x0blegacyState\x12=\n\x0csubmitted_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0bsubmittedAt\x12=\n\nstarted_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\tstartedAt\x12@\n\x0etask_summaries\x18\t \x03(\x0b\x32\x19.workflows.v1.TaskSummaryR\rtaskSummaries\x12\x33\n\rautomation_id\x18\n \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x32\n\x08progress\x18\x0b \x03(\x0b\x32\x16.workflows.v1.ProgressR\x08progress\x12,\n\x05state\x18\x0c \x01(\x0e\x32\x16.workflows.v1.JobStateR\x05state\x12\x45\n\x0f\x65xecution_stats\x18\r \x01(\x0b\x32\x1c.workflows.v1.ExecutionStatsR\x0e\x65xecutionStatsJ\x04\x08\x04\x10\x05\"\xaf\x03\n\x0e\x45xecutionStats\x12M\n\x15\x66irst_task_started_at\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x12\x66irstTaskStartedAt\x12K\n\x14last_task_stopped_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x11lastTaskStoppedAt\x12<\n\x0c\x63ompute_time\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0b\x63omputeTime\x12<\n\x0c\x65lapsed_time\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x0b\x65lapsedTime\x12 \n\x0bparallelism\x18\x05 \x01(\x01R\x0bparallelism\x12\x1f\n\x0btotal_tasks\x18\x06 \x01(\x04R\ntotalTasks\x12\x42\n\x0etasks_by_state\x18\x07 \x03(\x0b\x32\x1c.workflows.v1.TaskStateCountR\x0ctasksByState\"U\n\x0eTaskStateCount\x12-\n\x05state\x18\x01 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12\x14\n\x05\x63ount\x18\x02 \x01(\x04R\x05\x63ount\"\x9f\x02\n\x0bTaskSummary\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x18\n\x07\x64isplay\x18\x02 \x01(\tR\x07\x64isplay\x12-\n\x05state\x18\x03 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12+\n\tparent_id\x18\x04 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x08parentId\x12\x39\n\nstarted_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartedAt\x12\x39\n\nstopped_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstoppedAtJ\x04\x08\x05\x10\x06\"S\n\x08Progress\x12\x1d\n\x05label\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x18\x64R\x05label\x12\x14\n\x05total\x18\x02 \x01(\x04R\x05total\x12\x12\n\x04\x64one\x18\x03 \x01(\x04R\x04\x64one\"\xa2\x03\n\x04Task\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12<\n\nidentifier\x18\x02 \x01(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\nidentifier\x12-\n\x05state\x18\x03 \x01(\x0e\x32\x17.workflows.v1.TaskStateR\x05state\x12\x1b\n\x05input\x18\x04 \x01(\x0c\x42\x05\xaa\x01\x02\x08\x01R\x05input\x12\x1f\n\x07\x64isplay\x18\x05 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x07\x64isplay\x12#\n\x03job\x18\x06 \x01(\x0b\x32\x11.workflows.v1.JobR\x03job\x12+\n\tparent_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x08parentId\x12-\n\ndepends_on\x18\x08 \x03(\x0b\x32\x0e.tilebox.v1.IDR\tdependsOn\x12-\n\x05lease\x18\t \x01(\x0b\x32\x17.workflows.v1.TaskLeaseR\x05lease\x12\x1f\n\x0bretry_count\x18\n \x01(\x03R\nretryCount\"d\n\x0eTaskIdentifier\x12\x1e\n\x04name\x18\x01 \x01(\tB\n\xbaH\x07r\x05 \x01(\x80\x02R\x04name\x12\x32\n\x07version\x18\x02 \x01(\tB\x18\xbaH\x15r\x13 \x01\x32\x0f^v(\\d+)\\.(\\d+)$R\x07version\"Q\n\x0fTaskIdentifiers\x12>\n\x0bidentifiers\x18\x01 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\x0bidentifiers\"1\n\x05Tasks\x12(\n\x05tasks\x18\x01 \x03(\x0b\x32\x12.workflows.v1.TaskR\x05tasks\"\x98\x02\n\x14SingleTaskSubmission\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\x12<\n\nidentifier\x18\x02 \x01(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\nidentifier\x12!\n\x07\x64isplay\x18\x04 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07\x64isplay\x12\x32\n\x0c\x64\x65pendencies\x18\x05 \x03(\x03\x42\x0e\xbaH\x0b\x92\x01\x08\"\x06\"\x04\x18?(\x00R\x0c\x64\x65pendencies\x12(\n\x0bmax_retries\x18\x06 \x01(\x03\x42\x07\xbaH\x04\"\x02(\x00R\nmaxRetries\x12\x1e\n\x05input\x18\x03 \x01(\x0c\x42\x08\xbaH\x05z\x03\x18\x80\x10R\x05input\"\x91\x02\n\x0fTaskSubmissions\x12N\n\x0btask_groups\x18\x01 \x03(\x0b\x32!.workflows.v1.TaskSubmissionGroupB\n\xbaH\x07\x92\x01\x04\x08\x01\x10@R\ntaskGroups\x12.\n\x13\x63luster_slug_lookup\x18\x02 \x03(\tR\x11\x63lusterSlugLookup\x12I\n\x11identifier_lookup\x18\x03 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\x10identifierLookup\x12\x33\n\x0e\x64isplay_lookup\x18\x04 \x03(\tB\x0c\xbaH\t\x92\x01\x06\"\x04r\x02\x10\x01R\rdisplayLookup\"\xd2\t\n\x13TaskSubmissionGroup\x12?\n\x1c\x64\x65pendencies_on_other_groups\x18\x01 \x03(\rR\x19\x64\x65pendenciesOnOtherGroups\x12\'\n\x06inputs\x18\x02 \x03(\x0c\x42\x0f\xbaH\x0c\x92\x01\t\x08\x01\"\x05z\x03\x18\x80\x10R\x06inputs\x12/\n\x13identifier_pointers\x18\x03 \x03(\x04R\x12identifierPointers\x12\x32\n\x15\x63luster_slug_pointers\x18\x04 \x03(\x04R\x13\x63lusterSlugPointers\x12)\n\x10\x64isplay_pointers\x18\x05 \x03(\x04R\x0f\x64isplayPointers\x12,\n\x12max_retries_values\x18\x06 \x03(\x03R\x10maxRetriesValues\x12\'\n\x0foptional_values\x18\x07 \x03(\x08R\x0eoptionalValues:\xe9\x06\xbaH\xe5\x06\x1a\xa6\x01\n,task_submission_group.identifiers_size_match\x12?The number of inputs must match the number of task identifiers.\x1a\x35this.inputs.size() == this.identifier_pointers.size()\x1a\xa7\x01\n.task_submission_group.cluster_slugs_size_match\x12The number of optional values must match the number of inputs.\x1aUthis.optional_values.size() == 0 || this.inputs.size() == this.optional_values.size()\"\xa9\x01\n\tTaskLease\x12/\n\x05lease\x18\x01 \x01(\x0b\x32\x19.google.protobuf.DurationR\x05lease\x12k\n%recommended_wait_until_next_extension\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR!recommendedWaitUntilNextExtension*\x8d\x01\n\x0eLegacyJobState\x12 \n\x1cLEGACY_JOB_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17LEGACY_JOB_STATE_QUEUED\x10\x01\x12\x1c\n\x18LEGACY_JOB_STATE_STARTED\x10\x02\x12\x1e\n\x1aLEGACY_JOB_STATE_COMPLETED\x10\x03*\xb3\x01\n\x08JobState\x12\x19\n\x15JOB_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13JOB_STATE_SUBMITTED\x10\x01\x12\x15\n\x11JOB_STATE_RUNNING\x10\x02\x12\x15\n\x11JOB_STATE_STARTED\x10\x03\x12\x17\n\x13JOB_STATE_COMPLETED\x10\x04\x12\x14\n\x10JOB_STATE_FAILED\x10\x05\x12\x16\n\x12JOB_STATE_CANCELED\x10\x06*\xbe\x01\n\tTaskState\x12\x1a\n\x16TASK_STATE_UNSPECIFIED\x10\x00\x12\x15\n\x11TASK_STATE_QUEUED\x10\x01\x12\x16\n\x12TASK_STATE_RUNNING\x10\x02\x12\x17\n\x13TASK_STATE_COMPUTED\x10\x03\x12\x15\n\x11TASK_STATE_FAILED\x10\x04\x12\x16\n\x12TASK_STATE_SKIPPED\x10\x05\x12\x1e\n\x1aTASK_STATE_FAILED_OPTIONAL\x10\x06\x42s\n\x10\x63om.workflows.v1B\tCoreProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -68,36 +68,36 @@ _globals['_TASKSUBMISSIONGROUP'].fields_by_name['inputs']._serialized_options = b'\272H\014\222\001\t\010\001\"\005z\003\030\200\020' _globals['_TASKSUBMISSIONGROUP']._loaded_options = None _globals['_TASKSUBMISSIONGROUP']._serialized_options = b'\272H\345\006\032\246\001\n,task_submission_group.identifiers_size_match\022?The number of inputs must match the number of task identifiers.\0325this.inputs.size() == this.identifier_pointers.size()\032\247\001\n.task_submission_group.cluster_slugs_size_match\022The number of optional values must match the number of inputs.\032Uthis.optional_values.size() == 0 || this.inputs.size() == this.optional_values.size()' - _globals['_LEGACYJOBSTATE']._serialized_start=4313 - _globals['_LEGACYJOBSTATE']._serialized_end=4454 - _globals['_JOBSTATE']._serialized_start=4457 - _globals['_JOBSTATE']._serialized_end=4636 - _globals['_TASKSTATE']._serialized_start=4639 - _globals['_TASKSTATE']._serialized_end=4829 - _globals['_CLUSTER']._serialized_start=156 - _globals['_CLUSTER']._serialized_end=256 - _globals['_JOB']._serialized_start=259 - _globals['_JOB']._serialized_end=872 - _globals['_EXECUTIONSTATS']._serialized_start=875 - _globals['_EXECUTIONSTATS']._serialized_end=1306 - _globals['_TASKSTATECOUNT']._serialized_start=1308 - _globals['_TASKSTATECOUNT']._serialized_end=1393 - _globals['_TASKSUMMARY']._serialized_start=1396 - _globals['_TASKSUMMARY']._serialized_end=1683 - _globals['_PROGRESS']._serialized_start=1685 - _globals['_PROGRESS']._serialized_end=1768 - _globals['_TASK']._serialized_start=1771 - _globals['_TASK']._serialized_end=2189 - _globals['_TASKIDENTIFIER']._serialized_start=2191 - _globals['_TASKIDENTIFIER']._serialized_end=2291 - _globals['_TASKS']._serialized_start=2293 - _globals['_TASKS']._serialized_end=2342 - _globals['_SINGLETASKSUBMISSION']._serialized_start=2345 - _globals['_SINGLETASKSUBMISSION']._serialized_end=2625 - _globals['_TASKSUBMISSIONS']._serialized_start=2628 - _globals['_TASKSUBMISSIONS']._serialized_end=2901 - _globals['_TASKSUBMISSIONGROUP']._serialized_start=2904 - _globals['_TASKSUBMISSIONGROUP']._serialized_end=4138 - _globals['_TASKLEASE']._serialized_start=4141 - _globals['_TASKLEASE']._serialized_end=4310 + _globals['_LEGACYJOBSTATE']._serialized_start=4294 + _globals['_LEGACYJOBSTATE']._serialized_end=4435 + _globals['_JOBSTATE']._serialized_start=4438 + _globals['_JOBSTATE']._serialized_end=4617 + _globals['_TASKSTATE']._serialized_start=4620 + _globals['_TASKSTATE']._serialized_end=4810 + _globals['_JOB']._serialized_start=157 + _globals['_JOB']._serialized_end=770 + _globals['_EXECUTIONSTATS']._serialized_start=773 + _globals['_EXECUTIONSTATS']._serialized_end=1204 + _globals['_TASKSTATECOUNT']._serialized_start=1206 + _globals['_TASKSTATECOUNT']._serialized_end=1291 + _globals['_TASKSUMMARY']._serialized_start=1294 + _globals['_TASKSUMMARY']._serialized_end=1581 + _globals['_PROGRESS']._serialized_start=1583 + _globals['_PROGRESS']._serialized_end=1666 + _globals['_TASK']._serialized_start=1669 + _globals['_TASK']._serialized_end=2087 + _globals['_TASKIDENTIFIER']._serialized_start=2089 + _globals['_TASKIDENTIFIER']._serialized_end=2189 + _globals['_TASKIDENTIFIERS']._serialized_start=2191 + _globals['_TASKIDENTIFIERS']._serialized_end=2272 + _globals['_TASKS']._serialized_start=2274 + _globals['_TASKS']._serialized_end=2323 + _globals['_SINGLETASKSUBMISSION']._serialized_start=2326 + _globals['_SINGLETASKSUBMISSION']._serialized_end=2606 + _globals['_TASKSUBMISSIONS']._serialized_start=2609 + _globals['_TASKSUBMISSIONS']._serialized_end=2882 + _globals['_TASKSUBMISSIONGROUP']._serialized_start=2885 + _globals['_TASKSUBMISSIONGROUP']._serialized_end=4119 + _globals['_TASKLEASE']._serialized_start=4122 + _globals['_TASKLEASE']._serialized_end=4291 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.pyi index 1c8257b..8e0d178 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/core_pb2.pyi @@ -56,16 +56,6 @@ TASK_STATE_FAILED: TaskState TASK_STATE_SKIPPED: TaskState TASK_STATE_FAILED_OPTIONAL: TaskState -class Cluster(_message.Message): - __slots__ = ("slug", "display_name", "deletable") - SLUG_FIELD_NUMBER: _ClassVar[int] - DISPLAY_NAME_FIELD_NUMBER: _ClassVar[int] - DELETABLE_FIELD_NUMBER: _ClassVar[int] - slug: str - display_name: str - deletable: bool - def __init__(self, slug: _Optional[str] = ..., display_name: _Optional[str] = ..., deletable: bool = ...) -> None: ... - class Job(_message.Message): __slots__ = ("id", "name", "trace_parent", "canceled", "legacy_state", "submitted_at", "started_at", "task_summaries", "automation_id", "progress", "state", "execution_stats") ID_FIELD_NUMBER: _ClassVar[int] @@ -178,6 +168,12 @@ class TaskIdentifier(_message.Message): version: str def __init__(self, name: _Optional[str] = ..., version: _Optional[str] = ...) -> None: ... +class TaskIdentifiers(_message.Message): + __slots__ = ("identifiers",) + IDENTIFIERS_FIELD_NUMBER: _ClassVar[int] + identifiers: _containers.RepeatedCompositeFieldContainer[TaskIdentifier] + def __init__(self, identifiers: _Optional[_Iterable[_Union[TaskIdentifier, _Mapping]]] = ...) -> None: ... + class Tasks(_message.Message): __slots__ = ("tasks",) TASKS_FIELD_NUMBER: _ClassVar[int] diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py index c98edc1..ed0577b 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/diagram_pb2_grpc.py @@ -5,7 +5,7 @@ from tilebox.workflows.workflows.v1 import diagram_pb2 as workflows_dot_v1_dot_diagram__pb2 -class DiagramServiceStub(object): +class DiagramServiceStub: """The diagram service """ @@ -22,7 +22,7 @@ def __init__(self, channel): _registered_method=True) -class DiagramServiceServicer(object): +class DiagramServiceServicer: """The diagram service """ @@ -48,7 +48,7 @@ def add_DiagramServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class DiagramService(object): +class DiagramService: """The diagram service """ diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.py index ca3a942..ad84cfe 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.py @@ -31,7 +31,7 @@ from tilebox.workflows.workflows.v1 import diagram_pb2 as workflows_dot_v1_dot_diagram__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16workflows/v1/job.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\x1a\x17workflows/v1/core.proto\x1a\x1aworkflows/v1/diagram.proto\"\xce\x03\n\x10SubmitJobRequest\x12\x33\n\x05tasks\x18\x05 \x01(\x0b\x32\x1d.workflows.v1.TaskSubmissionsR\x05tasks\x12\"\n\x08job_name\x18\x02 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07jobName\x12*\n\x0ctrace_parent\x18\x03 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0btraceParent\x12\x33\n\rautomation_id\x18\x04 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12R\n\x0clegacy_tasks\x18\x01 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionB\x0b\x18\x01\xbaH\x06\x92\x01\x03\x10\xe8\x07R\x0blegacyTasks:\xab\x01\xbaH\xa7\x01\x1a\xa4\x01\n!submit_job_request.tasks_required\x12$At least one task must be submitted.\x1aY(this.tasks != null && this.tasks.task_groups.size() > 0) || this.legacy_tasks.size() > 0\">\n\rGetJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"F\n\x15GetJobProgressRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"@\n\x0fRetryJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"F\n\x10RetryJobResponse\x12\x32\n\x15num_tasks_rescheduled\x18\x01 \x01(\x03R\x13numTasksRescheduled\"A\n\x10\x43\x61ncelJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"\x13\n\x11\x43\x61ncelJobResponse\"\xec\x01\n\x13VisualizeJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x42\n\x0erender_options\x18\x02 \x01(\x0b\x32\x1b.workflows.v1.RenderOptionsR\rrenderOptions\x12\x38\n\x05theme\x18\x03 \x01(\x0e\x32\".workflows.v1.WorkflowDiagramThemeR\x05theme\x12(\n\x10include_job_name\x18\x04 \x01(\x08R\x0eincludeJobName\"\xe9\x02\n\x0cQueryFilters\x12=\n\rtime_interval\x18\x01 \x01(\x0b\x32\x18.tilebox.v1.TimeIntervalR\x0ctimeInterval\x12\x37\n\x0bid_interval\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.IDIntervalR\nidInterval\x12\x35\n\x0e\x61utomation_ids\x18\x03 \x03(\x0b\x32\x0e.tilebox.v1.IDR\rautomationIds\x12.\n\x06states\x18\x04 \x03(\x0e\x32\x16.workflows.v1.JobStateR\x06states\x12\x1b\n\x04name\x18\x05 \x01(\tB\x07\xbaH\x04r\x02\x18\x64R\x04name\x12\x38\n\x0btask_states\x18\x06 \x03(\x0e\x32\x17.workflows.v1.TaskStateR\ntaskStates:#\xbaH \"\x1e\n\rtime_interval\n\x0bid_interval\x10\x00\"{\n\x10QueryJobsRequest\x12\x34\n\x07\x66ilters\x18\x01 \x01(\x0b\x32\x1a.workflows.v1.QueryFiltersR\x07\x66ilters\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"v\n\x11QueryJobsResponse\x12%\n\x04jobs\x18\x01 \x03(\x0b\x32\x11.workflows.v1.JobR\x04jobs\x12:\n\tnext_page\x18\x03 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"G\n\x16GetJobPrototypeRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"w\n\x17GetJobPrototypeResponse\x12\x41\n\nroot_tasks\x18\x01 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\trootTasks\x12\x19\n\x08job_name\x18\x02 \x01(\tR\x07jobName\"\xc6\x01\n\x0f\x43loneJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12`\n\x14root_tasks_overrides\x18\x02 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionB\n\xbaH\x07\x92\x01\x04\x08\x01\x10@R\x12rootTasksOverrides\x12\"\n\x08job_name\x18\x03 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07jobName*\xd4\x01\n\x14WorkflowDiagramTheme\x12&\n\"WORKFLOW_DIAGRAM_THEME_UNSPECIFIED\x10\x00\x12 \n\x1cWORKFLOW_DIAGRAM_THEME_LIGHT\x10\x01\x12\x1f\n\x1bWORKFLOW_DIAGRAM_THEME_DARK\x10\x02\x12(\n$WORKFLOW_DIAGRAM_THEME_CONSOLE_LIGHT\x10\x03\x12\'\n#WORKFLOW_DIAGRAM_THEME_CONSOLE_DARK\x10\x04\x32\x9f\x05\n\nJobService\x12>\n\tSubmitJob\x12\x1e.workflows.v1.SubmitJobRequest\x1a\x11.workflows.v1.Job\x12\x38\n\x06GetJob\x12\x1b.workflows.v1.GetJobRequest\x1a\x11.workflows.v1.Job\x12H\n\x0eGetJobProgress\x12#.workflows.v1.GetJobProgressRequest\x1a\x11.workflows.v1.Job\x12I\n\x08RetryJob\x12\x1d.workflows.v1.RetryJobRequest\x1a\x1e.workflows.v1.RetryJobResponse\x12L\n\tCancelJob\x12\x1e.workflows.v1.CancelJobRequest\x1a\x1f.workflows.v1.CancelJobResponse\x12H\n\x0cVisualizeJob\x12!.workflows.v1.VisualizeJobRequest\x1a\x15.workflows.v1.Diagram\x12L\n\tQueryJobs\x12\x1e.workflows.v1.QueryJobsRequest\x1a\x1f.workflows.v1.QueryJobsResponse\x12^\n\x0fGetJobPrototype\x12$.workflows.v1.GetJobPrototypeRequest\x1a%.workflows.v1.GetJobPrototypeResponse\x12<\n\x08\x43loneJob\x12\x1d.workflows.v1.CloneJobRequest\x1a\x11.workflows.v1.JobBr\n\x10\x63om.workflows.v1B\x08JobProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16workflows/v1/job.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\x1a\x17workflows/v1/core.proto\x1a\x1aworkflows/v1/diagram.proto\"\xce\x03\n\x10SubmitJobRequest\x12\x33\n\x05tasks\x18\x05 \x01(\x0b\x32\x1d.workflows.v1.TaskSubmissionsR\x05tasks\x12\"\n\x08job_name\x18\x02 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07jobName\x12*\n\x0ctrace_parent\x18\x03 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0btraceParent\x12\x33\n\rautomation_id\x18\x04 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12R\n\x0clegacy_tasks\x18\x01 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionB\x0b\x18\x01\xbaH\x06\x92\x01\x03\x10\xe8\x07R\x0blegacyTasks:\xab\x01\xbaH\xa7\x01\x1a\xa4\x01\n!submit_job_request.tasks_required\x12$At least one task must be submitted.\x1aY(this.tasks != null && this.tasks.task_groups.size() > 0) || this.legacy_tasks.size() > 0\">\n\rGetJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"F\n\x15GetJobProgressRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"@\n\x0fRetryJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"F\n\x10RetryJobResponse\x12\x32\n\x15num_tasks_rescheduled\x18\x01 \x01(\x03R\x13numTasksRescheduled\"A\n\x10\x43\x61ncelJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"\x13\n\x11\x43\x61ncelJobResponse\"\xec\x01\n\x13VisualizeJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x42\n\x0erender_options\x18\x02 \x01(\x0b\x32\x1b.workflows.v1.RenderOptionsR\rrenderOptions\x12\x38\n\x05theme\x18\x03 \x01(\x0e\x32\".workflows.v1.WorkflowDiagramThemeR\x05theme\x12(\n\x10include_job_name\x18\x04 \x01(\x08R\x0eincludeJobName\"\xe9\x02\n\x0cQueryFilters\x12=\n\rtime_interval\x18\x01 \x01(\x0b\x32\x18.tilebox.v1.TimeIntervalR\x0ctimeInterval\x12\x37\n\x0bid_interval\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.IDIntervalR\nidInterval\x12\x35\n\x0e\x61utomation_ids\x18\x03 \x03(\x0b\x32\x0e.tilebox.v1.IDR\rautomationIds\x12.\n\x06states\x18\x04 \x03(\x0e\x32\x16.workflows.v1.JobStateR\x06states\x12\x1b\n\x04name\x18\x05 \x01(\tB\x07\xbaH\x04r\x02\x18\x64R\x04name\x12\x38\n\x0btask_states\x18\x06 \x03(\x0e\x32\x17.workflows.v1.TaskStateR\ntaskStates:#\xbaH \"\x1e\n\rtime_interval\n\x0bid_interval\x10\x00\"\xbd\x01\n\x10QueryJobsRequest\x12\x34\n\x07\x66ilters\x18\x01 \x01(\x0b\x32\x1a.workflows.v1.QueryFiltersR\x07\x66ilters\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12@\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x19.tilebox.v1.SortDirectionR\rsortDirection\"v\n\x11QueryJobsResponse\x12%\n\x04jobs\x18\x01 \x03(\x0b\x32\x11.workflows.v1.JobR\x04jobs\x12:\n\tnext_page\x18\x03 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"G\n\x16GetJobPrototypeRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\"w\n\x17GetJobPrototypeResponse\x12\x41\n\nroot_tasks\x18\x01 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\trootTasks\x12\x19\n\x08job_name\x18\x02 \x01(\tR\x07jobName\"\xc6\x01\n\x0f\x43loneJobRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12`\n\x14root_tasks_overrides\x18\x02 \x03(\x0b\x32\".workflows.v1.SingleTaskSubmissionB\n\xbaH\x07\x92\x01\x04\x08\x01\x10@R\x12rootTasksOverrides\x12\"\n\x08job_name\x18\x03 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x07jobName*\xd4\x01\n\x14WorkflowDiagramTheme\x12&\n\"WORKFLOW_DIAGRAM_THEME_UNSPECIFIED\x10\x00\x12 \n\x1cWORKFLOW_DIAGRAM_THEME_LIGHT\x10\x01\x12\x1f\n\x1bWORKFLOW_DIAGRAM_THEME_DARK\x10\x02\x12(\n$WORKFLOW_DIAGRAM_THEME_CONSOLE_LIGHT\x10\x03\x12\'\n#WORKFLOW_DIAGRAM_THEME_CONSOLE_DARK\x10\x04\x32\x9f\x05\n\nJobService\x12>\n\tSubmitJob\x12\x1e.workflows.v1.SubmitJobRequest\x1a\x11.workflows.v1.Job\x12\x38\n\x06GetJob\x12\x1b.workflows.v1.GetJobRequest\x1a\x11.workflows.v1.Job\x12H\n\x0eGetJobProgress\x12#.workflows.v1.GetJobProgressRequest\x1a\x11.workflows.v1.Job\x12I\n\x08RetryJob\x12\x1d.workflows.v1.RetryJobRequest\x1a\x1e.workflows.v1.RetryJobResponse\x12L\n\tCancelJob\x12\x1e.workflows.v1.CancelJobRequest\x1a\x1f.workflows.v1.CancelJobResponse\x12H\n\x0cVisualizeJob\x12!.workflows.v1.VisualizeJobRequest\x1a\x15.workflows.v1.Diagram\x12L\n\tQueryJobs\x12\x1e.workflows.v1.QueryJobsRequest\x1a\x1f.workflows.v1.QueryJobsResponse\x12^\n\x0fGetJobPrototype\x12$.workflows.v1.GetJobPrototypeRequest\x1a%.workflows.v1.GetJobPrototypeResponse\x12<\n\x08\x43loneJob\x12\x1d.workflows.v1.CloneJobRequest\x1a\x11.workflows.v1.JobBr\n\x10\x63om.workflows.v1B\x08JobProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -73,8 +73,8 @@ _globals['_CLONEJOBREQUEST'].fields_by_name['root_tasks_overrides']._serialized_options = b'\272H\007\222\001\004\010\001\020@' _globals['_CLONEJOBREQUEST'].fields_by_name['job_name']._loaded_options = None _globals['_CLONEJOBREQUEST'].fields_by_name['job_name']._serialized_options = b'\272H\004r\002\020\001' - _globals['_WORKFLOWDIAGRAMTHEME']._serialized_start=2320 - _globals['_WORKFLOWDIAGRAMTHEME']._serialized_end=2532 + _globals['_WORKFLOWDIAGRAMTHEME']._serialized_start=2387 + _globals['_WORKFLOWDIAGRAMTHEME']._serialized_end=2599 _globals['_SUBMITJOBREQUEST']._serialized_start=250 _globals['_SUBMITJOBREQUEST']._serialized_end=712 _globals['_GETJOBREQUEST']._serialized_start=714 @@ -93,16 +93,16 @@ _globals['_VISUALIZEJOBREQUEST']._serialized_end=1313 _globals['_QUERYFILTERS']._serialized_start=1316 _globals['_QUERYFILTERS']._serialized_end=1677 - _globals['_QUERYJOBSREQUEST']._serialized_start=1679 - _globals['_QUERYJOBSREQUEST']._serialized_end=1802 - _globals['_QUERYJOBSRESPONSE']._serialized_start=1804 - _globals['_QUERYJOBSRESPONSE']._serialized_end=1922 - _globals['_GETJOBPROTOTYPEREQUEST']._serialized_start=1924 - _globals['_GETJOBPROTOTYPEREQUEST']._serialized_end=1995 - _globals['_GETJOBPROTOTYPERESPONSE']._serialized_start=1997 - _globals['_GETJOBPROTOTYPERESPONSE']._serialized_end=2116 - _globals['_CLONEJOBREQUEST']._serialized_start=2119 - _globals['_CLONEJOBREQUEST']._serialized_end=2317 - _globals['_JOBSERVICE']._serialized_start=2535 - _globals['_JOBSERVICE']._serialized_end=3206 + _globals['_QUERYJOBSREQUEST']._serialized_start=1680 + _globals['_QUERYJOBSREQUEST']._serialized_end=1869 + _globals['_QUERYJOBSRESPONSE']._serialized_start=1871 + _globals['_QUERYJOBSRESPONSE']._serialized_end=1989 + _globals['_GETJOBPROTOTYPEREQUEST']._serialized_start=1991 + _globals['_GETJOBPROTOTYPEREQUEST']._serialized_end=2062 + _globals['_GETJOBPROTOTYPERESPONSE']._serialized_start=2064 + _globals['_GETJOBPROTOTYPERESPONSE']._serialized_end=2183 + _globals['_CLONEJOBREQUEST']._serialized_start=2186 + _globals['_CLONEJOBREQUEST']._serialized_end=2384 + _globals['_JOBSERVICE']._serialized_start=2602 + _globals['_JOBSERVICE']._serialized_end=3273 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.pyi index a305562..0d60341 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2.pyi @@ -104,12 +104,14 @@ class QueryFilters(_message.Message): def __init__(self, time_interval: _Optional[_Union[_query_pb2.TimeInterval, _Mapping]] = ..., id_interval: _Optional[_Union[_query_pb2.IDInterval, _Mapping]] = ..., automation_ids: _Optional[_Iterable[_Union[_id_pb2.ID, _Mapping]]] = ..., states: _Optional[_Iterable[_Union[_core_pb2.JobState, str]]] = ..., name: _Optional[str] = ..., task_states: _Optional[_Iterable[_Union[_core_pb2.TaskState, str]]] = ...) -> None: ... class QueryJobsRequest(_message.Message): - __slots__ = ("filters", "page") + __slots__ = ("filters", "page", "sort_direction") FILTERS_FIELD_NUMBER: _ClassVar[int] PAGE_FIELD_NUMBER: _ClassVar[int] + SORT_DIRECTION_FIELD_NUMBER: _ClassVar[int] filters: QueryFilters page: _query_pb2.Pagination - def __init__(self, filters: _Optional[_Union[QueryFilters, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + sort_direction: _query_pb2.SortDirection + def __init__(self, filters: _Optional[_Union[QueryFilters, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[_query_pb2.SortDirection, str]] = ...) -> None: ... class QueryJobsResponse(_message.Message): __slots__ = ("jobs", "next_page") diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2_grpc.py index 6d49f82..6308e9e 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/job_pb2_grpc.py @@ -7,7 +7,7 @@ from tilebox.workflows.workflows.v1 import job_pb2 as workflows_dot_v1_dot_job__pb2 -class JobServiceStub(object): +class JobServiceStub: """A service for interacting with jobs. """ @@ -64,7 +64,7 @@ def __init__(self, channel): _registered_method=True) -class JobServiceServicer(object): +class JobServiceServicer: """A service for interacting with jobs. """ @@ -178,7 +178,7 @@ def add_JobServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class JobService(object): +class JobService: """A service for interacting with jobs. """ diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/task_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/task_pb2_grpc.py index 078f35b..6646265 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/task_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/task_pb2_grpc.py @@ -6,7 +6,7 @@ from tilebox.workflows.workflows.v1 import task_pb2 as workflows_dot_v1_dot_task__pb2 -class TaskServiceStub(object): +class TaskServiceStub: """A service for task runners to communicate with the workflows service. """ @@ -33,7 +33,7 @@ def __init__(self, channel): _registered_method=True) -class TaskServiceServicer(object): +class TaskServiceServicer: """A service for task runners to communicate with the workflows service. """ @@ -95,7 +95,7 @@ def add_TaskServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class TaskService(object): +class TaskService: """A service for task runners to communicate with the workflows service. """ diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.py index 91ff39f..abb7f56 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.py @@ -30,7 +30,7 @@ from tilebox.datasets.tilebox.v1 import query_pb2 as tilebox_dot_v1_dot_query__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/telemetry.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xbb\x01\n\x13QueryJobLogsRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12\x42\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x1b.workflows.v1.SortDirectionR\rsortDirection\"\xd2\x01\n\x1aQueryLogsInIntervalRequest\x12=\n\rtime_interval\x18\x01 \x01(\x0b\x32\x18.tilebox.v1.TimeIntervalR\x0ctimeInterval\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12\x42\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x1b.workflows.v1.SortDirectionR\rsortDirection\"\x9f\x01\n\x11PaginatedLogsData\x12N\n\rresource_logs\x18\x01 \x03(\x0b\x32).opentelemetry.proto.logs.v1.ResourceLogsR\x0cresourceLogs\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"\xbc\x01\n\x14QueryJobSpansRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12\x42\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x1b.workflows.v1.SortDirectionR\rsortDirection\"\xa4\x01\n\x12PaginatedSpansData\x12R\n\x0eresource_spans\x18\x01 \x03(\x0b\x32+.opentelemetry.proto.trace.v1.ResourceSpansR\rresourceSpans\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage*l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02\x32\xa4\x02\n\x15TelemetryQueryService\x12R\n\x0cQueryJobLogs\x12!.workflows.v1.QueryJobLogsRequest\x1a\x1f.workflows.v1.PaginatedLogsData\x12`\n\x13QueryLogsInInterval\x12(.workflows.v1.QueryLogsInIntervalRequest\x1a\x1f.workflows.v1.PaginatedLogsData\x12U\n\rQueryJobSpans\x12\".workflows.v1.QueryJobSpansRequest\x1a .workflows.v1.PaginatedSpansDataBs\n\x10\x63om.workflows.v1B\x0eTelemetryProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1b\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/telemetry.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&opentelemetry/proto/logs/v1/logs.proto\x1a(opentelemetry/proto/trace/v1/trace.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xb9\x01\n\x13QueryJobLogsRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12@\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x19.tilebox.v1.SortDirectionR\rsortDirection\"\xd0\x01\n\x1aQueryLogsInIntervalRequest\x12=\n\rtime_interval\x18\x01 \x01(\x0b\x32\x18.tilebox.v1.TimeIntervalR\x0ctimeInterval\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12@\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x19.tilebox.v1.SortDirectionR\rsortDirection\"\x9f\x01\n\x11PaginatedLogsData\x12N\n\rresource_logs\x18\x01 \x03(\x0b\x32).opentelemetry.proto.logs.v1.ResourceLogsR\x0cresourceLogs\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"\xba\x01\n\x14QueryJobSpansRequest\x12-\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\x05jobId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\x12@\n\x0esort_direction\x18\x03 \x01(\x0e\x32\x19.tilebox.v1.SortDirectionR\rsortDirection\"\xa4\x01\n\x12PaginatedSpansData\x12R\n\x0eresource_spans\x18\x01 \x03(\x0b\x32+.opentelemetry.proto.trace.v1.ResourceSpansR\rresourceSpans\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage2\xa4\x02\n\x15TelemetryQueryService\x12R\n\x0cQueryJobLogs\x12!.workflows.v1.QueryJobLogsRequest\x1a\x1f.workflows.v1.PaginatedLogsData\x12`\n\x13QueryLogsInInterval\x12(.workflows.v1.QueryLogsInIntervalRequest\x1a\x1f.workflows.v1.PaginatedLogsData\x12U\n\rQueryJobSpans\x12\".workflows.v1.QueryJobSpansRequest\x1a .workflows.v1.PaginatedSpansDataBs\n\x10\x63om.workflows.v1B\x0eTelemetryProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1b\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -52,18 +52,16 @@ _globals['_QUERYJOBSPANSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' _globals['_PAGINATEDSPANSDATA'].fields_by_name['next_page']._loaded_options = None _globals['_PAGINATEDSPANSDATA'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' - _globals['_SORTDIRECTION']._serialized_start=1158 - _globals['_SORTDIRECTION']._serialized_end=1266 _globals['_QUERYJOBLOGSREQUEST']._serialized_start=236 - _globals['_QUERYJOBLOGSREQUEST']._serialized_end=423 - _globals['_QUERYLOGSININTERVALREQUEST']._serialized_start=426 - _globals['_QUERYLOGSININTERVALREQUEST']._serialized_end=636 - _globals['_PAGINATEDLOGSDATA']._serialized_start=639 - _globals['_PAGINATEDLOGSDATA']._serialized_end=798 - _globals['_QUERYJOBSPANSREQUEST']._serialized_start=801 - _globals['_QUERYJOBSPANSREQUEST']._serialized_end=989 - _globals['_PAGINATEDSPANSDATA']._serialized_start=992 - _globals['_PAGINATEDSPANSDATA']._serialized_end=1156 - _globals['_TELEMETRYQUERYSERVICE']._serialized_start=1269 - _globals['_TELEMETRYQUERYSERVICE']._serialized_end=1561 + _globals['_QUERYJOBLOGSREQUEST']._serialized_end=421 + _globals['_QUERYLOGSININTERVALREQUEST']._serialized_start=424 + _globals['_QUERYLOGSININTERVALREQUEST']._serialized_end=632 + _globals['_PAGINATEDLOGSDATA']._serialized_start=635 + _globals['_PAGINATEDLOGSDATA']._serialized_end=794 + _globals['_QUERYJOBSPANSREQUEST']._serialized_start=797 + _globals['_QUERYJOBSPANSREQUEST']._serialized_end=983 + _globals['_PAGINATEDSPANSDATA']._serialized_start=986 + _globals['_PAGINATEDSPANSDATA']._serialized_end=1150 + _globals['_TELEMETRYQUERYSERVICE']._serialized_start=1153 + _globals['_TELEMETRYQUERYSERVICE']._serialized_end=1445 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.pyi index 401f4f6..f0b182a 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2.pyi @@ -5,7 +5,6 @@ from opentelemetry.proto.trace.v1 import trace_pb2 as _trace_pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 from tilebox.datasets.tilebox.v1 import query_pb2 as _query_pb2 from google.protobuf.internal import containers as _containers -from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from collections.abc import Iterable as _Iterable, Mapping as _Mapping @@ -13,15 +12,6 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor -class SortDirection(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = () - SORT_DIRECTION_UNSPECIFIED: _ClassVar[SortDirection] - SORT_DIRECTION_ASCENDING: _ClassVar[SortDirection] - SORT_DIRECTION_DESCENDING: _ClassVar[SortDirection] -SORT_DIRECTION_UNSPECIFIED: SortDirection -SORT_DIRECTION_ASCENDING: SortDirection -SORT_DIRECTION_DESCENDING: SortDirection - class QueryJobLogsRequest(_message.Message): __slots__ = ("job_id", "page", "sort_direction") JOB_ID_FIELD_NUMBER: _ClassVar[int] @@ -29,8 +19,8 @@ class QueryJobLogsRequest(_message.Message): SORT_DIRECTION_FIELD_NUMBER: _ClassVar[int] job_id: _id_pb2.ID page: _query_pb2.Pagination - sort_direction: SortDirection - def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[SortDirection, str]] = ...) -> None: ... + sort_direction: _query_pb2.SortDirection + def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[_query_pb2.SortDirection, str]] = ...) -> None: ... class QueryLogsInIntervalRequest(_message.Message): __slots__ = ("time_interval", "page", "sort_direction") @@ -39,8 +29,8 @@ class QueryLogsInIntervalRequest(_message.Message): SORT_DIRECTION_FIELD_NUMBER: _ClassVar[int] time_interval: _query_pb2.TimeInterval page: _query_pb2.Pagination - sort_direction: SortDirection - def __init__(self, time_interval: _Optional[_Union[_query_pb2.TimeInterval, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[SortDirection, str]] = ...) -> None: ... + sort_direction: _query_pb2.SortDirection + def __init__(self, time_interval: _Optional[_Union[_query_pb2.TimeInterval, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[_query_pb2.SortDirection, str]] = ...) -> None: ... class PaginatedLogsData(_message.Message): __slots__ = ("resource_logs", "next_page") @@ -57,8 +47,8 @@ class QueryJobSpansRequest(_message.Message): SORT_DIRECTION_FIELD_NUMBER: _ClassVar[int] job_id: _id_pb2.ID page: _query_pb2.Pagination - sort_direction: SortDirection - def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[SortDirection, str]] = ...) -> None: ... + sort_direction: _query_pb2.SortDirection + def __init__(self, job_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ..., sort_direction: _Optional[_Union[_query_pb2.SortDirection, str]] = ...) -> None: ... class PaginatedSpansData(_message.Message): __slots__ = ("resource_spans", "next_page") diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py index 768cc14..c39a307 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/telemetry_pb2_grpc.py @@ -5,7 +5,7 @@ from tilebox.workflows.workflows.v1 import telemetry_pb2 as workflows_dot_v1_dot_telemetry__pb2 -class TelemetryQueryServiceStub(object): +class TelemetryQueryServiceStub: """TelemetryQueryService is the service definition for querying telemetry data about workflows such as logs, traces and metrics. """ @@ -33,7 +33,7 @@ def __init__(self, channel): _registered_method=True) -class TelemetryQueryServiceServicer(object): +class TelemetryQueryServiceServicer: """TelemetryQueryService is the service definition for querying telemetry data about workflows such as logs, traces and metrics. """ @@ -82,7 +82,7 @@ def add_TelemetryQueryServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class TelemetryQueryService(object): +class TelemetryQueryService: """TelemetryQueryService is the service definition for querying telemetry data about workflows such as logs, traces and metrics. """ diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.py new file mode 100644 index 0000000..b202340 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: workflows/v1/worker.proto +# Protobuf Python Version: 6.30.0 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 30, + 0, + '', + 'workflows/v1/worker.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from tilebox.datasets.buf.validate import validate_pb2 as buf_dot_validate_dot_validate__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.workflows.workflows.v1 import automation_pb2 as workflows_dot_v1_dot_automation__pb2 +from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 +from tilebox.workflows.workflows.v1 import task_pb2 as workflows_dot_v1_dot_task__pb2 +from tilebox.workflows.workflows.v1 import workflows_pb2 as workflows_dot_v1_dot_workflows__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19workflows/v1/worker.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x13tilebox/v1/id.proto\x1a\x1dworkflows/v1/automation.proto\x1a\x17workflows/v1/core.proto\x1a\x17workflows/v1/task.proto\x1a\x1cworkflows/v1/workflows.proto\"\x99\x02\n\x17InitializeRunnerRequest\x12+\n\trunner_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x08runnerId\x12!\n\x0ctrace_parent\x18\x02 \x01(\tR\x0btraceParent\x12/\n\x07\x63luster\x18\x03 \x01(\x0b\x32\x15.workflows.v1.ClusterR\x07\x63luster\x12\x32\n\x08workflow\x18\x04 \x01(\x0b\x32\x16.workflows.v1.WorkflowR\x08workflow\x12I\n\x0e\x61pi_connection\x18\x05 \x01(\x0b\x32\".workflows.v1.TileboxAPIConnectionR\rapiConnection\">\n\x14TileboxAPIConnection\x12\x10\n\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n\x05token\x18\x02 \x01(\tR\x05token\"\x1a\n\x18InitializeRunnerResponse\"\xbb\x01\n\x13\x45xecuteTaskResponse\x12?\n\rcomputed_task\x18\x01 \x01(\x0b\x32\x1a.workflows.v1.ComputedTaskR\x0c\x63omputedTask\x12@\n\x0b\x66\x61iled_task\x18\x02 \x01(\x0b\x32\x1f.workflows.v1.TaskFailedRequestR\nfailedTask:!\xbaH\x1e\"\x1c\n\rcomputed_task\n\x0b\x66\x61iled_task2\xce\x02\n\rWorkerService\x12L\n\x13ListRegisteredTasks\x12\x16.google.protobuf.Empty\x1a\x1d.workflows.v1.TaskIdentifiers\x12\x63\n\x10InitializeWorker\x12%.workflows.v1.InitializeRunnerRequest\x1a&.workflows.v1.InitializeRunnerResponse\"\x00\x12\x46\n\x0b\x45xecuteTask\x12\x12.workflows.v1.Task\x1a!.workflows.v1.ExecuteTaskResponse\"\x00\x12\x42\n\x0eShutdownWorker\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x42p\n\x10\x63om.workflows.v1B\x0bWorkerProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1b\x08\x65\x64itionsp\xe8\x07') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.worker_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\020com.workflows.v1B\013WorkerProtoP\001\242\002\003WXX\252\002\014Workflows.V1\312\002\014Workflows\\V1\342\002\030Workflows\\V1\\GPBMetadata\352\002\rWorkflows::V1' + _globals['_EXECUTETASKRESPONSE']._loaded_options = None + _globals['_EXECUTETASKRESPONSE']._serialized_options = b'\272H\036\"\034\n\rcomputed_task\n\013failed_task' + _globals['_INITIALIZERUNNERREQUEST']._serialized_start=234 + _globals['_INITIALIZERUNNERREQUEST']._serialized_end=515 + _globals['_TILEBOXAPICONNECTION']._serialized_start=517 + _globals['_TILEBOXAPICONNECTION']._serialized_end=579 + _globals['_INITIALIZERUNNERRESPONSE']._serialized_start=581 + _globals['_INITIALIZERUNNERRESPONSE']._serialized_end=607 + _globals['_EXECUTETASKRESPONSE']._serialized_start=610 + _globals['_EXECUTETASKRESPONSE']._serialized_end=797 + _globals['_WORKERSERVICE']._serialized_start=800 + _globals['_WORKERSERVICE']._serialized_end=1134 +# @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.pyi new file mode 100644 index 0000000..8c0fc4d --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2.pyi @@ -0,0 +1,47 @@ +from tilebox.datasets.buf.validate import validate_pb2 as _validate_pb2 +from google.protobuf import empty_pb2 as _empty_pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 +from tilebox.workflows.workflows.v1 import automation_pb2 as _automation_pb2 +from tilebox.workflows.workflows.v1 import core_pb2 as _core_pb2 +from tilebox.workflows.workflows.v1 import task_pb2 as _task_pb2 +from tilebox.workflows.workflows.v1 import workflows_pb2 as _workflows_pb2 +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from collections.abc import Mapping as _Mapping +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class InitializeRunnerRequest(_message.Message): + __slots__ = ("runner_id", "trace_parent", "cluster", "workflow", "api_connection") + RUNNER_ID_FIELD_NUMBER: _ClassVar[int] + TRACE_PARENT_FIELD_NUMBER: _ClassVar[int] + CLUSTER_FIELD_NUMBER: _ClassVar[int] + WORKFLOW_FIELD_NUMBER: _ClassVar[int] + API_CONNECTION_FIELD_NUMBER: _ClassVar[int] + runner_id: _id_pb2.ID + trace_parent: str + cluster: _workflows_pb2.Cluster + workflow: _workflows_pb2.Workflow + api_connection: TileboxAPIConnection + def __init__(self, runner_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., trace_parent: _Optional[str] = ..., cluster: _Optional[_Union[_workflows_pb2.Cluster, _Mapping]] = ..., workflow: _Optional[_Union[_workflows_pb2.Workflow, _Mapping]] = ..., api_connection: _Optional[_Union[TileboxAPIConnection, _Mapping]] = ...) -> None: ... + +class TileboxAPIConnection(_message.Message): + __slots__ = ("url", "token") + URL_FIELD_NUMBER: _ClassVar[int] + TOKEN_FIELD_NUMBER: _ClassVar[int] + url: str + token: str + def __init__(self, url: _Optional[str] = ..., token: _Optional[str] = ...) -> None: ... + +class InitializeRunnerResponse(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class ExecuteTaskResponse(_message.Message): + __slots__ = ("computed_task", "failed_task") + COMPUTED_TASK_FIELD_NUMBER: _ClassVar[int] + FAILED_TASK_FIELD_NUMBER: _ClassVar[int] + computed_task: _task_pb2.ComputedTask + failed_task: _task_pb2.TaskFailedRequest + def __init__(self, computed_task: _Optional[_Union[_task_pb2.ComputedTask, _Mapping]] = ..., failed_task: _Optional[_Union[_task_pb2.TaskFailedRequest, _Mapping]] = ...) -> None: ... diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2_grpc.py new file mode 100644 index 0000000..87ce98f --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/worker_pb2_grpc.py @@ -0,0 +1,220 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 +from tilebox.workflows.workflows.v1 import worker_pb2 as workflows_dot_v1_dot_worker__pb2 + + +class WorkerServiceStub: + """WorkerService defines the RPC interface between a runner and a a worker runtime that can execute workflow tasks. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ListRegisteredTasks = channel.unary_unary( + '/workflows.v1.WorkerService/ListRegisteredTasks', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=workflows_dot_v1_dot_core__pb2.TaskIdentifiers.FromString, + _registered_method=True) + self.InitializeWorker = channel.unary_unary( + '/workflows.v1.WorkerService/InitializeWorker', + request_serializer=workflows_dot_v1_dot_worker__pb2.InitializeRunnerRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_worker__pb2.InitializeRunnerResponse.FromString, + _registered_method=True) + self.ExecuteTask = channel.unary_unary( + '/workflows.v1.WorkerService/ExecuteTask', + request_serializer=workflows_dot_v1_dot_core__pb2.Task.SerializeToString, + response_deserializer=workflows_dot_v1_dot_worker__pb2.ExecuteTaskResponse.FromString, + _registered_method=True) + self.ShutdownWorker = channel.unary_unary( + '/workflows.v1.WorkerService/ShutdownWorker', + request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + _registered_method=True) + + +class WorkerServiceServicer: + """WorkerService defines the RPC interface between a runner and a a worker runtime that can execute workflow tasks. + """ + + def ListRegisteredTasks(self, request, context): + """ListRegisteredTasks is called by the runner to discover the task identifiers that a worker runtime can execute. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def InitializeWorker(self, request, context): + """InitializeWorker is called by the runner to initialize a worker runtime with some known metadata about the + runner. This allows us to add those metadata to any logs and traces emitted by the worker. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ExecuteTask(self, request, context): + """ExecuteTask is called by the runner to execute a task on the worker runtime. The worker runtime will execute the + task, and then respond with the result of the task execution, which can either be a computed task, or a failed + task. If the worker runtime crashes or becomes unreachable during task execution, the runner will treat the task + as failed with an unknown error. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ShutdownWorker(self, request, context): + """Gracefully shuts down the worker runtime. After receiving this request, the worker runtime will + cleanly shut down. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_WorkerServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ListRegisteredTasks': grpc.unary_unary_rpc_method_handler( + servicer.ListRegisteredTasks, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=workflows_dot_v1_dot_core__pb2.TaskIdentifiers.SerializeToString, + ), + 'InitializeWorker': grpc.unary_unary_rpc_method_handler( + servicer.InitializeWorker, + request_deserializer=workflows_dot_v1_dot_worker__pb2.InitializeRunnerRequest.FromString, + response_serializer=workflows_dot_v1_dot_worker__pb2.InitializeRunnerResponse.SerializeToString, + ), + 'ExecuteTask': grpc.unary_unary_rpc_method_handler( + servicer.ExecuteTask, + request_deserializer=workflows_dot_v1_dot_core__pb2.Task.FromString, + response_serializer=workflows_dot_v1_dot_worker__pb2.ExecuteTaskResponse.SerializeToString, + ), + 'ShutdownWorker': grpc.unary_unary_rpc_method_handler( + servicer.ShutdownWorker, + request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, + response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'workflows.v1.WorkerService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('workflows.v1.WorkerService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class WorkerService: + """WorkerService defines the RPC interface between a runner and a a worker runtime that can execute workflow tasks. + """ + + @staticmethod + def ListRegisteredTasks(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkerService/ListRegisteredTasks', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + workflows_dot_v1_dot_core__pb2.TaskIdentifiers.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def InitializeWorker(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkerService/InitializeWorker', + workflows_dot_v1_dot_worker__pb2.InitializeRunnerRequest.SerializeToString, + workflows_dot_v1_dot_worker__pb2.InitializeRunnerResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ExecuteTask(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkerService/ExecuteTask', + workflows_dot_v1_dot_core__pb2.Task.SerializeToString, + workflows_dot_v1_dot_worker__pb2.ExecuteTaskResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ShutdownWorker(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkerService/ShutdownWorker', + google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, + google_dot_protobuf_dot_empty__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py index 8a3bc8c..82ef8f5 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py @@ -23,10 +23,13 @@ from tilebox.datasets.buf.validate import validate_pb2 as buf_dot_validate_dot_validate__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/workflows.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x17workflows/v1/core.proto\"3\n\x14\x43reateClusterRequest\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\"6\n\x11GetClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"B\n\x14\x44\x65leteClusterRequest\x12*\n\x0c\x63luster_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0b\x63lusterSlug\"\x17\n\x15\x44\x65leteClusterResponse\"\x15\n\x13ListClustersRequest\"I\n\x14ListClustersResponse\x12\x31\n\x08\x63lusters\x18\x01 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters2\xd5\x02\n\x10WorkflowsService\x12J\n\rCreateCluster\x12\".workflows.v1.CreateClusterRequest\x1a\x15.workflows.v1.Cluster\x12\x44\n\nGetCluster\x12\x1f.workflows.v1.GetClusterRequest\x1a\x15.workflows.v1.Cluster\x12X\n\rDeleteCluster\x12\".workflows.v1.DeleteClusterRequest\x1a#.workflows.v1.DeleteClusterResponse\x12U\n\x0cListClusters\x12!.workflows.v1.ListClustersRequest\x1a\".workflows.v1.ListClustersResponseBx\n\x10\x63om.workflows.v1B\x0eWorkflowsProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/workflows.proto\x12\x0cworkflows.v1\x1a\x1b\x62uf/validate/validate.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\"\xa9\x01\n\x07\x43luster\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12!\n\x0c\x64isplay_name\x18\x03 \x01(\tR\x0b\x64isplayName\x12\x1c\n\tdeletable\x18\x04 \x01(\x08R\tdeletable\x12\x43\n\x11\x64\x65ployed_releases\x18\x05 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\x10\x64\x65ployedReleasesJ\x04\x08\x01\x10\x02\"3\n\x14\x43reateClusterRequest\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\"6\n\x11GetClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"B\n\x14\x44\x65leteClusterRequest\x12*\n\x0c\x63luster_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0b\x63lusterSlug\"\x17\n\x15\x44\x65leteClusterResponse\"\x15\n\x13ListClustersRequest\"I\n\x14ListClustersResponse\x12\x31\n\x08\x63lusters\x18\x01 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"B\n\x12GetWorkflowRequest\x12,\n\rworkflow_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0cworkflowSlug\"\xc6\x01\n\x1dPublishWorkflowReleaseRequest\x12,\n\rworkflow_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0cworkflowSlug\x12\x37\n\x0b\x61rtifact_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x06\xbaH\x03\xc8\x01\x01R\nartifactId\x12>\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentB\x06\xbaH\x03\xc8\x01\x01R\x07\x63ontent\"\x16\n\x14ListWorkflowsRequest\"M\n\x15ListWorkflowsResponse\x12\x34\n\tworkflows\x18\x01 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\tworkflows\"V\n\x15\x43reateWorkflowRequest\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\"\x95\x01\n\x08Workflow\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x04 \x01(\tR\x0b\x64\x65scription\x12\x39\n\x08releases\x18\x05 \x03(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x08releasesJ\x04\x08\x01\x10\x02\"\xd8\x01\n\x0fWorkflowRelease\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x32\n\x08\x61rtifact\x18\x02 \x01(\x0b\x32\x16.workflows.v1.ArtifactR\x08\x61rtifact\x12\x36\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentR\x07\x63ontent\x12\x39\n\ncreated_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\"Y\n\x08\x41rtifact\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12-\n\x06\x64igest\x18\x02 \x01(\tB\x15\xbaH\x12r\x10\x32\x0e^[a-f0-9]{64}$R\x06\x64igest\"\x8a\x02\n\x0eReleaseContent\x12\x37\n\x0b\x66ingerprint\x18\x01 \x01(\tB\x15\xbaH\x12r\x10\x32\x0e^[a-f0-9]{64}$R\x0b\x66ingerprint\x12<\n\x05tasks\x18\x02 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierB\x08\xbaH\x05\x92\x01\x02\x08\x01R\x05tasks\x12(\n\x05\x66iles\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x05\x66iles\x12,\n\x12runner_object_path\x18\x04 \x01(\tR\x10runnerObjectPath\x12)\n\x10\x63ommand_override\x18\x05 \x03(\tR\x0f\x63ommandOverride\"h\n\x04Path\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\x12\x1c\n\tdirectory\x18\x02 \x01(\x08R\tdirectory\x12.\n\x08\x63hildren\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x08\x63hildren\"\xb0\x01\n\x1c\x44\x65ployWorkflowReleaseRequest\x12,\n\rworkflow_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12\x33\n\rcluster_slugs\x18\x03 \x03(\tB\x0e\xbaH\x0b\x92\x01\x08\x08\x00\"\x04r\x02 \x01R\x0c\x63lusterSlugs\"\x8b\x01\n\x1d\x44\x65ployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"\xb2\x01\n\x1eUndeployWorkflowReleaseRequest\x12,\n\rworkflow_slug\x18\x01 \x01(\tB\x07\xbaH\x04r\x02\x10\x01R\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12\x33\n\rcluster_slugs\x18\x03 \x03(\tB\x0e\xbaH\x0b\x92\x01\x08\x08\x01\"\x04r\x02 \x01R\x0c\x63lusterSlugs\"\x8d\x01\n\x1fUndeployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters2\x97\x07\n\x10WorkflowsService\x12J\n\rCreateCluster\x12\".workflows.v1.CreateClusterRequest\x1a\x15.workflows.v1.Cluster\x12\x44\n\nGetCluster\x12\x1f.workflows.v1.GetClusterRequest\x1a\x15.workflows.v1.Cluster\x12X\n\rDeleteCluster\x12\".workflows.v1.DeleteClusterRequest\x1a#.workflows.v1.DeleteClusterResponse\x12U\n\x0cListClusters\x12!.workflows.v1.ListClustersRequest\x1a\".workflows.v1.ListClustersResponse\x12M\n\x0e\x43reateWorkflow\x12#.workflows.v1.CreateWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12X\n\rListWorkflows\x12\".workflows.v1.ListWorkflowsRequest\x1a#.workflows.v1.ListWorkflowsResponse\x12G\n\x0bGetWorkflow\x12 .workflows.v1.GetWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12\x64\n\x16PublishWorkflowRelease\x12+.workflows.v1.PublishWorkflowReleaseRequest\x1a\x1d.workflows.v1.WorkflowRelease\x12p\n\x15\x44\x65ployWorkflowRelease\x12*.workflows.v1.DeployWorkflowReleaseRequest\x1a+.workflows.v1.DeployWorkflowReleaseResponse\x12v\n\x17UndeployWorkflowRelease\x12,.workflows.v1.UndeployWorkflowReleaseRequest\x1a-.workflows.v1.UndeployWorkflowReleaseResponseBx\n\x10\x63om.workflows.v1B\x0eWorkflowsProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -38,18 +41,72 @@ _globals['_CREATECLUSTERREQUEST'].fields_by_name['name']._serialized_options = b'\272H\004r\002\020\001' _globals['_DELETECLUSTERREQUEST'].fields_by_name['cluster_slug']._loaded_options = None _globals['_DELETECLUSTERREQUEST'].fields_by_name['cluster_slug']._serialized_options = b'\272H\004r\002\020\001' - _globals['_CREATECLUSTERREQUEST']._serialized_start=100 - _globals['_CREATECLUSTERREQUEST']._serialized_end=151 - _globals['_GETCLUSTERREQUEST']._serialized_start=153 - _globals['_GETCLUSTERREQUEST']._serialized_end=207 - _globals['_DELETECLUSTERREQUEST']._serialized_start=209 - _globals['_DELETECLUSTERREQUEST']._serialized_end=275 - _globals['_DELETECLUSTERRESPONSE']._serialized_start=277 - _globals['_DELETECLUSTERRESPONSE']._serialized_end=300 - _globals['_LISTCLUSTERSREQUEST']._serialized_start=302 - _globals['_LISTCLUSTERSREQUEST']._serialized_end=323 - _globals['_LISTCLUSTERSRESPONSE']._serialized_start=325 - _globals['_LISTCLUSTERSRESPONSE']._serialized_end=398 - _globals['_WORKFLOWSSERVICE']._serialized_start=401 - _globals['_WORKFLOWSSERVICE']._serialized_end=742 + _globals['_GETWORKFLOWREQUEST'].fields_by_name['workflow_slug']._loaded_options = None + _globals['_GETWORKFLOWREQUEST'].fields_by_name['workflow_slug']._serialized_options = b'\272H\004r\002\020\001' + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._loaded_options = None + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._serialized_options = b'\272H\004r\002\020\001' + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['artifact_id']._loaded_options = None + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['artifact_id']._serialized_options = b'\272H\003\310\001\001' + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['content']._loaded_options = None + _globals['_PUBLISHWORKFLOWRELEASEREQUEST'].fields_by_name['content']._serialized_options = b'\272H\003\310\001\001' + _globals['_CREATEWORKFLOWREQUEST'].fields_by_name['name']._loaded_options = None + _globals['_CREATEWORKFLOWREQUEST'].fields_by_name['name']._serialized_options = b'\272H\004r\002\020\001' + _globals['_ARTIFACT'].fields_by_name['digest']._loaded_options = None + _globals['_ARTIFACT'].fields_by_name['digest']._serialized_options = b'\272H\022r\0202\016^[a-f0-9]{64}$' + _globals['_RELEASECONTENT'].fields_by_name['fingerprint']._loaded_options = None + _globals['_RELEASECONTENT'].fields_by_name['fingerprint']._serialized_options = b'\272H\022r\0202\016^[a-f0-9]{64}$' + _globals['_RELEASECONTENT'].fields_by_name['tasks']._loaded_options = None + _globals['_RELEASECONTENT'].fields_by_name['tasks']._serialized_options = b'\272H\005\222\001\002\010\001' + _globals['_DEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._loaded_options = None + _globals['_DEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._serialized_options = b'\272H\004r\002\020\001' + _globals['_DEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['cluster_slugs']._loaded_options = None + _globals['_DEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['cluster_slugs']._serialized_options = b'\272H\013\222\001\010\010\000\"\004r\002 \001' + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._loaded_options = None + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['workflow_slug']._serialized_options = b'\272H\004r\002\020\001' + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['cluster_slugs']._loaded_options = None + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST'].fields_by_name['cluster_slugs']._serialized_options = b'\272H\013\222\001\010\010\001\"\004r\002 \001' + _globals['_CLUSTER']._serialized_start=184 + _globals['_CLUSTER']._serialized_end=353 + _globals['_CREATECLUSTERREQUEST']._serialized_start=355 + _globals['_CREATECLUSTERREQUEST']._serialized_end=406 + _globals['_GETCLUSTERREQUEST']._serialized_start=408 + _globals['_GETCLUSTERREQUEST']._serialized_end=462 + _globals['_DELETECLUSTERREQUEST']._serialized_start=464 + _globals['_DELETECLUSTERREQUEST']._serialized_end=530 + _globals['_DELETECLUSTERRESPONSE']._serialized_start=532 + _globals['_DELETECLUSTERRESPONSE']._serialized_end=555 + _globals['_LISTCLUSTERSREQUEST']._serialized_start=557 + _globals['_LISTCLUSTERSREQUEST']._serialized_end=578 + _globals['_LISTCLUSTERSRESPONSE']._serialized_start=580 + _globals['_LISTCLUSTERSRESPONSE']._serialized_end=653 + _globals['_GETWORKFLOWREQUEST']._serialized_start=655 + _globals['_GETWORKFLOWREQUEST']._serialized_end=721 + _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_start=724 + _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_end=922 + _globals['_LISTWORKFLOWSREQUEST']._serialized_start=924 + _globals['_LISTWORKFLOWSREQUEST']._serialized_end=946 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_start=948 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_end=1025 + _globals['_CREATEWORKFLOWREQUEST']._serialized_start=1027 + _globals['_CREATEWORKFLOWREQUEST']._serialized_end=1113 + _globals['_WORKFLOW']._serialized_start=1116 + _globals['_WORKFLOW']._serialized_end=1265 + _globals['_WORKFLOWRELEASE']._serialized_start=1268 + _globals['_WORKFLOWRELEASE']._serialized_end=1484 + _globals['_ARTIFACT']._serialized_start=1486 + _globals['_ARTIFACT']._serialized_end=1575 + _globals['_RELEASECONTENT']._serialized_start=1578 + _globals['_RELEASECONTENT']._serialized_end=1844 + _globals['_PATH']._serialized_start=1846 + _globals['_PATH']._serialized_end=1950 + _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=1953 + _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=2129 + _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=2132 + _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=2271 + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=2274 + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=2452 + _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=2455 + _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=2596 + _globals['_WORKFLOWSSERVICE']._serialized_start=2599 + _globals['_WORKFLOWSSERVICE']._serialized_end=3518 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi index ef216ea..6725c51 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi @@ -1,4 +1,7 @@ from tilebox.datasets.buf.validate import validate_pb2 as _validate_pb2 +from google.protobuf import empty_pb2 as _empty_pb2 +from google.protobuf import timestamp_pb2 as _timestamp_pb2 +from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 from tilebox.workflows.workflows.v1 import core_pb2 as _core_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor @@ -8,6 +11,18 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor +class Cluster(_message.Message): + __slots__ = ("slug", "display_name", "deletable", "deployed_releases") + SLUG_FIELD_NUMBER: _ClassVar[int] + DISPLAY_NAME_FIELD_NUMBER: _ClassVar[int] + DELETABLE_FIELD_NUMBER: _ClassVar[int] + DEPLOYED_RELEASES_FIELD_NUMBER: _ClassVar[int] + slug: str + display_name: str + deletable: bool + deployed_releases: _containers.RepeatedCompositeFieldContainer[Workflow] + def __init__(self, slug: _Optional[str] = ..., display_name: _Optional[str] = ..., deletable: bool = ..., deployed_releases: _Optional[_Iterable[_Union[Workflow, _Mapping]]] = ...) -> None: ... + class CreateClusterRequest(_message.Message): __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] @@ -37,5 +52,131 @@ class ListClustersRequest(_message.Message): class ListClustersResponse(_message.Message): __slots__ = ("clusters",) CLUSTERS_FIELD_NUMBER: _ClassVar[int] - clusters: _containers.RepeatedCompositeFieldContainer[_core_pb2.Cluster] - def __init__(self, clusters: _Optional[_Iterable[_Union[_core_pb2.Cluster, _Mapping]]] = ...) -> None: ... + clusters: _containers.RepeatedCompositeFieldContainer[Cluster] + def __init__(self, clusters: _Optional[_Iterable[_Union[Cluster, _Mapping]]] = ...) -> None: ... + +class GetWorkflowRequest(_message.Message): + __slots__ = ("workflow_slug",) + WORKFLOW_SLUG_FIELD_NUMBER: _ClassVar[int] + workflow_slug: str + def __init__(self, workflow_slug: _Optional[str] = ...) -> None: ... + +class PublishWorkflowReleaseRequest(_message.Message): + __slots__ = ("workflow_slug", "artifact_id", "content") + WORKFLOW_SLUG_FIELD_NUMBER: _ClassVar[int] + ARTIFACT_ID_FIELD_NUMBER: _ClassVar[int] + CONTENT_FIELD_NUMBER: _ClassVar[int] + workflow_slug: str + artifact_id: _id_pb2.ID + content: ReleaseContent + def __init__(self, workflow_slug: _Optional[str] = ..., artifact_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., content: _Optional[_Union[ReleaseContent, _Mapping]] = ...) -> None: ... + +class ListWorkflowsRequest(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class ListWorkflowsResponse(_message.Message): + __slots__ = ("workflows",) + WORKFLOWS_FIELD_NUMBER: _ClassVar[int] + workflows: _containers.RepeatedCompositeFieldContainer[Workflow] + def __init__(self, workflows: _Optional[_Iterable[_Union[Workflow, _Mapping]]] = ...) -> None: ... + +class CreateWorkflowRequest(_message.Message): + __slots__ = ("name", "description") + NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + name: str + description: str + def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ...) -> None: ... + +class Workflow(_message.Message): + __slots__ = ("slug", "name", "description", "releases") + SLUG_FIELD_NUMBER: _ClassVar[int] + NAME_FIELD_NUMBER: _ClassVar[int] + DESCRIPTION_FIELD_NUMBER: _ClassVar[int] + RELEASES_FIELD_NUMBER: _ClassVar[int] + slug: str + name: str + description: str + releases: _containers.RepeatedCompositeFieldContainer[WorkflowRelease] + def __init__(self, slug: _Optional[str] = ..., name: _Optional[str] = ..., description: _Optional[str] = ..., releases: _Optional[_Iterable[_Union[WorkflowRelease, _Mapping]]] = ...) -> None: ... + +class WorkflowRelease(_message.Message): + __slots__ = ("id", "artifact", "content", "created_at") + ID_FIELD_NUMBER: _ClassVar[int] + ARTIFACT_FIELD_NUMBER: _ClassVar[int] + CONTENT_FIELD_NUMBER: _ClassVar[int] + CREATED_AT_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + artifact: Artifact + content: ReleaseContent + created_at: _timestamp_pb2.Timestamp + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., artifact: _Optional[_Union[Artifact, _Mapping]] = ..., content: _Optional[_Union[ReleaseContent, _Mapping]] = ..., created_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... + +class Artifact(_message.Message): + __slots__ = ("id", "digest") + ID_FIELD_NUMBER: _ClassVar[int] + DIGEST_FIELD_NUMBER: _ClassVar[int] + id: _id_pb2.ID + digest: str + def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., digest: _Optional[str] = ...) -> None: ... + +class ReleaseContent(_message.Message): + __slots__ = ("fingerprint", "tasks", "files", "runner_object_path", "command_override") + FINGERPRINT_FIELD_NUMBER: _ClassVar[int] + TASKS_FIELD_NUMBER: _ClassVar[int] + FILES_FIELD_NUMBER: _ClassVar[int] + RUNNER_OBJECT_PATH_FIELD_NUMBER: _ClassVar[int] + COMMAND_OVERRIDE_FIELD_NUMBER: _ClassVar[int] + fingerprint: str + tasks: _containers.RepeatedCompositeFieldContainer[_core_pb2.TaskIdentifier] + files: _containers.RepeatedCompositeFieldContainer[Path] + runner_object_path: str + command_override: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, fingerprint: _Optional[str] = ..., tasks: _Optional[_Iterable[_Union[_core_pb2.TaskIdentifier, _Mapping]]] = ..., files: _Optional[_Iterable[_Union[Path, _Mapping]]] = ..., runner_object_path: _Optional[str] = ..., command_override: _Optional[_Iterable[str]] = ...) -> None: ... + +class Path(_message.Message): + __slots__ = ("path", "directory", "children") + PATH_FIELD_NUMBER: _ClassVar[int] + DIRECTORY_FIELD_NUMBER: _ClassVar[int] + CHILDREN_FIELD_NUMBER: _ClassVar[int] + path: str + directory: bool + children: _containers.RepeatedCompositeFieldContainer[Path] + def __init__(self, path: _Optional[str] = ..., directory: bool = ..., children: _Optional[_Iterable[_Union[Path, _Mapping]]] = ...) -> None: ... + +class DeployWorkflowReleaseRequest(_message.Message): + __slots__ = ("workflow_slug", "release_id", "cluster_slugs") + WORKFLOW_SLUG_FIELD_NUMBER: _ClassVar[int] + RELEASE_ID_FIELD_NUMBER: _ClassVar[int] + CLUSTER_SLUGS_FIELD_NUMBER: _ClassVar[int] + workflow_slug: str + release_id: _id_pb2.ID + cluster_slugs: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, workflow_slug: _Optional[str] = ..., release_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., cluster_slugs: _Optional[_Iterable[str]] = ...) -> None: ... + +class DeployWorkflowReleaseResponse(_message.Message): + __slots__ = ("release", "clusters") + RELEASE_FIELD_NUMBER: _ClassVar[int] + CLUSTERS_FIELD_NUMBER: _ClassVar[int] + release: WorkflowRelease + clusters: _containers.RepeatedCompositeFieldContainer[Cluster] + def __init__(self, release: _Optional[_Union[WorkflowRelease, _Mapping]] = ..., clusters: _Optional[_Iterable[_Union[Cluster, _Mapping]]] = ...) -> None: ... + +class UndeployWorkflowReleaseRequest(_message.Message): + __slots__ = ("workflow_slug", "release_id", "cluster_slugs") + WORKFLOW_SLUG_FIELD_NUMBER: _ClassVar[int] + RELEASE_ID_FIELD_NUMBER: _ClassVar[int] + CLUSTER_SLUGS_FIELD_NUMBER: _ClassVar[int] + workflow_slug: str + release_id: _id_pb2.ID + cluster_slugs: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, workflow_slug: _Optional[str] = ..., release_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., cluster_slugs: _Optional[_Iterable[str]] = ...) -> None: ... + +class UndeployWorkflowReleaseResponse(_message.Message): + __slots__ = ("release", "clusters") + RELEASE_FIELD_NUMBER: _ClassVar[int] + CLUSTERS_FIELD_NUMBER: _ClassVar[int] + release: WorkflowRelease + clusters: _containers.RepeatedCompositeFieldContainer[Cluster] + def __init__(self, release: _Optional[_Union[WorkflowRelease, _Mapping]] = ..., clusters: _Optional[_Iterable[_Union[Cluster, _Mapping]]] = ...) -> None: ... diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py index 4d354a5..73a4cff 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py @@ -2,11 +2,10 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 from tilebox.workflows.workflows.v1 import workflows_pb2 as workflows_dot_v1_dot_workflows__pb2 -class WorkflowsServiceStub(object): +class WorkflowsServiceStub: """A service for managing workflows. """ @@ -19,12 +18,12 @@ def __init__(self, channel): self.CreateCluster = channel.unary_unary( '/workflows.v1.WorkflowsService/CreateCluster', request_serializer=workflows_dot_v1_dot_workflows__pb2.CreateClusterRequest.SerializeToString, - response_deserializer=workflows_dot_v1_dot_core__pb2.Cluster.FromString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.Cluster.FromString, _registered_method=True) self.GetCluster = channel.unary_unary( '/workflows.v1.WorkflowsService/GetCluster', request_serializer=workflows_dot_v1_dot_workflows__pb2.GetClusterRequest.SerializeToString, - response_deserializer=workflows_dot_v1_dot_core__pb2.Cluster.FromString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.Cluster.FromString, _registered_method=True) self.DeleteCluster = channel.unary_unary( '/workflows.v1.WorkflowsService/DeleteCluster', @@ -36,9 +35,39 @@ def __init__(self, channel): request_serializer=workflows_dot_v1_dot_workflows__pb2.ListClustersRequest.SerializeToString, response_deserializer=workflows_dot_v1_dot_workflows__pb2.ListClustersResponse.FromString, _registered_method=True) + self.CreateWorkflow = channel.unary_unary( + '/workflows.v1.WorkflowsService/CreateWorkflow', + request_serializer=workflows_dot_v1_dot_workflows__pb2.CreateWorkflowRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.Workflow.FromString, + _registered_method=True) + self.ListWorkflows = channel.unary_unary( + '/workflows.v1.WorkflowsService/ListWorkflows', + request_serializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse.FromString, + _registered_method=True) + self.GetWorkflow = channel.unary_unary( + '/workflows.v1.WorkflowsService/GetWorkflow', + request_serializer=workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.Workflow.FromString, + _registered_method=True) + self.PublishWorkflowRelease = channel.unary_unary( + '/workflows.v1.WorkflowsService/PublishWorkflowRelease', + request_serializer=workflows_dot_v1_dot_workflows__pb2.PublishWorkflowReleaseRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.WorkflowRelease.FromString, + _registered_method=True) + self.DeployWorkflowRelease = channel.unary_unary( + '/workflows.v1.WorkflowsService/DeployWorkflowRelease', + request_serializer=workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseResponse.FromString, + _registered_method=True) + self.UndeployWorkflowRelease = channel.unary_unary( + '/workflows.v1.WorkflowsService/UndeployWorkflowRelease', + request_serializer=workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseResponse.FromString, + _registered_method=True) -class WorkflowsServiceServicer(object): +class WorkflowsServiceServicer: """A service for managing workflows. """ @@ -66,18 +95,54 @@ def ListClusters(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def CreateWorkflow(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListWorkflows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetWorkflow(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def PublishWorkflowRelease(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeployWorkflowRelease(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UndeployWorkflowRelease(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_WorkflowsServiceServicer_to_server(servicer, server): rpc_method_handlers = { 'CreateCluster': grpc.unary_unary_rpc_method_handler( servicer.CreateCluster, request_deserializer=workflows_dot_v1_dot_workflows__pb2.CreateClusterRequest.FromString, - response_serializer=workflows_dot_v1_dot_core__pb2.Cluster.SerializeToString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.Cluster.SerializeToString, ), 'GetCluster': grpc.unary_unary_rpc_method_handler( servicer.GetCluster, request_deserializer=workflows_dot_v1_dot_workflows__pb2.GetClusterRequest.FromString, - response_serializer=workflows_dot_v1_dot_core__pb2.Cluster.SerializeToString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.Cluster.SerializeToString, ), 'DeleteCluster': grpc.unary_unary_rpc_method_handler( servicer.DeleteCluster, @@ -89,6 +154,36 @@ def add_WorkflowsServiceServicer_to_server(servicer, server): request_deserializer=workflows_dot_v1_dot_workflows__pb2.ListClustersRequest.FromString, response_serializer=workflows_dot_v1_dot_workflows__pb2.ListClustersResponse.SerializeToString, ), + 'CreateWorkflow': grpc.unary_unary_rpc_method_handler( + servicer.CreateWorkflow, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.CreateWorkflowRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.Workflow.SerializeToString, + ), + 'ListWorkflows': grpc.unary_unary_rpc_method_handler( + servicer.ListWorkflows, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse.SerializeToString, + ), + 'GetWorkflow': grpc.unary_unary_rpc_method_handler( + servicer.GetWorkflow, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.Workflow.SerializeToString, + ), + 'PublishWorkflowRelease': grpc.unary_unary_rpc_method_handler( + servicer.PublishWorkflowRelease, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.PublishWorkflowReleaseRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.WorkflowRelease.SerializeToString, + ), + 'DeployWorkflowRelease': grpc.unary_unary_rpc_method_handler( + servicer.DeployWorkflowRelease, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseResponse.SerializeToString, + ), + 'UndeployWorkflowRelease': grpc.unary_unary_rpc_method_handler( + servicer.UndeployWorkflowRelease, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'workflows.v1.WorkflowsService', rpc_method_handlers) @@ -97,7 +192,7 @@ def add_WorkflowsServiceServicer_to_server(servicer, server): # This class is part of an EXPERIMENTAL API. -class WorkflowsService(object): +class WorkflowsService: """A service for managing workflows. """ @@ -117,7 +212,7 @@ def CreateCluster(request, target, '/workflows.v1.WorkflowsService/CreateCluster', workflows_dot_v1_dot_workflows__pb2.CreateClusterRequest.SerializeToString, - workflows_dot_v1_dot_core__pb2.Cluster.FromString, + workflows_dot_v1_dot_workflows__pb2.Cluster.FromString, options, channel_credentials, insecure, @@ -144,7 +239,7 @@ def GetCluster(request, target, '/workflows.v1.WorkflowsService/GetCluster', workflows_dot_v1_dot_workflows__pb2.GetClusterRequest.SerializeToString, - workflows_dot_v1_dot_core__pb2.Cluster.FromString, + workflows_dot_v1_dot_workflows__pb2.Cluster.FromString, options, channel_credentials, insecure, @@ -208,3 +303,165 @@ def ListClusters(request, timeout, metadata, _registered_method=True) + + @staticmethod + def CreateWorkflow(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/CreateWorkflow', + workflows_dot_v1_dot_workflows__pb2.CreateWorkflowRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.Workflow.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ListWorkflows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/ListWorkflows', + workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def GetWorkflow(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/GetWorkflow', + workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.Workflow.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def PublishWorkflowRelease(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/PublishWorkflowRelease', + workflows_dot_v1_dot_workflows__pb2.PublishWorkflowReleaseRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.WorkflowRelease.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def DeployWorkflowRelease(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/DeployWorkflowRelease', + workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.DeployWorkflowReleaseResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def UndeployWorkflowRelease(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/UndeployWorkflowRelease', + workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.UndeployWorkflowReleaseResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True)