Skip to content

Convert xrm images to tiffs #837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephen-riggs merged 11 commits into from
Jul 3, 2026
Merged

Convert xrm images to tiffs #837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
927cc32
Read xrm files for metadata on the client
stephen-riggs Jun 23, 2026
d5db21e
Move around the sxt logic
stephen-riggs Jun 24, 2026
1b5c534
Add dummy router for xrm position info
stephen-riggs Jun 24, 2026
2ce7055
Change router name in main
stephen-riggs Jun 24, 2026
e05e440
Revert db changes for later pr
stephen-riggs Jun 24, 2026
efaffae
Another revert
stephen-riggs Jun 24, 2026
f0535e5
Merge branch 'main' into sxt-metadata
stephen-riggs Jun 24, 2026
ef04d93
Improved path structuring and tests
stephen-riggs Jul 3, 2026
e0e427f
Test for roi mag
stephen-riggs Jul 3, 2026
124ba36
Break out ole read function
stephen-riggs Jul 3, 2026
c12f36f
Wrongly introduced indexes
stephen-riggs Jul 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 183 additions & 28 deletions src/murfey/client/contexts/sxt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def _find_reference(txrm_file: Path) -> Path | None:
return None


def _get_ole_header_value(ole_file, title: str, dtype: np.dtype):
return np.frombuffer(ole_file.openstream(title).getvalue(), dtype)


class SXTContext(Context):
def __init__(
self,
Expand Down Expand Up @@ -156,69 +160,221 @@ def post_transfer(
environment=environment,
**kwargs,
)
metadata: dict[str, Any] = {}

if transferred_file.suffix == ".xrm" and environment:
Comment thread
stephen-riggs marked this conversation as resolved.
# Make sure we have a dcg for this grid
dcg_tag = ensure_dcg_exists(
collection_type="sxt",
metadata_source=self._basepath,
environment=environment,
machine_config=self._machine_config,
token=self._token,
)

data_suffixes = [".txrm"]
with OleFileIO(str(transferred_file)) as xrm_ole:
if xrm_ole.exists("ImageInfo/XPosition") and xrm_ole.exists(
"ImageInfo/YPosition"
):
x_tiles = _get_ole_header_value(
xrm_ole, "ImageInfo/XPosition", np.float32
).tolist()
y_tiles = _get_ole_header_value(
xrm_ole, "ImageInfo/YPosition", np.float32
).tolist()
metadata["x_position"] = x_tiles[int(len(x_tiles) / 2)]
metadata["y_position"] = y_tiles[int(len(y_tiles) / 2)]

if transferred_file.suffix in data_suffixes and environment:
if xrm_ole.exists("ImageInfo/PixelSize"):
metadata["pixel_size"] = _get_ole_header_value(
xrm_ole, "ImageInfo/PixelSize", np.float32
).tolist()[0]

if xrm_ole.exists("ImageInfo/ImageHeight"):
metadata["height"] = _get_ole_header_value(
xrm_ole, "ImageInfo/ImageHeight", np.int32
).tolist()[0]

if xrm_ole.exists("ImageInfo/ImageWidth"):
metadata["width"] = _get_ole_header_value(
xrm_ole, "ImageInfo/ImageWidth", np.int32
).tolist()[0]

# Find images which are not mosaics (txrm spec typos this as mosiac)
if xrm_ole.exists("ImageInfo/MosiacRows") and xrm_ole.exists(
"ImageInfo/MosiacColumns"
):
metadata["mosaic_rows"] = _get_ole_header_value(
xrm_ole, "ImageInfo/MosiacRows", np.int32
)[0]
metadata["mosaic_columns"] = _get_ole_header_value(
xrm_ole, "ImageInfo/MosiacColumns", np.int32
)[0]
metadata["mosaic_size"] = int(
metadata["mosaic_rows"] * metadata["mosaic_columns"]
)

source = _get_source(transferred_file, environment=environment)
if source:
image_path = _file_transferred_to(
environment,
source,
transferred_file,
Path(self._machine_config.get("rsync_basepath", "")),
)
if (
environment.visit
in Path(environment.default_destinations[source]).parts
):
# Split either side of the raw directory
visit_idx = Path(
environment.default_destinations[source]
).parts.index(environment.visit)
Comment thread
stephen-riggs marked this conversation as resolved.
destination_base = "/".join(
Path(environment.default_destinations[source]).parts[
: visit_idx + 1
]
)
destination_extra = "/".join(
Path(environment.default_destinations[source]).parts[
visit_idx + 2 :
]
)
else:
destination_base = str(
Path(environment.default_destinations[source])
/ environment.visit
)
destination_extra = ""
converted_file_path = (
Path(self._machine_config.get("rsync_basepath", ""))
/ destination_base
/ self._machine_config.get("processed_directory_name", "")
/ self._machine_config.get("processed_extra_directory", "")
/ destination_extra
/ f"{transferred_file.relative_to(source).stem}_Annotated.tiff"
)
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow_sxt.router",
function_name="convert_xrm_to_tiff",
token=self._token,
instrument_name=environment.instrument_name,
data={
"xrm_path": str(image_path),
"tiff_path": str(converted_file_path),
},
)

if (
metadata.get("mosaic_size", 1) > 0
and metadata.get("pixel_size", 0) > 0.1
):
# Large pixel size, this is an atlas
dcg_data = {
"experiment_type_id": 44, # Atlas
Comment thread
stephen-riggs marked this conversation as resolved.
"tag": dcg_tag,
"atlas": str(converted_file_path),
"atlas_pixel_size": round(metadata.get("pixel_size", 0), 2),
"atlas_x_stage_position": metadata.get("x_position", None),
"atlas_y_stage_position": metadata.get("y_position", None),
"atlas_height": int(
metadata.get("height", 0) * metadata["mosaic_rows"]
),
"atlas_width": int(
metadata.get("width", 0) * metadata["mosaic_columns"]
),
}
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow.router",
function_name="register_dc_group",
token=self._token,
instrument_name=environment.instrument_name,
visit_name=environment.visit,
session_id=environment.murfey_session,
data=dcg_data,
)
elif metadata.get("mosaic_size", 1) > 0:
# Other mosaic images are of grid squares
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow_sxt.router",
function_name="register_sxt_roi",
token=self._token,
instrument_name=environment.instrument_name,
visit_name=environment.visit,
session_id=environment.murfey_session,
data={
"tag": dcg_tag,
"name": transferred_file.stem,
"x_stage_position": metadata.get("x_position", None),
"y_stage_position": metadata.get("y_position", None),
"pixel_size": round(metadata.get("pixel_size", 0), 2),
"height": int(
metadata.get("height", 0) * metadata["mosaic_rows"]
),
"width": int(
metadata.get("width", 0) * metadata["mosaic_columns"]
),
"image": str(converted_file_path),
},
)

elif transferred_file.suffix == ".txrm" and environment:
source = _get_source(transferred_file, environment)
if not source:
logger.warning(f"No source found for file {transferred_file}")
return False

# Read the tilt angles and pixel size from the txrm
angles: list = []
metadata: dict[str, Any] = {
"source": str(self._basepath),
"tilt_series_tag": transferred_file.stem,
}
metadata["source"] = str(self._basepath)
metadata["tilt_series_tag"] = transferred_file.stem
with OleFileIO(str(transferred_file)) as txrm_ole:
if txrm_ole.exists("ReferenceData/Image"):
metadata["has_reference"] = True

if txrm_ole.exists("ImageInfo/Angles"):
angles = np.frombuffer(
txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32
angles = _get_ole_header_value(
txrm_ole, "ImageInfo/Angles", np.float32
).tolist()
metadata["minimum_angle"] = min(angles)
metadata["maximum_angle"] = max(angles)

if txrm_ole.exists("ImageInfo/PixelSize"):
pixel_size_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/PixelSize").getvalue(),
np.float32,
pixel_size_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/PixelSize", np.float32
).tolist()
metadata["pixel_size"] = pixel_size_txrm[0] * 1e4

if txrm_ole.exists("ImageInfo/ImageWidth"):
image_width_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32
image_width_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/ImageWidth", np.int32
).tolist()
metadata["image_size_x"] = image_width_txrm[0]

if txrm_ole.exists("ImageInfo/ImageHeight"):
image_height_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(),
np.int32,
image_height_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/ImageHeight", np.int32
).tolist()
metadata["image_size_y"] = image_height_txrm[0]

if txrm_ole.exists("ImageInfo/ExpTimes"):
exposure_time_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32
exposure_time_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/ExpTimes", np.float32
).tolist()
metadata["exposure_time"] = exposure_time_txrm[0]

if txrm_ole.exists("ImageInfo/XrayMagnification"):
magnification_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(),
np.float32,
magnification_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/XrayMagnification", np.float32
).tolist()
metadata["magnification"] = magnification_txrm[0]

if txrm_ole.exists("ImageInfo/ImagesTaken"):
tilt_count_txrm = np.frombuffer(
txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(),
np.int32,
tilt_count_txrm = _get_ole_header_value(
txrm_ole, "ImageInfo/ImagesTaken", np.int32
).tolist()
metadata["tilt_series_length"] = tilt_count_txrm[0]

Expand All @@ -235,9 +391,8 @@ def post_transfer(
.split("\x00")
if i
]
axis_values = np.frombuffer(
txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(),
np.float32,
axis_values = _get_ole_header_value(
txrm_ole, "PositionInfo/MotorPositions", np.float32
)
if "Energy" in axis_names:
energy_index = list(np.array(axis_names) == "Energy").index(
Expand Down Expand Up @@ -302,7 +457,7 @@ def post_transfer(
reference_file_transferred_to = None
capture_post(
base_url=str(environment.url.geturl()),
router_name="workflow.sxt_router",
router_name="workflow_sxt.router",
function_name="process_sxt_tilt_series",
token=self._token,
instrument_name=environment.instrument_name,
Expand Down
33 changes: 4 additions & 29 deletions src/murfey/server/api/workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,6 @@
motion_corrected_mrc,
)
from murfey.util.tomo import midpoint
from murfey.workflows.sxt.process_sxt_tilt_series import (
SXTTiltSeriesInfo,
)
from murfey.workflows.tomo.tomo_metadata import register_search_map_in_database

logger = getLogger("murfey.server.api.workflow")
Expand All @@ -100,6 +97,10 @@ class DCGroupParameters(BaseModel):
atlas: str = ""
sample: Optional[int] = None
atlas_pixel_size: float = 0
atlas_x_stage_position: float | None = None
atlas_y_stage_position: float | None = None
atlas_width: int | None = None
atlas_height: int | None = None
create_smartem_grid: bool = False
acquisition_uuid: Optional[str] = None

Expand Down Expand Up @@ -1071,32 +1072,6 @@ def _add_tilt():
_add_tilt()


sxt_router = APIRouter(
prefix="/workflow/sxt",
dependencies=[Depends(validate_instrument_token)],
tags=["Workflows: Soft x-ray tomography"],
)


@sxt_router.post("/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series")
def process_sxt_tilt_series(
visit_name: str,
session_id: MurfeySessionID,
tilt_series_info: SXTTiltSeriesInfo,
db=murfey_db,
):
if _transport_object:
_transport_object.send(
_transport_object.feedback_queue,
{
"register": "sxt.process_tilt_series",
"session_id": session_id,
"visit_name": visit_name,
"tilt_series_info": tilt_series_info.model_dump(mode="json"),
},
)


correlative_router = APIRouter(
prefix="/workflow/correlative",
dependencies=[Depends(validate_instrument_token)],
Expand Down
Loading