diff --git a/src/relion/__init__.py b/src/relion/__init__.py index 575b5bc9..305d58ef 100644 --- a/src/relion/__init__.py +++ b/src/relion/__init__.py @@ -95,7 +95,6 @@ def __init__( raise ValueError(f"path {self.basepath} is not a directory") self._data_pipeline = Graph("DataPipeline", []) self._db_model = DBModel(database) - self._drift_cache = {} if run_options is None: self.run_options = RelionItOptions() else: @@ -159,7 +158,7 @@ def motioncorrection(self): """access the motion correction stage of the project. Returns a dictionary-like object with job names as keys, and lists of MCMicrograph namedtuples as values.""" - return MotionCorr(self.basepath / "MotionCorr", self._drift_cache) + return MotionCorr(self.basepath / "MotionCorr") @property @functools.lru_cache(maxsize=1) diff --git a/src/relion/_parser/motioncorrection.py b/src/relion/_parser/motioncorrection.py index ced46d19..63b2b966 100644 --- a/src/relion/_parser/motioncorrection.py +++ b/src/relion/_parser/motioncorrection.py @@ -2,6 +2,10 @@ import logging from collections import namedtuple +from pathlib import Path + +import numpy as np +import plotly.express as px from relion._parser.jobtype import JobType @@ -16,8 +20,9 @@ "total_motion", "early_motion", "late_motion", + "average_motion_per_frame", "micrograph_timestamp", - "drift_data", + "drift_plot_full_path", ], ) @@ -57,12 +62,8 @@ class MotionCorr(JobType): - def __init__(self, path, drift_cache=None): + def __init__(self, path): super().__init__(path) - if drift_cache is None: - self._drift_cache = {} - else: - self._drift_cache = drift_cache def __eq__(self, other): if isinstance(other, MotionCorr): # check this @@ -102,7 +103,11 @@ def _load_job_directory(self, jobdir): micrograph_list = [] for j in range(len(micrograph_name)): - drift_data, movie_name = self.collect_drift_data(micrograph_name[j], jobdir) + ( + number_of_frames, + drift_plot_full_path, + movie_name, + ) = self.collect_drift_data(micrograph_name[j], jobdir) if movie_name: try: movie_creation_time = ( @@ -125,55 +130,31 @@ def _load_job_directory(self, jobdir): accum_motion_total[j], accum_motion_early[j], accum_motion_late[j], + float(accum_motion_total[j]) / number_of_frames, movie_creation_time, - drift_data, + drift_plot_full_path, ) ) return micrograph_list def collect_drift_data(self, mic_name, jobdir): - drift_data = [] drift_star_file_path = mic_name.split(jobdir + "/")[-1].replace("mrc", "star") - if self._drift_cache.get(jobdir): - if self._drift_cache[jobdir].get(mic_name): - try: - if ( - self._drift_cache[jobdir][mic_name].file_size - == (self._basepath / jobdir / drift_star_file_path) - .stat() - .st_size - ): - return ( - self._drift_cache[jobdir][mic_name].data, - self._drift_cache[jobdir][mic_name].movie_name, - ) - except FileNotFoundError: - logger.debug( - "Could not find expected file containing drift data", - exc_info=True, - ) - return [], "" - else: - self._drift_cache[jobdir] = {} try: drift_star_file = self._read_star_file(jobdir, drift_star_file_path) except (FileNotFoundError, OSError, RuntimeError, ValueError): - return drift_data, "" + return 1, "", "" try: info_table = self._find_table_from_column_name( "_rlnMicrographFrameNumber", drift_star_file ) movie_table = 0 except (FileNotFoundError, OSError, RuntimeError, ValueError): - return drift_data, "" + return 1, "", "" if info_table is None: logger.debug( f"_rlnMicrographFrameNumber or _rlnMicrographMovieName not found in file {drift_star_file}" ) - return drift_data, "" - frame_numbers = self.parse_star_file( - "_rlnMicrographFrameNumber", drift_star_file, info_table - ) + return 1, "", "" deltaxs = self.parse_star_file( "_rlnMicrographShiftX", drift_star_file, info_table ) @@ -183,17 +164,16 @@ def collect_drift_data(self, mic_name, jobdir): movie_name = self.parse_star_file_pair( "_rlnMicrographMovieName", drift_star_file, movie_table ) - for f, dx, dy in zip(frame_numbers, deltaxs, deltays): - drift_data.append(MCMicrographDrift(int(f), float(dx), float(dy))) try: - self._drift_cache[jobdir][mic_name] = MCDriftCacheRecord( - drift_data, - (self._basepath / jobdir / drift_star_file_path).stat().st_size, - movie_name, + fig = px.scatter( + x=np.array(deltaxs, dtype=float), y=np.array(deltays, dtype=float) ) + drift_plot_name = Path(mic_name).stem + "_drift_plot.json" + drift_plot_full_path = Path(mic_name).parent / drift_plot_name + fig.write_json(drift_plot_full_path) except FileNotFoundError: - return [], "" - return drift_data, movie_name + return 1, "", "" + return len(deltaxs), drift_plot_full_path, movie_name @staticmethod def for_cache(mcmicrograph): @@ -220,12 +200,10 @@ def db_unpack(micrograph_list): "total_motion": micrograph.total_motion, "early_motion": micrograph.early_motion, "late_motion": micrograph.late_motion, - "average_motion_per_frame": ( - float(micrograph.total_motion) / len(micrograph.drift_data) - ), + "average_motion_per_frame": micrograph.average_motion_per_frame, "image_number": micrograph.micrograph_number, "micrograph_snapshot_full_path": micrograph.micrograph_snapshot_full_path, - "drift_data": micrograph.drift_data, + "drift_plot_full_path": micrograph.drift_plot_full_path, "created_time_stamp": micrograph.micrograph_timestamp, } for micrograph in micrograph_list diff --git a/src/relion/dbmodel/modeltables.py b/src/relion/dbmodel/modeltables.py index 7a9266b0..77b9f88c 100644 --- a/src/relion/dbmodel/modeltables.py +++ b/src/relion/dbmodel/modeltables.py @@ -189,7 +189,6 @@ class MotionCorrectionTable(Table): def __init__(self): columns, prim_key = parse_sqlalchemy_table(sqlalchemy.MotionCorrection) columns.append("created_time_stamp") - columns.append("drift_data") # columns.append("job_string") super().__init__( columns, diff --git a/src/relion/zocalo/wrapper.py b/src/relion/zocalo/wrapper.py index 01488ebb..765433a6 100644 --- a/src/relion/zocalo/wrapper.py +++ b/src/relion/zocalo/wrapper.py @@ -579,10 +579,8 @@ def _( unsent_appended: Optional[dict] = None, ): row = table.get_row_by_primary_key(primary_key) - drift_data = row["drift_data"] - buffered = ["motion_correction_id", "drift_data"] + buffered = ["motion_correction_id"] buffer_store = row["motion_correction_id"] - drift_frames = [(frame.frame, frame.deltaX, frame.deltaY) for frame in drift_data] if resend: results = { "ispyb_command": "buffer", @@ -590,7 +588,6 @@ def _( "buffer_command": { "ispyb_command": "insert_motion_correction", **{k: v for k, v in row.items() if k not in buffered}, - "drift_frames": drift_frames, }, } else: @@ -599,7 +596,6 @@ def _( "buffer_command": { "ispyb_command": "insert_motion_correction", **{k: v for k, v in row.items() if k not in buffered}, - "drift_frames": drift_frames, }, "buffer_store": buffer_store, }