From 99f8d875cd1fd36d4f0d5cece2f7e906438347f8 Mon Sep 17 00:00:00 2001 From: xingjian_kang Date: Wed, 25 Feb 2026 19:27:42 +0100 Subject: [PATCH 1/2] modified the video handler --- exact/exact/images/models.py | 131 ++++++++++--------- exact/exact/images/views.py | 2 +- exact/util/video_handler.py | 235 +++++++++++++++++++++++++++++++++++ 3 files changed, 309 insertions(+), 59 deletions(-) create mode 100644 exact/util/video_handler.py diff --git a/exact/exact/images/models.py b/exact/exact/images/models.py index de09cfcd..c6ff9ae3 100644 --- a/exact/exact/images/models.py +++ b/exact/exact/images/models.py @@ -27,7 +27,8 @@ from openslide import OpenSlide, open_slide from czifile import czi2tif from util.cellvizio import ReadableCellVizioMKTDataset # just until data access is pip installable - +# for video handling +from util.video_handler import ReadableMP4Dataset from PIL import Image as PIL_Image from datetime import datetime @@ -192,69 +193,83 @@ def save_file(self, path:Path): os.remove(str(path_temp)) self.filename = path.name # Videos - elif Path(path).suffix.lower() in [".avi", ".mp4"]: - dtype_to_format = { - 'uint8': 'uchar', - 'int8': 'char', - 'uint16': 'ushort', - 'int16': 'short', - 'uint32': 'uint', - 'int32': 'int', - 'float32': 'float', - 'float64': 'double', - 'complex64': 'complex', - 'complex128': 'dpcomplex', - } - - folder_path = Path(self.image_set.root_path()) / path.stem - os.makedirs(str(folder_path), exist_ok =True) - os.chmod(str(folder_path), 0o777) - self.save() # initially save - - cap = cv2.VideoCapture(str(Path(path))) - frame_id = 0 - while cap.isOpened(): - ret, frame = cap.read() - if not ret: - # if video has just one frame copy file to top layer - if frame_id == 1: - copy_path = Path(path).with_suffix('.tiff') - shutil.copyfile(str(target_file), str(copy_path)) - self.filename = copy_path.name - break - - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - height, width, bands = frame.shape - linear = frame.reshape(width * height * bands) - - vi = pyvips.Image.new_from_memory(np.ascontiguousarray(linear.data), width, height, bands, - dtype_to_format[str(frame.dtype)]) - if dtype_to_format[str(frame.dtype)] not in ["uchar"]: - vi = vi.scaleimage() - - height, width, channels = vi.height, vi.width, vi.bands - self.channels = channels - - target_file = folder_path / "{}_{}_{}".format(1, frame_id + 1, path.name) #z-axis frame image - vi.tiffsave(str(target_file), tile=True, compression='lzw', bigtiff=True, pyramid=True, tile_width=256, tile_height=256) - - # save first frame as default file for thumbnail etc. - if frame_id == 0: - self.filename = target_file.name - - # save FrameDescription object for each frame + elif Path(path).suffix.lower().endswith(".mp4"): + reader = ReadableMP4Dataset(str(path)) + self.frames = reader.nFrames + self.width, self.height = reader.dimensions + self.channels = 3 + self.filename = path.name + self.save() + for frame_id in range(reader.nFrames): FrameDescription.objects.create( Image=self, frame_id=frame_id, - file_path=target_file, + file_path=self.filename, frame_type=FrameType.TIMESERIES, - description='%.2f s (%d)' % ((float(frame_id-1)/cap.get(cv2.CAP_PROP_FPS)), frame_id) + description=reader.frame_descriptors[frame_id] ) - - - frame_id += 1 + # dtype_to_format = { + # 'uint8': 'uchar', + # 'int8': 'char', + # 'uint16': 'ushort', + # 'int16': 'short', + # 'uint32': 'uint', + # 'int32': 'int', + # 'float32': 'float', + # 'float64': 'double', + # 'complex64': 'complex', + # 'complex128': 'dpcomplex', + # } + + # folder_path = Path(self.image_set.root_path()) / path.stem + # os.makedirs(str(folder_path), exist_ok =True) + # os.chmod(str(folder_path), 0o777) + # self.save() # initially save + + # cap = cv2.VideoCapture(str(Path(path))) + # frame_id = 0 + # while cap.isOpened(): + # ret, frame = cap.read() + # if not ret: + # # if video has just one frame copy file to top layer + # if frame_id == 1: + # copy_path = Path(path).with_suffix('.tiff') + # shutil.copyfile(str(target_file), str(copy_path)) + # self.filename = copy_path.name + # break + + # frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + # height, width, bands = frame.shape + # linear = frame.reshape(width * height * bands) + + # vi = pyvips.Image.new_from_memory(np.ascontiguousarray(linear.data), width, height, bands, + # dtype_to_format[str(frame.dtype)]) + # if dtype_to_format[str(frame.dtype)] not in ["uchar"]: + # vi = vi.scaleimage() + + # height, width, channels = vi.height, vi.width, vi.bands + # self.channels = channels + + # target_file = folder_path / "{}_{}_{}".format(1, frame_id + 1, path.name) #z-axis frame image + # vi.tiffsave(str(target_file), tile=True, compression='lzw', bigtiff=True, pyramid=True, tile_width=256, tile_height=256) + + # # save first frame as default file for thumbnail etc. + # if frame_id == 0: + # self.filename = target_file.name + + # # save FrameDescription object for each frame + # FrameDescription.objects.create( + # Image=self, + # frame_id=frame_id, + # file_path=target_file, + # frame_type=FrameType.TIMESERIES, + # description='%.2f s (%d)' % ((float(frame_id-1)/cap.get(cv2.CAP_PROP_FPS)), frame_id) + # ) + + + # frame_id += 1 - self.frames = frame_id + # self.frames = frame_id # check if file is philips iSyntax diff --git a/exact/exact/images/views.py b/exact/exact/images/views.py index fe1fec71..6116351f 100644 --- a/exact/exact/images/views.py +++ b/exact/exact/images/views.py @@ -303,7 +303,7 @@ def upload_image(request, imageset_id): image_set=imageset).first() print('Image:',image) if image is None: - + os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, 'wb') as out: for chunk in f.chunks(): out.write(chunk) diff --git a/exact/util/video_handler.py b/exact/util/video_handler.py new file mode 100644 index 00000000..042683c4 --- /dev/null +++ b/exact/util/video_handler.py @@ -0,0 +1,235 @@ +""" +Scripts for MP4 files's support + +""" +import threading +from collections import OrderedDict + +import openslide +from openslide import OpenSlideError +import numpy as np +import cv2 +from PIL import Image +try : + from util.enums import FrameType +except ImportError: + from enums import FrameType + + +class ReadableMP4Dataset(openslide.ImageSlide): + def __init__(self, filename, cache_size=32, max_cache_bytes=None): + self.slide_path = filename + + self._cap = None + self._cap_lock = threading.RLock() + self._frame_cache = OrderedDict() + self._cache_size = cache_size + + self._max_cache_bytes = max_cache_bytes # optional based on memory + self._cache_bytes = 0 + + cap = cv2.VideoCapture(filename) + if not cap.isOpened(): + raise OpenSlideError(f"Could not open video file: {filename}") + # Get video properties + self._width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self._height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.numberOfLayers = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # total number of frames + self.fps = cap.get(cv2.CAP_PROP_FPS) + + cap.release() + + self._dimensions = (self._width, self._height) + + def __reduce__(self): + return (self.__class__, (self.slide_path,)) + + def close(self): + with self._cap_lock: + if self._cap is not None: + self._cap.release() + self._cap = None + + self._frame_cache.clear() + self._cache_bytes = 0 + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @property + def properties(self): + return { + openslide.PROPERTY_NAME_BACKGROUND_COLOR: '000000', + openslide.PROPERTY_NAME_MPP_X: 0, + openslide.PROPERTY_NAME_MPP_Y: 0, + openslide.PROPERTY_NAME_OBJECTIVE_POWER: 1, + openslide.PROPERTY_NAME_VENDOR: 'MP4' + } + + + @property + def dimensions(self): + return self._dimensions + + @property + def frame_descriptors(self) -> list[str]: + """ returns a list of strings, used as descriptor for each frame + """ + return ['%.2f' % (x/self.fps) for x in range(self.nFrames)] + + @property + def frame_type(self): + return FrameType.TIMESERIES + + @property + def default_frame(self) -> list[str]: + return 0 + + @property + def nFrames(self): + return self.numberOfLayers + + @property + def level_dimensions(self): + return (self.dimensions,) + + @property + def level_count(self): + return 1 + + def _get_capture(self): + if self._cap is None or not self._cap.isOpened(): + self._cap = cv2.VideoCapture(self.slide_path) + return self._cap + + def _frame_num_bytes(self, frame_arr: np.ndarray) -> int: + """ + + Calculate the number of bytes used by a frame array. + :param frame_arr: Description + """ + try: + return int(frame_arr.nbytes) + except Exception: + return 0 + + def _evict_if_needed(self): + """ + Evict frames by LRU + """ + + # Evict frames if exceeding max frame count + while self._cache_size is not None and len(self._frame_cache) > self._cache_size: + old_idx, old_frame = self._frame_cache.popitem(last=False) + self._cache_bytes -= self._frame_num_bytes(old_frame) + # Evict based on byte size + if self._max_cache_bytes is not None: + while self._cache_bytes > self._max_cache_bytes and len(self._frame_cache) > 0: + old_idx, old_frame = self._frame_cache.popitem(last=False) + self._cache_bytes -= self._frame_num_bytes(old_frame) + + def _read_frame(self, frame_idx: int): + """ + Before reading the frame, check the cache first with thread safety. + Followed by LRU cache eviction policy. + + :param frame_idx: + """ + with self._cap_lock: + cached = self._frame_cache.get(frame_idx) + if cached is not None: + self._frame_cache.move_to_end(frame_idx) + return cached + + cap = self._get_capture() + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx) + success, img = cap.read() + if not success: + return None + # Convert BGR to RGBA + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGBA) + self._frame_cache[frame_idx] = img_rgb + self._frame_cache.move_to_end(frame_idx, last=True) + self._cache_bytes += self._frame_num_bytes(img_rgb) + + self._evict_if_needed() + return img_rgb + + + def get_thumbnail(self, size): + return self.read_region((0,0),0, self.dimensions).resize(size) + + def read_region(self, location, level, size, frame=0): + + """ + Reads a region from a specific video frame. + Return a PIL.Image containing the contents of the region. + Reference: https://github.com/DeepMicroscopy/Exact/commit/4d52b614fa41328bf08367d99e088c1e838fb05a + + + location: (x, y) tuple giving the top left pixel in the level 0 + reference frame. + level: the level number. + size: (width, height) tuple giving the region size. + frame: the frame index to read from the video. + + """ + if level != 0: + raise OpenSlideError("Only level 0 is supported for video files.") + + if any(s < 0 for s in size): + raise OpenSlideError(f"Size {size} must be non-negative") + + # Clamp frame index + frame = max(0, min(frame, self.numberOfLayers - 1)) + img_rgb = self._read_frame(frame) + if img_rgb is None: + # Return a transparent tile if frame read fails + return Image.new("RGBA", size, (0, 0, 0, 0)) + + x, y = location + w, h = size + + # Create the transparent canvas + tile = Image.new("RGBA", size, (0, 0, 0, 0)) + + # Calculate crop boundaries within the source image + img_h, img_w = img_rgb.shape[:2] + + # Source coordinates + src_x1 = max(0, min(x, img_w)) + src_y1 = max(0, min(y, img_h)) + src_x2 = max(0, min(x + w, img_w)) + src_y2 = max(0, min(y + h, img_h)) + + # Destination coordinates (where to paste on the tile) + dst_x1 = max(0, -x) if x < 0 else 0 + dst_y1 = max(0, -y) if y < 0 else 0 + + # Extract the crop using numpy slicing + crop_data = img_rgb[src_y1:src_y2, src_x1:src_x2] + + if crop_data.size > 0: + crop_img = Image.fromarray(crop_data) + tile.paste(crop_img, (dst_x1, dst_y1)) + + return tile + + def get_duration(self): + """Get the length of the video in seconds.""" + return self.numberOfLayers / self.fps if self.fps > 0 else 0 + + def time_to_frame(self, time_seconds: float) -> int: + """Convert time in seconds to frame index.""" + frame_idx = int(time_seconds * self.fps) + return max(0, min(frame_idx, self.numberOfFrames - 1)) + + def frame_to_time(self, frame_idx: int) -> float: + """Convert frame index to time in seconds.""" + return frame_idx / self.fps if self.fps > 0 else 0 \ No newline at end of file From de0b47062b9a532f11336638232c78c0ba89ffbf Mon Sep 17 00:00:00 2001 From: xingjian_kang Date: Wed, 25 Feb 2026 19:59:01 +0100 Subject: [PATCH 2/2] add avi support --- exact/exact/images/models.py | 69 ++---------------------------------- exact/util/slide_server.py | 19 ++++++---- exact/util/video_handler.py | 2 +- 3 files changed, 16 insertions(+), 74 deletions(-) diff --git a/exact/exact/images/models.py b/exact/exact/images/models.py index c6ff9ae3..20620992 100644 --- a/exact/exact/images/models.py +++ b/exact/exact/images/models.py @@ -28,7 +28,7 @@ from czifile import czi2tif from util.cellvizio import ReadableCellVizioMKTDataset # just until data access is pip installable # for video handling -from util.video_handler import ReadableMP4Dataset +from util.video_handler import ReadableVideoDataset from PIL import Image as PIL_Image from datetime import datetime @@ -193,8 +193,8 @@ def save_file(self, path:Path): os.remove(str(path_temp)) self.filename = path.name # Videos - elif Path(path).suffix.lower().endswith(".mp4"): - reader = ReadableMP4Dataset(str(path)) + elif Path(path).suffix.lower() in [".avi", ".mp4"]: + reader = ReadableVideoDataset(str(path)) self.frames = reader.nFrames self.width, self.height = reader.dimensions self.channels = 3 @@ -208,69 +208,6 @@ def save_file(self, path:Path): frame_type=FrameType.TIMESERIES, description=reader.frame_descriptors[frame_id] ) - # dtype_to_format = { - # 'uint8': 'uchar', - # 'int8': 'char', - # 'uint16': 'ushort', - # 'int16': 'short', - # 'uint32': 'uint', - # 'int32': 'int', - # 'float32': 'float', - # 'float64': 'double', - # 'complex64': 'complex', - # 'complex128': 'dpcomplex', - # } - - # folder_path = Path(self.image_set.root_path()) / path.stem - # os.makedirs(str(folder_path), exist_ok =True) - # os.chmod(str(folder_path), 0o777) - # self.save() # initially save - - # cap = cv2.VideoCapture(str(Path(path))) - # frame_id = 0 - # while cap.isOpened(): - # ret, frame = cap.read() - # if not ret: - # # if video has just one frame copy file to top layer - # if frame_id == 1: - # copy_path = Path(path).with_suffix('.tiff') - # shutil.copyfile(str(target_file), str(copy_path)) - # self.filename = copy_path.name - # break - - # frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - # height, width, bands = frame.shape - # linear = frame.reshape(width * height * bands) - - # vi = pyvips.Image.new_from_memory(np.ascontiguousarray(linear.data), width, height, bands, - # dtype_to_format[str(frame.dtype)]) - # if dtype_to_format[str(frame.dtype)] not in ["uchar"]: - # vi = vi.scaleimage() - - # height, width, channels = vi.height, vi.width, vi.bands - # self.channels = channels - - # target_file = folder_path / "{}_{}_{}".format(1, frame_id + 1, path.name) #z-axis frame image - # vi.tiffsave(str(target_file), tile=True, compression='lzw', bigtiff=True, pyramid=True, tile_width=256, tile_height=256) - - # # save first frame as default file for thumbnail etc. - # if frame_id == 0: - # self.filename = target_file.name - - # # save FrameDescription object for each frame - # FrameDescription.objects.create( - # Image=self, - # frame_id=frame_id, - # file_path=target_file, - # frame_type=FrameType.TIMESERIES, - # description='%.2f s (%d)' % ((float(frame_id-1)/cap.get(cv2.CAP_PROP_FPS)), frame_id) - # ) - - - # frame_id += 1 - - # self.frames = frame_id - # check if file is philips iSyntax elif Path(path).suffix.lower().endswith(".isyntax"): diff --git a/exact/util/slide_server.py b/exact/util/slide_server.py index 25e82e88..3a13556e 100644 --- a/exact/util/slide_server.py +++ b/exact/util/slide_server.py @@ -21,6 +21,7 @@ from PIL import Image import numpy as np from util.cellvizio import ReadableCellVizioMKTDataset +from util.video_handler import ReadableVideoDataset from openslide import OpenSlideError import tifffile from util.tiffzstack import OMETiffSlide, OMETiffZStack @@ -454,12 +455,6 @@ class JPEGEXIFFileType(FileType): extensions = ['jpg','jpeg'] handler = ImageSlideWrapper -class MP4MovieFileType(FileType): - extensions = ['mp4'] - magic_number = b'\x66\x74\x79\x70' - magic_number_offset = 4 - handler = MovieWrapperCV2 - class PNGFileType(FileType): magic_number = b'\x89\x50\x4e\x47' extensions = ['png'] @@ -491,9 +486,19 @@ class MKTFileType(FileType): extensions = 'mkt' handler = ReadableCellVizioMKTDataset +class VideoMP4FileType(FileType): + magic_number = b'\x66\x74\x79\x70' + extensions = ['mp4'] + magic_number_offset = 4 + handler = ReadableVideoDataset +class VideoAVIFileType(FileType): + magic_number = b'\x52\x49\x46\x46' # RIFF + magic_number_offset = 0 + extensions = ['avi'] + handler = ReadableVideoDataset -SupportedFileTypes = [MKTFileType, MP4MovieFileType, DicomFileType, MiraxFileType, PhilipsISyntaxFileType, PNGFileType, JPEGEXIFFileType, JPEGJFIFFileType, OlympusVSIFileType, NormalTiffFileType, BigTiffFileType, ZeissCZIFile] +SupportedFileTypes = [MKTFileType, VideoMP4FileType, VideoAVIFileType, DicomFileType, MiraxFileType, PhilipsISyntaxFileType, PNGFileType, JPEGEXIFFileType, JPEGJFIFFileType, OlympusVSIFileType, NormalTiffFileType, BigTiffFileType, ZeissCZIFile] diff --git a/exact/util/video_handler.py b/exact/util/video_handler.py index 042683c4..a6d5157e 100644 --- a/exact/util/video_handler.py +++ b/exact/util/video_handler.py @@ -16,7 +16,7 @@ from enums import FrameType -class ReadableMP4Dataset(openslide.ImageSlide): +class ReadableVideoDataset(openslide.ImageSlide): def __init__(self, filename, cache_size=32, max_cache_bytes=None): self.slide_path = filename