Commit cd7fc36a authored by Julia Garriga Ferrer's avatar Julia Garriga Ferrer
Browse files

Merge branch 'data_copy' into 'master'

Data copy

See merge request !123
parents e6a322b1 caf6bbb6
Pipeline #48144 passed with stage
in 2 minutes and 6 seconds
......@@ -100,21 +100,22 @@ class Dataset():
"""
def __init__(self, _dir, data=None, first_filename=None, filenames=None,
dims=None, treated="treated_data", in_memory=True):
dims=None, in_memory=True, copy_files=False):
self._data = None
self._frames_intensity = []
self._dir = _dir
self._treated = treated
self._lock = threading.Lock()
self.running_data = None
self.moments_dims = {}
self.operations_state = numpy.zeros(len(Operation))
if not os.path.isdir(self.dir + "/" + treated):
try:
os.mkdir(self.dir + "/" + treated)
except PermissionError:
raise PermissionError("Add a directory in the Treated Data tab with WRITE permission")
self._dir = _dir
if copy_files:
self._dir += "/treated"
if not os.path.isdir(self._dir):
try:
os.mkdir(self._dir)
except PermissionError:
raise PermissionError("Add a directory in the Treated Data tab with WRITE permission")
if dims is None:
self.__dims = AcquisitionDims()
......@@ -129,7 +130,7 @@ class Dataset():
self._data = data
else:
if filenames is None and first_filename is None:
filenames = sorted([x for x in glob.glob(self.dir + "/*") if os.path.isfile(x)])
filenames = sorted([x for x in glob.glob(_dir + "/*") if os.path.isfile(x)])
self.filenames = filenames
self.first_filename = first_filename
......@@ -449,7 +450,7 @@ class Dataset():
return None, None
def apply_background_subtraction(self, background=None, method="median", indices=None,
step=None, chunk_shape=[100, 100]):
step=None, chunk_shape=[100, 100], _dir=None):
"""
Applies background subtraction to the data and saves the new data
into disk.
......@@ -475,9 +476,12 @@ class Dataset():
the new urls.
:rtype: Dataset
"""
bs_dir = self.dir + "/" + self.treated + "/bs/"
if not os.path.isdir(bs_dir):
os.mkdir(bs_dir)
_dir = self.dir if _dir is None else _dir
if not os.path.isdir(_dir):
os.mkdir(_dir)
method = Method.from_value(method)
self.running_data = self.get_data(indices)
......@@ -501,7 +505,7 @@ class Dataset():
if self._in_memory:
new_data = background_subtraction(self.running_data, bg_data, method).view(Data)
new_data.save(bs_dir + "/data_bs")
new_data.save(_dir + "/data")
urls = new_data.urls
else:
bg = numpy.zeros(self.running_data[0].shape, self.running_data.dtype)
......@@ -559,7 +563,7 @@ class Dataset():
return
urls = self.running_data.apply_funcs([(background_subtraction_2D, [bg])],
save=bs_dir + "/data_bs", text="Applying background subtraction",
save=_dir + "/data", text="Applying background subtraction",
operation=Operation.BS)
if urls is None:
return
......@@ -578,9 +582,9 @@ class Dataset():
new_data = Data(urls.reshape(self.data.urls.shape), self.data.metadata,
self._in_memory)
return Dataset(_dir=bs_dir, data=new_data, dims=self.__dims, in_memory=self._in_memory)
return Dataset(_dir=_dir, data=new_data, dims=self.__dims, in_memory=self._in_memory)
def apply_hot_pixel_removal(self, kernel=3, indices=None):
def apply_hot_pixel_removal(self, kernel=3, indices=None, _dir=None):
"""
Applies hot pixel removal to Data, and saves the new data
into disk.
......@@ -598,17 +602,18 @@ class Dataset():
self.running_data = self.get_data(indices)
_dir = self.dir + "/" + self.treated + "/hp/"
_dir = self.dir if _dir is None else _dir
if not os.path.isdir(_dir):
os.mkdir(_dir)
if self._in_memory:
new_data = hot_pixel_removal_3D(self.running_data, kernel).view(Data)
new_data.save(_dir + "/data_hp")
new_data.save(_dir + "/data")
urls = new_data.urls
else:
urls = self.running_data.apply_funcs([(hot_pixel_removal_2D, [kernel])],
save=_dir + "/data_hp", text="Applying hot pixel removal",
save=_dir + "/data", text="Applying hot pixel removal",
operation=Operation.HP)
if urls is None:
return
......@@ -624,7 +629,7 @@ class Dataset():
return Dataset(_dir=_dir, data=new_data, dims=self.__dims, in_memory=self._in_memory)
def apply_threshold_removal(self, bottom=None, top=None, indices=None):
def apply_threshold_removal(self, bottom=None, top=None, indices=None, _dir=None):
"""
Applies bottom threshold to Data, and saves the new data
into disk.
......@@ -642,17 +647,18 @@ class Dataset():
self.running_data = self.get_data(indices)
_dir = self.dir + "/" + self.treated + "/threshold/"
_dir = self.dir if _dir is None else _dir
if not os.path.isdir(_dir):
os.mkdir(_dir)
if self._in_memory:
new_data = threshold_removal(self.running_data, bottom, top).view(Data)
new_data.save(_dir + "/data_threshold")
new_data.save(_dir + "/data")
urls = new_data.urls
else:
urls = self.running_data.apply_funcs([(threshold_removal, [bottom, top])],
save=_dir + "/data_threshold", text="Applying threshold",
save=_dir + "/data", text="Applying threshold",
operation=Operation.THRESHOLD)
if urls is None:
return
......@@ -689,17 +695,17 @@ class Dataset():
:rtype: Dataset
"""
if roi_dir is None:
roi_dir = self.dir + "/" + self.treated + "/roi/"
self.running_data = self.get_data(indices)
roi_dir = self.dir if roi_dir is None else roi_dir
if not os.path.isdir(roi_dir):
os.mkdir(roi_dir)
self.running_data = self.get_data(indices)
if self._in_memory:
new_data = apply_3D_ROI(self.running_data, origin, size, center).view(Data)
new_data.save(roi_dir + "/roi")
new_data.save(roi_dir + "/data")
else:
urls = self.running_data.apply_funcs([(apply_2D_ROI, [origin, size, center])],
save=roi_dir + "/data_roi", text="Applying roi",
save=roi_dir + "/data", text="Applying roi",
operation=Operation.ROI)
if urls is None:
return
......@@ -727,7 +733,8 @@ class Dataset():
"""
return shift_detection(self.get_data(indices, dimension), h_max, h_step)
def apply_shift(self, shift, dimension=None, shift_approach="fft", indices=None, callback=None):
def apply_shift(self, shift, dimension=None, shift_approach="fft", indices=None,
callback=None, _dir=None):
"""
Apply shift of the data or part of it and save new data into disk.
......@@ -749,9 +756,10 @@ class Dataset():
if not numpy.any(shift):
return self
shift_dir = self.dir + "/" + self.treated + "/shift/"
if not os.path.isdir(shift_dir):
os.mkdir(shift_dir)
_dir = self.dir if _dir is None else _dir
if not os.path.isdir(_dir):
os.mkdir(_dir)
data = self.get_data(indices, dimension)
self._lock.acquire()
......@@ -768,7 +776,7 @@ class Dataset():
for i in range(len(data)):
if not self.operations_state[Operation.SHIFT]:
return
filename = shift_dir + "/data_shift" + str(i).zfill(4) + ".npy"
filename = _dir + "/data" + str(i).zfill(4) + ".npy"
img = apply_shift(data[i], shift[:, i], shift_approach)
numpy.save(filename, img)
urls.append(DataUrl(file_path=filename, scheme='fabio'))
......@@ -799,7 +807,7 @@ class Dataset():
for i in range(len(data)):
if not self.operations_state[Operation.SHIFT]:
return
filename = shift_dir + "/data_shift" + str(i).zfill(4) + ".npy"
filename = _dir + "/data" + str(i).zfill(4) + ".npy"
img = apply_shift(data[i], shift[:, i], shift_approach)
numpy.save(filename, img)
urls.append(DataUrl(file_path=filename, scheme='fabio'))
......@@ -815,7 +823,7 @@ class Dataset():
self._lock.release()
data = Data(new_urls.reshape(self.data.urls.shape), self.data.metadata, in_memory=self._in_memory)
return Dataset(_dir=shift_dir, data=data, dims=self.__dims, in_memory=self._in_memory)
return Dataset(_dir=_dir, data=data, dims=self.__dims, in_memory=self._in_memory)
def find_and_apply_shift(self, dimension=None, h_max=0.5, h_step=0.01, shift_approach="fft",
indices=None, callback=None):
......@@ -920,7 +928,7 @@ class Dataset():
:return: (H, W): The components matrix and the mixing matrix.
"""
bss_dir = self.dir + "/" + self.treated + "/bss/"
bss_dir = self.dir + "/bss/"
if not os.path.isdir(bss_dir):
os.mkdir(bss_dir)
if self._in_memory:
......@@ -972,7 +980,7 @@ class Dataset():
:return: (H, W): The components matrix and the mixing matrix.
"""
bss_dir = self.dir + "/" + self.treated + "/bss/"
bss_dir = self.dir + "/bss/"
if not os.path.isdir(bss_dir):
os.mkdir(bss_dir)
......@@ -1013,7 +1021,7 @@ class Dataset():
:return: (H, W): The components matrix and the mixing matrix.
"""
bss_dir = self.dir + "/" + self.treated + "/bss/"
bss_dir = self.dir + "/bss/"
if not os.path.isdir(bss_dir):
os.mkdir(bss_dir)
......@@ -1107,7 +1115,8 @@ class Dataset():
return self.moments_dims
def apply_fit(self, indices=None, dimension=None, int_thresh=None, chunk_shape=[100, 100]):
def apply_fit(self, indices=None, dimension=None, int_thresh=None,
chunk_shape=[100, 100], _dir=None):
"""
Fits the data around axis 0 and saves the new data into disk.
......@@ -1126,7 +1135,10 @@ class Dataset():
the new urls.
:rtype: Dataset
"""
_dir = self.dir + "/" + self.treated + "/fit/"
_dir = self.dir if _dir is None else _dir
_dir += "/fit"
if not os.path.isdir(_dir):
os.mkdir(_dir)
......@@ -1251,8 +1263,8 @@ class Dataset():
Create copy of the dataset. The data numpy array is also copied using
deep copy. The rest of the attributes are the same.
"""
dataset = type(self)(self.dir, data=self.data, filenames=self.filenames,
first_filename=self.first_filename, treated=self.treated)
dataset = type(self)(self.dir, data=self.data, dims=self.__dims,
in_memory=self.in_memory, copy_files=True)
dataset.dims = copy.deepcopy(self.__dims, memo)
return dataset
......
......@@ -26,7 +26,7 @@
__authors__ = ["J. Garriga"]
__license__ = "MIT"
__date__ = "21/01/2021"
__date__ = "19/05/2021"
import logging
import os
......@@ -97,12 +97,12 @@ class DatasetSelectionWidget(qt.QTabWidget):
_dir = _dir if _dir != "" else os.path.dirname(self._rawFilenameData.getFilename())
self._dataset = Dataset(_dir=_dir,
first_filename=self._rawFilenameData.getFilename(),
in_memory=not self._inDisk)
in_memory=not self._inDisk, copy_files=True)
else:
_dir = _dir if _dir != "" else self._rawFilesData.getDir()
self._dataset = Dataset(_dir=_dir,
filenames=self._rawFilesData.getFiles(),
in_memory=not self._inDisk)
in_memory=not self._inDisk, copy_files=True)
dark_filename = self._darkFilenameData.getFilename()
if dark_filename == "":
......@@ -111,9 +111,9 @@ class DatasetSelectionWidget(qt.QTabWidget):
if not os.path.isdir(_dir + "/dark"):
os.mkdir(_dir + "/dark")
self.bg_dataset = Dataset(_dir=_dir + "/dark",
first_filename=dark_filename)
first_filename=dark_filename, copy_files=True)
else:
self.bg_dataset = Dataset(_dir=os.path.dirname(dark_filename))
self.bg_dataset = Dataset(_dir=os.path.dirname(dark_filename), copy_files=True)
return True
except Exception as e:
raise e
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment