From 0e3350fa1eb1f31fc321d9b0f788a5a89bdd0ee2 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 19 Apr 2022 17:04:30 +0200 Subject: [PATCH 01/33] datacollection plans --- daiquiri/core/metadata/__init__.py | 3 +- .../core/metadata/ispyalchemy/__init__.py | 1 + daiquiri/core/metadata/ispyalchemy/dc.py | 103 ++++++++++++++++++ .../core/metadata/ispyalchemy/updates.sql | 3 + daiquiri/core/schema/metadata.py | 8 +- tests/api/test_metadata_api.py | 5 + 6 files changed, 120 insertions(+), 3 deletions(-) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index c6836d9be..1b27db4b5 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -21,7 +21,7 @@ from daiquiri.core.metadata.xrf import ( generate_composite_image, shape_map, ) - +from daiquiri.core.schema.validators import OneOf from daiquiri.core.schema import ErrorSchema, MessageSchema from daiquiri.core.schema.metadata import ( paginated, @@ -497,6 +497,7 @@ class DataCollectionPlansResource(CoreResource): "session": fields.Str(), "subsampleid": fields.Int(), "sampleid": fields.Int(), + "status": OneOf(["executed", "pending"]), }, out=[ [200, paginated(DataCollectionPlanSchema), "List of datacollection plans"] diff --git a/daiquiri/core/metadata/ispyalchemy/__init__.py b/daiquiri/core/metadata/ispyalchemy/__init__.py index fed771e16..4fc28fae4 100644 --- a/daiquiri/core/metadata/ispyalchemy/__init__.py +++ b/daiquiri/core/metadata/ispyalchemy/__init__.py @@ -103,6 +103,7 @@ class IspyalchemyMetaDataHandler(MetaDataHandler): "ContainerQueue", "ContainerQueueSample", "BLSample", + "BLSample_has_DataCollectionPlan", "BLSubSample", "Position", "Positioner", diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index cc722a66d..cfd3611de 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -23,6 +23,10 @@ class DCHandler(IspyalchemyHandler): "get_sampleactions", "get_scanqualityindicators", "add_scanqualityindicators", + "get_datacollectionplans", + "add_datacollectionplan", + "update_datacollectionplan", + "remove_datacollectionplan", ] def get_datacollections(self, datacollectionid=None, **kwargs): @@ -541,3 +545,102 @@ class DCHandler(IspyalchemyHandler): ses.commit() return f"{sqi.datacollectionid}-{sqi.imagenumber}" + + def get_datacollectionplans( + self, + datacollectionplanid=None, + blsampleid=None, + status=None, + no_context=False, + **kwargs, + ): + with self.session_scope() as ses: + datacollectionplans = ( + ses.query( + self.DiffractionPlan.diffractionplanid.label( + "datacollectionplanid" + ), + self.DiffractionPlan.scanparameters, + self.BLSample.blsampleid.label("sampleid"), + self.BLSample.name, + self.Protein.acronym.label("component"), + self.Protein.proteinid.label("componentid"), + self.BLSample_has_DataCollectionPlan.planorder, + self.DataCollection.datacollectionid, + self.ContainerQueueSample.containerqueueid, + func.IF( + self.DataCollection.datacollectionid, "executed", "pending" + ).label("status"), + func.IF( + self.ContainerQueueSample.containerqueuesampleid, True, False + ).label("queued"), + ) + .outerjoin( + self.BLSample_has_DataCollectionPlan, + self.DiffractionPlan.diffractionplanid + == self.BLSample_has_DataCollectionPlan.datacollectionplanid, + ) + .outerjoin( + self.BLSample, + self.BLSample.blsampleid + == self.BLSample_has_DataCollectionPlan.blsampleid, + ) + .outerjoin( + self.ContainerQueueSample, + self.ContainerQueueSample.datacollectionplanid + == self.DiffractionPlan.diffractionplanid, + ) + .outerjoin( + self.Crystal, self.Crystal.crystalid == self.BLSample.crystalid + ) + .outerjoin( + self.Protein, self.Crystal.proteinid == self.Protein.proteinid + ) + .outerjoin( + self.DataCollection, + self.DataCollection.datacollectionplanid + == self.DiffractionPlan.diffractionplanid, + ) + ) + + if not no_context: + datacollectionplans = datacollectionplans.filter( + self.Proposal.proposalid == g.blsession.get("proposalid") + ) + + if blsampleid: + datacollectionplans = datacollectionplans.filter( + self.BLSample_has_DataCollectionPlan.blsampleid == blsampleid + ) + + if status: + if status == "completed": + datacollectionplans = datacollectionplans.filter( + self.DataCollection.datacollectionid != None # noqa: E711 + ) + else: + datacollectionplans = datacollectionplans.filter( + self.DataCollection.datacollectionid == None # noqa: E711 + ) + + if datacollectionplanid: + datacollectionplans = datacollectionplans.filter( + self.DiffractionPlan.diffractionplanid == datacollectionplanid + ) + datacollectionplan = datacollectionplans.first() + if datacollectionplan: + return datacollectionplan._asdict() + + else: + print(datacollectionplans) + datacollectionplans = [r._asdict() for r in datacollectionplans.all()] + return {"total": len(datacollectionplans), "rows": datacollectionplans} + + def add_datacollectionplan(self, **kwargs): + return super().add_datacollectionplan(**kwargs) + + def update_datacollectionplan(self, datacollectionplanid, **kwargs): + return super().update_datacollectionplan(datacollectionplanid, **kwargs) + + def remove_datacollectionplan(self, datacollectionplanid, **kwargs): + return super().remove_datacollectionplan(datacollectionplanid, **kwargs) diff --git a/daiquiri/core/metadata/ispyalchemy/updates.sql b/daiquiri/core/metadata/ispyalchemy/updates.sql index e69de29bb..111957c22 100644 --- a/daiquiri/core/metadata/ispyalchemy/updates.sql +++ b/daiquiri/core/metadata/ispyalchemy/updates.sql @@ -0,0 +1,3 @@ +ALTER TABLE DiffractionPlan + ADD `scanParameters` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`scanParameters`)); + \ No newline at end of file diff --git a/daiquiri/core/schema/metadata.py b/daiquiri/core/schema/metadata.py index 13d4b377a..0bfe4b307 100644 --- a/daiquiri/core/schema/metadata.py +++ b/daiquiri/core/schema/metadata.py @@ -233,10 +233,14 @@ class DataCollectionPlanSchema(Schema): datacollectionplanid = fields.Int() sampleid = fields.Int() subsampleid = fields.Int() + datacollectionid = fields.Int() + containerqueueid = fields.Int() + queued = fields.Bool() + status = fields.Str() experimentkind = fields.Str() - boxsizex = fields.Float() - boxsizey = fields.Float() energy = fields.Float() + exposuretime = fields.Float() + scanparameters = fields.Dict() class ScanQualityIndicatorsSchema(Schema): diff --git a/tests/api/test_metadata_api.py b/tests/api/test_metadata_api.py index d2741bae1..fbdbab72c 100644 --- a/tests/api/test_metadata_api.py +++ b/tests/api/test_metadata_api.py @@ -243,3 +243,8 @@ def test_get_scanqualityindicators(auth_client, with_session): dcid = 1 res = auth_client.get(f"/api/metadata/datacollections/sqis/{dcid}") assert res.status_code == 200 + + +def test_get_datacollectionplans(auth_client, with_session): + res = auth_client.get("/api/metadata/datacollections/plans") + assert res.status_code == 200 -- GitLab From eb64742c9d63ec4d8e7ccd76e07b0a1a29686234 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 26 Apr 2022 18:06:49 +0200 Subject: [PATCH 02/33] add add dcplan, decode json --- daiquiri/core/metadata/__init__.py | 3 +- daiquiri/core/metadata/ispyalchemy/dc.py | 108 +++++++++++++++++++++-- daiquiri/core/schema/metadata.py | 15 +++- tests/api/test_metadata_api.py | 16 +++- 4 files changed, 129 insertions(+), 13 deletions(-) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index 1b27db4b5..cf5bd774d 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -39,6 +39,7 @@ from daiquiri.core.schema.metadata import ( SampleImageSchema, DataCollectionSchema, DataCollectionAttachmentSchema, + NewDataCollectionPlanSchema, DataCollectionPlanSchema, ScanQualityIndicatorsSchema, XRFMapSchema, @@ -508,7 +509,7 @@ class DataCollectionPlansResource(CoreResource): return self._parent.get_datacollectionplans(**kwargs), 200 @marshal( - inp=DataCollectionPlanSchema, + inp=NewDataCollectionPlanSchema, out=[ [200, DataCollectionPlanSchema(), "New Datacollection plan"], [400, ErrorSchema(), "Could not create datacollection plan"], diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index cfd3611de..52e88712b 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import json from flask import g @@ -561,6 +562,10 @@ class DCHandler(IspyalchemyHandler): "datacollectionplanid" ), self.DiffractionPlan.scanparameters, + self.DiffractionPlan.experimentkind, + self.DiffractionPlan.recordtimestamp.label("timestamp"), + self.DiffractionPlan.energy, + self.DiffractionPlan.exposuretime, self.BLSample.blsampleid.label("sampleid"), self.BLSample.name, self.Protein.acronym.label("component"), @@ -629,18 +634,107 @@ class DCHandler(IspyalchemyHandler): ) datacollectionplan = datacollectionplans.first() if datacollectionplan: - return datacollectionplan._asdict() + return self._decode_scanparameters(datacollectionplan._asdict()) else: - print(datacollectionplans) - datacollectionplans = [r._asdict() for r in datacollectionplans.all()] + datacollectionplans = [ + self._decode_scanparameters(r._asdict()) + for r in datacollectionplans.all() + ] return {"total": len(datacollectionplans), "rows": datacollectionplans} - def add_datacollectionplan(self, **kwargs): - return super().add_datacollectionplan(**kwargs) + def _decode_scanparameters(self, datacollectionplan): + datacollectionplan["scanparameters"] = json.loads( + datacollectionplan["scanparameters"] + ) + return datacollectionplan + + def add_datacollectionplan( + self, + *, + scanparameters, + sampleid, + energy=None, + exposuretime=None, + experimentkind=None, + planorder=None, + no_context=False, + ): + with self.session_scope() as ses: + datacollectionplan = self.DiffractionPlan( + scanparameters=json.dumps(scanparameters), + experimentkind=experimentkind, + energy=energy, + exposuretime=exposuretime, + ) + + ses.add(datacollectionplan) + ses.commit() + + sample_has_datacollectionplan = self.BLSample_has_DataCollectionPlan( + blsampleid=sampleid, + datacollectionplanid=datacollectionplan.diffractionplanid, + planorder=planorder, + ) + + ses.add(sample_has_datacollectionplan) + ses.commit() + + return self.get_datacollectionplans( + datacollectionplanid=datacollectionplan.diffractionplanid, + no_context=no_context, + ) + + def update_datacollectionplan( + self, datacollectionplanid, no_context=False, **kwargs + ): + with self.session_scope() as ses: + chk = self.get_datacollectionplans( + datacollectionplanid=datacollectionplanid, + no_context=kwargs.get("no_context"), + ) + if chk: + datacollectionplan = ( + ses.query(self.DiffractionPlan) + .filter( + self.DiffractionPlan.diffractionplanid == datacollectionplanid + ) + .first() + ) + + updatable = [ + "scanparameters", + "experimentkind", + "exposuretime", + "energy", + ] + for kw in kwargs: + if kw in updatable: + val = kwargs[kw] + + setattr(datacollectionplan, kw, val) - def update_datacollectionplan(self, datacollectionplanid, **kwargs): - return super().update_datacollectionplan(datacollectionplanid, **kwargs) + sample_has_datacollectionplan = ( + ses.query(self.BLSample_has_DataCollectionPlan) + .filter( + self.BLSample_has_DataCollectionPlan.datacollectionplanid + == datacollectionplanid, + ) + .first() + ) + + updatable_in_sample = ["planorder"] + for kw in kwargs: + if kw in updatable_in_sample: + val = kwargs[kw] + + setattr(sample_has_datacollectionplan, kw, val) + + ses.commit() + + return self.get_datacollectionplans( + datacollectionplanid=datacollectionplanid, no_context=no_context + ) def remove_datacollectionplan(self, datacollectionplanid, **kwargs): return super().remove_datacollectionplan(datacollectionplanid, **kwargs) diff --git a/daiquiri/core/schema/metadata.py b/daiquiri/core/schema/metadata.py index 0bfe4b307..f4c643239 100644 --- a/daiquiri/core/schema/metadata.py +++ b/daiquiri/core/schema/metadata.py @@ -229,17 +229,24 @@ class DataCollectionAttachmentSchema(Schema): filetype = fields.Str() -class DataCollectionPlanSchema(Schema): +class NewDataCollectionPlanSchema(Schema): + sampleid = fields.Int(required=True) + experimentkind = fields.Str() + energy = fields.Float() + exposuretime = fields.Float() + planorder = fields.Int() + scanparameters = fields.Dict(required=True) + + +class DataCollectionPlanSchema(NewDataCollectionPlanSchema): datacollectionplanid = fields.Int() + timestamp = fields.DateTime() sampleid = fields.Int() subsampleid = fields.Int() datacollectionid = fields.Int() containerqueueid = fields.Int() queued = fields.Bool() status = fields.Str() - experimentkind = fields.Str() - energy = fields.Float() - exposuretime = fields.Float() scanparameters = fields.Dict() diff --git a/tests/api/test_metadata_api.py b/tests/api/test_metadata_api.py index fbdbab72c..cf4e31988 100644 --- a/tests/api/test_metadata_api.py +++ b/tests/api/test_metadata_api.py @@ -245,6 +245,20 @@ def test_get_scanqualityindicators(auth_client, with_session): assert res.status_code == 200 -def test_get_datacollectionplans(auth_client, with_session): +def test_get_datacollection_plans(auth_client, with_session): res = auth_client.get("/api/metadata/datacollections/plans") assert res.status_code == 200 + + +def test_add_datacollection_plan(auth_client, with_session): + data = { + "scanparameters": {"actor": "testactor", "param1": 1.454, "param2": 5.4}, + "sampleid": 1, + } + + res = auth_client.post("/api/metadata/datacollections/plans", payload=data) + print("RES", res.json) + assert res.status_code == 201 + + for k in data: + assert res.json[k] == data[k] -- GitLab From 7bfeca3c0d9205ad7cdafa4fe35d4fbfbf5990b9 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 07:45:55 +0200 Subject: [PATCH 03/33] dont load when None --- daiquiri/core/metadata/ispyalchemy/dc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 52e88712b..82b287914 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -644,9 +644,10 @@ class DCHandler(IspyalchemyHandler): return {"total": len(datacollectionplans), "rows": datacollectionplans} def _decode_scanparameters(self, datacollectionplan): - datacollectionplan["scanparameters"] = json.loads( - datacollectionplan["scanparameters"] - ) + if datacollectionplan["scanparameters"]: + datacollectionplan["scanparameters"] = json.loads( + datacollectionplan["scanparameters"] + ) return datacollectionplan def add_datacollectionplan( -- GitLab From 7edae306e9ccf234230684172cf8e754b4fbe94e Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 08:15:20 +0200 Subject: [PATCH 04/33] debug --- tests/api/test_metadata_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/api/test_metadata_api.py b/tests/api/test_metadata_api.py index cf4e31988..5ffc5c7a4 100644 --- a/tests/api/test_metadata_api.py +++ b/tests/api/test_metadata_api.py @@ -257,7 +257,6 @@ def test_add_datacollection_plan(auth_client, with_session): } res = auth_client.post("/api/metadata/datacollections/plans", payload=data) - print("RES", res.json) assert res.status_code == 201 for k in data: -- GitLab From b167920becbfcac75186b6a79dc56f771f3d1aad Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 08:15:37 +0200 Subject: [PATCH 05/33] return planid with dc --- daiquiri/core/metadata/ispyalchemy/dc.py | 5 +++++ daiquiri/core/schema/metadata.py | 1 + 2 files changed, 6 insertions(+) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 82b287914..aacbe9caf 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -84,6 +84,7 @@ class DCHandler(IspyalchemyHandler): self.GridInfo.patchesy, self.GridInfo.dx_mm.label("dx_mm"), self.GridInfo.dy_mm.label("dy_mm"), + self.DataCollection.datacollectionplanid ) .join( self.DataCollectionGroup, @@ -100,6 +101,10 @@ class DCHandler(IspyalchemyHandler): self.GridInfo.datacollectionid == self.DataCollection.datacollectionid, ) + .outerjoin( + self.DiffractionPlan, + self.DataCollection.datacollectionplanid == self.DiffractionPlan.diffractionplanid + ) ) if kwargs.get("datacollectiongroupid"): diff --git a/daiquiri/core/schema/metadata.py b/daiquiri/core/schema/metadata.py index f4c643239..47d3b4e18 100644 --- a/daiquiri/core/schema/metadata.py +++ b/daiquiri/core/schema/metadata.py @@ -182,6 +182,7 @@ class DataCollectionSchema(Schema): sample = fields.Str() subsampleid = fields.Int() starttime = fields.DateTime() + datacollectionplanid = fields.Int() endtime = fields.DateTime() duration = fields.Int() experimenttype = fields.Str() -- GitLab From 0b5483a4d7aecef0c74e53b2b390eb99875154b0 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 08:19:09 +0200 Subject: [PATCH 06/33] style --- daiquiri/core/metadata/ispyalchemy/dc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index aacbe9caf..c86af0aee 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -84,7 +84,7 @@ class DCHandler(IspyalchemyHandler): self.GridInfo.patchesy, self.GridInfo.dx_mm.label("dx_mm"), self.GridInfo.dy_mm.label("dy_mm"), - self.DataCollection.datacollectionplanid + self.DataCollection.datacollectionplanid, ) .join( self.DataCollectionGroup, @@ -103,7 +103,8 @@ class DCHandler(IspyalchemyHandler): ) .outerjoin( self.DiffractionPlan, - self.DataCollection.datacollectionplanid == self.DiffractionPlan.diffractionplanid + self.DataCollection.datacollectionplanid + == self.DiffractionPlan.diffractionplanid, ) ) -- GitLab From b14556eafaa3fe70ed2d53f8375bb3366d9fe015 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 17:07:17 +0200 Subject: [PATCH 07/33] correct join --- daiquiri/core/metadata/ispyalchemy/dc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index c86af0aee..8660b8d4c 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -616,7 +616,7 @@ class DCHandler(IspyalchemyHandler): if not no_context: datacollectionplans = datacollectionplans.filter( - self.Proposal.proposalid == g.blsession.get("proposalid") + self.Protein.proposalid == g.blsession.get("proposalid") ) if blsampleid: -- GitLab From 289aa00510d4b42a67d9911b423406971cbbf895 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 17:09:14 +0200 Subject: [PATCH 08/33] allow queueing with existing dcplan --- daiquiri/core/metadata/ispyalchemy/sample.py | 42 +++++++++++--------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/sample.py b/daiquiri/core/metadata/ispyalchemy/sample.py index 9c44aef1c..08972a525 100644 --- a/daiquiri/core/metadata/ispyalchemy/sample.py +++ b/daiquiri/core/metadata/ispyalchemy/sample.py @@ -536,19 +536,22 @@ class SampleHandler(IspyalchemyHandler): return True - def queue_subsample(self, subsampleid, **kwargs): + def queue_subsample(self, subsampleid, datacollectionplanid=None, **kwargs): with self.session_scope() as ses: subsample = self.get_subsamples(subsampleid=subsampleid) if subsample: - dp = self.DiffractionPlan( - # steps_x=kwargs.get("steps_x"), - # steps_y=kwargs.get("steps_y"), - experimentkind=kwargs.get("experimenttype"), - exposuretime=kwargs.get("exposuretime"), - ) + if not datacollectionplanid: + dp = self.DiffractionPlan( + # steps_x=kwargs.get("steps_x"), + # steps_y=kwargs.get("steps_y"), + experimentkind=kwargs.get("experimenttype"), + exposuretime=kwargs.get("exposuretime"), + ) - ses.add(dp) - ses.commit() + ses.add(dp) + ses.commit() + + datacollectionplanid = dp.diffractionplanid cq = ( ses.query(self.ContainerQueue.containerqueueid) @@ -574,7 +577,7 @@ class SampleHandler(IspyalchemyHandler): cqs = self.ContainerQueueSample( containerqueueid=cq.containerqueueid, - datacollectionplanid=dp.diffractionplanid, + datacollectionplanid=datacollectionplanid, blsubsampleid=subsampleid, ) @@ -613,17 +616,20 @@ class SampleHandler(IspyalchemyHandler): return True - def queue_sample(self, sampleid, **kwargs): + def queue_sample(self, sampleid, datacollectionplanid=None, **kwargs): with self.session_scope() as ses: sample = self.get_samples(sampleid=sampleid) if sample: - dp = self.DiffractionPlan( - experimentkind=kwargs.get("experimenttype"), - exposuretime=kwargs.get("exposuretime"), - ) + if not datacollectionplanid: + dp = self.DiffractionPlan( + experimentkind=kwargs.get("experimenttype"), + exposuretime=kwargs.get("exposuretime"), + ) - ses.add(dp) - ses.commit() + ses.add(dp) + ses.commit() + + datacollectionplanid = dp.diffractionplanid cq = ( ses.query(self.ContainerQueue.containerqueueid) @@ -645,7 +651,7 @@ class SampleHandler(IspyalchemyHandler): cqs = self.ContainerQueueSample( containerqueueid=cq.containerqueueid, - datacollectionplanid=dp.diffractionplanid, + datacollectionplanid=datacollectionplanid, blsampleid=sampleid, ) -- GitLab From 166bb406dfa037c5c742f5c554b26db38be6a07e Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 17:09:52 +0200 Subject: [PATCH 09/33] allow saving dc plan with executing actor --- daiquiri/core/components/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/daiquiri/core/components/__init__.py b/daiquiri/core/components/__init__.py index c3c7d756d..f737b6866 100644 --- a/daiquiri/core/components/__init__.py +++ b/daiquiri/core/components/__init__.py @@ -44,6 +44,15 @@ def actor(name, **kwargs): def actor_wrapper(name, **actkw): def decorator(fn): def wrapper(self, *args, **kwargs): + if "save" in kwargs: + kwargs.pop("save") + datacollectionplan = self._parent._metadata.add_datacollectionplan( + sampleid=kwargs.pop("sampleid"), + scanparameters={"actor": name, **kwargs}, + ) + log.get("user").info(f"New actor created '{name}'", type="actor") + return {"datacollectionplanid": datacollectionplan["datacollectionplanid"]}, 200 + if "preprocess" in actkw: #  Allow preprocessors to throw errors try: -- GitLab From 5dbb6658578136f552e3ef84f94aebe04881168b Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 17:10:07 +0200 Subject: [PATCH 10/33] allow queuing actor from plan (wip) --- daiquiri/core/components/samplescan.py | 48 ++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 57e9b5a3b..735d6adc6 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -1,8 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from datetime import datetime +import requests -from flask import g +from flask import g, url_for, request from daiquiri.core import require_control, require_staff, marshal from daiquiri.core.components import ( @@ -12,6 +13,7 @@ from daiquiri.core.components import ( ComponentActorKilled, ) from daiquiri.core.components.dcutilsmixin import DCUtilsMixin +from daiquiri.core.schema import ErrorSchema, MessageSchema from daiquiri.core.schema.samplescan import ScanConfigSchema, AvailableScansSchema from daiquiri.core.schema.metadata import paginated @@ -21,6 +23,19 @@ import logging logger = logging.getLogger(__name__) +class QueueScanResource(ComponentResource): + @marshal( + out=[ + [200, MessageSchema(), "Scan successfully queued"], + [400, ErrorSchema(), "Could not queue scan"], + [404, ErrorSchema(), "No such data collection plan"], + ] + ) + def post(self, datacollectionplanid): + """Queue a scan from a datacollectionplan""" + return self._parent.queue_scan(datacollectionplanid) + + class AvailableScansResource(ComponentResource): @marshal(out=[[200, paginated(AvailableScansSchema), "A list of available scans"]]) def get(self, **kwargs): @@ -35,6 +50,7 @@ class Samplescan(Component, DCUtilsMixin): def setup(self): self._scan_actors = [] self.register_route(AvailableScansResource, "") + self.register_route(QueueScanResource, "/queue/") self._generate_scan_actors() @@ -57,7 +73,9 @@ class Samplescan(Component, DCUtilsMixin): ( kwargs["containerqueuesampleid"], kwargs["datacollectionplanid"], - ) = self._parent._metadata.queue_sample(kwargs["sampleid"]) + ) = self._parent._metadata.queue_sample( + kwargs["sampleid"], kwargs.get("datacollectionplanid") + ) kwargs["before_scan_starts"] = self._parent.before_scan_starts kwargs["update_datacollection"] = self._parent.update_datacollection kwargs["open_attachment"] = self._parent._open_dc_attachment @@ -172,3 +190,29 @@ class Samplescan(Component, DCUtilsMixin): containerqueuesampleid=actor["containerqueuesampleid"], no_context=True, ) + + def queue_scan(self, datacollectionplanid): + """Queue a data collection plan""" + datacollectionplan = self._metadata.get_datacollectionplans( + datacollectionplanid=datacollectionplanid + ) + if not datacollectionplan: + return {"error": "Data collection plan not found"}, 404 + + params = datacollectionplan["scanparameters"] + actor = params.pop("actor") + response = requests.post( + request.url_root + url_for(f"{self.__class__.__name__}.{actor}"), + json={ + "datacollectionplanid": datacollectionplanid, + "sampleid": datacollectionplan["sampleid"], + **params, + }, + headers={"Authorization": request.headers["Authorization"]}, + verify=False, + ) + + if response.status_code == 200: + return {"message": response.json()} + else: + return {"error": f"{response.status_code}: {response.json()}"}, 400 -- GitLab From 97466add12cb936f2ab452695807ffa5d462d2b3 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Mon, 2 May 2022 17:11:47 +0200 Subject: [PATCH 11/33] style --- daiquiri/core/components/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/daiquiri/core/components/__init__.py b/daiquiri/core/components/__init__.py index f737b6866..450521896 100644 --- a/daiquiri/core/components/__init__.py +++ b/daiquiri/core/components/__init__.py @@ -51,7 +51,14 @@ def actor_wrapper(name, **actkw): scanparameters={"actor": name, **kwargs}, ) log.get("user").info(f"New actor created '{name}'", type="actor") - return {"datacollectionplanid": datacollectionplan["datacollectionplanid"]}, 200 + return ( + { + "datacollectionplanid": datacollectionplan[ + "datacollectionplanid" + ] + }, + 200, + ) if "preprocess" in actkw: #  Allow preprocessors to throw errors -- GitLab From 2243e02a875513dbd1e243c2b4879a58ec6f85ed Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:01:45 +0200 Subject: [PATCH 12/33] correct message --- daiquiri/core/components/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daiquiri/core/components/__init__.py b/daiquiri/core/components/__init__.py index 450521896..da4a3e9d0 100644 --- a/daiquiri/core/components/__init__.py +++ b/daiquiri/core/components/__init__.py @@ -50,7 +50,10 @@ def actor_wrapper(name, **actkw): sampleid=kwargs.pop("sampleid"), scanparameters={"actor": name, **kwargs}, ) - log.get("user").info(f"New actor created '{name}'", type="actor") + log.get("user").info( + f"New data collection plan saved for actor '{name}' with id '{datacollectionplan['datacollectionplanid']}'", + type="actor", + ) return ( { "datacollectionplanid": datacollectionplan[ -- GitLab From 53d3ea7c253274637bc0dfa14b4689911ff9ea7e Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:11:11 +0200 Subject: [PATCH 13/33] correct logic --- daiquiri/core/components/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daiquiri/core/components/__init__.py b/daiquiri/core/components/__init__.py index da4a3e9d0..72cc70ec4 100644 --- a/daiquiri/core/components/__init__.py +++ b/daiquiri/core/components/__init__.py @@ -44,7 +44,7 @@ def actor(name, **kwargs): def actor_wrapper(name, **actkw): def decorator(fn): def wrapper(self, *args, **kwargs): - if "save" in kwargs: + if kwargs.get("save"): kwargs.pop("save") datacollectionplan = self._parent._metadata.add_datacollectionplan( sampleid=kwargs.pop("sampleid"), -- GitLab From 887fa746be6b323f12c2e533e775c97a23b3e9cb Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:12:16 +0200 Subject: [PATCH 14/33] use dc function --- daiquiri/core/metadata/ispyalchemy/sample.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/sample.py b/daiquiri/core/metadata/ispyalchemy/sample.py index 08972a525..e2a0cb55c 100644 --- a/daiquiri/core/metadata/ispyalchemy/sample.py +++ b/daiquiri/core/metadata/ispyalchemy/sample.py @@ -536,7 +536,9 @@ class SampleHandler(IspyalchemyHandler): return True - def queue_subsample(self, subsampleid, datacollectionplanid=None, **kwargs): + def queue_subsample( + self, subsampleid, datacollectionplanid=None, scanparameters=None, **kwargs + ): with self.session_scope() as ses: subsample = self.get_subsamples(subsampleid=subsampleid) if subsample: @@ -621,15 +623,8 @@ class SampleHandler(IspyalchemyHandler): sample = self.get_samples(sampleid=sampleid) if sample: if not datacollectionplanid: - dp = self.DiffractionPlan( - experimentkind=kwargs.get("experimenttype"), - exposuretime=kwargs.get("exposuretime"), - ) - - ses.add(dp) - ses.commit() - - datacollectionplanid = dp.diffractionplanid + dp = self.add_datacollectionplan(sampleid=sampleid, **kwargs) + datacollectionplanid = dp["datacollectionplanid"] cq = ( ses.query(self.ContainerQueue.containerqueueid) @@ -658,7 +653,7 @@ class SampleHandler(IspyalchemyHandler): ses.add(cqs) ses.commit() - return cqs.containerqueuesampleid, dp.diffractionplanid + return cqs.containerqueuesampleid, datacollectionplanid def unqueue_sample(self, sampleid, **kwargs): with self.session_scope() as ses: -- GitLab From b88d02851eae58441fbf28c21026de79417b486a Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:12:24 +0200 Subject: [PATCH 15/33] export decode --- daiquiri/core/metadata/ispyalchemy/dc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 8660b8d4c..60879f775 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -28,6 +28,7 @@ class DCHandler(IspyalchemyHandler): "add_datacollectionplan", "update_datacollectionplan", "remove_datacollectionplan", + "_decode_scanparameters" ] def get_datacollections(self, datacollectionid=None, **kwargs): -- GitLab From 8ff97cdda719176dd34628891e84f6b53ca8ffa2 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:12:43 +0200 Subject: [PATCH 16/33] set dc fns on sample handler --- daiquiri/core/metadata/ispyalchemy/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daiquiri/core/metadata/ispyalchemy/__init__.py b/daiquiri/core/metadata/ispyalchemy/__init__.py index 4fc28fae4..63d1fddbf 100644 --- a/daiquiri/core/metadata/ispyalchemy/__init__.py +++ b/daiquiri/core/metadata/ispyalchemy/__init__.py @@ -147,6 +147,9 @@ class IspyalchemyMetaDataHandler(MetaDataHandler): for m in SampleHandler.exported: setattr(DCHandler, m, getattr(SampleHandler, m)) + for m in DCHandler.exported: + setattr(SampleHandler, m, getattr(DCHandler, m)) + super().setup() @contextmanager -- GitLab From dfed9d44a846973867ba9db36a3e7431f18c4798 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 11:13:02 +0200 Subject: [PATCH 17/33] use internal calls rather than self request --- daiquiri/core/components/samplescan.py | 96 ++++++++++++++++---------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 735d6adc6..2f94b5522 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -6,6 +6,7 @@ import requests from flask import g, url_for, request from daiquiri.core import require_control, require_staff, marshal +from daiquiri.core.logging import log from daiquiri.core.components import ( Component, ComponentResource, @@ -57,32 +58,34 @@ class Samplescan(Component, DCUtilsMixin): def reload(self): self._generate_scan_actors() + def preprocess(self, **kwargs): + sample = self._metadata.get_samples(kwargs["sampleid"]) + if not sample: + raise AttributeError(f"No such sample {kwargs['sampleid']}") + kwargs["sampleid"] = sample["sampleid"] + kwargs["sample"] = sample["name"] + kwargs["sessionid"] = g.blsession.get("sessionid") + ( + kwargs["containerqueuesampleid"], + kwargs["datacollectionplanid"], + ) = self._metadata.queue_sample( + kwargs["sampleid"], + kwargs.get("datacollectionplanid"), + scanparameters=kwargs, + ) + kwargs["before_scan_starts"] = self.before_scan_starts + kwargs["update_datacollection"] = self.update_datacollection + kwargs["open_attachment"] = self._open_dc_attachment + kwargs["add_scanqualityindicators"] = self.add_scanqualityindicators + kwargs["enqueue"] = kwargs.get("enqueue", True) + return kwargs + def _generate_scan_actors(self): """Dynamically generate scan actor resources""" def post(self, **kwargs): pass - def preprocess(self, **kwargs): - sample = self._parent._metadata.get_samples(kwargs["sampleid"]) - if not sample: - raise AttributeError(f"No such sample {kwargs['sampleid']}") - kwargs["sampleid"] = sample["sampleid"] - kwargs["sample"] = sample["name"] - kwargs["sessionid"] = g.blsession.get("sessionid") - ( - kwargs["containerqueuesampleid"], - kwargs["datacollectionplanid"], - ) = self._parent._metadata.queue_sample( - kwargs["sampleid"], kwargs.get("datacollectionplanid") - ) - kwargs["before_scan_starts"] = self._parent.before_scan_starts - kwargs["update_datacollection"] = self._parent.update_datacollection - kwargs["open_attachment"] = self._parent._open_dc_attachment - kwargs["add_scanqualityindicators"] = self._parent.add_scanqualityindicators - kwargs["enqueue"] = kwargs.get("enqueue", True) - return kwargs - for scan in self._config["scans"]: name = scan["actor"] @@ -107,7 +110,7 @@ class Samplescan(Component, DCUtilsMixin): act_res = type( name, (ComponentResource,), - {"post": fn, "preprocess": preprocess}, + {"post": fn, "preprocess": self.preprocess}, ) self.register_actor_route(act_res, f"/{name}") @@ -199,20 +202,43 @@ class Samplescan(Component, DCUtilsMixin): if not datacollectionplan: return {"error": "Data collection plan not found"}, 404 - params = datacollectionplan["scanparameters"] + if datacollectionplan["datacollectionid"]: + return { + "error": f"That data collection plan has already been executed: datacollectionid: {datacollectionplan['datacollectionid']}" + }, 400 + + params = { + "sampleid": datacollectionplan["sampleid"], + **datacollectionplan["scanparameters"], + } actor = params.pop("actor") - response = requests.post( - request.url_root + url_for(f"{self.__class__.__name__}.{actor}"), - json={ - "datacollectionplanid": datacollectionplanid, - "sampleid": datacollectionplan["sampleid"], - **params, - }, - headers={"Authorization": request.headers["Authorization"]}, - verify=False, - ) - if response.status_code == 200: - return {"message": response.json()} + # TODO: This all needs factoring out of `actor_wrapper` to make + # it reusable + schema = self.actor_schema(actor) + schema_inst = schema() + schema_inst.load(params) + + params["datacollectionplanid"] = datacollectionplan["datacollectionplanid"] + params = self.preprocess(**params) + + actkw = {} + if hasattr(self, "actor_success"): + actkw["success"] = self.actor_success + + if hasattr(self, "actor_started"): + actkw["start"] = self.actor_started + + if hasattr(self, "actor_error"): + actkw["error"] = self.actor_error + + if hasattr(self, "actor_remove"): + actkw["remove"] = self.actor_remove + + uuid = self.actor(actor, actargs=params, **actkw) + + if uuid: + log.get("user").info(f"New actor created '{actor}'", type="actor") + return {"message": f"Actor created with uuid {uuid}"} else: - return {"error": f"{response.status_code}: {response.json()}"}, 400 + return {"error": "Could not create actor"}, 400 -- GitLab From ebf84c88a826f6beb73e63013fcc0179841f5ba8 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 13:48:46 +0200 Subject: [PATCH 18/33] style --- daiquiri/core/components/samplescan.py | 9 ++++++--- daiquiri/core/metadata/ispyalchemy/dc.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 2f94b5522..351b22f10 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -203,9 +203,12 @@ class Samplescan(Component, DCUtilsMixin): return {"error": "Data collection plan not found"}, 404 if datacollectionplan["datacollectionid"]: - return { - "error": f"That data collection plan has already been executed: datacollectionid: {datacollectionplan['datacollectionid']}" - }, 400 + return ( + { + "error": f"That data collection plan has already been executed: datacollectionid: {datacollectionplan['datacollectionid']}" + }, + 400, + ) params = { "sampleid": datacollectionplan["sampleid"], diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 60879f775..20f240d68 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -28,7 +28,7 @@ class DCHandler(IspyalchemyHandler): "add_datacollectionplan", "update_datacollectionplan", "remove_datacollectionplan", - "_decode_scanparameters" + "_decode_scanparameters", ] def get_datacollections(self, datacollectionid=None, **kwargs): -- GitLab From 770ec7cff2526349b47d33b884ba38f961e51dd3 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 17:12:16 +0200 Subject: [PATCH 19/33] validation, lint tidy --- daiquiri/core/components/samplescan.py | 40 +++++++++++++++----------- daiquiri/core/schema/__init__.py | 4 +++ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 351b22f10..64eb9bb6d 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from datetime import datetime -import requests -from flask import g, url_for, request +from flask import g +from marshmallow import ValidationError from daiquiri.core import require_control, require_staff, marshal from daiquiri.core.logging import log @@ -14,7 +14,7 @@ from daiquiri.core.components import ( ComponentActorKilled, ) from daiquiri.core.components.dcutilsmixin import DCUtilsMixin -from daiquiri.core.schema import ErrorSchema, MessageSchema +from daiquiri.core.schema import ErrorSchema, MessageSchema, ValidationErrorSchema from daiquiri.core.schema.samplescan import ScanConfigSchema, AvailableScansSchema from daiquiri.core.schema.metadata import paginated @@ -30,6 +30,7 @@ class QueueScanResource(ComponentResource): [200, MessageSchema(), "Scan successfully queued"], [400, ErrorSchema(), "Could not queue scan"], [404, ErrorSchema(), "No such data collection plan"], + [422, ValidationErrorSchema(), "Data collection plan is not valid"], ] ) def post(self, datacollectionplanid): @@ -214,30 +215,35 @@ class Samplescan(Component, DCUtilsMixin): "sampleid": datacollectionplan["sampleid"], **datacollectionplan["scanparameters"], } + + if "actor" not in params: + return ( + { + "error": "No actor specified within the selected data collection plan" + }, + 400, + ) + actor = params.pop("actor") # TODO: This all needs factoring out of `actor_wrapper` to make # it reusable schema = self.actor_schema(actor) schema_inst = schema() - schema_inst.load(params) + try: + schema_inst.load(params) + except ValidationError as err: + return {"messages": err.messages}, 422 params["datacollectionplanid"] = datacollectionplan["datacollectionplanid"] params = self.preprocess(**params) - actkw = {} - if hasattr(self, "actor_success"): - actkw["success"] = self.actor_success - - if hasattr(self, "actor_started"): - actkw["start"] = self.actor_started - - if hasattr(self, "actor_error"): - actkw["error"] = self.actor_error - - if hasattr(self, "actor_remove"): - actkw["remove"] = self.actor_remove - + actkw = { + "success": self.actor_success, + "start": self.actor_started, + "error": self.actor_error, + "remove": self.actor_remove, + } uuid = self.actor(actor, actargs=params, **actkw) if uuid: diff --git a/daiquiri/core/schema/__init__.py b/daiquiri/core/schema/__init__.py index c9c7e352e..65ae3327a 100644 --- a/daiquiri/core/schema/__init__.py +++ b/daiquiri/core/schema/__init__.py @@ -27,6 +27,10 @@ class MessageSchema(MarshmallowSchema): message = fields.Str() +class ValidationErrorSchema(MarshmallowSchema): + messages = fields.Dict() + + class SchemasListSchema(MarshmallowSchema): name = fields.Str() -- GitLab From 09b7b7c308e8c3e6ecf6d52b6e7843df8d102dcc Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 17:12:30 +0200 Subject: [PATCH 20/33] fix updating plans, add test --- daiquiri/core/metadata/__init__.py | 3 ++- daiquiri/core/metadata/ispyalchemy/dc.py | 3 +++ tests/api/test_metadata_api.py | 34 ++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index cf5bd774d..e22f406e8 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -540,10 +540,11 @@ class DataCollectionPlanResource(CoreResource): return {"error": "No such datacollection plan"}, 404 @marshal( + inp=DataCollectionPlanSchema, out=[ [200, DataCollectionPlanSchema(), "Updated datacollection plan"], [404, ErrorSchema(), "No such datacollection plan"], - ] + ], ) def patch(self, datacollectionplanid, **kwargs): """Update a datacollection plan""" diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 20f240d68..1cae99944 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -720,6 +720,9 @@ class DCHandler(IspyalchemyHandler): if kw in updatable: val = kwargs[kw] + if kw == "scanparameters": + val = json.dumps(val) + setattr(datacollectionplan, kw, val) sample_has_datacollectionplan = ( diff --git a/tests/api/test_metadata_api.py b/tests/api/test_metadata_api.py index 5ffc5c7a4..c84c61f10 100644 --- a/tests/api/test_metadata_api.py +++ b/tests/api/test_metadata_api.py @@ -261,3 +261,37 @@ def test_add_datacollection_plan(auth_client, with_session): for k in data: assert res.json[k] == data[k] + + +def test_update_datacollection_plan(auth_client, with_session): + scanparameters = {"actor": "testactor", "param1": 1.454, "param2": 5.4} + data = { + "scanparameters": {"actor": "testactor", "param1": 1.454, "param2": 5.4}, + "sampleid": 1, + } + + res = auth_client.post("/api/metadata/datacollections/plans", payload=data) + assert res.status_code == 201 + + plan = res.json + + new_scanparameters = scanparameters.copy() + new_scanparameters["param1"] = 4 + + res = auth_client.patch( + f"/api/metadata/datacollections/plans/{plan['datacollectionplanid']}", + payload={"scanparameters": new_scanparameters}, + ) + + assert res.status_code == 200 + new_plan = res.json + + assert new_plan["scanparameters"] == new_scanparameters + + new_planorder = 3 + res = auth_client.patch( + f"/api/metadata/datacollections/plans/{plan['datacollectionplanid']}", + payload={"planorder": new_planorder}, + ) + + assert res.json["planorder"] == new_planorder -- GitLab From 65601a376c415f6a0229023d73d9108edd47b59d Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 3 May 2022 17:39:25 +0200 Subject: [PATCH 21/33] dont delete plan on unqueue if its associated to a sample --- daiquiri/core/metadata/ispyalchemy/sample.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/sample.py b/daiquiri/core/metadata/ispyalchemy/sample.py index e2a0cb55c..b6af742e0 100644 --- a/daiquiri/core/metadata/ispyalchemy/sample.py +++ b/daiquiri/core/metadata/ispyalchemy/sample.py @@ -682,8 +682,20 @@ class SampleHandler(IspyalchemyHandler): ses.commit() if dp: - ses.delete(dp) - ses.commit() + sample_has_datacollectionplan = ( + ses.query(self.BLSample_has_DataCollectionPlan) + .filter( + self.BLSample_has_DataCollectionPlan.datacollectionplanid + == dp.diffractionplanid, + ) + .first() + ) + + # If this plan is associated to a sample it was created using the plans routes + # rather than automatically, unqueuing should preserve the plan + if not sample_has_datacollectionplan: + ses.delete(dp) + ses.commit() return True -- GitLab From e912477ab27273db5b374b1614102b8c3fc14e47 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 11:33:53 +0200 Subject: [PATCH 22/33] add ordering, fix filter, add delete and test --- daiquiri/core/metadata/__init__.py | 19 ++++++---- daiquiri/core/metadata/ispyalchemy/dc.py | 48 +++++++++++++++++++++++- tests/api/test_metadata_api.py | 13 +++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index e22f406e8..e930c78c5 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -500,6 +500,7 @@ class DataCollectionPlansResource(CoreResource): "sampleid": fields.Int(), "status": OneOf(["executed", "pending"]), }, + ordered=True, out=[ [200, paginated(DataCollectionPlanSchema), "List of datacollection plans"] ], @@ -558,17 +559,21 @@ class DataCollectionPlanResource(CoreResource): out=[ [200, MessageSchema(), "Datacollection plan deleted"], [404, ErrorSchema(), "No such datacollection plan"], + [400, ErrorSchema(), "Could not remove datacollection plan"], ] ) def delete(self, datacollectionplanid, **kwargs): """Delete a datacollection plan""" - subsample = self._parent.remove_datacollectionplan( - datacollectionplanid, **kwargs - ) - if subsample: - return {"message": "Datacollection plan deleted"}, 200 - else: - return {"error": "No such datacollection plan"}, 404 + try: + dcplan = self._parent.remove_datacollectionplan( + datacollectionplanid, **kwargs + ) + if dcplan: + return {"message": "Datacollection plan deleted"}, 200 + else: + return {"error": "No such datacollection plan"}, 404 + except RuntimeError as e: + return {"error": e.args[0]}, 400 class ScanQualityIndicatorsResource(CoreResource): diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 1cae99944..b81cf0351 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -626,7 +626,7 @@ class DCHandler(IspyalchemyHandler): ) if status: - if status == "completed": + if status == "executed": datacollectionplans = datacollectionplans.filter( self.DataCollection.datacollectionid != None # noqa: E711 ) @@ -644,6 +644,17 @@ class DCHandler(IspyalchemyHandler): return self._decode_scanparameters(datacollectionplan._asdict()) else: + datacollectionplans = order( + datacollectionplans, + { + "sampleid": self.BLSample.blsampleid, + "planorder": self.BLSample_has_DataCollectionPlan.planorder, + "datacollectionplanid": self.DiffractionPlan.diffractionplanid, + }, + default=["datacollectionplanid", "desc"], + **kwargs, + ) + datacollectionplans = [ self._decode_scanparameters(r._asdict()) for r in datacollectionplans.all() @@ -748,4 +759,37 @@ class DCHandler(IspyalchemyHandler): ) def remove_datacollectionplan(self, datacollectionplanid, **kwargs): - return super().remove_datacollectionplan(datacollectionplanid, **kwargs) + with self.session_scope() as ses: + chk = self.get_datacollectionplans( + datacollectionplanid=datacollectionplanid, + no_context=kwargs.get("no_context"), + ) + + if chk: + if chk["datacollectionid"]: + raise RuntimeError( + "Data collection plan has been executed, it cannot be deleted" + ) + + if chk["queued"]: + raise RuntimeError( + "Data collection plan has been queued, it cannot be deleted without being unqueued" + ) + + dp, sample_has_datacollectionplan = ( + ses.query( + self.DiffractionPlan, self.BLSample_has_DataCollectionPlan + ) + .filter( + self.DiffractionPlan.diffractionplanid == datacollectionplanid + ) + .first() + ) + + if sample_has_datacollectionplan: + ses.delete(sample_has_datacollectionplan) + + ses.delete(dp) + ses.commit() + + return True diff --git a/tests/api/test_metadata_api.py b/tests/api/test_metadata_api.py index c84c61f10..65f8ffdce 100644 --- a/tests/api/test_metadata_api.py +++ b/tests/api/test_metadata_api.py @@ -295,3 +295,16 @@ def test_update_datacollection_plan(auth_client, with_session): ) assert res.json["planorder"] == new_planorder + + +def test_remove_datacollection_plan(auth_client, with_session): + res = auth_client.get("/api/metadata/datacollections/plans") + assert res.status_code == 200 + + rows = res.json["rows"] + dp = rows[0] + + res = auth_client.delete( + f"/api/metadata/datacollections/plans/{dp['datacollectionplanid']}" + ) + assert res.status_code == 200 -- GitLab From 1bc8bd6802cc856a6c37de0a4a031f584fdb949e Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 11:55:39 +0200 Subject: [PATCH 23/33] return the correct `sample has plan` --- daiquiri/core/metadata/ispyalchemy/dc.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index b81cf0351..433aa3b8a 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -776,18 +776,26 @@ class DCHandler(IspyalchemyHandler): "Data collection plan has been queued, it cannot be deleted without being unqueued" ) - dp, sample_has_datacollectionplan = ( - ses.query( - self.DiffractionPlan, self.BLSample_has_DataCollectionPlan + dp = ( + ses.query(self.DiffractionPlan,) + .filter( + self.DiffractionPlan.diffractionplanid == datacollectionplanid, ) + .first() + ) + + sample_has_datacollectionplan = ( + ses.query(self.BLSample_has_DataCollectionPlan) .filter( - self.DiffractionPlan.diffractionplanid == datacollectionplanid + self.BLSample_has_DataCollectionPlan.datacollectionplanid + == datacollectionplanid ) .first() ) if sample_has_datacollectionplan: ses.delete(sample_has_datacollectionplan) + ses.commit() ses.delete(dp) ses.commit() -- GitLab From 362efc44f1a62518e3eb46e104d5c3ed6664102f Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 11:57:33 +0200 Subject: [PATCH 24/33] emit event on add / update / delete dcplan --- daiquiri/core/metadata/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index e930c78c5..fd9e6a8cd 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -991,6 +991,7 @@ class MetaDataHandler(CoreBase): "sample", "subsample", "datacollection", + "datacollectionplan", "sampleimage", "xrf_map", "xrf_composite", -- GitLab From 09f7e4f9046be63dfcf3bbc173e63eab8f7d11ab Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:02:23 +0200 Subject: [PATCH 25/33] save actor and component --- daiquiri/core/components/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/daiquiri/core/components/__init__.py b/daiquiri/core/components/__init__.py index 72cc70ec4..2e8a2d428 100644 --- a/daiquiri/core/components/__init__.py +++ b/daiquiri/core/components/__init__.py @@ -44,8 +44,12 @@ def actor(name, **kwargs): def actor_wrapper(name, **actkw): def decorator(fn): def wrapper(self, *args, **kwargs): + kwargs["actor"] = name + kwargs["component"] = fn.__module__ + if kwargs.get("save"): kwargs.pop("save") + kwargs.pop("enqueue", None) datacollectionplan = self._parent._metadata.add_datacollectionplan( sampleid=kwargs.pop("sampleid"), scanparameters={"actor": name, **kwargs}, -- GitLab From 15287be1609148b9c67fdc499a77e837e83299fb Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:02:45 +0200 Subject: [PATCH 26/33] update error message, pop component --- daiquiri/core/components/samplescan.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 64eb9bb6d..9a7a6d461 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -206,7 +206,7 @@ class Samplescan(Component, DCUtilsMixin): if datacollectionplan["datacollectionid"]: return ( { - "error": f"That data collection plan has already been executed: datacollectionid: {datacollectionplan['datacollectionid']}" + "error": f"That data collection plan has already been executed `datacollectionid`: {datacollectionplan['datacollectionid']}" }, 400, ) @@ -225,6 +225,7 @@ class Samplescan(Component, DCUtilsMixin): ) actor = params.pop("actor") + component = params.pop("component") # TODO: This all needs factoring out of `actor_wrapper` to make # it reusable -- GitLab From 45673f7822cafd2c38184f6778e9416f0b76baa8 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:02:55 +0200 Subject: [PATCH 27/33] save scanparameters for imageviewer too --- daiquiri/core/components/imageviewer/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daiquiri/core/components/imageviewer/__init__.py b/daiquiri/core/components/imageviewer/__init__.py index 373ddcb7e..ba4ae6125 100644 --- a/daiquiri/core/components/imageviewer/__init__.py +++ b/daiquiri/core/components/imageviewer/__init__.py @@ -327,7 +327,9 @@ class Imageviewer(Component, DCUtilsMixin): ( kwargs["containerqueuesampleid"], kwargs["datacollectionplanid"], - ) = self._parent._metadata.queue_subsample(kwargs["subsampleid"]) + ) = self._parent._metadata.queue_subsample( + kwargs["subsampleid"], scanparameters=kwargs + ) kwargs["absol"] = self._parent.get_absolute(kwargs["subsampleid"]) kwargs["before_scan_starts"] = self._parent.before_scan_starts kwargs["update_datacollection"] = self._parent.update_datacollection -- GitLab From 014e4414f89e74be32c0f4173fea2e0b7950d215 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:03:19 +0200 Subject: [PATCH 28/33] make queuing subsample consistent --- daiquiri/core/metadata/ispyalchemy/dc.py | 17 ++++++++-------- daiquiri/core/metadata/ispyalchemy/sample.py | 21 +++++--------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index 433aa3b8a..bf14e53d6 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -672,7 +672,7 @@ class DCHandler(IspyalchemyHandler): self, *, scanparameters, - sampleid, + sampleid=None, energy=None, exposuretime=None, experimentkind=None, @@ -690,14 +690,15 @@ class DCHandler(IspyalchemyHandler): ses.add(datacollectionplan) ses.commit() - sample_has_datacollectionplan = self.BLSample_has_DataCollectionPlan( - blsampleid=sampleid, - datacollectionplanid=datacollectionplan.diffractionplanid, - planorder=planorder, - ) + if sampleid: + sample_has_datacollectionplan = self.BLSample_has_DataCollectionPlan( + blsampleid=sampleid, + datacollectionplanid=datacollectionplan.diffractionplanid, + planorder=planorder, + ) - ses.add(sample_has_datacollectionplan) - ses.commit() + ses.add(sample_has_datacollectionplan) + ses.commit() return self.get_datacollectionplans( datacollectionplanid=datacollectionplan.diffractionplanid, diff --git a/daiquiri/core/metadata/ispyalchemy/sample.py b/daiquiri/core/metadata/ispyalchemy/sample.py index b6af742e0..8b0ddfa2a 100644 --- a/daiquiri/core/metadata/ispyalchemy/sample.py +++ b/daiquiri/core/metadata/ispyalchemy/sample.py @@ -536,24 +536,13 @@ class SampleHandler(IspyalchemyHandler): return True - def queue_subsample( - self, subsampleid, datacollectionplanid=None, scanparameters=None, **kwargs - ): + def queue_subsample(self, subsampleid, datacollectionplanid=None, **kwargs): with self.session_scope() as ses: subsample = self.get_subsamples(subsampleid=subsampleid) if subsample: if not datacollectionplanid: - dp = self.DiffractionPlan( - # steps_x=kwargs.get("steps_x"), - # steps_y=kwargs.get("steps_y"), - experimentkind=kwargs.get("experimenttype"), - exposuretime=kwargs.get("exposuretime"), - ) - - ses.add(dp) - ses.commit() - - datacollectionplanid = dp.diffractionplanid + dp = self.add_datacollectionplan(no_context=True, **kwargs) + datacollectionplanid = dp["datacollectionplanid"] cq = ( ses.query(self.ContainerQueue.containerqueueid) @@ -586,7 +575,7 @@ class SampleHandler(IspyalchemyHandler): ses.add(cqs) ses.commit() - return cqs.containerqueuesampleid, dp.diffractionplanid + return cqs.containerqueuesampleid, datacollectionplanid def unqueue_subsample(self, subsampleid, **kwargs): with self.session_scope() as ses: @@ -623,7 +612,7 @@ class SampleHandler(IspyalchemyHandler): sample = self.get_samples(sampleid=sampleid) if sample: if not datacollectionplanid: - dp = self.add_datacollectionplan(sampleid=sampleid, **kwargs) + dp = self.add_datacollectionplan(no_context=True, **kwargs) datacollectionplanid = dp["datacollectionplanid"] cq = ( -- GitLab From 1f7d71c968ad2bc8943ed08f00c34abaf6ae400d Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:12:34 +0200 Subject: [PATCH 29/33] lint --- daiquiri/core/components/samplescan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daiquiri/core/components/samplescan.py b/daiquiri/core/components/samplescan.py index 9a7a6d461..9d4dd204c 100644 --- a/daiquiri/core/components/samplescan.py +++ b/daiquiri/core/components/samplescan.py @@ -225,7 +225,7 @@ class Samplescan(Component, DCUtilsMixin): ) actor = params.pop("actor") - component = params.pop("component") + params.pop("component") # TODO: This all needs factoring out of `actor_wrapper` to make # it reusable -- GitLab From fa50d0992366e7408059228f72f57b427c25b619 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 14:12:58 +0200 Subject: [PATCH 30/33] tidy query --- daiquiri/core/metadata/ispyalchemy/dc.py | 20 +++++++------------- daiquiri/core/schema/metadata.py | 4 +++- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index bf14e53d6..d11dae50f 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -102,11 +102,6 @@ class DCHandler(IspyalchemyHandler): self.GridInfo.datacollectionid == self.DataCollection.datacollectionid, ) - .outerjoin( - self.DiffractionPlan, - self.DataCollection.datacollectionplanid - == self.DiffractionPlan.diffractionplanid, - ) ) if kwargs.get("datacollectiongroupid"): @@ -574,12 +569,11 @@ class DCHandler(IspyalchemyHandler): self.DiffractionPlan.energy, self.DiffractionPlan.exposuretime, self.BLSample.blsampleid.label("sampleid"), - self.BLSample.name, + self.BLSample.name.label("sample"), self.Protein.acronym.label("component"), - self.Protein.proteinid.label("componentid"), self.BLSample_has_DataCollectionPlan.planorder, self.DataCollection.datacollectionid, - self.ContainerQueueSample.containerqueueid, + self.ContainerQueueSample.containerqueuesampleid, func.IF( self.DataCollection.datacollectionid, "executed", "pending" ).label("status"), @@ -592,16 +586,16 @@ class DCHandler(IspyalchemyHandler): self.DiffractionPlan.diffractionplanid == self.BLSample_has_DataCollectionPlan.datacollectionplanid, ) - .outerjoin( - self.BLSample, - self.BLSample.blsampleid - == self.BLSample_has_DataCollectionPlan.blsampleid, - ) .outerjoin( self.ContainerQueueSample, self.ContainerQueueSample.datacollectionplanid == self.DiffractionPlan.diffractionplanid, ) + .outerjoin( + self.BLSample, + self.BLSample.blsampleid + == self.BLSample_has_DataCollectionPlan.blsampleid, + ) .outerjoin( self.Crystal, self.Crystal.crystalid == self.BLSample.crystalid ) diff --git a/daiquiri/core/schema/metadata.py b/daiquiri/core/schema/metadata.py index 47d3b4e18..78486df52 100644 --- a/daiquiri/core/schema/metadata.py +++ b/daiquiri/core/schema/metadata.py @@ -243,9 +243,11 @@ class DataCollectionPlanSchema(NewDataCollectionPlanSchema): datacollectionplanid = fields.Int() timestamp = fields.DateTime() sampleid = fields.Int() + sample = fields.Str() + component = fields.Str() subsampleid = fields.Int() datacollectionid = fields.Int() - containerqueueid = fields.Int() + containerqueuesampleid = fields.Int() queued = fields.Bool() status = fields.Str() scanparameters = fields.Dict() -- GitLab From ce4442a5640931ea64e259d4a285fa877ed01e1a Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Wed, 4 May 2022 15:00:13 +0200 Subject: [PATCH 31/33] add queued filter, support returning plans for subsamples --- daiquiri/core/metadata/__init__.py | 1 + daiquiri/core/metadata/ispyalchemy/dc.py | 27 +++++++++++++++++++++--- daiquiri/core/schema/metadata.py | 2 ++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/daiquiri/core/metadata/__init__.py b/daiquiri/core/metadata/__init__.py index fd9e6a8cd..ac1579ec7 100644 --- a/daiquiri/core/metadata/__init__.py +++ b/daiquiri/core/metadata/__init__.py @@ -499,6 +499,7 @@ class DataCollectionPlansResource(CoreResource): "subsampleid": fields.Int(), "sampleid": fields.Int(), "status": OneOf(["executed", "pending"]), + "queued": fields.Bool(), }, ordered=True, out=[ diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index d11dae50f..c5c468636 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -5,7 +5,7 @@ import json from flask import g import sqlalchemy -from sqlalchemy import func +from sqlalchemy import func, or_ from sqlalchemy.sql.expression import cast from daiquiri.core.metadata.ispyalchemy.handler import IspyalchemyHandler @@ -554,6 +554,7 @@ class DCHandler(IspyalchemyHandler): datacollectionplanid=None, blsampleid=None, status=None, + queued=None, no_context=False, **kwargs, ): @@ -569,6 +570,7 @@ class DCHandler(IspyalchemyHandler): self.DiffractionPlan.energy, self.DiffractionPlan.exposuretime, self.BLSample.blsampleid.label("sampleid"), + self.BLSubSample.blsubsampleid.label("subsampleid"), self.BLSample.name.label("sample"), self.Protein.acronym.label("component"), self.BLSample_has_DataCollectionPlan.planorder, @@ -580,6 +582,9 @@ class DCHandler(IspyalchemyHandler): func.IF( self.ContainerQueueSample.containerqueuesampleid, True, False ).label("queued"), + func.IF( + self.BLSample_has_DataCollectionPlan.blsampleid, True, False, + ).label("linked"), ) .outerjoin( self.BLSample_has_DataCollectionPlan, @@ -591,10 +596,20 @@ class DCHandler(IspyalchemyHandler): self.ContainerQueueSample.datacollectionplanid == self.DiffractionPlan.diffractionplanid, ) + .outerjoin( + self.BLSubSample, + self.BLSubSample.blsubsampleid + == self.ContainerQueueSample.blsubsampleid, + ) .outerjoin( self.BLSample, - self.BLSample.blsampleid - == self.BLSample_has_DataCollectionPlan.blsampleid, + or_( + self.BLSample.blsampleid + == self.BLSample_has_DataCollectionPlan.blsampleid, + self.BLSample.blsampleid + == self.ContainerQueueSample.blsampleid, + self.BLSample.blsampleid == self.BLSubSample.blsampleid, + ), ) .outerjoin( self.Crystal, self.Crystal.crystalid == self.BLSample.crystalid @@ -629,6 +644,12 @@ class DCHandler(IspyalchemyHandler): self.DataCollection.datacollectionid == None # noqa: E711 ) + if queued: + datacollectionplans = datacollectionplans.filter( + self.ContainerQueueSample.containerqueuesampleid + != None # noqa: E711 + ) + if datacollectionplanid: datacollectionplans = datacollectionplans.filter( self.DiffractionPlan.diffractionplanid == datacollectionplanid diff --git a/daiquiri/core/schema/metadata.py b/daiquiri/core/schema/metadata.py index 78486df52..2589d42b6 100644 --- a/daiquiri/core/schema/metadata.py +++ b/daiquiri/core/schema/metadata.py @@ -243,6 +243,7 @@ class DataCollectionPlanSchema(NewDataCollectionPlanSchema): datacollectionplanid = fields.Int() timestamp = fields.DateTime() sampleid = fields.Int() + subsampleid = fields.Int() sample = fields.Str() component = fields.Str() subsampleid = fields.Int() @@ -250,6 +251,7 @@ class DataCollectionPlanSchema(NewDataCollectionPlanSchema): containerqueuesampleid = fields.Int() queued = fields.Bool() status = fields.Str() + linked = fields.Bool() scanparameters = fields.Dict() -- GitLab From 3af744e5ad41f501898b8f6d3e1e02f078100d9e Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 28 Jun 2022 17:20:15 +0200 Subject: [PATCH 32/33] style --- daiquiri/core/metadata/ispyalchemy/dc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/dc.py b/daiquiri/core/metadata/ispyalchemy/dc.py index c5c468636..dfc5a9d4b 100644 --- a/daiquiri/core/metadata/ispyalchemy/dc.py +++ b/daiquiri/core/metadata/ispyalchemy/dc.py @@ -583,7 +583,9 @@ class DCHandler(IspyalchemyHandler): self.ContainerQueueSample.containerqueuesampleid, True, False ).label("queued"), func.IF( - self.BLSample_has_DataCollectionPlan.blsampleid, True, False, + self.BLSample_has_DataCollectionPlan.blsampleid, + True, + False, ).label("linked"), ) .outerjoin( @@ -793,7 +795,9 @@ class DCHandler(IspyalchemyHandler): ) dp = ( - ses.query(self.DiffractionPlan,) + ses.query( + self.DiffractionPlan, + ) .filter( self.DiffractionPlan.diffractionplanid == datacollectionplanid, ) -- GitLab From 472f79c9ce60e69b9adbfc8d27aeff501739ed81 Mon Sep 17 00:00:00 2001 From: Stuart Fisher Date: Tue, 30 Aug 2022 13:33:27 +0200 Subject: [PATCH 33/33] db merged --- daiquiri/core/metadata/ispyalchemy/updates.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/daiquiri/core/metadata/ispyalchemy/updates.sql b/daiquiri/core/metadata/ispyalchemy/updates.sql index 111957c22..e69de29bb 100644 --- a/daiquiri/core/metadata/ispyalchemy/updates.sql +++ b/daiquiri/core/metadata/ispyalchemy/updates.sql @@ -1,3 +0,0 @@ -ALTER TABLE DiffractionPlan - ADD `scanParameters` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`scanParameters`)); - \ No newline at end of file -- GitLab