Skip to content

Add endpoint to retrieve datasets from ICAT

We would need ICAT+ to query ICAT and retrieve data in a way that will be easily parsed by the UI. Today we are doing this by getting the datasets per parameter and then build the datasets that it is quite heavy for the browser. Query is:

SELECT dataset.id, dataset.name, dataset.startDate, dataset.endDate, investigation.name, parameterType.name, parameter.stringValue, parameter.numericValue, dataset.location FROM DatasetParameter as parameter JOIN parameter.dataset dataset JOIN parameter.type parameterType JOIN dataset.investigation investigation where investigation.id = :investigationId

And then we parse the data with:

  let datasets = this.parametersToDatasetObject(this.groupBy(res.data, 0));

  parametersToDatasetObject = function (parametersGroupedByInvestigationId) {
    let datasets = {};

    for (var datasetId in parametersGroupedByInvestigationId) {
      for (var index in parametersGroupedByInvestigationId[datasetId]) {
        var param = parametersGroupedByInvestigationId[datasetId][index];
        var key = param[5];
        var value = param[6];
        if (datasets[datasetId] == null) {
          datasets[datasetId] = {};
        }
        datasets[datasetId][key] = value;
        /** Some metadata at level of dataset to be added as parameters of the dataset */
        datasets[datasetId]["id"] = param[0];
        datasets[datasetId]["name"] = param[1];
        datasets[datasetId]["startDate"] = param[2];
        datasets[datasetId]["endDate"] = param[3];
        datasets[datasetId]["investigationName"] = param[4];
        datasets[datasetId]["location"] = param[8];
        datasets[datasetId]["investigationId"] = param[9];
      }
    }

    var array = [];
    for (var ds in datasets) {
      array.push(datasets[ds]);
    }
    return array;
  };

Structure reveived from ICAT is:

[[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_vertical_offset","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_name"," p1 p2 s1 s2 s3 s4 s5 s6",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_name"," s1",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_horizontal_gap","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_blade_down","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSource_mode","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_vertical_gap","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","Sample_name","test01",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_horizontal_offset","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_horizontal_offset","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_horizontal_offset","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_name"," p1",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_blade_front","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_blade_front","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_blade_up","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector01_name"," detector01",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector02_name"," detector02",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","definition","KMAP",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_vertical_offset","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_blade_back","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentInsertionDevice_taper_value","ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_horizontal_gap","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","SamplePositioners_name"," toto1",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_vertical_gap","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_blade_up","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector01Positioners_name"," att11 att12 att13 att21",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSource_current","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_horizontal_gap","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","Sample_description","est",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_blade_back","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector02Positioners_value","ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentOpticsPositioners_value","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentAttenuatorPositioners_name"," att11 att12 att13 att21",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentInsertionDevice_gap_name"," U35a U27b U35b U27c",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_vertical_offset","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector01Positioners_value","ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentAttenuatorPositioners_value","ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentOpticsPositioners_name","  monobof mononrj monoth mir1rz mir1rzf mir2crv mir2ty mirboff mirrz mirty mirty1 mirty2 mirtz monotz monoty monorz cc2pit dcm2pf dcm2pit dcm2rlf dcm2rol dcm2ty mir1rz mir1rzf mm1rz mm2tx mm2ty mmboff mmrz mmty mmty2 mmtz tf1rz tf1y tf1z1 tf1z2",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_blade_front","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitPrimary_blade_down","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","Notes_note_00","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","Notes_note_01","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_blade_down","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentInsertionDevice_taper_name"," U35a U27c",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentDetector02Positioners_name"," att11 att12 att13 att21",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlits_blade_back","ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_blade_up","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentInsertionDevice_gap_value","ERROR ERROR ERROR ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"],[63335951,"test01_2017_10_26_172625","2017-10-26T17:26:25.464+02:00","2017-10-26T17:27:44.439+02:00","id010100","InstrumentSlitSecondary_vertical_gap","ERROR",null,"/data/id01/inhouse/metadata/id010100/id01/test01/test01_2017_10_26_172625"]]
Edited by Alejandro De Maria Antolinos