From a0c3b9c376cfdc21ce575b151060f59dd50c266f Mon Sep 17 00:00:00 2001
From: payno <henri.payno@esrf.fr>
Date: Tue, 9 Apr 2019 08:27:11 +0200
Subject: [PATCH] [CI] set back the luigi notebook for testing

---
 .gitlab-ci.yml                     |  3 +-
 doc/tutorials/workflow_luigi.ipynb | 55 ++++++++++++------------------
 2 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a0d8e0fed..4872c07c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -118,7 +118,7 @@ test:test-tomwer-tutorials:
   script:
     - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a jupyter nbconvert --to notebook --execute doc/tutorials/workflow_operation.ipynb
 
-#
+
 #test:test-luigi:
 #  type: test
 #  image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
@@ -141,4 +141,5 @@ test:test-tomwer-tutorials:
 #    - python -m pip install .
 #    - python -m pip install luigi
 #  script:
+#    - luigid --background
 #    - /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a jupyter nbconvert --to notebook --execute doc/tutorials/workflow_luigi.ipynb
diff --git a/doc/tutorials/workflow_luigi.ipynb b/doc/tutorials/workflow_luigi.ipynb
index 2b6106d0e..04b7bde08 100644
--- a/doc/tutorials/workflow_luigi.ipynb
+++ b/doc/tutorials/workflow_luigi.ipynb
@@ -176,7 +176,9 @@
     "import luigi\n",
     "import tempfile\n",
     "import os\n",
+    "import logging\n",
     "\n",
+    "_logger = logging.getLogger('root_logger')\n",
     "global server_client\n",
     "\n",
     "## get some data to check Task process\n",
@@ -219,7 +221,7 @@
     "    print('launching darkrefs task...')\n",
     "    assert luigi.interface.run(params) is True\n",
     "\n",
-    "    ## make sure processing was correct\n",
+    "    # make sure processing was correct\n",
     "    assert os.path.exists(dark_file_0)\n",
     "    assert os.path.exists(dark_file_1)\n",
     "    assert os.path.exists(refHST_file_0)\n",
@@ -333,7 +335,7 @@
     "                \"start\",\n",
     "                parameters_files_dir,\n",
     "                observed_dir],\n",
-    "               shell=True)"
+    "                shell=True)"
    ]
   },
   {
@@ -379,7 +381,7 @@
     "                \"stop\",\n",
     "                parameters_files_dir,\n",
     "                observed_dir],\n",
-    "               shell=True)"
+    "                shell=True)"
    ]
   },
   {
@@ -434,7 +436,6 @@
     "from tomwer.core.utils import DownloadDataset\n",
     "import tempfile\n",
     "import os\n",
-    "import shutil\n",
     "\n",
     "original_folder = tempfile.mkdtemp(prefix='orignal_')\n",
     "buffer_folder = tempfile.mkdtemp(prefix='buffer_')\n",
@@ -455,11 +456,11 @@
     "    assert not os.path.exists(refHST_file_0)\n",
     "    assert not os.path.exists(refHST_file_1)\n",
     "    node1 = Node(luigi_task='esrf.tomwer.FolderTransfertTask',\n",
-    "                 properties={'dest_dir':buffer_folder})\n",
+    "                 properties={'dest_dir': buffer_folder})\n",
     "    node2 = Node(luigi_task='esrf.tomwer.DarkRefsTask')\n",
     "    node3 = Node(luigi_task='esrf.tomwer.FtseriesTask')\n",
     "    node4 = Node(luigi_task='esrf.tomwer.FolderTransfertTask',\n",
-    "                 properties={'dest_dir':final_folder})\n",
+    "                 properties={'dest_dir': final_folder})\n",
     "\n",
     "    nodes = [node1, node2, node3, node4]\n",
     "\n",
@@ -507,23 +508,27 @@
    "source": [
     "import tomwer.luigi\n",
     "import shutil\n",
+    "import tempfile\n",
     "import os\n",
     "\n",
-    "for _folder in ('/tmp/esrf_tomwer', '/tmp/luigi'):\n",
-    "    if os.path.exists(_folder):\n",
-    "        shutil.rmtree(_folder)\n",
+    "def clear_luigi_histo():\n",
+    "    for _folder in ('/tmp/esrf_tomwer', '/tmp/luigi'):\n",
+    "        if os.path.exists(_folder):\n",
+    "            shutil.rmtree(_folder)\n",
     "\n",
     "scan = os.path.join(original_folder, 'D2_H2_T2_h_')\n",
     "\n",
     "assert len(os.listdir(final_folder)) is 0\n",
     "\n",
-    "properties_folder = '/nobackup/linazimov/payno/dev/install_esrforange/orangeProject/orange/properties_folder'\n",
-    "shutil.rmtree(properties_folder)\n",
-    "os.mkdir(properties_folder)\n",
+    "properties_folder = tempfile.mkdtemp()\n",
+    "\n",
     "\n",
     "tomwer.luigi.exec_(scheme, properties_folder=properties_folder,\n",
     "                   scan=scan, name='test_process')\n",
-    "\n",
+    "print('----------')\n",
+    "print(os.listdir(original_folder))\n",
+    "print('----------')\n",
+    "print(os.listdir(final_folder))\n",
     "assert len(os.listdir(final_folder)) is 1\n",
     "assert len(os.listdir(os.path.join(final_folder, 'D2_H2_T2_h_'))) > 1\n",
     "\n",
@@ -577,7 +582,7 @@
     "                \"start\",\n",
     "                parameters_files_dir,\n",
     "                observed_dir],\n",
-    "               shell=True)"
+    "                shell=True)"
    ]
   },
   {
@@ -600,7 +605,7 @@
     "                \"create_scan\",\n",
     "                 parameters_files_dir,\n",
     "                 observed_dir],\n",
-    "               shell=True)"
+    "                 shell=True)"
    ]
   },
   {
@@ -653,22 +658,6 @@
    "source": [
     "## The special case of the data list"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# TODO"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -680,7 +669,7 @@
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 3.0
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
@@ -691,5 +680,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 0
 }
-- 
GitLab