diff --git a/exercises/demos/2_data_flow/foreach_task.json b/demos/2_data_flow/foreach_task.json
similarity index 100%
rename from exercises/demos/2_data_flow/foreach_task.json
rename to demos/2_data_flow/foreach_task.json
diff --git a/exercises/demos/2_data_flow/foreach_task.yaml b/demos/2_data_flow/foreach_task.yaml
similarity index 100%
rename from exercises/demos/2_data_flow/foreach_task.yaml
rename to demos/2_data_flow/foreach_task.yaml
diff --git a/exercises/demos/2_data_flow/python_function_task.json b/demos/2_data_flow/pytask.json
similarity index 100%
rename from exercises/demos/2_data_flow/python_function_task.json
rename to demos/2_data_flow/pytask.json
diff --git a/exercises/demos/2_data_flow/python_function_task.yaml b/demos/2_data_flow/pytask.yaml
similarity index 100%
rename from exercises/demos/2_data_flow/python_function_task.yaml
rename to demos/2_data_flow/pytask.yaml
diff --git a/exercises/demos/3_files_and_commands/capitalA.png b/demos/3_files_and_commands/capitalA.png
similarity index 100%
rename from exercises/demos/3_files_and_commands/capitalA.png
rename to demos/3_files_and_commands/capitalA.png
diff --git a/exercises/demos/3_files_and_commands/command_line_task.json b/demos/3_files_and_commands/command_line_task.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/command_line_task.json
rename to demos/3_files_and_commands/command_line_task.json
diff --git a/exercises/demos/3_files_and_commands/command_line_task.yaml b/demos/3_files_and_commands/command_line_task.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/command_line_task.yaml
rename to demos/3_files_and_commands/command_line_task.yaml
diff --git a/exercises/demos/3_files_and_commands/command_line_task_wf.json b/demos/3_files_and_commands/command_line_task_wf.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/command_line_task_wf.json
rename to demos/3_files_and_commands/command_line_task_wf.json
diff --git a/exercises/demos/3_files_and_commands/command_line_task_wf.yaml b/demos/3_files_and_commands/command_line_task_wf.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/command_line_task_wf.yaml
rename to demos/3_files_and_commands/command_line_task_wf.yaml
diff --git a/exercises/demos/3_files_and_commands/file_transfer_task.json b/demos/3_files_and_commands/file_transfer_task.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/file_transfer_task.json
rename to demos/3_files_and_commands/file_transfer_task.json
diff --git a/exercises/demos/3_files_and_commands/file_transfer_task.yaml b/demos/3_files_and_commands/file_transfer_task.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/file_transfer_task.yaml
rename to demos/3_files_and_commands/file_transfer_task.yaml
diff --git a/exercises/demos/3_files_and_commands/foreach_task_cmd.json b/demos/3_files_and_commands/foreach_task_cmd.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/foreach_task_cmd.json
rename to demos/3_files_and_commands/foreach_task_cmd.json
diff --git a/exercises/demos/3_files_and_commands/foreach_task_cmd.yaml b/demos/3_files_and_commands/foreach_task_cmd.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/foreach_task_cmd.yaml
rename to demos/3_files_and_commands/foreach_task_cmd.yaml
diff --git a/exercises/demos/3_files_and_commands/image_animate.json b/demos/3_files_and_commands/image_animate.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_animate.json
rename to demos/3_files_and_commands/image_animate.json
diff --git a/exercises/demos/3_files_and_commands/image_animate.yaml b/demos/3_files_and_commands/image_animate.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_animate.yaml
rename to demos/3_files_and_commands/image_animate.yaml
diff --git a/exercises/demos/3_files_and_commands/image_mirror_horizontally.json b/demos/3_files_and_commands/image_mirror_horizontally.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_mirror_horizontally.json
rename to demos/3_files_and_commands/image_mirror_horizontally.json
diff --git a/exercises/demos/3_files_and_commands/image_mirror_horizontally.yaml b/demos/3_files_and_commands/image_mirror_horizontally.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_mirror_horizontally.yaml
rename to demos/3_files_and_commands/image_mirror_horizontally.yaml
diff --git a/exercises/demos/3_files_and_commands/image_mirror_vertically.json b/demos/3_files_and_commands/image_mirror_vertically.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_mirror_vertically.json
rename to demos/3_files_and_commands/image_mirror_vertically.json
diff --git a/exercises/demos/3_files_and_commands/image_mirror_vertically.yaml b/demos/3_files_and_commands/image_mirror_vertically.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_mirror_vertically.yaml
rename to demos/3_files_and_commands/image_mirror_vertically.yaml
diff --git a/exercises/demos/3_files_and_commands/image_montage+swirl.json b/demos/3_files_and_commands/image_montage+swirl.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_montage+swirl.json
rename to demos/3_files_and_commands/image_montage+swirl.json
diff --git a/exercises/demos/3_files_and_commands/image_montage.json b/demos/3_files_and_commands/image_montage.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_montage.json
rename to demos/3_files_and_commands/image_montage.json
diff --git a/exercises/demos/3_files_and_commands/image_montage.yaml b/demos/3_files_and_commands/image_montage.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_montage.yaml
rename to demos/3_files_and_commands/image_montage.yaml
diff --git a/exercises/demos/3_files_and_commands/image_montage_list.json b/demos/3_files_and_commands/image_montage_list.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_montage_list.json
rename to demos/3_files_and_commands/image_montage_list.json
diff --git a/exercises/demos/3_files_and_commands/image_montage_list.yaml b/demos/3_files_and_commands/image_montage_list.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_montage_list.yaml
rename to demos/3_files_and_commands/image_montage_list.yaml
diff --git a/exercises/demos/3_files_and_commands/image_rotation.json b/demos/3_files_and_commands/image_rotation.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_rotation.json
rename to demos/3_files_and_commands/image_rotation.json
diff --git a/exercises/demos/3_files_and_commands/image_rotation.yaml b/demos/3_files_and_commands/image_rotation.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_rotation.yaml
rename to demos/3_files_and_commands/image_rotation.yaml
diff --git a/exercises/demos/3_files_and_commands/image_swirl.json b/demos/3_files_and_commands/image_swirl.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_swirl.json
rename to demos/3_files_and_commands/image_swirl.json
diff --git a/exercises/demos/3_files_and_commands/image_swirl.yaml b/demos/3_files_and_commands/image_swirl.yaml
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_swirl.yaml
rename to demos/3_files_and_commands/image_swirl.yaml
diff --git a/exercises/demos/3_files_and_commands/image_swirl_list.json b/demos/3_files_and_commands/image_swirl_list.json
similarity index 100%
rename from exercises/demos/3_files_and_commands/image_swirl_list.json
rename to demos/3_files_and_commands/image_swirl_list.json
diff --git a/exercises/demos/3_files_and_commands/tile-0.png b/demos/3_files_and_commands/tile-0.png
similarity index 100%
rename from exercises/demos/3_files_and_commands/tile-0.png
rename to demos/3_files_and_commands/tile-0.png
diff --git a/exercises/demos/3_files_and_commands/tile-1.png b/demos/3_files_and_commands/tile-1.png
similarity index 100%
rename from exercises/demos/3_files_and_commands/tile-1.png
rename to demos/3_files_and_commands/tile-1.png
diff --git a/exercises/demos/3_files_and_commands/tile-2.png b/demos/3_files_and_commands/tile-2.png
similarity index 100%
rename from exercises/demos/3_files_and_commands/tile-2.png
rename to demos/3_files_and_commands/tile-2.png
diff --git a/exercises/demos/3_files_and_commands/tile-3.png b/demos/3_files_and_commands/tile-3.png
similarity index 100%
rename from exercises/demos/3_files_and_commands/tile-3.png
rename to demos/3_files_and_commands/tile-3.png
diff --git a/docs/agenda.rst b/docs/agenda.rst
index ce4d5a5372a57d384ca85c4d4aaac699a28a75bd..0f21380c6d1d8d77033a3c6abf73d859bdb97d9e 100644
--- a/docs/agenda.rst
+++ b/docs/agenda.rst
@@ -1,13 +1,14 @@
Basic procedures
================
- * Compose Fireworks and workflows
- * Add fireworks to LaunchPad: ``lpad add``
- * Validate workflows ``lpad add -c``, ``lpad check_wflow``
- * View workflows ``lpad check_wflow -g``
- * Execution: ``rlaunch``
- * Query fireworks and workflows: ``lpad get_fws``, ``lpad get_wflows``
- * Monitoring: ``lpad report``, ``lpad track_fws``
+ * Compose workflows
+ * Validate workflows
+ * Add workflows to LaunchPad
+ * Visualize workflows
+ * Launch fireworks
+ * Query fireworks and workflows
+ * Rerun fireworks
+ * Remove workflows
Exercise 1: Managing control flow
=================================
@@ -36,15 +37,16 @@ Exercise 4: Extending a workflow
* Extension of example from Exercise 3
* Example: Image swirl
-Exercise 5: Productive use
+Exercise 5: Writing a Firetask
+==============================
+ * Extension of example from Exercise 2
+
+Exercise 6: Advanced and productive use
+=======================================
+
* Best practices: separate configs, launches, templates, inputs
* Using FireWorks with a batch system on HPC clusters
* Recovery from failure
- * Duplicates
+ * Duplicates detection
* Heterogeneous and distributed computing: _category, _fworker, etc.
* ...
-
-Exercise 6: Writing a Firetask
-==============================
- * Extension of example from Exercise 2
-
diff --git a/docs/basics.rst b/docs/basics.rst
index c5bce57b5bd292e0e2cc7c6e92e66514068f852b..e2b01a994b8d9df5735d2c156d10d25a908bb1c1 100644
--- a/docs/basics.rst
+++ b/docs/basics.rst
@@ -19,7 +19,7 @@ All workflow definitions in all exercises will be based on JSON and YAML.
Exercise 5 will introduce to writing a custom Firetask for which
Python will be used. Again, JSON and YAML will be used to define the workflows
using custom Firetasks. For each exercise there are one or more initial
-examples in the **exercises/demos** folder. We recommend trying these examples
+examples in the **demos** folder. Trying these examples is recommend
before starting solving the problems.
**NOTE:** Most of the examples here will be presented in YAML (more readable and
@@ -71,9 +71,9 @@ Here is a short example for a workflow demonstrating the usage of the
metadata: {}
name: Simple coffee workflow
-Open a text editor, such as ``vi``, ``nano``, ``gedit`` or ``emacs``, and save the
-example above as **workflow.yaml**. To convert to JSON you can use the following
-command::
+Open a text editor, such as ``vi``, ``nano``, ``gedit`` or ``emacs``, and save
+the example above as **workflow.yaml**. To convert to JSON you can use the
+following command::
yaml2json < workflow.yaml > workflow.json
@@ -125,7 +125,8 @@ If a workflow has been added without such a check, it can be checked later with:
Visualize workflows
-------------------
-Already added workflows can be converted into DOT format and viewed graphically::
+Already added workflows can be converted into graphviz DOT format and viewed
+graphically::
lpad check_wflow -i <firework ID> [-g <controlflow | dataflow | combined>] [-f <filename>]
@@ -136,12 +137,10 @@ can be viewed::
evince workflow.pdf
-.. _execution:
-
-Execute workflows
------------------
+Launch fireworks
+----------------
-The workflow engine of FireWorks is called FireWorker. Multiple FireWorkers can be
+Fireworks can be launched by so-called FireWorker. Multiple FireWorkers can be
running on different resources where individual Fireworks can be executed by the
rocket launcher ``rlaunch`` which has three modes of operation: *singleshot*,
*rapidfire* and *multi*.
@@ -172,33 +171,39 @@ To suppress verbose information on the screen the *-s* flag can be added::
rlaunch -s rapidfire
-Monitor workflows
------------------
+Query workflows
+---------------
+
+To query workflows available on the LaunchPad use the command
+``lpad get_wflows``::
-To query workflows available on the LaunchPad use the command ``lpad get_wflows``::
+ lpad get_wflows [[-i <firework ID>]|[-q <query>]] [[-d <more|all>]|[-t]]
- lpad get_wflows [-d <more|all>]
+The ID of any firework included in the workflow can be used to query a
+specific workflow. Alternatively, workflows can be filtered using a pymongo
+query after the **-q** flag.
To query individual Fireworks use the command::
- lpad get_fws [-i <firework ID> [-d <more|all>]]
+ lpad get_fws [-i <firework ID>] [-d <more|all>]
-**NOTE:** The query from the command line is recommended in this tutorial.
+Adding the flag **-o yaml** after **lpad** will produce the output in YAML
+instead of JSON. To obtain a more detailed help on a specifc lpad command you
+can the online help::
-Alternatively the web GUI can be used::
+ lpad <lpad command> --help
- lpad webgui
+The following will provide a full list of commands and **lpad** options::
-**NOTE:** Make sure that a web browser is configured in your terminal session and an
-X server is running on your machine and it is configured and tunneled properly
-in your terminal session.
+ lpad --help
Remove a workflow from LaunchPad
--------------------------------
-A selection of workflows can be deleted from LaunchPad using the ``lpad delete_wflows``
-command. For example to delete workflows including Fireworks with given IDs::
+A selection of workflows can be deleted from LaunchPad using the
+``lpad delete_wflows`` command. For example to delete workflows including
+Fireworks with given IDs::
lpad delete_wflows -i <firework IDs>
diff --git a/docs/setup.rst b/docs/setup.rst
index 1fb25a70a775aca172c03515cca4935ed5030363..849ed94bfd87894d905941191c476ecc6fa9b8fa 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -4,22 +4,45 @@ Setup
Prerequisites
-------------
-Python
-~~~~~~
+The tutorial might still work with Python 2 but it is recommended to use
+Python 3.
-For this tutorial it is recommended to use the anaconda python distribution.
-Download anaconda (Python 3.6 version) from https://www.continuum.io/downloads
-for Linux x86_64 and follow the installation instructions on the download page.
-Make sure that python from this installation is in your ``$PATH``. In addition
-the following python packages must be installed::
+Install Python
+~~~~~~~~~~~~~~
+
+If you already have python on your computer you can skip this step.
+
+It is recommended to use the anaconda python distribution. Download anaconda
+(Python 3 version) from https://www.anaconda.com/distribution/ and follow the
+installation instructions on the download page.
+
+After the installation you should make sure that python from this installation
+is in your ``$PATH``.
+
+Virtual Environment
+~~~~~~~~~~~~~~~~~~~
+
+If you do not want to modify your python installation or if you use an
+installation for which you do not have write permissions you can set up a
+virtual environment and activate it::
+
+ python -m venv $HOME/python3-fireworks-tutorial
+ . $HOME/python3-fireworks-tutorial/bin/activate
+
+Install packages
+~~~~~~~~~~~~~~~~
+
+The following python packages must be installed::
pip install --upgrade pip
pip install pjson
pip install pyaml
- pip install future
pip install python-igraph
+ pip install fireworks
+
+**NOTE:** For the igraph package a C compiler must be installed on your computer.
-**Hint:** If later, during usage of the igraph library, an error like this::
+**NOTE:** If later, during usage of the igraph package, an error like this::
ImportError: /home/gks/anaconda3/lib/python3.6/site-packages/igraph/_igraph.cpython-36m-x86_64-linux-gnu.so: undefined symbol: _ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEC1Ev
@@ -28,47 +51,77 @@ occurs, then the libgcc package has to be installed/upgraded::
conda install libgcc
-MongoDB
-~~~~~~~
+Setting up a FireServer
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The tutorial requires a FireServer which is accessible from your computer.
+
+Remote FireServer
+'''''''''''''''''
+
+If you intend to run the rocket launchers on a remote computing cluster then
+you can configure a launchpad file for an existing FireServer.
+In this case you should create a launchpad configuration file named
+**launchpad.yaml** in the folder **$HOME/.fireworks** (create this folder if
+necessary). The file must have the following contents::
+
+ host: <hostname of the FireServer>
+ port: 27017
+ name: <database name>
+ username: <your username>
+ password: <your password>
+ logdir: null
+ strm_lvl: INFO
+ ssl: true
+ ssl_ca_certs: <path to root CA certificate>
+
+The root CA certficate can be downloaded from this URL
+http://www.gridka.de/ca/dd4b34ea.pem and installed locally.
-If your system is Ubuntu and you have administrator permissions you can install
+The hostname, username and password you will get from the instructors. In the
+folder **$HOME/.fireworks** the fireworks configuration file
+**FW_config.yaml** must be created with the following content::
+
+ echo LAUNCHPAD_LOC: $HOME/.fireworks/launchpad.yaml >> .fireworks/FW_config.yaml
+
+This configuration has to made on all resources that will use the FireServer, i.e.
+your local computer, login nodes and compute nodes of the clusters etc.
+
+
+Local FireServer
+''''''''''''''''
+
+If you have Linux and administrator permissions on your computer you can install
MongoDB system-wide with the following command::
sudo apt-get install mongodb
-The server is automatically started after the installation is completed.
+The server is automatically started after the installation is completed. In
+this case you do not need fireworks and launchpad configuration files. However,
+you cannot launch fireworks on a remote FireWorker machine in this mode.
For general instructions please read the installation manual
https://docs.mongodb.com/manual/administration/install-on-linux/
-Install FireWorks
------------------
-
-The following instructions are for installation in your ``$HOME`` directory.
-You might like to create another directory for this tutorial and replace
-``$HOME`` with the relevant path.
-
-::
-
- cd $HOME
- git clone https://github.com/ikondov/fireworks
- cd fireworks
- python setup.py develop --user
- export PYTHONPATH=$PWD:$PYTHONPATH
- export PATH=$HOME/.local/bin:$PATH
Install the tutorial
--------------------
-::
+
+The tutorial is available for download on the SCC Gitlab repository::
cd $HOME
- git clone https://git.scc.kit.edu/jk7683/gridka-school-fireworks
- cd gridka-school-fireworks
+ git clone https://git.scc.kit.edu/jk7683/grk2450-fireworks-tutorial
+ cd grk2450-fireworks-tutorial
export PYTHONPATH=$PWD/lib:$PYTHONPATH
export PATH=$PWD/bin:$PATH
-Further packages
-----------------
+**NOTE:** The folder **grk2450-fireworks-tutorial/lib** must also be available
+in **$PYTHONPATH** on the FireWorker, i.e. the machine where fireworks will be
+launched.
+
+
+Install further packages
+------------------------
In order to visualize the workflows graphically two packages must be installed::
@@ -82,7 +135,3 @@ If editors like ``vi`` or ``nano`` are not preferred, more advanced editors may
be installed, e.g.::
sudo apt-get install gedit emacs
-
-For the web GUI of lpad a web browser must be installed, e.g.::
-
- sudo apt-get install firefox