Merge branch 'develop' into mo-march-queue-doc

.github/workflows/create-tag.yml

@@ -8,6 +8,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+      with:
+        token: ${{ secrets.CI_TOKEN }}
     - name: Set up Python 3.10
       uses: actions/setup-python@v4
       with:
@@ -23,7 +25,7 @@ jobs:
           git config --global user.name "Marcus Oskarsson"
           git add -A
           git commit -m "[skip ci] Bumped minor version"
-          git push
+          git push -f
           poetry build
     - name: Publish package to PyPI
       id: publish-pacakge

deprecated/HardwareObjects/mockup/MultiCollectMockup.py

@@ -157,10 +157,15 @@ def set_detector_filenames(
     ):
         return
 
-    def prepare_oscillation(self, start, osc_range, exptime, npass):
+    def prepare_oscillation(
+        self, start, osc_range, exptime, number_of_images, shutterless, first_frame
+    ):
         return (start, start + osc_range)
 
-    def do_oscillation(self, start, end, exptime, npass):
+
+    def do_oscillation(
+        self, start, end, exptime, number_of_images, shutterless, first_frame
+    ):
         gevent.sleep(exptime)
 
     def start_acquisition(self, exptime, npass, first_frame):

docs/source/dev/configuration_files.md

@@ -0,0 +1,136 @@
+# Configuration files
+
+The MXCuBE core is organised as a tree of *HardwareObjects*,
+and configuration is organised with a file per HardwareObject.
+The (mandatory) topmost object in the tree is the *Beamline* object,
+which is configured in the `beamline_config.yml` file.
+This in turn contains the names of other configuration files,
+recursively, which may be a mixture of YAML and XML configuration files.
+The Beamline object can be accessed as an attribute of the HardwareRepository module
+(`HardwareRepository.beamline`).
+As of 2024-03-14 MXCuBE is in the middle of a change-over to a new system for configuration
+that uses YAML files instead of XML files,
+which has different ways of accessing the configuration data from inside the code.
+HardwareObjects can be configured to contain both other hardware objects and properties.
+The former are identified relative to the container by a role.
+
+## Finding the files
+Configuration files are searched for by name in a series of directories given as a lookup path.
+
+In mxcubeqt this is specified either with the `--coreConfigPath` command line parameter to MXCuBE,
+or through the `MXCUBE_CORE_CONFIG_PATH` environment variable.
+
+In mxcubeweb the configuration lookup path is specified with the `--repository` option to `mxcube-server`,
+the default being `mxcubeweb/test/HardwareObjectsMockup.xml/`,
+where the mxcubeweb mock configuration resides.
+
+There is a set of configuration data under the `mxcubecore/mxcubecore/configuration` directory.
+This includes the `mockup/` directory
+with `qt/` and `gphl/` subdirectories for mock mxcubeqt operation.
+There is also a directory with configuration for each beamline,
+but these are mostly not up to date.
+The actual beamline configuration files are mostly held in a separate (and non-comitted) directory at the beamlines.
+
+## Yaml-configured objects
+### Code and file structure
+Each YAML-configured object has a `name` attribute,
+which is equal to the role that identifies the object within the containing object
+(the name of the Beamline object is `beamline`).
+
+YAML-configured objects must be subclasses of the `BaseHardwareObjects.ConfiguredObject` class.
+HardwareObjects proper (which excludes e.g. `Beamline` and procedures)
+are subclasses of `BaseHardwareObjects.HadwareObjectYaml`.
+The most complete example is the Beamline object and the comments in `beamline_config.yml`
+are the best guide to the syntax of YAML configuration files.
+It is a key principle of YAML-configured classes that **all** attributes
+added in the configuration must match a pre-defined attribute coded in the class.
+This means that you can look in the class code to see which attributes are available.
+The only exception is the `_initialise_class` attribute at the start of the file.
+This dictionary contains the import name of the class that is to be created,
+and optionally parameters to be passed to the `init()` method of that class.
+The `_objects` attribute in the file gives the HardwareObjects that are contained in
+(i.e. children of) the object.
+The dictionary key is the role name, and the value is the name of the configuration file.
+Each `role_name` must match a read-only property coded in the body of the class,
+and must be added to the `__content_roles` list of the class by the class code.
+Note that classes are loaded and initialised in the order given by this list,
+so that there is a reproducible loading order.
+Contained objects can be defined as procedures, so that they are added to the list of procedures.
+Each YAML-configured class has an `_init()` method that is executed immediately after the object is created,
+and an `init()` function that is executed after configured parameters and contained objects have been loaded.
+
+### Accessing configuration data
+The Beamline object (`HardwareRepository.beamline`) is a YAML-configured object,
+and is the starting point for finding other hardware objects.
+These may in turn contain other objects, so you can do e.g.
+`HardwareRepository.beamline.detector.distance` to get the detector distance motor object.
+Configured properties are similarly accessed as simple attributes, e,g, `beamline.default_acquisition_parameters`.
+Each `ConfiguredObject` has three special properties and one function to deal with the objects contained within it.
+These are:
+
+- `all_roles`: a list of the roles (attribute names) of contained HardwareObjects, in loading order;
+- `all_objects_by_role`: an ordered dictionary of contained HardwareObjects;
+- `procedures` an ordered dictionary of HardwareObjects for procedures;
+- `replace_object()`: a method to replace an existing configured object at runtime with a new object.
+
+## XML-configured objects
+### Code and file structure
+XML-configured objects have a `name()` method,
+that returns the name of the configuration file used to specify it (without the `.xml` suffix).
+It is this name that is used in internal data structures and a number of access functions.
+
+XML-configured objects must be subclasses of the `BaseHardwareObjects.HardwareObject` class.
+A good example is `mxcubecore/configuration/mockup/detector-mockup.xml`
+(note that `hwrid` is an alias for what is normally written as `href`).
+In XML configuration contained objects are given using the "object" element,
+with the `href` attribute giving the configuration file name to pick up
+(you can use a similar syntax to redirect the topmost element to another file)
+and the `role` attribute giving the role name.
+Simple properties are given as contained XML elements,
+and complex properties (dictionaries) are given as elements of type 'object' without a 'href' attribute.
+
+
+The configuration data are kept in complex internal data structures,
+with links back to the original XML.
+
+The important methods can be found in the `BaseHardwareObjects,HardwareObjectNode` class.
+XML-configured files have no limits on the attributes or objects they can contain.
+This leads to greater flexibility, since you can add a new attribute when needed without modifying the class code;
+it also means that there is no way to check which attributes are supported without looking into the configuration files,
+and gives more scope for local and potentially conflicting implementations.
+The functions have quite complex behaviour that amounts to overloading.
+
+### Accessing configuration data
+
+The recommended way to access contained objects is through the `get_object_by_role` function,
+since it works on role names rather than the less predictable file names.
+As implemented the function will look recursively in contained objects for a given role name
+if the topmost object does not contain it.
+The `get_roles` method returns a list of roles that are defined on the object itself.
+
+You can get and set the values of simple properties by normal `obj.attr` syntax,
+which will also get you normal, non-property attributes.
+The `get_properties` method returns a dictionary of all properties and their values,
+and the `get_property` method behaves as `get_properties().get`.
+Direct setting of properties internally calls the `set_property` function,
+and this function automatically converts strings to `int`, `float` or `bool` if possible.
+
+There are additional ways of accessing contained objects.
+`get_objects` and `has_object` take as input the object name
+As currently coded (was it always thus?) the name is equal to the role name used to add the object.
+An XML-configured object is also coded to mimic a Python list and dictionary of contained objects,
+so that `anObject[ii]`
+(`ii` being an integer) returns the `ii`'th contained object,
+whereas `anObject[key]` (key being a string) returns the contained object defined by the name (i.e. the role name).
+
+For XML-configured HardwareObjects (but not for YAML:-configured ones)
+there are two additional ways of getting hold of HardwareObjects.
+
+Beamline.get_hardware_object lets you get a HO from a dotted list of rolenames (e.g. 'detector.distance')
+This is essentially a convenience function to avoid repeated get_object_by_role calls.
+For YAML-configured objects the same could be done by direct attribute access.
+
+HardwareReposotory.get_hardware_object, on the other hand,
+lets you access hardware objects by the name of the configuration file,
+loading the file if it has not been loaded already.
+Use of this function requires you to hardwire the names of configuration files in the code.

docs/source/dev/json-schema-generated-user-interface.md

@@ -10,7 +10,7 @@ We have a system for code-generated interfaces that can be plugged in to both a
 
 The system is used to generate parameter queries from mxcubecore (for now from the GΦL workflow). It includes a wide range of widgets; multiple nested layout fields; support for custom update functions triggered when values are edited, that can reset values of other fields, change widget colouring and pulldown enums; field value validation; and colouring and disabling of action for invalid values. There are several examples of pulldown menu contents being modified depending on the values of other fields. The system is started with a `PARAMETERS_NEEDED` on signals being sent from the mxcubecore side with the JSON schemas as parameters. The UI side responds by sending a `PARAMETER_RETURN_SIGNAL` with the complete dictionary of parameter values. There is also a control parameter, that determines whether the response matches a Continue or Cancel click (closing the UI popup), or whether it is a request for a UI update. In the latter case the update is sent from the mxcubecore side with a `PARAMETER_UPDATE_SIGNAL`.
 
-The form of the JSON schemas used has been agreed between Rasmus Foghy, Jean-Baptiste FLorial, an MArcus Oscarsson. Hopefully it fits with standard practices.
+The form of the JSON schemas used has been agreed between Rasmus Fogh, Jean-Baptiste Florial, and Marcus Oscarsson. Hopefully it fits with standard practices.
 
 The only really detailed documentation available is the Qt implementation, which serves as a worked example. The attached files show screenshots of the generated interfaces (with some examples of e.g. widget colouring); the JSON schemas that generate them can be found in the mxcubecore repository, in `mxcubecore/doc/json_schema_gui` as can the parameters returned, and some examples of the response to gui update requests.
 

mxcubecore/BaseHardwareObjects.py

@@ -123,7 +123,7 @@ def replace_object(self, role: str, new_object: object) -> None:
     # NB this function must be re-implemented in nested subclasses
     @property
     def all_roles(self) -> Tuple[str]:
-        """Tuple of all content object roles, indefinition and loading order
+        """Tuple of all content object roles, in definition and loading order
 
         Returns:
             Tuple[str]: Content object roles

mxcubecore/Command/exporter/ExporterClient.py

@@ -107,12 +107,22 @@ def execute(self, method, pars=None, timeout=-1):
             pars(str): parameters
             timeout(float): Timeout [s]
         """
+
+        # runScript returns results in a different way than any other
+        # comand, fix on MD side ?
+        if method == "runScript" and pars is not None:
+            pars = pars[0].split(",")
+
         cmd = "{} {} ".format(CMD_SYNC_CALL, method)
+
         if pars is not None:
-            for par in pars:
-                if isinstance(par, (list, tuple)):
-                    par = self.create_array_parameter(par)
-                cmd += str(par) + PARAMETER_SEPARATOR
+            if isinstance(pars, (list, tuple)):
+                for par in pars:
+                    if isinstance(par, (list, tuple)):
+                        par = self.create_array_parameter(par)
+                    cmd += str(par) + PARAMETER_SEPARATOR
+            else:
+                cmd += str(pars)
 
         ret = self.send_receive(cmd, timeout)
         return self.__process_return(ret)

mxcubecore/HardwareObjects/BeamlineActions.py

@@ -87,6 +87,7 @@ def __init__(self, name, hwobj):
         self.type = TWO_STATE_COMMAND_T
         self.argument_type = ARGUMENT_TYPE_LIST
         self._hwobj.connect("valueChanged", self._cmd_done)
+        self._running = False
 
     def _get_action(self):
         """Return which action has to be executed.
@@ -105,6 +106,7 @@ def __call__(self, *args, **kwargs):
         Args: None
         Kwargs: None
         """
+        self._running = True
         self.emit("commandBeginWaitReply", (str(self.name()),))
         value = self._get_action()
         self._hwobj.set_value(value, timeout=60)
@@ -122,9 +124,11 @@ def _cmd_done(self, state):
         else:
             if isinstance(res, gevent.GreenletExit):
                 self.emit("commandFailed", (str(self.name()),))
-            else:
+            elif self._running:
                 self.emit("commandReplyArrived", (str(self.name()), res))
 
+        self.running = False
+
     def value(self):
         """Return the current command vaue.
         Return:

mxcubecore/HardwareObjects/EDNACharacterisation.py

@@ -91,9 +91,6 @@ def _run_edna(self, input_file, results_file, process_directory):
                 logging.getLogger("queue_exec").info(
                     "Received characterisation results via XMLRPC"
                 )
-                # logging.getLogger("queue_exec").info([self.characterisationResult])
-                # with open("/tmp/EDNA_results.xml", "w") as f:
-                #     f.write(self.characterisationResult)
                 self.result = XSDataResultMXCuBE.parseString(
                     self.characterisationResult
                 )
@@ -265,8 +262,9 @@ def input_from_params(self, data_collection, char_params):
         for img_num in range(int(acquisition_parameters.num_images)):
             image_file = XSDataImage()
             path = XSDataString()
-            path.setValue(path_str % (img_num + 1))
-            image_file.setPath(path)
+            path.value = path_str % (img_num + 1)
+            image_file.path = path
+            image_file.number = XSDataInteger(img_num + 1)
             data_set.addImageFile(image_file)
 
         edna_input.addDataSet(data_set)
@@ -372,7 +370,11 @@ def dc_from_output(self, edna_result, reference_image_collection):
                 # and update the members the needs to be changed. Keeping
                 # the directories of the reference collection.
                 ref_pt = reference_image_collection.acquisitions[0].path_template
+
                 acq.path_template = copy.deepcopy(ref_pt)
+                acq.path_template.directory = "/".join(
+                    ref_pt.directory.split("/")[0:-2]
+                )
                 acq.path_template.wedge_prefix = "w" + str(i + 1)
                 acq.path_template.reference_image_prefix = str()
 

mxcubecore/HardwareObjects/EMBLFlexHCD.py

@@ -587,8 +587,6 @@ def _do_load(self, sample=None):
         return self._set_loaded_sample_and_prepare(loaded_sample, previous_sample)
 
     def _do_unload(self, sample=None):
-        HWR.beamline.diffractometer.set_phase("Transfer")
-
         self._execute_cmd_exporter(
             "unloadSample",
             sample.get_cell_no(),

mxcubecore/HardwareObjects/ESRF/ESRFBeam.py

@@ -100,6 +100,18 @@ def _get_aperture_size(self):
         return _size / 1000.0, _label
 
     def _get_complex_size(self):
+        """Get the size and the name of the definer in place.
+        Returns:
+            (float, str): Size [mm], label.
+        """
+        try:
+            value = self._complex.get_value()
+            return value.value[1], value.name
+        except AttributeError:
+            logging.getLogger("HWR").info("Could not read beam size")
+            return (-1, -1), "UNKNOWN"
+
+    def _get_complex_size_old(self):
         """Get the size and the name of the definer in place.
         Returns:
             (float, str): Size [mm], label.
@@ -167,7 +179,7 @@ def _beam_size_compare(size):
                 self.beam_height = _val[1]
             else:
                 self.beam_height = _val[0]
-        except ValueError:
+        except (ValueError, TypeError):
             return None, None, _shape, "none"
 
         return self.beam_width, self.beam_height, _shape, _name
@@ -201,7 +213,7 @@ def get_available_size(self):
                 "values": [_low_w, _high_w, _low_h, _high_h],
             }
 
-        return None
+        return []
 
     def _set_slits_size(self, size=None):
         """Move the slits to the desired position.