DM-53494: Add code needed to run submit processes as jobs.

doc/changes/DM-53494.feature.rst

@@ -0,0 +1 @@
+Added capabilities needed to run submit processes as batch jobs on same cluster with shared filesystems.  This included handling of lazy subdags.

doc/lsst.ctrl.bps.htcondor/userguide.rst

@@ -195,6 +195,12 @@ at the root level or inside a ``site`` section, but not inside a ``pipetask``,
 If your main workflow contains sub-workflow defined in individual DAG
 description files, they will use the same configuration as the main workflow.
 
+Miscellaneous
+^^^^^^^^^^^^^
+
+* ``saveHTCdot`` - true/false.  Whether condor_dagman outputs a DOT
+  representation of the workflow DAG.
+
 .. __: https://htcondor.readthedocs.io/en/latest/admin-manual/configuration-macros.html#dagman-configuration-file-entries
 
 .. .. _htc-plugin-authenticating:
@@ -601,6 +607,22 @@ For more information about expressions, see HTCondor documentation:
    2 held <submit dir>/*.nodes.log``.
 
 
+.. _htc_submit_as_batch:
+
+Submit Stages as Batch Jobs
+---------------------------
+
+When BPS uses batch jobs for the submission process, there isn't much
+different in how HTC handles them vs the payload jobs.  There will be
+a ``jobs`` subdir for each of them which will contain the standard
+output/error and HTCondor job log.  If you have to provision resources
+for the payload jobs, these new jobs will also need resources provisioned.
+
+Noticable differences in the HTCondor-level details include a second
+``*.dag`` file in the submit directory and a second ``condor_dagman``
+job in the queue.  This is subDAG handles the payload workflow and
+isn't created until the ``preparePayloadWorkflow`` job executes.
+
 .. _htc-plugin-troubleshooting:
 
 Troubleshooting

python/lsst/ctrl/bps/htcondor/check_group_status.sh

@@ -12,6 +12,6 @@ if ! [ -f "$1" ]; then
     exit 1
 fi
 
-group_status=$(<"$1")
+group_status=$(/usr/bin/cat "$1")
 echo "Read status ${group_status} from group status file ($1)" >&2
 exit ${group_status}

python/lsst/ctrl/bps/htcondor/common_utils.py

@@ -229,6 +229,7 @@ def _wms_id_to_cluster(wms_id):
     schedd_ad = None
     cluster_id = None
     id_type = _wms_id_type(wms_id)
+    _LOG.debug("id_type = %s", id_type.name)
     if id_type == WmsIdType.LOCAL:
         schedd_ad = coll.locate(htcondor.DaemonTypes.Schedd)
         cluster_id = int(float(wms_id))
@@ -244,7 +245,7 @@ def _wms_id_to_cluster(wms_id):
             cluster_id = int(float(job_id))
     elif id_type == WmsIdType.PATH:
         try:
-            job_info = read_dag_info(wms_id)
+            _, job_info = read_dag_info(wms_id)
         except (FileNotFoundError, PermissionError, OSError):
             pass
         else:

python/lsst/ctrl/bps/htcondor/dagman_configurator.py

@@ -66,6 +66,7 @@
         "dagman_debug": (str, ""),
         "dagman_node_record_info": (str, ""),
         "dagman_record_machine_attrs": (str, ""),
+        "dagman_manager_job_append_getenv": (str, ""),
     }
 )
 

python/lsst/ctrl/bps/htcondor/etc/htcondor_defaults.yaml

@@ -56,3 +56,60 @@ wmsConfig:
   # A boolean flag controlling whether DAGMan should generate submit files for
   # nested DAGs automatically.
   DAGMAN_GENERATE_SUBDAG_SUBMITS: true
+
+bpsMakeCommand: true
+#payloadCommand: >
+#  {setupEnv}
+#  logDir={jobLogDir};
+#  pwd;
+#  {gwjobCommand} & pJob=$!;
+#  prmon -i 5
+#  -f ${logDir}/memory_monitor_output.txt
+#  -j ${logDir}/memory_monitor_summary.json
+#  -p $pJob & mJob=$!;
+#  wait $pJob;
+#  ret=$?;
+#  wait $mJob;
+#  {jobCleanup}
+#  exit $ret;
+
+payloadCommand: >
+  echo -n BPS Beginning of job commands:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+  {setupEnv}
+  logDir={jobLogDir};
+  pwd;
+  echo -n BPS Before running command:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+  echo ========================================;
+  {gwjobCommand};
+  ret=$?;
+  echo ========================================;
+  echo -n BPS After running command:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+  echo Command exited with code: $ret;
+  {jobCleanup}
+  echo -n BPS End of job commands:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+  exit $ret;
+
+
+# Job environment setup
+softwarePath: "/cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/{LSST_VERSION}"
+
+customEnvSetup: ""
+setupEnv: >
+  unset PYTHONPATH;
+  echo Using stack in {softwarePath};
+  source {softwarePath}/loadLSST.bash;
+  setup lsst_distrib;
+  echo -n BPS After setup of lsst_distrib:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+  {customEnvSetup}
+  echo -n BPS After custom env setup:\ ;
+  /usr/bin/date +%Y-%m-%dT%H:%M:%S.%N%:z;
+
+# Other job variables
+jobInitDir: "`pwd`"
+jobLogDir: "{jobInitDir}"
+jobCleanup: ""

python/lsst/ctrl/bps/htcondor/htcondor_service.py

@@ -59,6 +59,7 @@
     htc_create_submit_from_file,
     htc_submit_dag,
     htc_version,
+    read_dag_info,
     read_dag_status,
     write_dag_info,
 )
@@ -180,7 +181,7 @@ def submit(self, workflow, **kwargs):
             _LOG.info("Submitting from directory: %s", os.getcwd())
             schedd_dag_info = htc_submit_dag(sub)
             if schedd_dag_info:
-                write_dag_info(f"{dag.name}.info.json", schedd_dag_info)
+                write_dag_info(schedd_dag_info)
 
                 _, dag_info = schedd_dag_info.popitem()
                 _, dag_ad = dag_info.popitem()
@@ -256,6 +257,14 @@ def restart(self, wms_workflow_id):
                 "Cannot determine the execution status of the workflow, continuing with restart regardless"
             )
 
+        # In the case of lazy DAGs, workflow summaries can change at
+        # runtime.  So read the workflow's info.json file before moving
+        # it to backup dir and use to update the summaries later before
+        # writing the new info.json file.
+        dag_info_filename, old_dag_schedd_info = read_dag_info(wms_path)
+        old_dag_info = next(iter(old_dag_schedd_info.values()))
+        old_dag_ad = next(iter(old_dag_info.values()))
+
         _LOG.info("Backing up select HTCondor files from previous run attempt")
         rescue_file = htc_backup_files(wms_path, subdir="backups")
         if (wms_path / "subdags").exists():
@@ -280,7 +289,12 @@ def restart(self, wms_workflow_id):
                 if schedd_dag_info:
                     dag_info = next(iter(schedd_dag_info.values()))
                     dag_ad = next(iter(dag_info.values()))
-                    write_dag_info(f"{dag_ad['bps_run']}.info.json", schedd_dag_info)
+
+                    # Just in case lazy DAGs, update the summaries.
+                    dag_ad["bps_job_summary"] = old_dag_ad["bps_job_summary"]
+                    dag_ad["bps_run_quanta"] = old_dag_ad["bps_run_quanta"]
+
+                    write_dag_info(schedd_dag_info, dag_info_filename)
                     run_id = f"{dag_ad['ClusterId']}.{dag_ad['ProcId']}"
                     run_name = dag_ad["bps_run"]
                 else:
@@ -583,3 +597,27 @@ def ping(self, pass_thru):
             status = 1
             message = f"Permission problem with {daemon_type} service."
         return status, message
+
+    def run_submission_checks(self):
+        """Check to run at start if running WMS specific submission steps.
+
+        Any exception other than NotImplementedError will halt submission.
+        Submit directory may not yet exist when this is called.
+        """
+        # Some early config sanity checks
+        found, value = self.config.search("bpsMakeCommand")
+        bps_make_command = value if found else True
+        if not bps_make_command:
+            found, value = self.config.search("payloadCommand", opt={"replaceVars": False})
+            if not found:
+                raise KeyError("Missing 'payloadCommand' in config while bpsMakeCommand=True")
+
+            if "setupEnv" in value:
+                found, value = self.config.search("setupEnv", opt={"replaceVars": False})
+                if not found:
+                    raise KeyError("Missing 'setupEnv' in config, but appears in payloadCommand")
+
+                if "lsstVersion" in value:
+                    found, value = self.config.search("lsstVersion", opt={"replaceVars": False})
+                    if not found:
+                        raise KeyError("Missing 'lsstVersion' in config, but appears in setupEnv in config")

python/lsst/ctrl/bps/htcondor/htcondor_workflow.py

@@ -35,9 +35,10 @@
 
 from lsst.ctrl.bps import (
     BaseWmsWorkflow,
+    BpsConfig,
 )
 
-from .prepare_utils import _generic_workflow_to_htcondor_dag
+from .prepare_utils import _generic_workflow_to_htcondor_dag, _update_job_summary
 
 _LOG = logging.getLogger(__name__)
 
@@ -87,3 +88,17 @@ def write(self, out_prefix):
 
         # Write down the workflow in HTCondor format.
         self.dag.write(out_prefix, job_subdir="jobs/{self.label}")
+
+    def add_to_parent_workflow(self, config: BpsConfig) -> None:
+        """Add self to parent workflow.
+
+        Parameters
+        ----------
+        config : `lsst.ctrl.bps.BpsConfig`
+            Configuration.
+        """
+        _update_job_summary(
+            self.name,
+            self.dag.graph["attr"]["bps_job_summary"],
+            config[".bps_defined.submitPath"],
+        )