canu: Improvements to SLURM resource limit control

[Jason Bacon <bacon%NetBSD.org@localhost>]

Module Name:	pkgsrc-wip
Committed By:	Jason Bacon <bacon%NetBSD.org@localhost>
Pushed By:	outpaddling
Date:		Tue May 29 14:40:58 2018 -0500
Changeset:	4000a5adfcdb12dcec4dd1f697875baa9779e695

Modified Files:
	canu/distinfo
	canu/patches/patch-pipelines_canu_Defaults.pm
	canu/patches/patch-pipelines_canu_Execution.pm

Log Message:
canu: Improvements to SLURM resource limit control

To see a diff of this commit:
https://wip.pkgsrc.org/cgi-bin/gitweb.cgi?p=pkgsrc-wip.git;a=commitdiff;h=4000a5adfcdb12dcec4dd1f697875baa9779e695

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

diffstat:
 canu/distinfo                                  |  4 +-
 canu/patches/patch-pipelines_canu_Defaults.pm  | 16 ++++++--
 canu/patches/patch-pipelines_canu_Execution.pm | 57 +++++++++++++++++++++++---
 3 files changed, 67 insertions(+), 10 deletions(-)

diffs:
diff --git a/canu/distinfo b/canu/distinfo
index 0cc03f7fef..522ae55c54 100644
--- a/canu/distinfo
+++ b/canu/distinfo
@@ -4,5 +4,5 @@ SHA1 (canu-1.7.tar.gz) = aaaafd8516be5e4d725af407bc84b3c6f178b2ac
 RMD160 (canu-1.7.tar.gz) = bd394bb6dcebd43d2793ca1aa9bd60ec6be75beb
 SHA512 (canu-1.7.tar.gz) = dbc1dca4df4117521143843bb781227f4dfde5ba7068e47b51ccf9e9dba45eb2629b5773531e4f362f1df05eab3f1a62088c5fa231d3624af23f11d97e7a4edf
 Size (canu-1.7.tar.gz) = 2533016 bytes
-SHA1 (patch-pipelines_canu_Defaults.pm) = e6136febb18e50c90f4935248ee84c0d18199fb1
-SHA1 (patch-pipelines_canu_Execution.pm) = 265a8a9794a879bc80624921ecb6ebf6e4a40b53
+SHA1 (patch-pipelines_canu_Defaults.pm) = 7364fae68e26140b31e89c5a7cd238d0922e5310
+SHA1 (patch-pipelines_canu_Execution.pm) = 91ba9e00a61af96a911c7601689b97e4fc9a725a
diff --git a/canu/patches/patch-pipelines_canu_Defaults.pm b/canu/patches/patch-pipelines_canu_Defaults.pm
index aba13962c4..8375d8709c 100644
--- a/canu/patches/patch-pipelines_canu_Defaults.pm
+++ b/canu/patches/patch-pipelines_canu_Defaults.pm
@@ -1,11 +1,21 @@
+$NetBSD$
+
+# Add slurm job limits
+
 --- pipelines/canu/Defaults.pm.orig	2018-02-27 13:46:07 UTC
 +++ pipelines/canu/Defaults.pm
-@@ -637,6 +637,9 @@ sub setExecDefaults ($$) {
+@@ -637,6 +637,15 @@ sub setExecDefaults ($$) {
      my $tag         = shift @_;
      my $name        = shift @_;
  
-+    $global{"slurmArrayConcurrentTasks"}  = undef;
-+    $synops{"slurmArrayConcurrentTasks"}  = "Max tasks that can run at once";
++    $global{"slurmCormhapTaskLimit"}  = undef;
++    $synops{"slurmCormhapTaskLimit"}  = "Max corhmap tasks that can run at once";
++
++    $global{"slurmArrayTaskLimit"}  = undef;
++    $synops{"slurmArrayTaskLimit"}  = "Max tasks that can run at once";
++
++    $global{"slurmArrayCoreLimit"}  = undef;
++    $synops{"slurmArrayCoreLimit"}  = "Max cores we can allocate at once";
 +
      $global{"gridOptions${tag}"}   = undef;
      $synops{"gridOptions${tag}"}   = "Grid engine options applied to $name jobs";
diff --git a/canu/patches/patch-pipelines_canu_Execution.pm b/canu/patches/patch-pipelines_canu_Execution.pm
index 5b709d0eb8..77415e4dce 100644
--- a/canu/patches/patch-pipelines_canu_Execution.pm
+++ b/canu/patches/patch-pipelines_canu_Execution.pm
@@ -1,3 +1,7 @@
+$NetBSD$
+
+# Add slurm job limits
+
 --- pipelines/canu/Execution.pm.orig	2018-02-27 13:46:07 UTC
 +++ pipelines/canu/Execution.pm
 @@ -303,10 +303,6 @@ sub skipStage ($$@) {
@@ -11,13 +15,56 @@
      return($installDir);
  }
  
-@@ -727,6 +723,9 @@ sub buildGridArray ($$$$) {
+@@ -694,8 +690,8 @@ sub submitScript ($$) {
  
-     $opt =~ s/ARRAY_NAME/$name/g;        #  Replace ARRAY_NAME with 'job name'
-     $opt =~ s/ARRAY_JOBS/$bgn-$end/g;    #  Replace ARRAY_JOBS with 'bgn-end'
-+    if ( defined(getGlobal("slurmArrayConcurrentTasks")) ) {
-+        $opt = $opt . '%' . getGlobal("slurmArrayConcurrentTasks");
+ 
+ 
+-sub buildGridArray ($$$$) {
+-    my ($name, $bgn, $end, $opt) = @_;
++sub buildGridArray (@) {
++    my ( $name, $bgn, $end, $opt, $thr ) = @_;
+     my  $off = 0;
+ 
+     #  In some grids (SGE)   this is the maximum size of an array job.
+@@ -725,8 +721,30 @@ sub buildGridArray ($$$$) {
+         $off = "-F \"$off\"";
+     }
+ 
+-    $opt =~ s/ARRAY_NAME/$name/g;        #  Replace ARRAY_NAME with 'job name'
+-    $opt =~ s/ARRAY_JOBS/$bgn-$end/g;    #  Replace ARRAY_JOBS with 'bgn-end'
++    if( $opt =~ m/(ARRAY_NAME)/ )
++    {
++       $opt =~ s/$1/$name/; # Replace ARRAY_NAME with 'job name'
++    }
++    elsif( $opt =~ m/(ARRAY_JOBS)/ )
++    {
++       $opt =~ s/$1/$bgn-$end/; # Replace ARRAY_JOBS with 'bgn-end'
++
++       if( lc( getGlobal( 'gridEngine' ) ) eq 'slurm' && $end > 1 )
++       {
++           if( $name =~ m/mhap/i && defined getGlobal( 'slurmCormhapTaskLimit' ) )
++           {
++               $opt .= '%' . getGlobal( 'slurmCormhapTaskLimit' );
++           }
++           elsif( defined getGlobal( 'slurmArrayTaskLimit' ) )
++           {
++               $opt .= '%' . getGlobal( 'slurmArrayTaskLimit' );
++           }
++           elsif( defined getGlobal( 'slurmArrayCoreLimit' ) )
++           {
++               $opt .= '%' . int( getGlobal( 'slurmArrayCoreLimit' ) / $thr );
++           }
++       }
 +    }
  
      return($opt, $off);
  }
+@@ -870,7 +888,7 @@ sub buildGridJob ($$$$$$$$$) {
+     my $jobNameT               = makeUniqueJobName($jobType, $asm);
+ 
+     my ($jobName,  $jobOff)    = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayName"));
+-    my ($arrayOpt, $arrayOff)  = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayOption"));
++    my ( $arrayOpt, $arrayOff ) = buildGridArray( $jobNameT, $bgnJob, $endJob, getGlobal( "gridEngineArrayOption" ), $thr );
+ 
+     my $outputOption           = buildOutputOption($path, $script);
+