canu: Add resource controls for SLURM

[Jason Bacon <bacon%NetBSD.org@localhost>]

Module Name:	pkgsrc-wip
Committed By:	Jason Bacon <bacon%NetBSD.org@localhost>
Pushed By:	outpaddling
Date:		Thu Jun 28 11:57:36 2018 -0500
Changeset:	75e20c7fc8b6feaceb60b8f7d3e2df5aba6c01df

Modified Files:
	canu/Makefile
	canu/distinfo
	canu/patches/patch-pipelines_canu_Defaults.pm
	canu/patches/patch-pipelines_canu_Execution.pm

Log Message:
canu: Add resource controls for SLURM

To see a diff of this commit:
https://wip.pkgsrc.org/cgi-bin/gitweb.cgi?p=pkgsrc-wip.git;a=commitdiff;h=75e20c7fc8b6feaceb60b8f7d3e2df5aba6c01df

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

diffstat:
 canu/Makefile                                  |  2 +-
 canu/distinfo                                  | 12 +++---
 canu/patches/patch-pipelines_canu_Defaults.pm  | 29 +++++++-------
 canu/patches/patch-pipelines_canu_Execution.pm | 54 ++++++++++++++++----------
 4 files changed, 55 insertions(+), 42 deletions(-)

diffs:
diff --git a/canu/Makefile b/canu/Makefile
index 4f16d4bc29..bc20404dc7 100644
--- a/canu/Makefile
+++ b/canu/Makefile
@@ -61,7 +61,7 @@ GCC_REQD+=	4.8
 WRKSRC=		${WRKDIR}/canu-${PV}/src
 MAKE_ENV+=	DESTDIR=${WRKSRC}
 
-PV=		1.7
+PV=		1.7.1
 
 TMP_INST=		${WRKSRC}${PREFIX}/${OPSYS}-${MACHINE_ARCH:S/x86_64/amd64/}
 SITE_PERL_REL=		lib/perl5/site_perl
diff --git a/canu/distinfo b/canu/distinfo
index 522ae55c54..e35f25e2cc 100644
--- a/canu/distinfo
+++ b/canu/distinfo
@@ -1,8 +1,8 @@
 $NetBSD$
 
-SHA1 (canu-1.7.tar.gz) = aaaafd8516be5e4d725af407bc84b3c6f178b2ac
-RMD160 (canu-1.7.tar.gz) = bd394bb6dcebd43d2793ca1aa9bd60ec6be75beb
-SHA512 (canu-1.7.tar.gz) = dbc1dca4df4117521143843bb781227f4dfde5ba7068e47b51ccf9e9dba45eb2629b5773531e4f362f1df05eab3f1a62088c5fa231d3624af23f11d97e7a4edf
-Size (canu-1.7.tar.gz) = 2533016 bytes
-SHA1 (patch-pipelines_canu_Defaults.pm) = 7364fae68e26140b31e89c5a7cd238d0922e5310
-SHA1 (patch-pipelines_canu_Execution.pm) = 91ba9e00a61af96a911c7601689b97e4fc9a725a
+SHA1 (canu-1.7.1.tar.gz) = 9f3d0d07d9cf3b5430d4321f152565cc5196cb86
+RMD160 (canu-1.7.1.tar.gz) = 14f3b48584e2ffce8de3910f348fb0b5802910e6
+SHA512 (canu-1.7.1.tar.gz) = 50b2219b284292248b001c72adf80dee2fde375bc1b84e6007842b89f0a5995b862cba1dcaf40284a8f77719da035a98b36005e2958cd4397553c0878c38910c
+Size (canu-1.7.1.tar.gz) = 2535055 bytes
+SHA1 (patch-pipelines_canu_Defaults.pm) = 987c1e370bc81381824f86001f70da66ea251524
+SHA1 (patch-pipelines_canu_Execution.pm) = 4c8ad9a3be08a8e5885e3ef35dca33e85c344547
diff --git a/canu/patches/patch-pipelines_canu_Defaults.pm b/canu/patches/patch-pipelines_canu_Defaults.pm
index 8375d8709c..0d67834a66 100644
--- a/canu/patches/patch-pipelines_canu_Defaults.pm
+++ b/canu/patches/patch-pipelines_canu_Defaults.pm
@@ -1,22 +1,23 @@
 $NetBSD$
 
-# Add slurm job limits
+# Add resource limits for SLURM
 
---- pipelines/canu/Defaults.pm.orig	2018-02-27 13:46:07 UTC
+--- pipelines/canu/Defaults.pm.orig	2018-06-22 08:20:52.000000000 +0000
 +++ pipelines/canu/Defaults.pm
-@@ -637,6 +637,15 @@ sub setExecDefaults ($$) {
-     my $tag         = shift @_;
-     my $name        = shift @_;
+@@ -812,6 +812,16 @@ sub setDefaults () {
+     setDefault("gridEngineArraySubmitID",             undef, "Grid engine configuration, not documented");
+     setDefault("gridEngineJobID",                     undef, "Grid engine configuration, not documented");
  
-+    $global{"slurmCormhapTaskLimit"}  = undef;
-+    $synops{"slurmCormhapTaskLimit"}  = "Max corhmap tasks that can run at once";
++    #####  Slurm-specific parameters for controlling the number of
++    #####  cores / tasks dispatched per step or globally (WIP)
 +
-+    $global{"slurmArrayTaskLimit"}  = undef;
-+    $synops{"slurmArrayTaskLimit"}  = "Max tasks that can run at once";
++    setDefault( 'slurmCormhapCoreLimit', undef, 'Maximum number of cores allocated for MHAP pre-computing and alignment within the correction phase' );
++    setDefault( 'slurmOvbCoreLimit', undef, 'Maximum number of single-core tasks dispatched for the ovlStore bucketizing step within the trimming phase' );
++    setDefault( 'slurmOvsCoreLimit', undef, 'Maximum number of single-core tasks dispatched for the ovlStore sorting step within the trimming phase' );
++    setDefault( 'slurmRedCoreLimit', undef, 'Maximum number of cores allocated for read error detection within the unitigging phase' );
++    setDefault( 'slurmArrayTaskLimit', undef, 'Maximum number of tasks permitted for each step throughout assembly' );
++    setDefault( 'slurmArrayCoreLimit', undef, 'Maximum number of cores allocated for each step throughout assembly' );
 +
-+    $global{"slurmArrayCoreLimit"}  = undef;
-+    $synops{"slurmArrayCoreLimit"}  = "Max cores we can allocate at once";
-+
-     $global{"gridOptions${tag}"}   = undef;
-     $synops{"gridOptions${tag}"}   = "Grid engine options applied to $name jobs";
+     #####  Grid Engine Pipeline
  
+     setDefault("useGrid", 1, "If 'true', enable grid-based execution; if 'false', run all jobs on the local machine; if 'remote', create jobs for grid execution but do not submit; default 'true'");
diff --git a/canu/patches/patch-pipelines_canu_Execution.pm b/canu/patches/patch-pipelines_canu_Execution.pm
index 77415e4dce..5bd8ee1624 100644
--- a/canu/patches/patch-pipelines_canu_Execution.pm
+++ b/canu/patches/patch-pipelines_canu_Execution.pm
@@ -1,8 +1,8 @@
 $NetBSD$
 
-# Add slurm job limits
+# Add resource limits for SLURM
 
---- pipelines/canu/Execution.pm.orig	2018-02-27 13:46:07 UTC
+--- pipelines/canu/Execution.pm.orig	2018-06-22 08:20:52.000000000 +0000
 +++ pipelines/canu/Execution.pm
 @@ -303,10 +303,6 @@ sub skipStage ($$@) {
  sub getInstallDirectory () {
@@ -26,7 +26,7 @@ $NetBSD$
      my  $off = 0;
  
      #  In some grids (SGE)   this is the maximum size of an array job.
-@@ -725,8 +721,30 @@ sub buildGridArray ($$$$) {
+@@ -725,8 +721,42 @@ sub buildGridArray ($$$$) {
          $off = "-F \"$off\"";
      }
  
@@ -34,32 +34,44 @@ $NetBSD$
 -    $opt =~ s/ARRAY_JOBS/$bgn-$end/g;    #  Replace ARRAY_JOBS with 'bgn-end'
 +    if( $opt =~ m/(ARRAY_NAME)/ )
 +    {
-+       $opt =~ s/$1/$name/; # Replace ARRAY_NAME with 'job name'
++	$opt =~ s/$1/$name/; # Replace ARRAY_NAME with 'job name'
 +    }
 +    elsif( $opt =~ m/(ARRAY_JOBS)/ )
 +    {
-+       $opt =~ s/$1/$bgn-$end/; # Replace ARRAY_JOBS with 'bgn-end'
++	$opt =~ s/$1/$bgn-$end/; # Replace ARRAY_JOBS with 'bgn-end'
 +
-+       if( lc( getGlobal( 'gridEngine' ) ) eq 'slurm' && $end > 1 )
-+       {
-+           if( $name =~ m/mhap/i && defined getGlobal( 'slurmCormhapTaskLimit' ) )
-+           {
-+               $opt .= '%' . getGlobal( 'slurmCormhapTaskLimit' );
-+           }
-+           elsif( defined getGlobal( 'slurmArrayTaskLimit' ) )
-+           {
-+               $opt .= '%' . getGlobal( 'slurmArrayTaskLimit' );
-+           }
-+           elsif( defined getGlobal( 'slurmArrayCoreLimit' ) )
-+           {
-+               $opt .= '%' . int( getGlobal( 'slurmArrayCoreLimit' ) / $thr );
-+           }
-+       }
++	if( lc( getGlobal( 'gridEngine' ) ) eq 'slurm' && $end > 1 )
++	{
++	    if( $name =~ m/^cormhap_/i && defined getGlobal( 'slurmCormhapCoreLimit' ) )
++	    {
++		$opt .= '%' . int( getGlobal( 'slurmCormhapCoreLimit' ) / $thr );
++	    }
++	    elsif( $name =~ m/^ovb_/i && defined getGlobal( 'slurmOvbCoreLimit' ) )
++	    {
++		$opt .= '%' . getGlobal( 'slurmOvbCoreLimit' );
++	    }
++	    elsif( $name =~ m/^ovs_/i && defined getGlobal( 'slurmOvsCoreLimit' ) )
++	    {
++		$opt .= '%' . getGlobal( 'slurmOvsCoreLimit' );
++	    }
++	    elsif( $name =~ m/^red_/i && defined getGlobal( 'slurmRedCoreLimit' ) )
++	    {
++		$opt .= '%' . int( getGlobal( 'slurmRedCoreLimit' ) / $thr );
++	    }
++	    elsif( defined getGlobal( 'slurmArrayTaskLimit' ) )
++	    {
++		$opt .= '%' . getGlobal( 'slurmArrayTaskLimit' );
++	    }
++	    elsif( defined getGlobal( 'slurmArrayCoreLimit' ) )
++	    {
++		$opt .= '%' . int( getGlobal( 'slurmArrayCoreLimit' ) / $thr );
++	    }
++	}
 +    }
  
      return($opt, $off);
  }
-@@ -870,7 +888,7 @@ sub buildGridJob ($$$$$$$$$) {
+@@ -870,7 +900,7 @@ sub buildGridJob ($$$$$$$$$) {
      my $jobNameT               = makeUniqueJobName($jobType, $asm);
  
      my ($jobName,  $jobOff)    = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayName"));