Mercurial > repos > siwaa > redelac_stics_g

changeset 14:812bb8097dac draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://forgemia.inra.fr/redelac commit d16b9ecdb4dd51b69a6aaa4ab298b82455fe0267"
author siwaa
date Tue, 12 Dec 2023 10:38:20 +0000
parents 3db1f57029a7
children 413cec4dfe5f
files redelacSticsInputGenerator.R redelacSticsInputGenerator.bash redelacSticsInputGenerator.xml
diffstat 3 files changed, 97 insertions(+), 104 deletions(-) [+]
line wrap: on
line diff
--- a/redelacSticsInputGenerator.R	Fri Oct 27 09:17:06 2023 +0000
+++ b/redelacSticsInputGenerator.R	Tue Dec 12 10:38:20 2023 +0000
@@ -3,6 +3,7 @@
 library(lubridate)
 library(parallel)
 library(doParallel)
+library(doFuture)
 
 sessionInfo()
 
@@ -21,9 +22,11 @@
 
 args = commandArgs(trailingOnly = TRUE)
 
-genUSMsXmlOneAfterOther <- (args[1] == "usmx_one_after_other")
+redelac <- (args[1] == "optim")
 parallel <- (args[2] == "parallel")
+print(paste("parallel:", parallel, args[2]))
 nbSlots <- as.integer(args[3])
+print(paste("nbslots:", nbSlots))
 
 # USMs file loading
 
@@ -72,92 +75,37 @@
 
 gen_sta_xml(param_df = sta_param, out_dir = workspace)
 
-if (genUSMsXmlOneAfterOther) {
-
-  durGenX <- lubridate::as.duration(0)
-  durGenT <- lubridate::as.duration(0)
-
-  if (parallel) {
-    cl <- makeCluster(nbSlots)
-    print(cl)
-    registerDoParallel(cl)
-    `%dordopar%` <- `%dopar%`
-  } else {
-    `%dordopar%` <- `%do%`
-  }
-
-  # looping on USMs
-  loopRes <- foreach::foreach(i = 1:nrow(usms_param),
-    .packages = c("SticsRFiles", "lubridate")) %dordopar% {
-
-    beforeGenXmlTimeI <- Sys.time()
-
-    usmxFileName = paste0("usms", as.character(i) , ".xml")
-
-    usmxFilePath  = file.path(workspace, usmxFileName)
-
-    gen_usms_xml(file = usmxFilePath,
-                 param_df = usms_param[i, ])
-
-    afterGenXmlTimeI <- Sys.time()
-    durGenXI <-
-      lubridate::as.duration(lubridate::interval(beforeGenXmlTimeI,
-                                                 afterGenXmlTimeI))
-
-    # one after the other generating txt folder
-
-    beforeGenTxtTimeI <- Sys.time()
-
-    gen_usms_xml2txt(
-      usms_file = usmxFileName,
-      javastics = javastics_path,
-      workspace = workspace,
-      out_dir = txt_path,
-      verbose = TRUE
-    )
-
-    afterGenTxtTimeI <- Sys.time()
-    durGenTI <-
-      lubridate::as.duration(lubridate::interval(beforeGenTxtTimeI,
-                                                 afterGenTxtTimeI))
-
-    return(c(durGenXI, durGenTI))
-  }
-
-  if (parallel) stopCluster(cl)
-
-  durDS <- as.data.frame(do.call(rbind, loopRes))
-  durGenX <- sum(durDS[1])
-  durGenT <- sum(durDS[2])
-
-} else {
-
-  beforeGenXmlTime <- Sys.time()
-
-  gen_usms_xml(file = file.path(workspace, "usms.xml"),
-               param_df = usms_param)
-
-  afterGenXmlTime <- Sys.time()
-  durGenX <-
-    lubridate::as.duration(lubridate::interval(beforeGenXmlTime,
-                                               afterGenXmlTime))
-
-  beforeGenTxtTime <- Sys.time()
-
-  gen_usms_xml2txt(
-    javastics = javastics_path,
-    workspace = workspace,
-    out_dir = txt_path,
-    verbose = TRUE
-  )
-
-  afterGenTxtTime <- Sys.time()
-  durGenT <-
-    lubridate::as.duration(lubridate::interval(beforeGenTxtTime,
-                                               afterGenTxtTime))
-
+if (parallel) {
+  doFuture::registerDoFuture()
+  future::plan(multisession, workers = nbSlots)
 }
 
+beforeGenXmlTime <- Sys.time()
+
+gen_usms_xml(file = file.path(workspace, "usms.xml"),
+             param_df = usms_param)
+
+afterGenXmlTime <- Sys.time()
+durGenX <-
+  lubridate::as.duration(lubridate::interval(beforeGenXmlTime,
+                                             afterGenXmlTime))
+beforeGenTxtTime <- Sys.time()
+
+gen_usms_xml2txt(
+  javastics = javastics_path,
+  workspace = workspace,
+  out_dir = txt_path,
+  usms_file = "usms.xml",
+  redelac = redelac,
+  parallel = parallel,
+  verbose = FALSE
+)
+
+afterGenTxtTime <- Sys.time()
+durGenT <-
+  lubridate::as.duration(lubridate::interval(beforeGenTxtTime,
+                                             afterGenTxtTime))
+					     
 endTime <- Sys.time()
 
 dur <-
@@ -167,8 +115,10 @@
 
 print("REDELACSticsInputGenerator Report")
 print("=================================")
-print(paste0("number of usms: ", nrow(usms_param)))
-print(paste("usms.xml one after the other:", genUSMsXmlOneAfterOther))
+print(paste("number of usms: ", nrow(usms_param)))
+print(paste("redelac optimization:", redelac)) 
+print(paste("process in parallel:", parallel))
+print(paste("nbslots:", nbSlots))
 
 sprintf(
   'overall duration: %g jour(s) %g heure(s) %g minute(s) %g seconde(s)',
--- a/redelacSticsInputGenerator.bash	Fri Oct 27 09:17:06 2023 +0000
+++ b/redelacSticsInputGenerator.bash	Tue Dec 12 10:38:20 2023 +0000
@@ -17,10 +17,10 @@
 cp $9 JavaSTICS-1.5.1-STICS-10.0.0/WS/Ini.csv &&
 cp ${10} JavaSTICS-1.5.1-STICS-10.0.0/WS/Station.csv &&
 cd JavaSTICS-1.5.1-STICS-10.0.0 &&
-if [[ "$11" != "noAdditionalPlantOrConfig" ]]
+(if [[ "$11" != "noAdditionalPlantOrConfig" ]]
 then
-   unzip -o ${10}
-fi
+   unzip -q -o ${11}
+fi) &&
 /usr/bin/time -p -o ./bashlog Rscript --verbose $4/redelacSticsInputGenerator.R $1 $2 $3 &&
 echo "Step 1, time to run the R script:" &&
 echo "=================================" && 
--- a/redelacSticsInputGenerator.xml	Fri Oct 27 09:17:06 2023 +0000
+++ b/redelacSticsInputGenerator.xml	Tue Dec 12 10:38:20 2023 +0000
@@ -1,7 +1,7 @@
-<tool id="redelacSticsInputGenerator" name="redelacSticsInputGenerator" version="1.0.7">
+<tool id="redelacSticsInputGenerator" name="redelacSticsInputGenerator" version="1.0.8">
   <description>to run generate Stics Inputs for REDELAC</description>
   <requirements>
-    <container type="singularity">docker://registry.forgemia.inra.fr/redelac/redelac-toolbox/sticsrpacks:0faa63e7838a2b86369026bf2e628842d7af9863</container>
+    <container type="singularity">docker://registry.forgemia.inra.fr/redelac/redelac-toolbox/sticsrpacks:fc777963cf7921da512ee8b091f4cebcfd1fbe19</container>
   </requirements>
   <command detect_errors="aggressive">
     <![CDATA[
@@ -10,12 +10,12 @@
 	     #else
                 additional=noAdditionalPlantOrConfig &&
              #end if
-	     #if str( $IterUsmX.IterUsmX ) == 'usmx_one_after_other':
-                parallel=${$IterUsmX.parallel} &&
+	     #if $REDELAC.optim == 'optim':
+                parallel=${$REDELAC.parallel} &&
              #else
-                parallel=parallel &&
+                parallel=sequential &&
              #end if
-	        bash $__tool_directory__/redelacSticsInputGenerator.bash ${IterUsmX.IterUsmX} \${parallel} \${GALAXY_SLOTS:-4} $__tool_directory__ ${Workspace} ${Climat} ${USMs} ${Tec} ${Ini} ${Station} \${additional} ${GeneratedWorkspace} ${compress}
+	        bash $__tool_directory__/redelacSticsInputGenerator.bash ${REDELAC.optim} \${parallel} \${GALAXY_SLOTS:-4} $__tool_directory__ ${Workspace} ${Climat} ${USMs} ${Tec} ${Ini} ${Station} \${additional} ${GeneratedWorkspace} ${compress}
      ]]>
   </command>
   <inputs>
@@ -26,15 +26,15 @@
     <param name="Ini" type="data" format="csv" label="Select the Ini sheet (csv)"/>
     <param name="Station" type="data" format="csv" label="Select the Station sheet (csv)"/>
     <param name="additional" type="data" optional="true" format="zip" label="[optional] Select a zipped archive with folder plant (and or) config (zip file)"/>    
-    <conditional name="IterUsmX">
-      <param name="IterUsmX" type="boolean" checked="true" truevalue="usmx_one_after_other" falsevalue="usmx_at_once"/>
-      <when value="usmx_one_after_other">
-	<param name="parallel" type="boolean" checked="true" truevalue="parallel" falsevalue="sequential"/>
+    <conditional name="REDELAC">
+      <param name="optim" type="boolean" checked="true" truevalue="optim" falsevalue="noptim" label="To speed up the xml2txt conversion process by generating file only once"/>
+      <when value="optim">
+	<param name="parallel" type="boolean" checked="true" truevalue="parallel" falsevalue="sequential" label="To parallelize the xml2txt conversion process" />
       </when>
-	<when value="usmx_at_once">
+	<when value="noptim">
       </when>
     </conditional>
-    <param name="compress" type="boolean" checked="true" truevalue="-6" falsevalue="-0"/>
+    <param name="compress" type="boolean" checked="true" truevalue="-6" falsevalue="-0" label="To compress or not the output(compressing does slow down the job)"/>
   </inputs>
   <outputs>
     <data format="zip" name="GeneratedWorkspace"
@@ -43,6 +43,49 @@
   </outputs>
   <tests>
     <test>
+      <param name="optim" value="false" ftype="boolean" />
+      <param name="Workspace" value="input/sticsBaseWs.zip" ftype="zip" />
+      <param name="Climat" value="input/flatMeteo.zip" ftype="zip" />    
+      <param name="USMs" value="input/USMsSheet.csv" ftype="csv" />
+      <param name="Tec" value="input/TecSheet.csv" ftype="csv" />
+      <param name="Ini" value="input/IniSheet.csv" ftype="csv"/>
+      <param name="Station" value="input/StationsSheet.csv" ftype="csv"/>
+      <output name="GeneratedWorkspace" ftype="zip" >
+	<assert_contents>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1973-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-NA-1973-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1974-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-NA-1974-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1975-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-NA-1975-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-1976-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-1976-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-1977-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-1977-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2000-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-2000-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2001-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-2001-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2002-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-2002-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2003-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-2003-PP" />
+	  </has_archive_member>
+	  <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2004-PP\/new_travail.usm">
+	    <has_text text="806_1893-637_NPP1-8-2-PP-ref-2004-PP" />
+	  </has_archive_member>
+	</assert_contents>	  
+      </output>
+    </test>
+    <test>
       <param name="parallel" value="false" ftype="boolean" />
       <param name="Workspace" value="input/sticsBaseWs.zip" ftype="zip" />
       <param name="Climat" value="input/flatMeteo.zip" ftype="zip" />    
@@ -174,9 +217,9 @@
     </test>
   </tests>
   <help>
-*The usage of this tool is restricted to the user of the stics_authorized group.*
+*The usage of this tool is restricted to the user of the redelac_authorized group.*
 
-*To be granted, apply to the Project Stics Team and then request to SIWAA*
+*To be granted, apply to the REDELAC Project Team and then request to SIWAA*
 
 Documentation
 -------------