Mercurial > repos > siwaa > redelac_stics_g
changeset 9:2a6d7d053ba7 draft
"planemo upload for repository https://forgemia.inra.fr/redelac commit adcac73e150f1f821b4b7f7673a490dc0fadc936"
| author | siwaa |
|---|---|
| date | Mon, 25 Sep 2023 16:25:51 +0000 |
| parents | 0625e6a639ea |
| children | f707c2fb725b |
| files | redelacSticsInputGenerator.R redelacSticsInputGenerator.bash redelacSticsInputGenerator.xml test-data/input/.~lock.USMsSheet.csv# |
| diffstat | 4 files changed, 128 insertions(+), 45 deletions(-) [+] |
line wrap: on
line diff
--- a/redelacSticsInputGenerator.R Wed Aug 30 08:23:12 2023 +0000 +++ b/redelacSticsInputGenerator.R Mon Sep 25 16:25:51 2023 +0000 @@ -1,7 +1,10 @@ library(SticsRFiles) library(dplyr) library(lubridate) +library(parallel) +library(doParallel) +sessionInfo() startTime <- Sys.time() @@ -19,6 +22,8 @@ args = commandArgs(trailingOnly = TRUE) genUSMsXmlOneAfterOther <- (args[1] == "usmx_one_after_other") +parallel <- (args[2] == "parallel") +nbSlots <- as.integer(args[3]) # USMs file loading @@ -68,97 +73,120 @@ gen_sta_xml(param_df = sta_param, out_dir = workspace) if (genUSMsXmlOneAfterOther) { + durGenX <- lubridate::as.duration(0) durGenT <- lubridate::as.duration(0) - + + if (parallel) { + cl <- makeCluster(nbSlots) + print(cl) + registerDoParallel(cl) + `%dordopar%` <- `%dopar%` + } else { + `%dordopar%` <- `%do%` + } + # looping on USMs - for (i in 1:nrow(usms_param)) { - row <- usms_param[i, ] - + loopRes <- foreach::foreach(i = 1:nrow(usms_param), + .packages = c("SticsRFiles", "lubridate")) %dordopar% { + beforeGenXmlTimeI <- Sys.time() - - gen_usms_xml(file = file.path(workspace, "usms.xml"), - param_df = row) - + + usmxFileName = paste0("usms", as.character(i) , ".xml") + + usmxFilePath = file.path(workspace, usmxFileName) + + gen_usms_xml(file = usmxFilePath, + param_df = usms_param[i, ]) + afterGenXmlTimeI <- Sys.time() durGenXI <- lubridate::as.duration(lubridate::interval(beforeGenXmlTimeI, afterGenXmlTimeI)) - + # one after the other generating txt folder - + beforeGenTxtTimeI <- Sys.time() - + gen_usms_xml2txt( + usms_file = usmxFileName, javastics = javastics_path, workspace = workspace, out_dir = txt_path, verbose = TRUE ) - + afterGenTxtTimeI <- Sys.time() durGenTI <- lubridate::as.duration(lubridate::interval(beforeGenTxtTimeI, afterGenTxtTimeI)) - - durGenX <- durGenX + durGenXI - durGenT <- durGenT + durGenTI + + return(c(durGenXI, durGenTI)) } + + if (parallel) stopCluster(cl) + + durDS <- as.data.frame(do.call(rbind, loopRes)) + durGenX <- sum(durDS[1]) + durGenT <- sum(durDS[2]) + } else { + beforeGenXmlTime <- Sys.time() - + gen_usms_xml(file = file.path(workspace, "usms.xml"), param_df = usms_param) - + afterGenXmlTime <- Sys.time() durGenX <- lubridate::as.duration(lubridate::interval(beforeGenXmlTime, afterGenXmlTime)) - + beforeGenTxtTime <- Sys.time() - + gen_usms_xml2txt( javastics = javastics_path, workspace = workspace, out_dir = txt_path, verbose = TRUE ) - + afterGenTxtTime <- Sys.time() durGenT <- lubridate::as.duration(lubridate::interval(beforeGenTxtTime, afterGenTxtTime)) - + } endTime <- Sys.time() + dur <- lubridate::as.period(lubridate::as.duration(lubridate::interval(startTime, endTime))) durGenXP <- lubridate::as.period(durGenX) durGenTP <- lubridate::as.period(durGenT) -print("REDELAC Report") -print("==============") +print("REDELACSticsInputGenerator Report") +print("=================================") print(paste0("number of usms: ", nrow(usms_param))) print(paste("usms.xml one after the other:", genUSMsXmlOneAfterOther)) + sprintf( 'overall duration: %g jour(s) %g heure(s) %g minute(s) %g seconde(s)', lubridate::day(dur), lubridate::hour(dur), lubridate::minute(dur), - lubridate::second(dur) -) + lubridate::second(dur)) + sprintf( 'inside gen_usms_xml duration: %g jour(s) %g heure(s) %g minute(s) %g seconde(s)', lubridate::day(durGenXP), lubridate::hour(durGenXP), lubridate::minute(durGenXP), - lubridate::second(durGenXP) -) + lubridate::second(durGenXP)) + sprintf( 'inside gen_usms_xml2txt duration: %g jour(s) %g heure(s) %g minute(s) %g seconde(s)', lubridate::day(durGenTP), lubridate::hour(durGenTP), lubridate::minute(durGenTP), - lubridate::second(durGenTP) -) + lubridate::second(durGenTP))
--- a/redelacSticsInputGenerator.bash Wed Aug 30 08:23:12 2023 +0000 +++ b/redelacSticsInputGenerator.bash Mon Sep 25 16:25:51 2023 +0000 @@ -1,20 +1,20 @@ -cp -r /home/rstudio/JavaSTICS-1.5.1-STICS-10.0.0/JavaSTICS-1.5.1-STICS-10.0.0 . && +cp -r /home/rstudio/JavaSTICS-1.5.1-STICS-10.0.0 . && rm -rf JavaSTICS-1.5.1-STICS-10.0.0/config && -cp -r $2/config JavaSTICS-1.5.1-STICS-10.0.0 && -unzip -q $3 -d TMP && +cp -r $4/config JavaSTICS-1.5.1-STICS-10.0.0 && +unzip -q $5 -d TMP && export WSname=$(ls TMP) && mv TMP/* JavaSTICS-1.5.1-STICS-10.0.0/WS && cp JavaSTICS-1.5.1-STICS-10.0.0/config/param_gen.xml JavaSTICS-1.5.1-STICS-10.0.0/WS && cp JavaSTICS-1.5.1-STICS-10.0.0/config/param_newform.xml JavaSTICS-1.5.1-STICS-10.0.0/WS && -unzip -q $4 -d TMPbis && +unzip -q $6 -d TMPbis && mv TMPbis/*/* JavaSTICS-1.5.1-STICS-10.0.0/WS && -cp $5 JavaSTICS-1.5.1-STICS-10.0.0/WS/USMs.csv && -cp $6 JavaSTICS-1.5.1-STICS-10.0.0/WS/Tec.csv && -cp $7 JavaSTICS-1.5.1-STICS-10.0.0/WS/Ini.csv && -cp $8 JavaSTICS-1.5.1-STICS-10.0.0/WS/Station.csv && +cp $7 JavaSTICS-1.5.1-STICS-10.0.0/WS/USMs.csv && +cp $8 JavaSTICS-1.5.1-STICS-10.0.0/WS/Tec.csv && +cp $9 JavaSTICS-1.5.1-STICS-10.0.0/WS/Ini.csv && +cp ${10} JavaSTICS-1.5.1-STICS-10.0.0/WS/Station.csv && cd JavaSTICS-1.5.1-STICS-10.0.0 && -Rscript --verbose $2/redelacSticsInputGenerator.R $1 && +Rscript --verbose $4/redelacSticsInputGenerator.R $1 $2 $3 && rm WS/1* && mv WS $WSname && zip -q -r $WSname $WSname && -cp $WSname.zip $9 +cp $WSname.zip ${11}
--- a/redelacSticsInputGenerator.xml Wed Aug 30 08:23:12 2023 +0000 +++ b/redelacSticsInputGenerator.xml Mon Sep 25 16:25:51 2023 +0000 @@ -1,12 +1,17 @@ -<tool id="redelacSticsInputGenerator" name="redelacSticsInputGenerator" version="1.0.2"> +<tool id="redelacSticsInputGenerator" name="redelacSticsInputGenerator" version="1.0.4"> <description>to run generate Stics Inputs for REDELAC</description> <requirements> - <container type="singularity">docker://registry.forgemia.inra.fr/redelac/redelac-toolbox/sticsrfiles:9a2d54c0674a6b902299dd22fdd37c2f67509fa2</container> + <container type="singularity">docker://registry.forgemia.inra.fr/redelac/redelac-toolbox/sticsrpacks:b782e0b7ce3af7e7a14a87ea750145ed1c2595a0</container> </requirements> <command detect_errors="aggressive"> <![CDATA[ - bash $__tool_directory__/redelacSticsInputGenerator.bash ${IterUsmX} $__tool_directory__ ${Workspace} ${Climat} ${USMs} ${Tec} ${Ini} ${Station} ${GeneratedWorkspace} - ]]>{ + #if str( $IterUsmX.IterUsmX ) == 'usmx_one_after_other': + parallel=${$IterUsmX.parallel} && + #else + parallel=parallel && + #end if + bash $__tool_directory__/redelacSticsInputGenerator.bash ${IterUsmX.IterUsmX} \${parallel} \${GALAXY_SLOTS:-4} $__tool_directory__ ${Workspace} ${Climat} ${USMs} ${Tec} ${Ini} ${Station} ${GeneratedWorkspace} + ]]> </command> <inputs> <param name="Workspace" type="data" format="zip" label="Select a stics Workspace (zip file)"/> @@ -16,7 +21,14 @@ <param name="Ini" type="data" format="csv" label="Select the Ini sheet (csv)"/> <param name="Station" type="data" format="csv" label="Select the Station sheet (csv)"/> <param name="Tag" type="text" value="" label="Outputs tag"/> - <param name="IterUsmX" type="boolean" checked="true" truevalue="usmx_one_after_other" falsevalue="usmx_at_once"/> + <conditional name="IterUsmX"> + <param name="IterUsmX" type="boolean" checked="true" truevalue="usmx_one_after_other" falsevalue="usmx_at_once"/> + <when value="usmx_one_after_other"> + <param name="parallel" type="boolean" checked="true" truevalue="parallel" falsevalue="sequential"/> + </when> + <when value="usmx_at_once"> + </when> + </conditional> </inputs> <outputs> <data format="zip" name="GeneratedWorkspace" @@ -25,6 +37,50 @@ </outputs> <tests> <test> + <param name="parallel" value="false" ftype="boolean" /> + <param name="Workspace" value="input/sticsBaseWs.zip" ftype="zip" /> + <param name="Climat" value="input/flatMeteo.zip" ftype="zip" /> + <param name="USMs" value="input/USMsSheet.csv" ftype="csv" /> + <param name="Tec" value="input/TecSheet.csv" ftype="csv" /> + <param name="Ini" value="input/IniSheet.csv" ftype="csv"/> + <param name="Station" value="input/StationsSheet.csv" ftype="csv"/> + <output name="GeneratedWorkspace" ftype="zip" > + <assert_contents> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1973-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-NA-1973-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1974-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-NA-1974-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-NA-1975-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-NA-1975-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-1976-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-1976-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-1977-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-1977-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2000-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-2000-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2001-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-2001-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2002-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-2002-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2003-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-2003-PP" /> + </has_archive_member> + <has_archive_member path="rot8_2\/txt_files\/806_1893-637_NPP1-8-2-PP-ref-2004-PP\/new_travail.usm"> + <has_text text="806_1893-637_NPP1-8-2-PP-ref-2004-PP" /> + </has_archive_member> + </assert_contents> + </output> + </test> + <test> + <param name="parallel" value="true" ftype="boolean" /> <param name="Workspace" value="input/sticsBaseWs.zip" ftype="zip" /> <param name="Climat" value="input/flatMeteo.zip" ftype="zip" /> <param name="USMs" value="input/USMsSheet.csv" ftype="csv" />