Mercurial > repos > siwaa > carboseq_s
annotate csopraSplitInput.py @ 11:097985b1b7c6 draft
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/f3192a3a79e9265b088c9c126972718cfb375643/toos/CarboSeqSimulator commit f3192a3a79e9265b088c9c126972718cfb375643-dirty"
| author | siwaa |
|---|---|
| date | Thu, 19 Sep 2024 11:11:46 +0000 |
| parents | 20886bc40659 |
| children | e455667fe22f |
| rev | line source |
|---|---|
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
1 import pandas as pd |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
2 import os |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
3 import shutil |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
4 import math |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
5 import sys |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
6 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
7 def split_csv(input_folder, output_folder, max_chunck_size): |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
8 unit_path = input_folder + "/units.csv" |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
9 soil_path = input_folder + "/soil.csv" |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
10 crop_path = input_folder + "/crop.csv" |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
11 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
12 os.mkdir(output_folder) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
13 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
14 dfs = pd.read_csv(soil_path) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
15 dfc = pd.read_csv(crop_path) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
16 |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
17 dfs['ID'] = dfs['ID'].astype('int') |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
18 dfc['ID'] = dfc['ID'].astype('int') |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
19 |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
20 IDs = dfs['ID'].unique() |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
21 |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
22 max_ID = len(IDs) |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
23 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
24 step = max_chunck_size |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
25 |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
26 nb_split = math.ceil(max_ID / max_chunck_size) |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
27 |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
28 for i in range(1, nb_split + 1): |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
29 |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
30 firstID = IDs[(i-1) * step] |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
31 lastID = IDs[min((i * step) - 1 , max_ID - 1)] |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
32 |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
33 splitIDs = IDs[(i-1) * step : min((i * step) , max_ID)] |
|
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
34 |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
35 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
36 dfSplit = pd.DataFrame() |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
37 dfSplit = dfs[dfs['ID'].isin(splitIDs)] |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
38 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
39 dfcSplit = pd.DataFrame() |
|
10
20886bc40659
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf/toos/CarboSeqSimulator commit b1cafd3eb61b631ee0b1f8f6c5ef2f9a0e2b4ccf-dirty"
siwaa
parents:
5
diff
changeset
|
40 dfcSplit = dfc[dfc['ID'].isin(splitIDs)] |
|
5
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
41 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
42 os.mkdir(os.path.join(output_folder, f"{input_folder}_{i}")) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
43 shutil.copy(unit_path, os.path.join(output_folder, f"{input_folder}_{i}")) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
44 dfSplit.to_csv(os.path.join(output_folder, f"{input_folder}_{i}", "soil.csv"), index=False) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
45 dfcSplit.to_csv(os.path.join(output_folder, f"{input_folder}_{i}", "crop.csv"), index=False) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
46 shutil.make_archive( os.path.join(output_folder,f"{input_folder}_{i}"), 'zip', os.path.join(output_folder, f"{input_folder}_{i}")) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
47 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
48 # Example usage |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
49 input_folder = "input" |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
50 output_folder = "outputs" |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
51 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
52 input_file = sys.argv[1] |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
53 max_chunck_size = int(sys.argv[2]) |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
54 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
55 shutil.unpack_archive(input_file, input_folder, 'zip') |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
56 |
|
3e7978af2ba2
"planemo upload for repository https://forgemia.inra.fr/carboseq/record-projet-carboseq/-/tree/5f761ea7dab0a6f8c978cbbe4946e16edc6d032e/toos/CarboSeqSimulator commit 5f761ea7dab0a6f8c978cbbe4946e16edc6d032e-dirty"
siwaa
parents:
diff
changeset
|
57 split_csv(input_folder, output_folder, max_chunck_size) |