Mercurial > repos > siwaa > wine_quality_train_eval

comparison fidle__init__.py @ 13:2f6c77865180 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://forgemia.inra.fr/nathalie.rousse/use/-/tree/dnn/DNN/galaxy-tools/wine_quality_train_eval commit 1906137478520a9f71a4d8996c0a8827f27aed16-dirty
author siwaa
date Thu, 05 Dec 2024 18:28:28 +0000
parents
children
comparison
equal deleted inserted replaced
12:60778af2dd78 13:2f6c77865180
1 # ==================================================================
2 # ______ _ _ _ __ __ _
3 # | ____(_) | | | | \/ | | |
4 # | |__ _ __| | | ___ | \ / | ___ __| |
5 # | __| | |/ _` | |/ _ \ | |\/| |/ _ \ / _` |
6 # | | | | (_| | | __/ | | | | (_) | (_| |
7 # |_| |_|\__,_|_|\___| |_| |_|\___/ \__,_|
8 #
9 # ==================================================================
10 # A simple module to host usefull functions for Fidle practical work
11 # Jean-Luc Parouty CNRS/MIAI/SIMaP 2022
12 # Contributed by Achille Mbogol Touye MIAI/SIMAP 2023 (PyTorch support)
13
14 import os,sys,platform
15 import glob
16 import shutil
17 import pathlib
18
19 from random import randint
20 import matplotlib
21
22 from IPython.display import display,HTML
23
24 import fidle.config
25 import fidle.scrawler
26 import fidle.utils
27
28 from fidle.Chrono import Chrono
29
30
31 __version__ = fidle.config.VERSION
32
33 __run_id = None
34 __run_dir = None
35 __datasets_dir = None
36
37 utils = fidle.utils
38 plot = fidle.scrawler
39 config = fidle.config
40 chrono = Chrono()
41
42
43 # -------------------------------------------------------------
44 # init
45 # -------------------------------------------------------------
46 #
47 '''
48 Initialization with parameters run_id and run_dir.
49 These two parameters can be overridden via environment variables
50 FIDLE_OVERRIDE_run_id and FIDLE_OVERRIDE_run_dir.
51 datasets_dir is retrieved via the environment variable <datasets_env_var>.
52 params:
53 run_id : Run id for the notebook (random number if None) (None)
54 run_dir : Run directory (./run/{run_id} if None) (None)
55 datasets_env_var : Name of env. var. specifying the location of the datasets (FIDLE_DATASETS_DIR)
56 return:
57 run_id, run_dir, datasets_dir
58 '''
59 def init(run_id=None, run_dir=None, datasets_env_var='FIDLE_DATASETS_DIR'):
60 global __run_id
61 global __run_dir
62 global __datasets_dir
63
64 # ---- run_id
65 #
66 # If None, we choose
67 __run_id = str(randint(10000,99999)) if run_id is None else run_id
68
69 # Overrided ?
70 __run_id = __get_override_env('run_id', __run_id)
71
72 # ---- run_dir
73 #
74 # If None, we set it
75 __run_dir = f'./run/{__run_id}' if run_dir is None else run_dir
76
77 # Override run_dir ?
78 __run_dir = __get_override_env('run_dir', __run_dir)
79
80 # Create run_dir
81 utils.mkdir(__run_dir)
82
83 # ---- Parameters from config.py
84 #
85 mplstyle = config.FIDLE_MPLSTYLE
86 cssfile = config.FIDLE_CSSFILE
87
88 # ---- Load matplotlib style and css
89 #
90 module_dir = pathlib.Path(__file__).parent
91 matplotlib.style.use(f'{module_dir}/{mplstyle}')
92 __load_cssfile(f'{module_dir}/{cssfile}')
93
94 # ---- Get datasets location
95 # From env var or by looking for
96 # Raise an exception if cannot be found.
97 #
98 __datasets_dir=utils.get_datasets_dir(datasets_env_var)
99
100 # ---- Update Keras cache
101 #
102 updated_keras = __update_keras_cache()
103
104 # ---- Update torch cache
105 #
106 updated_torch = __update_torch_cache()
107
108 # ---- Tensorflow log level
109 #
110 log_level = int(os.getenv('TF_CPP_MIN_LOG_LEVEL', 0 ))
111 str_level = ['Info + Warning + Error','Warning + Error','Error only'][log_level]
112
113 # ---- Today, now and hostname
114 #
115 chrono.start('__global__')
116 h = platform.uname()
117
118 # ---- Save figs
119 #
120 save_figs = os.getenv('FIDLE_SAVE_FIGS', str(config.SAVE_FIGS) )
121 save_figs = (save_figs.lower() == 'true')
122 figs_dir = f'{__run_dir}/figs'
123
124 plot.set_save_fig( save = save_figs,
125 figs_dir = figs_dir,
126 figs_name = f'fig_{__run_id}_',
127 figs_id = 0 )
128
129 # ---- Hello world
130 #
131 utils.display_md('<br>**FIDLE - Environment initialization**')
132 print('Version :', config.VERSION)
133 print('Run id :', __run_id)
134 print('Run dir :', __run_dir)
135 print('Datasets dir :', __datasets_dir)
136 print('Start time :', chrono.get_start('__global__'))
137 print('Hostname :', f'{h[1]} ({h[0]})')
138 print('Tensorflow log level :', str_level,f' (={log_level})')
139 print('Update keras cache :', updated_keras)
140 print('Update torch cache :', updated_torch)
141 print('Save figs :', figs_dir, f'({save_figs})')
142
143 # ---- Modules versions
144 #
145 for m in config.USED_MODULES:
146 if m in sys.modules:
147 print(f'{m:21s}:', sys.modules[m].__version__)
148
149 # ---- Overrided ?
150 #
151 if run_id != __run_id:
152 print(f'\n** run_id has been overrided from {run_id} to {__run_id}')
153
154
155 return __run_id, __run_dir, __datasets_dir
156
157 # ------------------------------------------------------------------
158 # Update keras cache
159 # ------------------------------------------------------------------
160 # Try to sync ~/.keras/cache with datasets/keras_cache
161 # because sometime, we cannot access to internet... (IDRIS..)
162 #
163 def __update_keras_cache():
164 updated = False
165 if os.path.isdir(f'{__datasets_dir}/keras_cache'):
166 from_dir = f'{__datasets_dir}/keras_cache/*.*'
167 to_dir = os.path.expanduser('~/.keras/datasets')
168 utils.mkdir(to_dir)
169 for pathname in glob.glob(from_dir):
170 filename=os.path.basename(pathname)
171 destname=f'{to_dir}/{filename}'
172 if not os.path.isfile(destname):
173 shutil.copy(pathname, destname)
174 updated=True
175 return updated
176
177
178 # ------------------------------------------------------------------
179 # Update torch cache
180 # ------------------------------------------------------------------
181 # Try to sync ~/data/MNIST/raw with datasets/torch_cache
182 # because sometime, we cannot access to internet... (IDRIS..)
183 #
184
185 def __update_torch_cache():
186 updated = False
187 if os.path.isdir(f'{__datasets_dir}/torch_cache/MNIST/raw'):
188 from_dir = f'{__datasets_dir}/torch_cache/MNIST/raw/*'
189 to_dir = os.getcwd() + '/data/MNIST/raw'
190 utils.mkdir(to_dir)
191 for pathname in glob.glob(from_dir):
192 filename=os.path.basename(pathname)
193 destname=f'{to_dir}/{filename}'
194 if not os.path.isfile(destname):
195 shutil.copy(pathname, destname)
196 updated=True
197 return updated
198
199
200 # ------------------------------------------------------------------
201 # Override
202 # ------------------------------------------------------------------
203 #
204
205 def override(*names, module_name='__main__', verbose=True, return_attributes=False):
206 '''
207 Try to override attributes given par name with environment variables.
208 Environment variables name must be : FIDLE_OVERRIDE_<NAME>
209 If no env variable is available for a given name, nothing is change.
210 If type is str, substitution is done with 'notebook_id' and 'datasets_dir'
211 Example : override('image_size','nb_epochs')
212 params:
213 names : list of attributes names as a str list
214 if empty, all attributes can be override
215 return :
216 dict {name=new value}
217 '''
218 # ---- Where to override
219 #
220 module=sys.modules[module_name]
221
222 # ---- No names : mean all
223 #
224 if len(names)==0:
225 names=[]
226 for name in dir(module):
227 if name.startswith('_'): continue
228 v=getattr(module,name)
229 if type(v) not in [str, int, float, bool, tuple, list, dict]: continue
230 names.append(name)
231
232 # ---- Search for names
233 #
234 overrides={}
235 for name in names:
236
237 # ---- Variable doesn't exist
238 #
239 if not hasattr(module,name):
240 print(f'** Warning : You try to override an inexistant variable ({name})')
241 continue
242
243 # ---- Get override environment variable name
244 #
245 env_value = __get_override_env(name, None)
246
247 # ---- Environment variable doesn't exist
248 #
249 if env_value is None: continue
250
251 # ---- Environment variable and variable exist both
252 #
253 value_old = getattr(module,name)
254 value_type = type(value_old)
255
256 if value_type in [ str ] :
257 new_value = env_value.format(datasets_dir=__datasets_dir, run_id=__run_id)
258
259 if value_type in [ int, float, bool, tuple, list, dict, type(None)]:
260 new_value = eval(env_value)
261
262 # ---- Override value
263 #
264 setattr(module,name,new_value)
265 overrides[name]=new_value
266
267 if verbose and len(overrides)>0:
268 print('** Overrided parameters : **')
269 for name,value in overrides.items():
270 print(f'{name:20s} : {value}')
271
272 if return_attributes:
273 return overrides
274
275
276 def __get_override_env(name, default_value=None):
277 env_name = f'FIDLE_OVERRIDE_{name}'
278 env_value = os.environ.get(env_name, default_value)
279 return env_value
280
281
282 # def __has_override_env(name):
283 # env_name = f'FIDLE_OVERRIDE_{name}'
284 # return (env_name in os.environ)
285
286
287
288 def __load_cssfile(cssfile):
289 if cssfile is None: return
290 styles = open(cssfile, "r").read()
291 display(HTML(styles))
292
293
294
295
296
297 def end():
298 chrono.stop('__global__')
299 end_time = chrono.get_end('__global__')
300 duration = chrono.get_delay('__global__', format='human')
301 site_url = "https://fidle.cnrs.fr"
302 md = f'**End time :** {end_time} \n'
303 md+= f'**Duration :** {duration} \n'
304 md+= f'This notebook ends here :-) \n'
305 md+= f'[{site_url}]({site_url})'
306 utils.display_md(md)
307
308