import json import numpy as np from datetime import datetime """ This class helps to generate a valid report.json file for the VirtualLab. - Input : JSON file - template of the report.json file. """ class ReportGenerator(): def __init__(self, path): self.path = path self._loadJson() def _loadJson(self): """Private function to load the json template - Create the timestamp - Clear all layer latency details in the template file """ f = open(self.path,"r") self.report = json.load(f) f.close() self.report["inference_latency"]["latency_per_layers"] = [] self.report["date"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") def addLayerLatency(self, layer_name:str = None, mean:float = None, std: float = None, min:float = None, max:float = None): """Add latency metrics for a layer arg 1: layer_name - layer description or name arg 2: mean - latency mean in seconds accorss all inference arg 3: std - standard deviation of the latency accorss all inference arg 4: min - Minimum latency recorded accoss all inference arg 5: max - Maximum latency recorded across all inference """ if mean is not None: assert mean >= 0, "Latency mean cannot be negative. Must be greater or equal than 0 and given in seconds" if std is not None: assert std >= 0, "Latency std cannot be negative. Must be greater or equal than." if min is not None: assert min >= 0, "Latency min cannot be negative. Must be greater or equal than 0 and given in seconds" if max is not None: assert max >= 0, "Latency max cannot be negative. Must be greater or equal than 0 and given in seconds" layer_info = { "layer_name": str(layer_name) if layer_name != None else None, "mean": float(mean) if mean != None else None, "std": float(std) if std != None else None, "min": float(min) if min != None else None, "max": float(max) if max != None else None } self.report["inference_latency"]["latency_per_layers"].append(layer_info) def addLatency(self, mean:float, std: float, min:float, max:float): """Add latency metrics for the model. arg 1: mean - latency mean in seconds accorss all inference arg 2: std - standard deviation of the latency accorss all inference arg 3: min - Minimum latency recorded accoss all inference arg 4: max - Maximum latency recorded across all inference """ if mean is not None: assert mean >= 0, "Latency mean cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["inference_latency"]["mean"] = float(mean) if std is not None: assert std >= 0, "Latency std cannot be negative. Must be greater or equal than." self.report["inference_latency"]["std"] = float(std) if min is not None: assert min >= 0, "Latency min cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["inference_latency"]["min"] = float(min) if max is not None: assert max >= 0, "Latency max cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["inference_latency"]["max"] = float(max) def addTroughput(self, troughput:float): """Add troughput metrics for the model. arg 1: troughput in inferences / second """ self.report["inference_latency"]["troughput"] = float(troughput) if troughput != None else None def addInfo(self, target:str, engine:str, benchmark_type: str, nb_inference:int): """Add general information arg 1: target - Target name arg 2: engine - Inference engine used arg 3: benchmark_type - This field contains the type of benchmark from which the measures from. (TYPE1 : random data , TYPE2 : dataset preprocessed , TYPE3: raw dataset). arg 4: nb_inference - Number of inference performed for the benchmark """ self.report["target"] = str(target) self.report["inference_engine"] = str(engine) self.report["benchmark_type"] = str(benchmark_type) self.report["nb_inference"] = int(nb_inference) def addNbInference(self, nb_inference:int): """Add general information arg 1: nb_inference - Number of inference performed for the benchmark """ self.report["nb_inference"] = int(nb_inference) def addInfoModel(self, model_file_name:str, model_size:int = None, nb_parameters_model:int = None): """Add model information arg 1: model_file_name - Name of the model used for the benchmark arg 2: model_size - Size in bytes of the model on the target (after any compression / optimisation performed by the engine) arg 3: nb_parameters_model - Number of parameters of the model. """ if model_size is not None: assert model_size >= 0, "Model size cannot be smaller than 0 bytes." self.report["model_size"] = int(model_size) if nb_parameters_model is not None: assert nb_parameters_model >= 0, "Number of parameter cannot be negative." self.report["nb_parameters_model"] = int(nb_parameters_model) self.report["model_file_name"] = str(model_file_name) def computeAndAddLayerLatency(self, layer_name:str, latency:list[float]): """Add compute and inference latency for a layer arg 1: layername - layer description or name arg 2: latency - Array of all the inference latency for the layer """ mean = np.mean(latency) std = np.std(latency) min = np.min(latency) max = np.max(latency) self.addLayerLatency(layer_name, mean, std, min, max) def computeAndAddLatency(self, latency:list[float]): """Compute and add the latency metrics of the model arg 1: latency - Array of all the inference latency for the model """ mean = np.mean(latency) std = np.std(latency) min = np.min(latency) max = np.max(latency) self.addLatency(mean, std, min, max) def addRAMInfo(self, ram_size:int = None, ram_peak:float = None): """Add RAM information arg 1: ram_size - Size of the RAM in bytes arg 2: ram_peak - Teak usage of the RAM in % """ if ram_peak is not None: assert ram_peak <= 100, "RAM usage greater than 1 (100%) - must be between 0 and 100" assert ram_peak >= 0, "RAM usage cannot be negative - must be between 0 and 100" self.report["ram_peak_usage"] = float(ram_peak) if ram_size is not None: assert ram_size >= 0, "RAM size cannot be negative. Must be greater or equal than 0 and given in bytes" self.report["ram_size"] = int(ram_size) def addFLASHInfo(self, flash_size:int = None, flash_usage:float = None): """Add FLASH information arg 1: flash_size - FLASH size in bytes arg 2: flash_usage - Percentage of the FLASH usage """ if flash_usage is not None: assert flash_usage <= 100, "FLASH usage greater than 1 (100%) - must be between 0 and 100" assert flash_usage >= 0, "FLASH usage cannot be negative - must be between 0 and 100" self.report["flash_usage"] = float(flash_usage) if flash_size is not None: assert flash_size >= 0, "FLASH size cannot be negative. Must be greater or equal than 0 and given in bytes" self.report["flash_size"] = int(flash_size) def addTemperature(self, ambiant_temp:float = None, system_temp:float = None): """Add Temperature metrics arg 1: ambiant_temp - Ambiant temperature in °C arg 2: flash_usage - System temperature in °C """ if ambiant_temp is not None: assert ambiant_temp >= -273.14, "Ambiant temperature cannot be smaller than 0 °K (-273.14 °C)" self.report["ambiant_temperature"] = float(ambiant_temp) if system_temp is not None: assert system_temp >= -273.14, "System temperature cannot be smaller than 0 °K (-273.14 °C)" self.report["temperature"] = float(system_temp) def addPowerInfo(self, power_consumption:float = None, energy_efficiency:float = None, GFLOPs:float = None): """Add power metrics arg 1: power_consumption - Power consumption of the system in Watt arg 2: energy_efficiency - Energy efficiency in Operation Per Watt (OPW). arg 3: GFLOPs """ if power_consumption is not None: assert power_consumption >= 0, "Power consuption cannot be negative" self.report["power_consumption"] = float(power_consumption) if energy_efficiency is not None: assert energy_efficiency >= 0, "Energy efficiency cannot be nagative" self.report["energy_efficiency"] = float(energy_efficiency) if GFLOPs is not None : assert GFLOPs >= 0, "GFLOPs cannot be negative" self.report["GFLOPs"] = float(GFLOPs) def addCPULoad(self, cpu_load:float): """Add CPU load metric arg 1: cpu_load - Percentage of the CPU usage. """ if cpu_load is not None: assert cpu_load >= 0, "CPU load cannot be negative - must be between 0 and 100 (usage % / nb core)" assert cpu_load <= 100, "CPU load cannot be bigger than 100% - must be between 0 and 100 (usage % / nb core)" self.report["load_cpu"] = float(cpu_load) def addAcceleratorLoad(self, accelerator_load:float): """Add load metrics for the hardware accelerator (GPU, NPU, ...) arg 1: accelerator_load - Percentage of the accelerator (GPU, NPU, ...) usage. """ if accelerator_load is not None: assert accelerator_load <= 100, "Accelerator load is greater than 1 (100%) - must be between 0 and 100" assert accelerator_load >= 0, "Accelerator load cannot be negative - must be between 0 and 100" self.report["load_accelerator"] = float(accelerator_load) def addPreprocessingTiming(self, mean:float, std: float, min:float, max:float): """Add preprocessing timing. arg 1: mean - mean in seconds accorss all preprocessing arg 2: std - standard deviation accorss all preprocessing arg 3: min - Minimum time recorded accoss all preprocessing arg 4: max - Maximum time recorded across all preprocessing """ if mean is not None: assert mean >= 0, "mean cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["preprocess_time"]["mean"] = float(mean) if std is not None: assert std >= 0, "std cannot be negative. Must be greater or equal than." self.report["preprocess_time"]["std"] = float(std) if min is not None: assert min >= 0, "min cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["preprocess_time"]["min"] = float(min) if max is not None: assert max >= 0, "max cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["preprocess_time"]["max"] = float(max) def addPostprocessingTiming(self, mean:float, std: float, min:float, max:float): """Add postprocessing timing. arg 1: mean - mean in seconds accorss all postprocessing arg 2: std - standard deviation accorss all postprocessing arg 3: min - Minimum time recorded accoss all postprocessing arg 4: max - Maximum time recorded across all postprocessing """ if mean is not None: assert mean >= 0, "mean cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["postprocess_time"]["mean"] = float(mean) if std is not None: assert std >= 0, "std cannot be negative. Must be greater or equal than." self.report["postprocess_time"]["std"] = float(std) if min is not None: assert min >= 0, "min cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["postprocess_time"]["min"] = float(min) if max is not None: assert max >= 0, "max cannot be negative. Must be greater or equal than 0 and given in seconds" self.report["postprocess_time"]["max"] = float(max) def addApplicationTiming(self, preprocess_time:list[float] = None , postprocess_time:list[float] = None): """Compute and add the latency metrics of the model arg 1: preprocess_time - Array of all the preprocess time. arg 2: postprocess_time - Array of all the postprocess time. """ if preprocess_time is not None : mean = np.mean(preprocess_time) std = np.std(preprocess_time) min = np.min(preprocess_time) max = np.max(preprocess_time) self.addPreprocessingTiming(mean, std, min, max) if postprocess_time is not None : mean = np.mean(postprocess_time) std = np.std(postprocess_time) min = np.min(postprocess_time) max = np.max(postprocess_time) self.addPostprocessingTiming(mean, std, min, max) def addModelPerformances(self, accuracy:float = None): """Add performances metrics of the model. arg 1: accuracy - Accuracy of the model (value between 0 and 1) """ if accuracy is not None: assert accuracy <= 1, "Accuracy greater than 1 (100%) - must be between 0 and 1" assert accuracy >= 0, "Accuracy cannot be negative - must be between 0 and 1" self.report["accuracy"] = float(accuracy) def save(self, path): """Save the report at the path location. - This function check if the minimum required metrics are present arg 1: path - file path to save """ # Test required metrics # assert self.report["model_file_name"] is not None, "Model file name must be set (use addInfoModel())" # Save report f = open(path, "w") json.dump(self.report,f,indent=4) f.close() if __name__ == '__main__': # Test and example of usage of the ReportGenerator class # Create the ReportGenerator object r = ReportGenerator("AI_Manager/report_template.json") # Add information # target, engine and benchmark_type will be overrided by the VLab. Use addNbInference for simplicity. r.addInfo(target="target name", engine="engine", benchmark_type="Type2", nb_inference=10) # or r.addNbInference(nb_inference=10) r.addInfoModel("Model file name", model_size=8096, nb_parameters_model=540) r.addFLASHInfo(flash_size=180150, flash_usage=0.36) r.addRAMInfo(ram_size=102, ram_peak=0.5) r.addPowerInfo(power_consumption=8, energy_efficiency=0.8) r.addTemperature(ambiant_temp=25.3, system_temp=57.6) r.addCPULoad(cpu_load=0.5) r.addAcceleratorLoad(accelerator_load=1) # Add latency metrics model = [100, 405, 452, 785, 546] layer1 = [10, 30, 20, 50, 62] r.computeAndAddLatency(latency=model) # or r.addLatency(mean=1, std=1, max=2, min=0) # Latency for layers r.computeAndAddLayerLatency(layer_name="layer1", latency=layer1) r.addLayerLatency(layer_name="Layer2", mean=0, std=0, min=0, max=0) r.addLayerLatency(layer_name="Layer3", mean=0) r.addLayerLatency(layer_name="Layer3", mean=0, std=0, min=None, max=0) r.addTroughput(troughput=29) # Application metrics r.addModelPerformances(accuracy=0.5) r.addApplicationTiming(preprocess_time=[6, 9, 5],postprocess_time=[6, 9, 7]) # Save the report r.save("AI_Manager/out/report.json")