mirror of
https://github.com/luxfi/fhe-ml.git
synced 2026-10-30 19:52:59 +00:00
178 lines
6.4 KiB
Python
178 lines
6.4 KiB
Python
#!/usr/bin/env python
|
|
# coding: utf-8
|
|
"""Used to convert output format from python-progress-tracker to new postgres DB format"""
|
|
|
|
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
from importlib.metadata import version
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
|
|
from common import benchmark_name_to_config
|
|
|
|
|
|
def is_git_diff(path: Union[None, Path, str]) -> bool:
|
|
"""Check if there is a diff in a repository."""
|
|
path = path if path is not None else "."
|
|
completed_process = subprocess.run(
|
|
["git", "diff", "HEAD"], capture_output=True, cwd=path, check=True
|
|
)
|
|
if completed_process.stderr:
|
|
raise ValueError("Check git diff raised an error:\n" f"{completed_process.stderr.decode()}")
|
|
return bool(completed_process.stdout)
|
|
|
|
|
|
def get_git_hash(path: Union[None, Path, str]) -> str:
|
|
"""Get git hash of repository."""
|
|
path = path if path is not None else "."
|
|
completed_process = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], capture_output=True, cwd=path, check=True
|
|
)
|
|
if completed_process.stderr:
|
|
raise ValueError("Check git hash raised an error:\n" f"{completed_process.stderr.decode()}")
|
|
return completed_process.stdout.decode().strip()
|
|
|
|
|
|
def get_git_hash_date(hash_str: str, path: Union[None, Path, str]) -> str:
|
|
"""Get repository git hash date."""
|
|
path = path if path is not None else "."
|
|
# We get the author date (%ai) and not the commit date (%ci)
|
|
# for more details please refer to https://git-scm.com/docs/git-show
|
|
completed_process = subprocess.run(
|
|
["git", "show", "-s", "--date=iso-strict", "--format=%ai", hash_str],
|
|
capture_output=True,
|
|
cwd=path,
|
|
check=True,
|
|
)
|
|
if completed_process.stderr:
|
|
raise ValueError("Check git hash raised an error:\n" f"{completed_process.stderr.decode()}")
|
|
print(completed_process.stdout.decode().strip())
|
|
return completed_process.stdout.decode().strip()
|
|
|
|
|
|
def git_iso_to_python_iso(date_str: str) -> str:
|
|
"""Transform git iso into Python iso."""
|
|
splitted = date_str.split()
|
|
return f"{splitted[0]}T{splitted[1]}{splitted[2][:3]}:{splitted[2][3:]}"
|
|
|
|
|
|
def find_element_in_zip(elements: List[Tuple[str, Any]], key: str) -> Any:
|
|
"""Find the element in a dict represented as a zip."""
|
|
for key_, value in elements:
|
|
if key_ == key:
|
|
return value
|
|
raise ValueError(f"Couldn't find key {key} in {[key for key, _ in elements]}")
|
|
|
|
|
|
def convert_to_new_postgres(
|
|
source: Path, target: Path, path_to_repository: Path, machine_name: Optional[str] = None
|
|
):
|
|
"""Convert json file generated via python-progress-tracker to new format."""
|
|
# Load from direct result of script
|
|
assert source.exists(), source
|
|
with open(source, "r", encoding="utf-8") as file:
|
|
progress = json.load(file)
|
|
|
|
# Get git information
|
|
assert not is_git_diff(path_to_repository)
|
|
current_git_hash = get_git_hash(path_to_repository)
|
|
current_git_hash_timestamp = datetime.datetime.fromisoformat(
|
|
git_iso_to_python_iso(get_git_hash_date(current_git_hash, path_to_repository))
|
|
).timestamp()
|
|
current_timestamp = datetime.datetime.now().timestamp()
|
|
|
|
session_data: Dict[str, Union[Dict, List]] = {}
|
|
|
|
# Create machine
|
|
session_data["machine"] = {
|
|
"machine_name": progress["machine"]["name"] if machine_name is None else machine_name,
|
|
"machine_specs": {
|
|
"cpu": find_element_in_zip(progress["machine"]["specs"], "CPU"),
|
|
"ram": find_element_in_zip(progress["machine"]["specs"], "RAM"),
|
|
"os": find_element_in_zip(progress["machine"]["specs"], "OS"),
|
|
},
|
|
}
|
|
|
|
# Create experiments
|
|
experiments = []
|
|
for experiment_name, experiment_values in progress["targets"].items():
|
|
model_name, dataset_name, hyper_args = benchmark_name_to_config(experiment_name)
|
|
experiment_representation = {}
|
|
experiment_representation["experiment_name"] = experiment_name
|
|
experiment_representation["experiment_metadata"] = {
|
|
"model_name": model_name,
|
|
"dataset_name": dataset_name,
|
|
"cml_version": version("luxfhe-ml"),
|
|
"cnp_version": version("concrete-python"),
|
|
}
|
|
experiment_representation["experiment_metadata"].update(hyper_args)
|
|
experiment_representation["git_hash"] = current_git_hash
|
|
experiment_representation["git_timestamp"] = current_git_hash_timestamp
|
|
experiment_representation["experiment_timestamp"] = current_timestamp
|
|
experiment_representation["metrics"] = []
|
|
if "measurements" in experiment_values:
|
|
for metric_name, metric_value in experiment_values["measurements"].items():
|
|
if math.isnan(metric_value): # Nan
|
|
metric_value = None
|
|
experiment_representation["metrics"].append(
|
|
{"metric_name": metric_name, "value": metric_value}
|
|
)
|
|
experiment_representation["metrics"].append(
|
|
{"metric_name": "working", "value": int(experiment_values["working"])}
|
|
)
|
|
experiments.append(experiment_representation)
|
|
session_data["experiments"] = experiments
|
|
|
|
# Dump modified file
|
|
with open(target, "w", encoding="utf-8") as file:
|
|
json.dump(session_data, file, indent=4)
|
|
|
|
|
|
def main():
|
|
"""Main function to convert json into new format."""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--source",
|
|
dest="source",
|
|
type=Path,
|
|
default=Path("./source.json"),
|
|
help="Path to json file to convert.",
|
|
)
|
|
parser.add_argument(
|
|
"--target",
|
|
dest="target",
|
|
type=Path,
|
|
default=Path("./target.json"),
|
|
help="Path to converted json file.",
|
|
)
|
|
parser.add_argument(
|
|
"--path_to_repository",
|
|
dest="path_to_repository",
|
|
type=Path,
|
|
default=Path("./"),
|
|
help="Path to repository used to run the benchmark",
|
|
)
|
|
parser.add_argument(
|
|
"--machine_name",
|
|
dest="machine_name",
|
|
type=str,
|
|
default=None,
|
|
help="Overwrite machine_name (default is None)",
|
|
)
|
|
args = parser.parse_args(sys.argv[1:])
|
|
convert_to_new_postgres(
|
|
source=args.source,
|
|
target=args.target,
|
|
path_to_repository=args.path_to_repository,
|
|
machine_name=args.machine_name,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|