mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
af15b0d553
* Add Flake and Black Lint * Change SDK APIs * Update E2E tests * Fix a few function parameters * Fix black format * Fix a few comments * Fix conftest location * Fix Job kind in tests * Fix client creation in test * Fix namespace arg in get_job_conditions * Update SDK examples with the latest changes * Rename SDK examples * Fix black action * Update checkout action version Co-authored-by: Yuki Iwai <[email protected]> * Use Black 23.9.1 version * Fix GitHub Action for Black * Add unit test to create PyTorchJob from func * Rename timeout to wait_timeout * Validate that Job is not set with other input parameters * Update black in developer guide * Remove pip_index_url validation * Use locals to verify input * Print Job info when E2E fails * Remove duplicated delete --------- Co-authored-by: Yuki Iwai <[email protected]>
84 lines
2.8 KiB
Python
Executable File
84 lines
2.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright 2021 The Kubeflow Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
This script is used for updating generated SDK files.
|
|
"""
|
|
|
|
import os
|
|
import fileinput
|
|
import re
|
|
|
|
__replacements = [
|
|
("import kubeflow.training", "from kubeflow.training.models import *"),
|
|
("kubeflow.training.models.v1\/.*.v1.", "V1"),
|
|
("kubeflow.training.models.kubeflow/org/v1/", "kubeflow_org_v1_"),
|
|
("\.kubeflow.org.v1\.", ".KubeflowOrgV1"),
|
|
]
|
|
|
|
sdk_dir = os.path.abspath(os.path.join(__file__, "../../..", "sdk/python"))
|
|
|
|
|
|
def main():
|
|
fix_test_files()
|
|
add_imports()
|
|
|
|
|
|
def fix_test_files() -> None:
|
|
"""
|
|
Fix invalid model imports in generated model tests
|
|
"""
|
|
test_folder_dir = os.path.join(sdk_dir, "test")
|
|
test_files = os.listdir(test_folder_dir)
|
|
for test_file in test_files:
|
|
print(f"Processing file {test_file}")
|
|
if test_file.endswith(".py"):
|
|
with fileinput.FileInput(
|
|
os.path.join(test_folder_dir, test_file), inplace=True
|
|
) as file:
|
|
for line in file:
|
|
print(_apply_regex(line), end="")
|
|
|
|
|
|
def add_imports() -> None:
|
|
with open(os.path.join(sdk_dir, "kubeflow/training/__init__.py"), "a") as f:
|
|
f.write("from kubeflow.training.api.training_client import TrainingClient\n")
|
|
f.write("from kubeflow.training.constants import constants\n")
|
|
with open(os.path.join(sdk_dir, "kubeflow/__init__.py"), "a") as f:
|
|
f.write("__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n")
|
|
|
|
# Add Kubernetes models to proper deserialization of Training models.
|
|
with open(os.path.join(sdk_dir, "kubeflow/training/models/__init__.py"), "r") as f:
|
|
new_lines = []
|
|
for line in f.readlines():
|
|
new_lines.append(line)
|
|
if line.startswith("from __future__ import absolute_import"):
|
|
new_lines.append("\n")
|
|
new_lines.append("# Import Kubernetes models.\n")
|
|
new_lines.append("from kubernetes.client import *\n")
|
|
with open(os.path.join(sdk_dir, "kubeflow/training/models/__init__.py"), "w") as f:
|
|
f.writelines(new_lines)
|
|
|
|
|
|
def _apply_regex(input_str: str) -> str:
|
|
for pattern, replacement in __replacements:
|
|
input_str = re.sub(pattern, replacement, input_str)
|
|
return input_str
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|