hanzo-ml/katib
1
0
Fork 0
mirror of https://github.com/hanzo-ml/katib.git synced 2026-09-24 00:31:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enhance pre-commit hooks with flake8 and black (#2407)

Browse Source 
* Add black formater and flake8 linter to pre-commit

Also add's the flake8 config file

Signed-off-by: Ignas Baranauskas <[email protected]>

* Fixes black formating

Signed-off-by: Ignas Baranauskas <[email protected]>

* Fixes flake8 linting errors

Signed-off-by: Ignas Baranauskas <[email protected]>

---------

Signed-off-by: Ignas Baranauskas <[email protected]>
This commit is contained in:
Ignas Baranauskas
2024-08-16 10:13:28 +00:00
committed by GitHub
parent 2e30a54fa5
commit bcb199ddc3
21 changed files with 819 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
+4
View File
@@ -0,0 +1,4 @@
[flake8]
max-line-length = 100
# E203 is ignored to avoid conflicts with Black's formatting, as it's not PEP 8 compliant
extend-ignore = W503, E203
.pre-commit-config.yaml
+10
View File
@@ -11,6 +11,16 @@ repos:
- id: isort
name: isort
entry: isort --profile google
- repo: https://github.com/psf/black
rev: 24.2.0
hooks:
- id: black
files: (sdk|examples)/.*
- repo: https://github.com/pycqa/flake8
rev: 7.1.1
hooks:
- id: flake8
files: (sdk|examples)/.*
exclude: |
(?x)^(
.*zz_generated.deepcopy.*|
examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py
+33 -55
View File
@@ -21,10 +21,12 @@
# This Experiment is similar to this:
# https://github.com/kubeflow/katib/blob/master/examples/v1beta1/kubeflow-training-operator/mpijob-horovod.yaml
# Check the training container source code here: https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod.
# Check the training container source code here:
# https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod.
# Note: To run this example, your Kubernetes cluster should run MPIJob operator.
# Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/
# Follow this guide to install MPIJob on your cluster:
# https://www.kubeflow.org/docs/components/training/mpi/
import kfp
from kfp import components
@@ -42,13 +44,12 @@ from kubeflow.katib import V1beta1TrialTemplate
@dsl.pipeline(
name="Launch Katib MPIJob Experiment",
description="An example to launch Katib Experiment with MPIJob"
description="An example to launch Katib Experiment with MPIJob",
)
def horovod_mnist_hpo(
experiment_name: str = "mpi-horovod-mnist",
experiment_namespace: str = "kubeflow-user-example-com",
):
# Trial count specification.
max_trial_count = 6
max_failed_trial_count = 3
@@ -64,12 +65,7 @@ def horovod_mnist_hpo(
# Algorithm specification.
algorithm = V1beta1AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
V1beta1AlgorithmSetting(
name="random_state",
value="10"
)
]
algorithm_settings=[V1beta1AlgorithmSetting(name="random_state", value="10")],
)
# Experiment search space.
@@ -78,19 +74,12 @@ def horovod_mnist_hpo(
V1beta1ParameterSpec(
name="lr",
parameter_type="double",
feasible_space=V1beta1FeasibleSpace(
min="0.001",
max="0.003"
),
feasible_space=V1beta1FeasibleSpace(min="0.001", max="0.003"),
),
V1beta1ParameterSpec(
name="num-steps",
parameter_type="int",
feasible_space=V1beta1FeasibleSpace(
min="50",
max="150",
step="10"
),
feasible_space=V1beta1FeasibleSpace(min="50", max="150", step="10"),
),
]
@@ -106,18 +95,14 @@ def horovod_mnist_hpo(
"replicas": 1,
"template": {
"metadata": {
"annotations": {
"sidecar.istio.io/inject": "false"
}
"annotations": {"sidecar.istio.io/inject": "false"}
},
"spec": {
"containers": [
{
"image": "docker.io/kubeflow/mpi-horovod-mnist",
"name": "mpi-launcher",
"command": [
"mpirun"
],
"command": ["mpirun"],
"args": [
"-np",
"2",
@@ -141,26 +126,21 @@ def horovod_mnist_hpo(
"--lr",
"${trialParameters.learningRate}",
"--num-steps",
"${trialParameters.numberSteps}"
"${trialParameters.numberSteps}",
],
"resources": {
"limits": {
"cpu": "500m",
"memory": "2Gi"
}
}
"limits": {"cpu": "500m", "memory": "2Gi"}
},
}
]
}
}
},
},
},
"Worker": {
"replicas": 2,
"template": {
"metadata": {
"annotations": {
"sidecar.istio.io/inject": "false"
}
"annotations": {"sidecar.istio.io/inject": "false"}
},
"spec": {
"containers": [
@@ -168,25 +148,20 @@ def horovod_mnist_hpo(
"image": "docker.io/kubeflow/mpi-horovod-mnist",
"name": "mpi-worker",
"resources": {
"limits": {
"cpu": "500m",
"memory": "4Gi"
}
}
"limits": {"cpu": "500m", "memory": "4Gi"}
},
}
]
}
}
}
}
}
},
},
},
},
},
}
# Configure parameters for the Trial template.
trial_template = V1beta1TrialTemplate(
primary_pod_labels={
"mpi-job-role": "launcher"
},
primary_pod_labels={"mpi-job-role": "launcher"},
primary_container_name="mpi-launcher",
success_condition='status.conditions.#(type=="Succeeded")#|#(status=="True")#',
failure_condition='status.conditions.#(type=="Failed")#|#(status=="True")#',
@@ -194,15 +169,15 @@ def horovod_mnist_hpo(
V1beta1TrialParameterSpec(
name="learningRate",
description="Learning rate for the training model",
reference="lr"
reference="lr",
),
V1beta1TrialParameterSpec(
name="numberSteps",
description="Number of training steps",
reference="num-steps"
reference="num-steps",
),
],
trial_spec=trial_spec
trial_spec=trial_spec,
)
# Create Experiment specification.
@@ -213,13 +188,15 @@ def horovod_mnist_hpo(
objective=objective,
algorithm=algorithm,
parameters=parameters,
trial_template=trial_template
trial_template=trial_template,
)
# Get the Katib launcher.
# Load component from the URL or from the file.
katib_experiment_launcher_op = components.load_component_from_url(
"https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml")
"https://raw.githubusercontent.com/kubeflow/pipelines/master/"
"components/kubeflow/katib-launcher/component.yaml"
)
# katib_experiment_launcher_op = components.load_component_from_file(
# "../../../components/kubeflow/katib-launcher/component.yaml"
# )
@@ -231,7 +208,8 @@ def horovod_mnist_hpo(
experiment_name=experiment_name,
experiment_namespace=experiment_namespace,
experiment_spec=ApiClient().sanitize_for_serialization(experiment_spec),
experiment_timeout_minutes=60)
experiment_timeout_minutes=60,
)
# Output container to print the results.
dsl.ContainerOp(
examples/v1beta1/trial-images/darts-cnn-cifar10/architect.py
+26 -18
View File
@@ -17,9 +17,8 @@ import copy
import torch
class Architect():
"""" Architect controls architecture of cell by computing gradients of alphas
"""
class Architect:
""" " Architect controls architecture of cell by computing gradients of alphas"""
def __init__(self, model, w_momentum, w_weight_decay, device):
self.model = model
@@ -48,25 +47,32 @@ class Architect():
# Compute gradient
gradients = torch.autograd.grad(loss, self.model.getWeights())
# Do virtual step (Update gradient)
# Below operations do not need gradient tracking
with torch.no_grad():
# dict key is not the value, but the pointer. So original network weight have to
# be iterated also.
for w, vw, g in zip(self.model.getWeights(), self.v_model.getWeights(), gradients):
m = w_optim.state[w].get("momentum_buffer", 0.) * self.w_momentum
if(self.device == 'cuda'):
vw.copy_(w - torch.cuda.FloatTensor(xi) * (m + g + self.w_weight_decay * w))
elif(self.device == 'cpu'):
vw.copy_(w - torch.FloatTensor(xi) * (m + g + self.w_weight_decay * w))
for w, vw, g in zip(
self.model.getWeights(), self.v_model.getWeights(), gradients
):
m = w_optim.state[w].get("momentum_buffer", 0.0) * self.w_momentum
if self.device == "cuda":
vw.copy_(
w
- torch.cuda.FloatTensor(xi) * (m + g + self.w_weight_decay * w)
)
elif self.device == "cpu":
vw.copy_(
w - torch.FloatTensor(xi) * (m + g + self.w_weight_decay * w)
)
# Sync alphas
for a, va in zip(self.model.getAlphas(), self.v_model.getAlphas()):
va.copy_(a)
def unrolled_backward(self, train_x, train_y, valid_x, valid_y, xi, w_optim):
""" Compute unrolled loss and backward its gradients
"""Compute unrolled loss and backward its gradients
Args:
xi: learning rate for virtual gradient step (same as model lr)
w_optim: weights optimizer - for virtual step
@@ -77,23 +83,23 @@ class Architect():
# Calculate unrolled loss
# Loss for validation with w'. L_valid(w')
loss = self.v_model.loss(valid_x, valid_y)
# Calculate gradient
v_alphas = tuple(self.v_model.getAlphas())
v_weights = tuple(self.v_model.getWeights())
v_grads = torch.autograd.grad(loss, v_alphas + v_weights)
dalpha = v_grads[:len(v_alphas)]
dws = v_grads[len(v_alphas):]
dalpha = v_grads[: len(v_alphas)]
dws = v_grads[len(v_alphas) :]
hessian = self.compute_hessian(dws, train_x, train_y)
# Update final gradient = dalpha - xi * hessian
with torch.no_grad():
for alpha, da, h in zip(self.model.getAlphas(), dalpha, hessian):
if(self.device == 'cuda'):
if self.device == "cuda":
alpha.grad = da - torch.cuda.FloatTensor(xi) * h
elif(self.device == 'cpu'):
elif self.device == "cpu":
alpha.grad = da - torch.cpu.FloatTensor(xi) * h
def compute_hessian(self, dws, train_x, train_y):
@@ -121,7 +127,7 @@ class Architect():
with torch.no_grad():
for p, dw in zip(self.model.getWeights(), dws):
# TODO (andreyvelich): Do we need this * 2.0 ?
p -= 2. * eps * dw
p -= 2.0 * eps * dw
loss = self.model.loss(train_x, train_y)
# dalpha { L_train(w-, alpha) }
@@ -132,5 +138,7 @@ class Architect():
for p, dw in zip(self.model.getWeights(), dws):
p += eps * dw
hessian = [(p-n) / (2. * eps) for p, n in zip(dalpha_positive, dalpha_negative)]
hessian = [
(p - n) / (2.0 * eps) for p, n in zip(dalpha_positive, dalpha_negative)
]
return hessian
examples/v1beta1/trial-images/darts-cnn-cifar10/model.py
+54 -16
View File
@@ -21,11 +21,20 @@ import torch.nn.functional as F
class Cell(nn.Module):
""" Cell for search
"""Cell for search
Each edge is mixed and continuous relaxed.
"""
def __init__(self, num_nodes, c_prev_prev, c_prev, c_cur, reduction_prev, reduction_cur, search_space):
def __init__(
self,
num_nodes,
c_prev_prev,
c_prev,
c_cur,
reduction_prev,
reduction_cur,
search_space,
):
"""
Args:
num_nodes: Number of intermediate cell nodes
@@ -45,7 +54,9 @@ class Cell(nn.Module):
if reduction_prev:
self.preprocess0 = FactorizedReduce(c_prev_prev, c_cur)
else:
self.preprocess0 = StdConv(c_prev_prev, c_cur, kernel_size=1, stride=1, padding=0)
self.preprocess0 = StdConv(
c_prev_prev, c_cur, kernel_size=1, stride=1, padding=0
)
self.preprocess1 = StdConv(c_prev, c_cur, kernel_size=1, stride=1, padding=0)
# Generate dag from mixed operations
@@ -54,7 +65,7 @@ class Cell(nn.Module):
for i in range(self.num_nodes):
self.dag_ops.append(nn.ModuleList())
# Include 2 input nodes
for j in range(2+i):
for j in range(2 + i):
# Reduction with stride = 2 must be only for the input node
stride = 2 if reduction_cur and j < 2 else 1
op = MixedOp(c_cur, stride, search_space)
@@ -66,7 +77,9 @@ class Cell(nn.Module):
states = [s0, s1]
for edges, w_list in zip(self.dag_ops, w_dag):
state_cur = sum(edges[i](s, w) for i, (s, w) in enumerate((zip(states, w_list))))
state_cur = sum(
edges[i](s, w) for i, (s, w) in enumerate((zip(states, w_list)))
)
states.append(state_cur)
state_out = torch.cat(states[2:], dim=1)
@@ -75,8 +88,17 @@ class Cell(nn.Module):
class NetworkCNN(nn.Module):
def __init__(self, init_channels, input_channels, num_classes,
num_layers, criterion, search_space, num_nodes, stem_multiplier):
def __init__(
self,
init_channels,
input_channels,
num_classes,
num_layers,
criterion,
search_space,
num_nodes,
stem_multiplier,
):
super(NetworkCNN, self).__init__()
self.init_channels = init_channels
@@ -87,11 +109,11 @@ class NetworkCNN(nn.Module):
self.num_nodes = num_nodes
self.stem_multiplier = stem_multiplier
c_cur = self.stem_multiplier*self.init_channels
c_cur = self.stem_multiplier * self.init_channels
self.stem = nn.Sequential(
nn.Conv2d(input_channels, c_cur, 3, padding=1, bias=False),
nn.BatchNorm2d(c_cur)
nn.BatchNorm2d(c_cur),
)
# In first Cell stem is used for s0 and s1
@@ -110,14 +132,24 @@ class NetworkCNN(nn.Module):
# For Network with two layers: First layer - Normal, Second - Reduction
# For Other Networks: [1/3, 2/3] Layers - Reduction cell with double channels
# Others - Normal cell
if ((self.num_layers == 2 and i == 1) or
(self.num_layers > 2 and i in [self.num_layers//3, 2*self.num_layers//3])):
if (self.num_layers == 2 and i == 1) or (
self.num_layers > 2
and i in [self.num_layers // 3, 2 * self.num_layers // 3]
):
c_cur *= 2
reduction_cur = True
else:
reduction_cur = False
cell = Cell(self.num_nodes, c_prev_prev, c_prev, c_cur, reduction_prev, reduction_cur, search_space)
cell = Cell(
self.num_nodes,
c_prev_prev,
c_prev,
c_cur,
reduction_prev,
reduction_cur,
search_space,
)
reduction_prev = reduction_cur
self.cells.append(cell)
@@ -134,9 +166,11 @@ class NetworkCNN(nn.Module):
self.alpha_reduce = nn.ParameterList()
for i in range(self.num_nodes):
self.alpha_normal.append(nn.Parameter(1e-3*torch.randn(i+2, num_ops)))
self.alpha_normal.append(nn.Parameter(1e-3 * torch.randn(i + 2, num_ops)))
if self.num_layers > 1:
self.alpha_reduce.append(nn.Parameter(1e-3*torch.randn(i+2, num_ops)))
self.alpha_reduce.append(
nn.Parameter(1e-3 * torch.randn(i + 2, num_ops))
)
# Setup alphas list
self.alphas = []
@@ -192,5 +226,9 @@ class NetworkCNN(nn.Module):
# concat all intermediate nodes
concat = range(2, 2 + self.num_nodes)
return search_space.genotype(normal=gene_normal, normal_concat=concat,
reduce=gene_reduce, reduce_concat=concat)
return search_space.genotype(
normal=gene_normal,
normal_concat=concat,
reduce=gene_reduce,
reduce_concat=concat,
)
examples/v1beta1/trial-images/darts-cnn-cifar10/operations.py
+62 -30
View File
@@ -16,18 +16,30 @@ import torch
import torch.nn as nn
OPS = {
'none': lambda channels, stride: Zero(stride),
'avg_pooling_3x3': lambda channels, stride: PoolBN('avg', channels, kernel_size=3, stride=stride, padding=1),
'max_pooling_3x3': lambda channels, stride: PoolBN('max', channels, kernel_size=3, stride=stride, padding=1),
'skip_connection': lambda channels, stride: Identity() if stride == 1 else FactorizedReduce(channels, channels),
'separable_convolution_3x3': lambda channels, stride: SepConv(channels, kernel_size=3, stride=stride, padding=1),
'separable_convolution_5x5': lambda channels, stride: SepConv(channels, kernel_size=5, stride=stride, padding=2),
"none": lambda channels, stride: Zero(stride),
"avg_pooling_3x3": lambda channels, stride: PoolBN(
"avg", channels, kernel_size=3, stride=stride, padding=1
),
"max_pooling_3x3": lambda channels, stride: PoolBN(
"max", channels, kernel_size=3, stride=stride, padding=1
),
"skip_connection": lambda channels, stride: (
Identity() if stride == 1 else FactorizedReduce(channels, channels)
),
"separable_convolution_3x3": lambda channels, stride: SepConv(
channels, kernel_size=3, stride=stride, padding=1
),
"separable_convolution_5x5": lambda channels, stride: SepConv(
channels, kernel_size=5, stride=stride, padding=2
),
# 3x3 -> 5x5
'dilated_convolution_3x3': lambda channels, stride: DilConv(channels,
kernel_size=3, stride=stride, padding=2, dilation=2),
"dilated_convolution_3x3": lambda channels, stride: DilConv(
channels, kernel_size=3, stride=stride, padding=2, dilation=2
),
# 5x5 -> 9x9
'dilated_convolution_5x5': lambda channels, stride: DilConv(channels,
kernel_size=5, stride=stride, padding=4, dilation=2),
"dilated_convolution_5x5": lambda channels, stride: DilConv(
channels, kernel_size=5, stride=stride, padding=4, dilation=2
),
}
@@ -42,9 +54,9 @@ class Zero(nn.Module):
def forward(self, x):
if self.stride == 1:
return x * 0.
return x * 0.0
# Resize by stride
return x[:, :, ::self.stride, ::self.stride] * 0.
return x[:, :, :: self.stride, :: self.stride] * 0.0
class PoolBN(nn.Module):
@@ -55,15 +67,14 @@ class PoolBN(nn.Module):
def __init__(self, pool_type, channels, kernel_size, stride, padding):
super(PoolBN, self).__init__()
if pool_type == "avg":
self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False)
self.pool = nn.AvgPool2d(
kernel_size, stride, padding, count_include_pad=False
)
elif pool_type == "max":
self.pool = nn.MaxPool2d(kernel_size, stride, padding)
self.bn = nn.BatchNorm2d(channels, affine=False)
self.net = nn.Sequential(
self.pool,
self.bn
)
self.net = nn.Sequential(self.pool, self.bn)
def forward(self, x):
# out = self.pool(x),
@@ -91,8 +102,12 @@ class FactorizedReduce(nn.Module):
def __init__(self, c_in, c_out):
super(FactorizedReduce, self).__init__()
self.relu = nn.ReLU()
self.conv1 = nn.Conv2d(c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False)
self.conv2 = nn.Conv2d(c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False)
self.conv1 = nn.Conv2d(
c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False
)
self.conv2 = nn.Conv2d(
c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False
)
self.bn = nn.BatchNorm2d(c_out, affine=False)
def forward(self, x):
@@ -105,7 +120,7 @@ class FactorizedReduce(nn.Module):
class StdConv(nn.Module):
""" Standard convolition
"""Standard convolition
ReLU - Conv - BN
"""
@@ -113,8 +128,15 @@ class StdConv(nn.Module):
super(StdConv, self).__init__()
self.net = nn.Sequential(
nn.ReLU(),
nn.Conv2d(c_in, c_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(c_out, affine=False)
nn.Conv2d(
c_in,
c_out,
kernel_size=kernel_size,
stride=stride,
padding=padding,
bias=False,
),
nn.BatchNorm2d(c_out, affine=False),
)
def forward(self, x):
@@ -122,7 +144,7 @@ class StdConv(nn.Module):
class DilConv(nn.Module):
""" (Dilated) depthwise separable conv
"""(Dilated) depthwise separable conv
ReLU - (Dilated) depthwise separable - Pointwise - BN
If dilation == 2, 3x3 conv => 5x5 receptive field
@@ -134,9 +156,20 @@ class DilConv(nn.Module):
self.net = nn.Sequential(
nn.ReLU(),
nn.Conv2d(channels, channels, kernel_size, stride, padding, dilation=dilation, groups=channels, bias=False),
nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(channels, affine=False)
nn.Conv2d(
channels,
channels,
kernel_size,
stride,
padding,
dilation=dilation,
groups=channels,
bias=False,
),
nn.Conv2d(
channels, channels, kernel_size=1, stride=1, padding=0, bias=False
),
nn.BatchNorm2d(channels, affine=False),
)
def forward(self, x):
@@ -144,7 +177,7 @@ class DilConv(nn.Module):
class SepConv(nn.Module):
""" Depthwise separable conv
"""Depthwise separable conv
DilConv (dilation=1) * 2
"""
@@ -152,7 +185,7 @@ class SepConv(nn.Module):
super(SepConv, self).__init__()
self.net = nn.Sequential(
DilConv(channels, kernel_size, stride=stride, padding=padding, dilation=1),
DilConv(channels, kernel_size, stride=1, padding=padding, dilation=1)
DilConv(channels, kernel_size, stride=1, padding=padding, dilation=1),
)
def forward(self, x):
@@ -160,8 +193,7 @@ class SepConv(nn.Module):
class MixedOp(nn.Module):
""" Mixed operation
"""
"""Mixed operation"""
def __init__(self, channels, stride, search_space):
super(MixedOp, self).__init__()
examples/v1beta1/trial-images/darts-cnn-cifar10/run_trial.py
+118 -39
View File
@@ -27,15 +27,27 @@ import utils
def main():
parser = argparse.ArgumentParser(description='TrainingContainer')
parser.add_argument('--algorithm-settings', type=str, default="", help="algorithm settings")
parser.add_argument('--search-space', type=str, default="", help="search space for the neural architecture search")
parser.add_argument('--num-layers', type=str, default="", help="number of layers of the neural network")
parser = argparse.ArgumentParser(description="TrainingContainer")
parser.add_argument(
"--algorithm-settings", type=str, default="", help="algorithm settings"
)
parser.add_argument(
"--search-space",
type=str,
default="",
help="search space for the neural architecture search",
)
parser.add_argument(
"--num-layers",
type=str,
default="",
help="number of layers of the neural network",
)
args = parser.parse_args()
# Get Algorithm Settings
algorithm_settings = args.algorithm_settings.replace("\'", "\"")
algorithm_settings = args.algorithm_settings.replace("'", '"')
algorithm_settings = json.loads(algorithm_settings)
print(">>> Algorithm settings")
for key, value in algorithm_settings.items():
@@ -69,7 +81,7 @@ def main():
stem_multiplier = int(algorithm_settings["stem_multiplier"])
# Get Search Space
search_space = args.search_space.replace("\'", "\"")
search_space = args.search_space.replace("'", '"')
search_space = json.loads(search_space)
search_space = SearchSpace(search_space)
@@ -103,16 +115,28 @@ def main():
criterion = nn.CrossEntropyLoss().to(device)
model = NetworkCNN(init_channels, input_channels, num_classes, num_layers,
criterion, search_space, num_nodes, stem_multiplier)
model = NetworkCNN(
init_channels,
input_channels,
num_classes,
num_layers,
criterion,
search_space,
num_nodes,
stem_multiplier,
)
model = model.to(device)
# Weights optimizer
w_optim = torch.optim.SGD(model.getWeights(), w_lr, momentum=w_momentum, weight_decay=w_weight_decay)
w_optim = torch.optim.SGD(
model.getWeights(), w_lr, momentum=w_momentum, weight_decay=w_weight_decay
)
# Alphas optimizer
alpha_optim = torch.optim.Adam(model.getAlphas(), alpha_lr, betas=(0.5, 0.999), weight_decay=alpha_weight_decay)
alpha_optim = torch.optim.Adam(
model.getAlphas(), alpha_lr, betas=(0.5, 0.999), weight_decay=alpha_weight_decay
)
# Split data to train/validation
num_train = len(train_data)
@@ -122,27 +146,30 @@ def main():
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
sampler=train_sampler,
num_workers=num_workers,
pin_memory=True)
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=batch_size,
sampler=train_sampler,
num_workers=num_workers,
pin_memory=True,
)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
sampler=valid_sampler,
num_workers=num_workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(
train_data,
batch_size=batch_size,
sampler=valid_sampler,
num_workers=num_workers,
pin_memory=True,
)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim,
num_epochs,
eta_min=w_lr_min)
w_optim, num_epochs, eta_min=w_lr_min
)
architect = Architect(model, w_momentum, w_weight_decay, device)
# Start training
best_top1 = 0.
best_top1 = 0.0
for epoch in range(num_epochs):
lr = lr_scheduler.get_last_lr()
@@ -151,14 +178,28 @@ def main():
# Training
print(">>> Training")
train(train_loader, valid_loader, model, architect, w_optim, alpha_optim,
lr, epoch, num_epochs, device, w_grad_clip, print_step)
train(
train_loader,
valid_loader,
model,
architect,
w_optim,
alpha_optim,
lr,
epoch,
num_epochs,
device,
w_grad_clip,
print_step,
)
lr_scheduler.step()
# Validation
print("\n>>> Validation")
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_step)
top1 = validate(
valid_loader, model, epoch, cur_step, num_epochs, device, print_step
)
# Print genotype
genotype = model.genotype(search_space)
@@ -173,18 +214,36 @@ def main():
print("\nBest-Genotype={}".format(str(best_genotype).replace(" ", "")))
def train(train_loader, valid_loader, model, architect, w_optim, alpha_optim,
lr, epoch, num_epochs, device, w_grad_clip, print_step):
def train(
train_loader,
valid_loader,
model,
architect,
w_optim,
alpha_optim,
lr,
epoch,
num_epochs,
device,
w_grad_clip,
print_step,
):
top1 = utils.AverageMeter()
top5 = utils.AverageMeter()
losses = utils.AverageMeter()
cur_step = epoch * len(train_loader)
model.train()
for step, ((train_x, train_y), (valid_x, valid_y)) in enumerate(zip(train_loader, valid_loader)):
for step, ((train_x, train_y), (valid_x, valid_y)) in enumerate(
zip(train_loader, valid_loader)
):
train_x, train_y = train_x.to(device, non_blocking=True), train_y.to(device, non_blocking=True)
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(device, non_blocking=True)
train_x, train_y = train_x.to(device, non_blocking=True), train_y.to(
device, non_blocking=True
)
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(
device, non_blocking=True
)
train_size = train_x.size(0)
@@ -213,12 +272,21 @@ def train(train_loader, valid_loader, model, architect, w_optim, alpha_optim,
print(
"Train: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} "
"Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format(
epoch+1, num_epochs, step, len(train_loader)-1, losses=losses,
top1=top1, top5=top5))
epoch + 1,
num_epochs,
step,
len(train_loader) - 1,
losses=losses,
top1=top1,
top5=top5,
)
)
cur_step += 1
print("Train: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch+1, num_epochs, top1.avg))
print(
"Train: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch + 1, num_epochs, top1.avg)
)
def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_step):
@@ -230,7 +298,9 @@ def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_ste
with torch.no_grad():
for step, (valid_x, valid_y) in enumerate(valid_loader):
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(device, non_blocking=True)
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(
device, non_blocking=True
)
valid_size = valid_x.size(0)
@@ -246,10 +316,19 @@ def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_ste
print(
"Validation: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} "
"Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format(
epoch+1, num_epochs, step, len(valid_loader)-1, losses=losses,
top1=top1, top5=top5))
epoch + 1,
num_epochs,
step,
len(valid_loader) - 1,
losses=losses,
top1=top1,
top5=top5,
)
)
print("Valid: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch+1, num_epochs, top1.avg))
print(
"Valid: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch + 1, num_epochs, top1.avg)
)
return top1.avg
examples/v1beta1/trial-images/darts-cnn-cifar10/search_space.py
+5 -3
View File
@@ -17,14 +17,16 @@ from collections import namedtuple
import torch
class SearchSpace():
class SearchSpace:
def __init__(self, search_space):
self.primitives = search_space
self.primitives.append("none")
print(">>> All Primitives")
print("{}\n".format(self.primitives))
self.genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat')
self.genotype = namedtuple(
"Genotype", "normal normal_concat reduce reduce_concat"
)
def parse(self, alpha, k):
"""
@@ -46,7 +48,7 @@ class SearchSpace():
"""
gene = []
assert self.primitives[-1] == 'none' # assume last PRIMITIVE is 'none'
assert self.primitives[-1] == "none" # assume last PRIMITIVE is 'none'
# 1) Convert the mixed op to discrete edge (single op) by choosing top-1 weight edge
# 2) Choose top-k edges per node by edge score (top-1 weight in edge)
examples/v1beta1/trial-images/darts-cnn-cifar10/utils.py
+10 -14
View File
@@ -16,21 +16,21 @@ import torchvision.datasets as dset
import torchvision.transforms as transforms
class AverageMeter():
""" Computes and stores the average and current value """
class AverageMeter:
"""Computes and stores the average and current value"""
def __init__(self):
self.reset()
def reset(self):
""" Reset all statistics """
"""Reset all statistics"""
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
""" Update statistics """
"""Update statistics"""
self.val = val
self.sum += val * n
self.count += n
@@ -38,7 +38,7 @@ class AverageMeter():
def accuracy(output, target, topk=(1,)):
""" Computes the precision@k for the specified values of k """
"""Computes the precision@k for the specified values of k"""
maxk = max(topk)
batch_size = target.size(0)
@@ -67,18 +67,14 @@ def get_dataset():
# Do preprocessing
MEAN = [0.49139968, 0.48215827, 0.44653124]
STD = [0.24703233, 0.24348505, 0.26158768]
transf = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
transf = [transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip()]
normalize = [
transforms.ToTensor(),
transforms.Normalize(MEAN, STD)
]
normalize = [transforms.ToTensor(), transforms.Normalize(MEAN, STD)]
train_transform = transforms.Compose(transf + normalize)
train_data = dataset_cls(root="./data", train=True, download=True, transform=train_transform)
train_data = dataset_cls(
root="./data", train=True, download=True, transform=train_transform
)
return input_channels, num_classes, train_data
examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py
+16 -28
View File
@@ -14,20 +14,11 @@
import json
from keras import backend as K
from keras.layers import Activation
from keras.layers import AveragePooling2D
from keras.layers import BatchNormalization
from keras.layers import concatenate
from keras.layers import Conv2D
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import GlobalAveragePooling2D
from keras.layers import Input
from keras.layers import MaxPooling2D
from keras.layers import ZeroPadding2D
from keras.models import Model
import numpy as np
from op_library import concat
from op_library import conv
from op_library import dw_conv
@@ -39,16 +30,14 @@ class ModelConstructor(object):
def __init__(self, arc_json, nn_json):
self.arch = json.loads(arc_json)
nn_config = json.loads(nn_json)
self.num_layers = nn_config['num_layers']
self.input_sizes = nn_config['input_sizes']
self.output_size = nn_config['output_sizes'][-1]
self.embedding = nn_config['embedding']
self.num_layers = nn_config["num_layers"]
self.input_sizes = nn_config["input_sizes"]
self.output_size = nn_config["output_sizes"][-1]
self.embedding = nn_config["embedding"]
def build_model(self):
# a list of the data all layers
all_layers = [0 for _ in range(self.num_layers + 1)]
# a list of all the dimensions of all layers
all_dims = [0 for _ in range(self.num_layers + 1)]
# ================= Stacking layers =================
# Input Layer. Layer 0
@@ -56,38 +45,37 @@ class ModelConstructor(object):
all_layers[0] = input_layer
# Intermediate Layers. Starting from layer 1.
for l in range(1, self.num_layers + 1):
for l_index in range(1, self.num_layers + 1):
input_layers = list()
opt = self.arch[l - 1][0]
opt = self.arch[l_index - 1][0]
opt_config = self.embedding[str(opt)]
skip = self.arch[l - 1][1:l+1]
skip = self.arch[l_index - 1][1 : l_index + 1]
# set up the connection to the previous layer first
input_layers.append(all_layers[l - 1])
input_layers.append(all_layers[l_index - 1])
# then add skip connections
for i in range(l - 1):
if l > 1 and skip[i] == 1:
for i in range(l_index - 1):
if l_index > 1 and skip[i] == 1:
input_layers.append(all_layers[i])
layer_input = concat(input_layers)
if opt_config['opt_type'] == 'convolution':
if opt_config["opt_type"] == "convolution":
layer_output = conv(layer_input, opt_config)
if opt_config['opt_type'] == 'separable_convolution':
if opt_config["opt_type"] == "separable_convolution":
layer_output = sp_conv(layer_input, opt_config)
if opt_config['opt_type'] == 'depthwise_convolution':
if opt_config["opt_type"] == "depthwise_convolution":
layer_output = dw_conv(layer_input, opt_config)
elif opt_config['opt_type'] == 'reduction':
elif opt_config["opt_type"] == "reduction":
layer_output = reduction(layer_input, opt_config)
all_layers[l] = layer_output
all_layers[l_index] = layer_output
# Final Layer
# Global Average Pooling, then Fully connected with softmax.
avgpooled = GlobalAveragePooling2D()(all_layers[self.num_layers])
dropped = Dropout(0.4)(avgpooled)
logits = Dense(units=self.output_size,
activation='softmax')(dropped)
logits = Dense(units=self.output_size, activation="softmax")(dropped)
# Encapsulate the model
self.model = Model(inputs=input_layer, outputs=logits)
examples/v1beta1/trial-images/enas-cnn-cifar10/RunTrial.py
+60 -33
View File
@@ -24,22 +24,42 @@ from tensorflow.keras.layers import Rescaling
from tensorflow.keras.utils import to_categorical
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='TrainingContainer')
parser.add_argument('--architecture', type=str, default="", metavar='N',
help='architecture of the neural network')
parser.add_argument('--nn_config', type=str, default="", metavar='N',
help='configurations and search space embeddings')
parser.add_argument('--num_epochs', type=int, default=10, metavar='N',
help='number of epoches that each child will be trained')
parser.add_argument('--num_gpus', type=int, default=1, metavar='N',
help='number of GPU that used for training')
parser = argparse.ArgumentParser(description="TrainingContainer")
parser.add_argument(
"--architecture",
type=str,
default="",
metavar="N",
help="architecture of the neural network",
)
parser.add_argument(
"--nn_config",
type=str,
default="",
metavar="N",
help="configurations and search space embeddings",
)
parser.add_argument(
"--num_epochs",
type=int,
default=10,
metavar="N",
help="number of epoches that each child will be trained",
)
parser.add_argument(
"--num_gpus",
type=int,
default=1,
metavar="N",
help="number of GPU that used for training",
)
args = parser.parse_args()
arch = args.architecture.replace("\'", "\"")
arch = args.architecture.replace("'", '"')
print(">>> arch received by trial")
print(arch)
nn_config = args.nn_config.replace("\'", "\"")
nn_config = args.nn_config.replace("'", '"')
print(">>> nn_config received by trial")
print(nn_config)
@@ -54,36 +74,40 @@ if __name__ == "__main__":
print("\n>>> Constructing Model...")
constructor = ModelConstructor(arch, nn_config)
num_physical_gpus = len(tf.config.experimental.list_physical_devices('GPU'))
num_physical_gpus = len(tf.config.experimental.list_physical_devices("GPU"))
if 1 <= num_gpus <= num_physical_gpus:
devices = ["/gpu:"+str(i) for i in range(num_physical_gpus)]
devices = ["/gpu:" + str(i) for i in range(num_physical_gpus)]
else:
num_physical_cpu = len(tf.config.experimental.list_physical_devices('CPU'))
devices = ["/cpu:"+str(j) for j in range(num_physical_cpu)]
num_physical_cpu = len(tf.config.experimental.list_physical_devices("CPU"))
devices = ["/cpu:" + str(j) for j in range(num_physical_cpu)]
strategy = tf.distribute.MirroredStrategy(devices)
with strategy.scope():
test_model = constructor.build_model()
test_model.summary()
test_model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(learning_rate=1e-3),
metrics=['accuracy'])
test_model.compile(
loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(learning_rate=1e-3),
metrics=["accuracy"],
)
print(">>> Model Constructed Successfully\n")
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train /= 255
x_test /= 255
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
augmentation = tf.keras.Sequential([
Rescaling(1./255),
RandomFlip('horizontal'),
RandomTranslation(height_factor=0.1, width_factor=0.1),
])
augmentation = tf.keras.Sequential(
[
Rescaling(1.0 / 255),
RandomFlip("horizontal"),
RandomTranslation(height_factor=0.1, width_factor=0.1),
]
)
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
train_dataset = train_dataset.map(lambda x, y: (augmentation(x, training=True), y))
@@ -93,11 +117,14 @@ if __name__ == "__main__":
print(">>> Data Loaded. Training starts.")
for e in range(num_epochs):
print("\nTotal Epoch {}/{}".format(e + 1, num_epochs))
history = test_model.fit(train_dataset,
steps_per_epoch=int(len(x_train) / 128) + 1,
epochs=1, verbose=1,
validation_data=(x_test, y_test))
print("Training-Accuracy={}".format(history.history['accuracy'][-1]))
print("Training-Loss={}".format(history.history['loss'][-1]))
print("Validation-Accuracy={}".format(history.history['val_accuracy'][-1]))
print("Validation-Loss={}".format(history.history['val_loss'][-1]))
history = test_model.fit(
train_dataset,
steps_per_epoch=int(len(x_train) / 128) + 1,
epochs=1,
verbose=1,
validation_data=(x_test, y_test),
)
print("Training-Accuracy={}".format(history.history["accuracy"][-1]))
print("Training-Loss={}".format(history.history["loss"][-1]))
print("Validation-Accuracy={}".format(history.history["val_accuracy"][-1]))
print("Validation-Loss={}".format(history.history["val_loss"][-1]))
examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py
+51 -46
View File
@@ -18,10 +18,7 @@ from keras.layers import AveragePooling2D
from keras.layers import BatchNormalization
from keras.layers import concatenate
from keras.layers import Conv2D
from keras.layers import Dense
from keras.layers import DepthwiseConv2D
from keras.layers import GlobalAveragePooling2D
from keras.layers import Input
from keras.layers import MaxPooling2D
from keras.layers import SeparableConv2D
from keras.layers import ZeroPadding2D
@@ -46,10 +43,13 @@ def concat(inputs):
diff = max_dim - total_dim[i][1]
half_diff = int(diff / 2)
if diff % 2 == 0:
padded_input[i] = ZeroPadding2D(padding=(half_diff, half_diff))(inputs[i])
padded_input[i] = ZeroPadding2D(padding=(half_diff, half_diff))(
inputs[i]
)
else:
padded_input[i] = ZeroPadding2D(padding=((half_diff, half_diff + 1),
(half_diff, half_diff + 1)))(inputs[i])
padded_input[i] = ZeroPadding2D(
padding=((half_diff, half_diff + 1), (half_diff, half_diff + 1))
)(inputs[i])
else:
padded_input[i] = inputs[i]
@@ -59,21 +59,22 @@ def concat(inputs):
def conv(x, config):
parameters = {
"num_filter": 64,
"filter_size": 3,
"stride": 1,
"num_filter": 64,
"filter_size": 3,
"stride": 1,
}
for k in parameters.keys():
if k in config:
parameters[k] = int(config[k])
activated = Activation('relu')(x)
activated = Activation("relu")(x)
conved = Conv2D(
filters=parameters['num_filter'],
kernel_size=parameters['filter_size'],
strides=parameters['stride'],
padding='same')(activated)
filters=parameters["num_filter"],
kernel_size=parameters["filter_size"],
strides=parameters["stride"],
padding="same",
)(activated)
result = BatchNormalization()(conved)
@@ -82,9 +83,9 @@ def conv(x, config):
def sp_conv(x, config):
parameters = {
"num_filter": 64,
"filter_size": 3,
"stride": 1,
"num_filter": 64,
"filter_size": 3,
"stride": 1,
"depth_multiplier": 1,
}
@@ -92,36 +93,39 @@ def sp_conv(x, config):
if k in config:
parameters[k] = int(config[k])
activated = Activation('relu')(x)
activated = Activation("relu")(x)
conved = SeparableConv2D(
filters=parameters['num_filter'],
kernel_size=parameters['filter_size'],
strides=parameters['stride'],
depth_multiplier=parameters['depth_multiplier'],
padding='same')(activated)
filters=parameters["num_filter"],
kernel_size=parameters["filter_size"],
strides=parameters["stride"],
depth_multiplier=parameters["depth_multiplier"],
padding="same",
)(activated)
result = BatchNormalization()(conved)
return result
def dw_conv(x, config):
parameters = {
"filter_size": 3,
"stride": 1,
"filter_size": 3,
"stride": 1,
"depth_multiplier": 1,
}
for k in parameters.keys():
if k in config:
parameters[k] = int(config[k])
activated = Activation('relu')(x)
activated = Activation("relu")(x)
conved = DepthwiseConv2D(
kernel_size=parameters['filter_size'],
strides=parameters['stride'],
depth_multiplier=parameters['depth_multiplier'],
padding='same')(activated)
kernel_size=parameters["filter_size"],
strides=parameters["stride"],
depth_multiplier=parameters["depth_multiplier"],
padding="same",
)(activated)
result = BatchNormalization()(conved)
@@ -134,31 +138,32 @@ def reduction(x, config):
# such situation is very likely to appear though
dim = K.int_shape(x)
if dim[1] == 1 or dim[2] == 1:
print("WARNING: One or more dimensions of the input of the reduction layer is 1. It cannot be further reduced. A identity layer will be used instead.")
print(
"WARNING: One or more dimensions of the input of the reduction layer is 1. "
"It cannot be further reduced. A identity layer will be used instead."
)
return x
parameters = {
'reduction_type': "max_pooling",
'pool_size': 2,
'stride': None,
"reduction_type": "max_pooling",
"pool_size": 2,
"stride": None,
}
if 'reduction_type' in config:
parameters['reduction_type'] = config['reduction_type']
if 'pool_size' in config:
parameters['pool_size'] = int(config['pool_size'])
if 'stride' in config:
parameters['stride'] = int(config['stride'])
if "reduction_type" in config:
parameters["reduction_type"] = config["reduction_type"]
if "pool_size" in config:
parameters["pool_size"] = int(config["pool_size"])
if "stride" in config:
parameters["stride"] = int(config["stride"])
if parameters['reduction_type'] == 'max_pooling':
if parameters["reduction_type"] == "max_pooling":
result = MaxPooling2D(
pool_size=parameters['pool_size'],
strides=parameters['stride']
pool_size=parameters["pool_size"], strides=parameters["stride"]
)(x)
elif parameters['reduction_type'] == 'avg_pooling':
elif parameters["reduction_type"] == "avg_pooling":
result = AveragePooling2D(
pool_size=parameters['pool_size'],
strides=parameters['stride']
pool_size=parameters["pool_size"], strides=parameters["stride"]
)(x)
return result
examples/v1beta1/trial-images/pytorch-mnist/mnist.py
+126 -57
View File
@@ -35,7 +35,7 @@ class Net(nn.Module):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 20, 5, 1)
self.conv2 = nn.Conv2d(20, 50, 5, 1)
self.fc1 = nn.Linear(4*4*50, 500)
self.fc1 = nn.Linear(4 * 4 * 50, 500)
self.fc2 = nn.Linear(500, 10)
def forward(self, x):
@@ -43,7 +43,7 @@ class Net(nn.Module):
x = F.max_pool2d(x, 2, 2)
x = F.relu(self.conv2(x))
x = F.max_pool2d(x, 2, 2)
x = x.view(-1, 4*4*50)
x = x.view(-1, 4 * 4 * 50)
x = F.relu(self.fc1(x))
x = self.fc2(x)
return F.log_softmax(x, dim=1)
@@ -60,10 +60,14 @@ def train(args, model, device, train_loader, optimizer, epoch):
optimizer.step()
if batch_idx % args.log_interval == 0:
msg = "Train Epoch: {} [{}/{} ({:.0f}%)]\tloss={:.4f}".format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.item())
epoch,
batch_idx * len(data),
len(train_loader.dataset),
100.0 * batch_idx / len(train_loader),
loss.item(),
)
logging.info(msg)
niter = epoch * len(train_loader) + batch_idx
niter = epoch * len(train_loader) + batch_idx # noqa: F841
def test(args, model, device, test_loader, epoch, hpt):
@@ -74,24 +78,31 @@ def test(args, model, device, test_loader, epoch, hpt):
for data, target in test_loader:
data, target = data.to(device), target.to(device)
output = model(data)
test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss
pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability
test_loss += F.nll_loss(
output, target, reduction="sum"
).item() # sum up batch loss
pred = output.max(1, keepdim=True)[
1
] # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
test_loss /= len(test_loader.dataset)
test_accuracy = float(correct) / len(test_loader.dataset)
logging.info("{{metricName: accuracy, metricValue: {:.4f}}};{{metricName: loss, metricValue: {:.4f}}}\n".format(
test_accuracy, test_loss))
logging.info(
"{{metricName: accuracy, metricValue: {:.4f}}};"
"{{metricName: loss, metricValue: {:.4f}}}\n".format(test_accuracy, test_loss)
)
if args.logger == "hypertune":
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='loss',
metric_value=test_loss,
global_step=epoch)
hyperparameter_metric_tag="loss", metric_value=test_loss, global_step=epoch
)
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='accuracy',
hyperparameter_metric_tag="accuracy",
metric_value=test_accuracy,
global_step=epoch)
global_step=epoch,
)
def should_distribute():
@@ -105,33 +116,82 @@ def is_distributed():
def main():
# Training settings
parser = argparse.ArgumentParser(description="PyTorch MNIST Example")
parser.add_argument("--batch-size", type=int, default=64, metavar="N",
help="input batch size for training (default: 64)")
parser.add_argument("--test-batch-size", type=int, default=1000, metavar="N",
help="input batch size for testing (default: 1000)")
parser.add_argument("--epochs", type=int, default=10, metavar="N",
help="number of epochs to train (default: 10)")
parser.add_argument("--lr", type=float, default=0.01, metavar="LR",
help="learning rate (default: 0.01)")
parser.add_argument("--momentum", type=float, default=0.5, metavar="M",
help="SGD momentum (default: 0.5)")
parser.add_argument("--no-cuda", action="store_true", default=False,
help="disables CUDA training")
parser.add_argument("--seed", type=int, default=1, metavar="S",
help="random seed (default: 1)")
parser.add_argument("--log-interval", type=int, default=10, metavar="N",
help="how many batches to wait before logging training status")
parser.add_argument("--log-path", type=str, default="",
help="Path to save logs. Print to StdOut if log-path is not set")
parser.add_argument("--save-model", action="store_true", default=False,
help="For Saving the current Model")
parser.add_argument("--logger", type=str, choices=["standard", "hypertune"],
help="Logger", default="standard")
parser.add_argument(
"--batch-size",
type=int,
default=64,
metavar="N",
help="input batch size for training (default: 64)",
)
parser.add_argument(
"--test-batch-size",
type=int,
default=1000,
metavar="N",
help="input batch size for testing (default: 1000)",
)
parser.add_argument(
"--epochs",
type=int,
default=10,
metavar="N",
help="number of epochs to train (default: 10)",
)
parser.add_argument(
"--lr",
type=float,
default=0.01,
metavar="LR",
help="learning rate (default: 0.01)",
)
parser.add_argument(
"--momentum",
type=float,
default=0.5,
metavar="M",
help="SGD momentum (default: 0.5)",
)
parser.add_argument(
"--no-cuda", action="store_true", default=False, help="disables CUDA training"
)
parser.add_argument(
"--seed", type=int, default=1, metavar="S", help="random seed (default: 1)"
)
parser.add_argument(
"--log-interval",
type=int,
default=10,
metavar="N",
help="how many batches to wait before logging training status",
)
parser.add_argument(
"--log-path",
type=str,
default="",
help="Path to save logs. Print to StdOut if log-path is not set",
)
parser.add_argument(
"--save-model",
action="store_true",
default=False,
help="For Saving the current Model",
)
parser.add_argument(
"--logger",
type=str,
choices=["standard", "hypertune"],
help="Logger",
default="standard",
)
if dist.is_available():
parser.add_argument("--backend", type=str, help="Distributed backend",
choices=[dist.Backend.GLOO, dist.Backend.NCCL, dist.Backend.MPI],
default=dist.Backend.GLOO)
parser.add_argument(
"--backend",
type=str,
help="Distributed backend",
choices=[dist.Backend.GLOO, dist.Backend.NCCL, dist.Backend.MPI],
default=dist.Backend.GLOO,
)
args = parser.parse_args()
# Use this format (%Y-%m-%dT%H:%M:%SZ) to record timestamp of the metrics.
@@ -140,16 +200,18 @@ def main():
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%dT%H:%M:%SZ",
level=logging.DEBUG)
level=logging.DEBUG,
)
else:
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%dT%H:%M:%SZ",
level=logging.DEBUG,
filename=args.log_path)
filename=args.log_path,
)
if args.logger == "hypertune" and args.log_path != "":
os.environ['CLOUD_ML_HP_METRIC_FILE'] = args.log_path
os.environ["CLOUD_ML_HP_METRIC_FILE"] = args.log_path
# For JSON logging
hpt = hypertune.HyperTune()
@@ -169,27 +231,34 @@ def main():
kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST("./data",
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor()
])),
batch_size=args.batch_size, shuffle=True, **kwargs)
datasets.FashionMNIST(
"./data",
train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()]),
),
batch_size=args.batch_size,
shuffle=True,
**kwargs,
)
test_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST("./data",
train=False,
transform=transforms.Compose([
transforms.ToTensor()
])),
batch_size=args.test_batch_size, shuffle=False, **kwargs)
datasets.FashionMNIST(
"./data", train=False, transform=transforms.Compose([transforms.ToTensor()])
),
batch_size=args.test_batch_size,
shuffle=False,
**kwargs,
)
model = Net().to(device)
if is_distributed():
Distributor = nn.parallel.DistributedDataParallel if use_cuda \
Distributor = (
nn.parallel.DistributedDataParallel
if use_cuda
else nn.parallel.DistributedDataParallelCPU
)
model = Distributor(model)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
@@ -198,7 +267,7 @@ def main():
train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader, epoch, hpt)
if (args.save_model):
if args.save_model:
torch.save(model.state_dict(), "mnist_cnn.pt")
examples/v1beta1/trial-images/tf-mnist-with-summaries/mnist.py
+100 -37
View File
@@ -25,9 +25,9 @@ from tensorflow.keras.layers import Flatten
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = Conv2D(32, 3, activation='relu')
self.conv1 = Conv2D(32, 3, activation="relu")
self.flatten = Flatten()
self.d1 = Dense(128, activation='relu')
self.d1 = Dense(128, activation="relu")
self.d2 = Dense(10)
def call(self, x):
@@ -37,7 +37,17 @@ class MyModel(Model):
return self.d2(x)
def train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summary_writer, train_loss, train_accuracy):
def train_step(
args,
model,
optimizer,
train_ds,
epoch,
loss_object,
train_summary_writer,
train_loss,
train_accuracy,
):
for step, (images, labels) in enumerate(train_ds):
with tf.GradientTape() as tape:
# training=True is only needed if there are layers with different
@@ -51,18 +61,25 @@ def train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summa
train_accuracy(labels, predictions)
if step % args.log_interval == 0:
print("Train Epoch: {} [{}/60000 ({:.0f}%)]\tloss={:.4f}, accuracy={:.4f}".format(
epoch + 1, step * args.batch_size, 100. * step * args.batch_size / 60000,
train_loss.result(), train_accuracy.result() * 100)
print(
"Train Epoch: {} [{}/60000 ({:.0f}%)]\tloss={:.4f}, accuracy={:.4f}".format(
epoch + 1,
step * args.batch_size,
100.0 * step * args.batch_size / 60000,
train_loss.result(),
train_accuracy.result() * 100,
)
)
with train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss.result(), step=epoch)
tf.summary.scalar('accuracy', train_accuracy.result(), step=epoch)
tf.summary.scalar("loss", train_loss.result(), step=epoch)
tf.summary.scalar("accuracy", train_accuracy.result(), step=epoch)
def test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy):
for (images, labels) in test_ds:
def test_step(
model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy
):
for images, labels in test_ds:
# training=False is only needed if there are layers with different
# behavior during training versus inference (e.g. Dropout).
predictions = model(images, training=False)
@@ -72,30 +89,53 @@ def test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss
test_accuracy(labels, predictions)
with test_summary_writer.as_default():
tf.summary.scalar('loss', test_loss.result(), step=epoch)
tf.summary.scalar('accuracy', test_accuracy.result(), step=epoch)
tf.summary.scalar("loss", test_loss.result(), step=epoch)
tf.summary.scalar("accuracy", test_accuracy.result(), step=epoch)
print("Test Loss: {:.4f}, Test Accuracy: {:.4f}\n".format(
test_loss.result(), test_accuracy.result() * 100)
print(
"Test Loss: {:.4f}, Test Accuracy: {:.4f}\n".format(
test_loss.result(), test_accuracy.result() * 100
)
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch-size', type=int, default=64,
help='input batch size for training (default: 64)')
parser.add_argument('--learning-rate', type=float, default=0.001,
help='learning rate (default: 0.001)')
parser.add_argument("--epochs", type=int, default=10, metavar="N",
help="number of epochs to train (default: 10)")
parser.add_argument("--log-interval", type=int, default=100, metavar="N",
help="how many batches to wait before logging training status (default: 100)")
parser.add_argument(
'--log-path',
"--batch-size",
type=int,
default=64,
help="input batch size for training (default: 64)",
)
parser.add_argument(
"--learning-rate",
type=float,
default=0.001,
help="learning rate (default: 0.001)",
)
parser.add_argument(
"--epochs",
type=int,
default=10,
metavar="N",
help="number of epochs to train (default: 10)",
)
parser.add_argument(
"--log-interval",
type=int,
default=100,
metavar="N",
help="how many batches to wait before logging training status (default: 100)",
)
parser.add_argument(
"--log-path",
type=str,
default=os.path.join(os.getenv('TEST_TMPDIR', '/tmp'),
'tensorflow/mnist/logs/mnist_with_summaries'),
help='Summaries log PATH')
default=os.path.join(
os.getenv("TEST_TMPDIR", "/tmp"),
"tensorflow/mnist/logs/mnist_with_summaries",
),
help="Summaries log PATH",
)
args = parser.parse_args()
# Setup dataset
@@ -105,12 +145,18 @@ def main():
# Add a channels dimension
x_train = x_train[..., tf.newaxis].astype("float32")
x_test = x_test[..., tf.newaxis].astype("float32")
train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(args.batch_size)
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(args.batch_size)
train_ds = (
tf.data.Dataset.from_tensor_slices((x_train, y_train))
.shuffle(10000)
.batch(args.batch_size)
)
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(
args.batch_size
)
# Setup tensorflow summaries
train_log_dir = os.path.join(args.log_path, 'train')
test_log_dir = os.path.join(args.log_path, 'test')
train_log_dir = os.path.join(args.log_path, "train")
test_log_dir = os.path.join(args.log_path, "test")
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir)
@@ -119,20 +165,37 @@ def main():
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
train_loss = tf.keras.metrics.Mean(name="train_loss")
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="train_accuracy")
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')
test_loss = tf.keras.metrics.Mean(name="test_loss")
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="test_accuracy")
for epoch in range(args.epochs):
# Reset the metrics at the start of the next epoch
train_summary_writer.flush()
test_summary_writer.flush()
train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summary_writer,
train_loss, train_accuracy)
test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy)
train_step(
args,
model,
optimizer,
train_ds,
epoch,
loss_object,
train_summary_writer,
train_loss,
train_accuracy,
)
test_step(
model,
test_ds,
epoch,
loss_object,
test_summary_writer,
test_loss,
test_accuracy,
)
if __name__ == "__main__":
hack/gen-python-sdk/post_gen.py
+63 -46
View File
@@ -18,14 +18,14 @@ import sys
IGNORE_LINES = [
"from kubeflow.katib.models.v1_unstructured_unstructured import V1UnstructuredUnstructured",
"from kubeflow.katib.models.v1_time import V1Time"
"from kubeflow.katib.models.v1_time import V1Time",
]
def _rewrite_helper(input_file, output_file, rewrite_rules):
rules = rewrite_rules or []
lines = []
with open(input_file, 'r') as f:
with open(input_file, "r") as f:
while True:
line = f.readline()
if not line:
@@ -34,11 +34,11 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
for rule in rules:
line = rule(line)
# Remove ignored lines.
if not any(l in line for l in IGNORE_LINES):
if not any(li in line for li in IGNORE_LINES):
lines.append(line)
# Add Katib APIs to the init file.
if (output_file == "sdk/python/v1beta1/kubeflow/katib/__init__.py"):
if output_file == "sdk/python/v1beta1/kubeflow/katib/__init__.py":
lines.append("# Import Katib API client.\n")
lines.append("from kubeflow.katib.api.katib_client import KatibClient\n")
lines.append("# Import Katib report metrics functions")
@@ -46,62 +46,79 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
lines.append("# Import Katib helper functions.\n")
lines.append("import kubeflow.katib.api.search as search\n")
lines.append("# Import Katib helper constants.\n")
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW\n")
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW_GPU\n")
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_PYTORCH\n")
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_MXNET\n")
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW\n"
)
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW_GPU\n"
)
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_PYTORCH\n"
)
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_MXNET\n"
)
# Add Kubernetes models to proper deserialization of Katib models.
if (output_file == "sdk/python/v1beta1/kubeflow/katib/models/__init__.py"):
if output_file == "sdk/python/v1beta1/kubeflow/katib/models/__init__.py":
lines.append("\n")
lines.append("# Import Kubernetes models.\n")
lines.append("from kubernetes.client import *\n")
with open(output_file, 'w') as f:
with open(output_file, "w") as f:
f.writelines(lines)
def update_python_sdk(src, dest, versions=('v1beta1')):
def update_python_sdk(src, dest, versions=("v1beta1")):
# tiny transformers to refine generated codes
rewrite_rules = [
# Models rules.
lambda l: l.replace('import katib', 'import kubeflow.katib'),
lambda l: l.replace('from katib', 'from kubeflow.katib'),
lambda line: line.replace("import katib", "import kubeflow.katib"),
lambda line: line.replace("from katib", "from kubeflow.katib"),
# For the api_client.py.
lambda l: l.replace('klass = getattr(katib.models, klass)', 'klass = getattr(kubeflow.katib.models, klass)'),
lambda line: line.replace(
"klass = getattr(katib.models, klass)",
"klass = getattr(kubeflow.katib.models, klass)",
),
# Doc rules.
lambda l: l.replace('[**datetime**](V1Time.md)', '**datetime**'),
lambda l: l.replace('[**object**](V1UnstructuredUnstructured.md)', '**object**'),
lambda l: l.replace('[**V1Container**](V1Container.md)',
'[**V1Container**](https://github.com/kubernetes-client/'
'python/blob/master/kubernetes/docs/V1Container.md)'),
lambda l: l.replace('[**V1ObjectMeta**](V1ObjectMeta.md)',
'[**V1ObjectMeta**](https://github.com/kubernetes-client/'
'python/blob/master/kubernetes/docs/V1ObjectMeta.md)'),
lambda l: l.replace('[**V1ListMeta**](V1ListMeta.md)',
'[**V1ListMeta**](https://github.com/kubernetes-client/'
'python/blob/master/kubernetes/docs/V1ListMeta.md)'),
lambda l: l.replace('[**V1HTTPGetAction**](V1HTTPGetAction.md)',
'[**V1HTTPGetAction**](https://github.com/kubernetes-client/'
'python/blob/master/kubernetes/docs/V1HTTPGetAction.md)')
lambda line: line.replace("[**datetime**](V1Time.md)", "**datetime**"),
lambda line: line.replace(
"[**object**](V1UnstructuredUnstructured.md)", "**object**"
),
lambda line: line.replace(
"[**V1Container**](V1Container.md)",
"[**V1Container**](https://github.com/kubernetes-client/"
"python/blob/master/kubernetes/docs/V1Container.md)",
),
lambda line: line.replace(
"[**V1ObjectMeta**](V1ObjectMeta.md)",
"[**V1ObjectMeta**](https://github.com/kubernetes-client/"
"python/blob/master/kubernetes/docs/V1ObjectMeta.md)",
),
lambda line: line.replace(
"[**V1ListMeta**](V1ListMeta.md)",
"[**V1ListMeta**](https://github.com/kubernetes-client/"
"python/blob/master/kubernetes/docs/V1ListMeta.md)",
),
lambda line: line.replace(
"[**V1HTTPGetAction**](V1HTTPGetAction.md)",
"[**V1HTTPGetAction**](https://github.com/kubernetes-client/"
"python/blob/master/kubernetes/docs/V1HTTPGetAction.md)",
),
]
# TODO (andreyvelich): Currently test can't be generated properly.
src_dirs = [
os.path.join(src, 'katib'),
os.path.join(src, 'katib', 'models'),
os.path.join(src, "katib"),
os.path.join(src, "katib", "models"),
# os.path.join(src, 'test'),
os.path.join(src, 'docs')
os.path.join(src, "docs"),
]
dest_dirs = [
os.path.join(dest, 'kubeflow', 'katib'),
os.path.join(dest, 'kubeflow', 'katib', 'models'),
os.path.join(dest, "kubeflow", "katib"),
os.path.join(dest, "kubeflow", "katib", "models"),
# os.path.join(dest, 'test'),
os.path.join(dest, 'docs')
os.path.join(dest, "docs"),
]
for src_dir, dest_dir in zip(src_dirs, dest_dirs):
@@ -128,13 +145,13 @@ def update_python_sdk(src, dest, versions=('v1beta1')):
update_buffer = []
# Get data from generated doc
with open(os.path.join(src, 'README.md'), 'r') as src_f:
with open(os.path.join(src, "README.md"), "r") as src_f:
anchor = 0
for line in src_f.readlines():
if line.startswith('## Documentation For Models'):
if line.startswith("## Documentation For Models"):
if anchor == 0:
anchor = 1
elif line.startswith('##') and anchor == 1:
elif line.startswith("##") and anchor == 1:
anchor = 2
if anchor == 0:
continue
@@ -148,24 +165,24 @@ def update_python_sdk(src, dest, versions=('v1beta1')):
update_buffer = update_buffer[:-1]
# Update README with new models
with open(os.path.join(dest, 'README.md'), 'r') as dest_f:
with open(os.path.join(dest, "README.md"), "r") as dest_f:
anchor = 0
for line in dest_f.readlines():
if line.startswith('## Documentation For Models'):
if line.startswith("## Documentation For Models"):
if anchor == 0:
buffer.extend(update_buffer)
anchor = 1
elif line.startswith('##') and anchor == 1:
elif line.startswith("##") and anchor == 1:
anchor = 2
if anchor == 1:
continue
buffer.append(line)
with open(os.path.join(dest, 'README.md'), 'w') as dest_f:
with open(os.path.join(dest, "README.md"), "w") as dest_f:
dest_f.writelines(buffer)
# Clear working dictionary
shutil.rmtree(src)
if __name__ == '__main__':
if __name__ == "__main__":
update_python_sdk(src=sys.argv[1], dest=sys.argv[2])
sdk/python/v1beta1/kubeflow/__init__.py
+1 -1
View File
@@ -1 +1 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
sdk/python/v1beta1/kubeflow/katib/api/katib_client.py
+45 -19
View File
@@ -106,7 +106,7 @@ class KatibClient(object):
namespace = namespace or self.namespace
experiment_name = None
if type(experiment) == models.V1beta1Experiment:
if type(experiment) is models.V1beta1Experiment:
if experiment.metadata.name is not None:
experiment_name = experiment.metadata.name
elif experiment.metadata.generate_name is not None:
@@ -137,7 +137,8 @@ class KatibClient(object):
except Exception as e:
if hasattr(e, "status") and e.status == 409:
raise Exception(
f"A Katib Experiment with the name {namespace}/{experiment_name} already exists."
f"A Katib Experiment with the name "
f"{namespace}/{experiment_name} already exists."
)
raise RuntimeError(
f"Failed to create Katib Experiment: {namespace}/{experiment_name}"
@@ -152,7 +153,8 @@ class KatibClient(object):
IPython.display.display(
IPython.display.HTML(
"Katib Experiment {} "
'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" target="_blank">here</a>'.format(
'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" '
'target="_blank">here</a>'.format(
experiment_name,
namespace,
experiment_name,
@@ -215,7 +217,8 @@ class KatibClient(object):
https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1EnvFromSource.md)
algorithm_name: Search algorithm for the HyperParameter tuning.
algorithm_settings: Settings for the search algorithm given.
For available fields, check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail.
For available fields, check this doc:
https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail.
objective_metric_name: Objective metric that Katib optimizes.
additional_metric_names: List of metrics that Katib collects from the
objective function in addition to objective metric.
@@ -223,7 +226,8 @@ class KatibClient(object):
Must be one of `minimize` or `maximize`.
objective_goal: Objective goal that Experiment should reach to be Succeeded.
max_trial_count: Maximum number of Trials to run. For the default
values check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec.
values check this doc:
https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec.
parallel_trial_count: Number of Trials that Experiment runs in parallel.
max_failed_trial_count: Maximum number of Trials allowed to fail.
resources_per_trial: A parameter that lets you specify how much
@@ -249,7 +253,7 @@ class KatibClient(object):
to the base image packages. These packages are installed before
executing the objective function.
pip_index_url: The PyPI url from which to install Python packages.
metrics_collector_config: Specify the config of metrics collector,
metrics_collector_config: Specify the config of metrics collector,
for example, `metrics_collector_config = {"kind": "Push"}`.
Currently, we only support `StdOut` and `Push` metrics collector.
@@ -330,10 +334,14 @@ class KatibClient(object):
# Otherwise, add value to the function input.
input_params[p_name] = p_value
# Wrap objective function to execute it from the file. For example
# Wrap objective function to execute it from the file. For example:
# def objective(parameters):
# print(f'Parameters are {parameters}')
# objective({'lr': '${trialParameters.lr}', 'epochs': '${trialParameters.epochs}', 'is_dist': False})
# objective({
# 'lr': '${trialParameters.lr}',
# 'epochs': '${trialParameters.epochs}',
# 'is_dist': False
# })
objective_code = f"{objective_code}\n{objective.__name__}({input_params})\n"
# Prepare execute script template.
@@ -385,7 +393,8 @@ class KatibClient(object):
)
# Add metrics collector to the Katib Experiment.
# Up to now, We only support parameter `kind`, of which default value is `StdOut`, to specify the kind of metrics collector.
# Up to now, we only support parameter `kind`, of which default value
# is `StdOut`, to specify the kind of metrics collector.
experiment.spec.metrics_collector_spec = models.V1beta1MetricsCollectorSpec(
collector=models.V1beta1CollectorSpec(kind=metrics_collector_config["kind"])
)
@@ -764,7 +773,9 @@ class KatibClient(object):
)
):
utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment
# Raise exception if Experiment is Failed.
@@ -784,7 +795,9 @@ class KatibClient(object):
)
):
utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment
# Check if Experiment reaches Running condition.
@@ -795,7 +808,9 @@ class KatibClient(object):
)
):
utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment
# Check if Experiment reaches Restarting condition.
@@ -806,7 +821,9 @@ class KatibClient(object):
)
):
utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment
# Check if Experiment reaches Succeeded condition.
@@ -817,18 +834,24 @@ class KatibClient(object):
)
):
utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"waiting for experiment: {namespace}/{name} "
f"to reach {expected_condition} condition\n\n\n"
)
return experiment
# Otherwise, print the current Experiment results and sleep for the pooling interval.
utils.print_experiment_status(experiment)
logger.debug(
f"Waiting for Experiment: {namespace}/{name} to reach {expected_condition} condition\n\n\n"
f"waiting for experiment: {namespace}/{name} "
f"to reach {expected_condition} condition\n\n\n"
)
time.sleep(polling_interval)
raise TimeoutError(
f"Timeout waiting for Experiment: {namespace}/{name} to reach {expected_condition} state"
f"Timeout waiting for Experiment: {namespace}/{name} "
f"to reach {expected_condition} state"
)
def edit_experiment_budget(
@@ -844,7 +867,8 @@ class KatibClient(object):
budget to resume Succeeded Experiments with `LongRunning` and `FromVolume`
resume policies.
Learn about resuming Experiments here: https://www.kubeflow.org/docs/components/katib/resume-experiment/
Learn about resuming Experiments here:
https://www.kubeflow.org/docs/components/katib/resume-experiment/
Args:
name: Name for the Experiment.
@@ -1257,10 +1281,12 @@ class KatibClient(object):
use the default Katib DB Manager address: `katib-db-manager.kubeflow:6789`.
If you run this API outside the cluster, you have to port-forward the
Katib DB Manager before getting the Trial metrics: `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`.
Katib DB Manager before getting the Trial metrics:
`kubectl port-forward svc/katib-db-manager -n kubeflow 6789`.
In that case, you can use this Katib DB Manager address: `localhost:6789`.
You can use `curl` to verify that Katib DB Manager is reachable: `curl <db-manager-address>`.
You can use `curl` to verify that Katib DB Manager is reachable:
`curl <db-manager-address>`.
Args:
name: Name for the Trial.
sdk/python/v1beta1/kubeflow/katib/api/katib_client_test.py
+25 -41
View File
@@ -38,16 +38,12 @@ def create_namespaced_custom_object_response(*args, **kwargs):
def generate_trial_template() -> V1beta1TrialTemplate:
trial_spec={
trial_spec = {
"apiVersion": "batch/v1",
"kind": "Job",
"spec": {
"template": {
"metadata": {
"annotations": {
"sidecar.istio.io/inject": "false"
}
},
"metadata": {"annotations": {"sidecar.istio.io/inject": "false"}},
"spec": {
"containers": [
{
@@ -60,13 +56,13 @@ def generate_trial_template() -> V1beta1TrialTemplate:
"--batch-size=64",
"--lr=${trialParameters.learningRate}",
"--momentum=${trialParameters.momentum}",
]
],
}
],
"restartPolicy": "Never"
}
"restartPolicy": "Never",
},
}
}
},
}
return V1beta1TrialTemplate(
@@ -75,15 +71,15 @@ def generate_trial_template() -> V1beta1TrialTemplate:
V1beta1TrialParameterSpec(
name="learningRate",
description="Learning rate for the training model",
reference="lr"
reference="lr",
),
V1beta1TrialParameterSpec(
name="momentum",
description="Momentum for the training model",
reference="momentum"
reference="momentum",
),
],
trial_spec=trial_spec
trial_spec=trial_spec,
)
@@ -106,60 +102,49 @@ def generate_experiment(
objective=objective_spec,
parameters=parameters,
trial_template=trial_template,
)
),
)
def create_experiment(
name: Optional[str] = None,
generate_name: Optional[str] = None
name: Optional[str] = None, generate_name: Optional[str] = None
) -> V1beta1Experiment:
experiment_namespace = "test"
if name is not None:
metadata = V1ObjectMeta(name=name, namespace=experiment_namespace)
elif generate_name is not None:
metadata = V1ObjectMeta(generate_name=generate_name, namespace=experiment_namespace)
metadata = V1ObjectMeta(
generate_name=generate_name, namespace=experiment_namespace
)
else:
metadata = V1ObjectMeta(namespace=experiment_namespace)
algorithm_spec=V1beta1AlgorithmSpec(
algorithm_name="random"
)
algorithm_spec = V1beta1AlgorithmSpec(algorithm_name="random")
objective_spec=V1beta1ObjectiveSpec(
objective_spec = V1beta1ObjectiveSpec(
type="minimize",
goal= 0.001,
goal=0.001,
objective_metric_name="loss",
)
parameters=[
parameters = [
V1beta1ParameterSpec(
name="lr",
parameter_type="double",
feasible_space=V1beta1FeasibleSpace(
min="0.01",
max="0.06"
),
feasible_space=V1beta1FeasibleSpace(min="0.01", max="0.06"),
),
V1beta1ParameterSpec(
name="momentum",
parameter_type="double",
feasible_space=V1beta1FeasibleSpace(
min="0.5",
max="0.9"
),
feasible_space=V1beta1FeasibleSpace(min="0.5", max="0.9"),
),
]
trial_template = generate_trial_template()
experiment = generate_experiment(
metadata,
algorithm_spec,
objective_spec,
parameters,
trial_template
metadata, algorithm_spec, objective_spec, parameters, trial_template
)
return experiment
@@ -246,15 +231,14 @@ def katib_client():
side_effect=create_namespaced_custom_object_response
)
),
), patch(
"kubernetes.config.load_kube_config",
return_value=Mock()
):
), patch("kubernetes.config.load_kube_config", return_value=Mock()):
client = KatibClient()
yield client
@pytest.mark.parametrize("test_name,kwargs,expected_output", test_create_experiment_data)
@pytest.mark.parametrize(
"test_name,kwargs,expected_output", test_create_experiment_data
)
def test_create_experiment(katib_client, test_name, kwargs, expected_output):
"""
test create_experiment function of katib client
sdk/python/v1beta1/kubeflow/katib/api/report_metrics.py
+7 -7
View File
@@ -37,7 +37,7 @@ def report_metrics(
For examle, `metrics = {"loss": 0.01, "accuracy": 0.99}`.
db-manager-address: Address for the Katib DB Manager in this format: `ip-address:port`.
timeout: Optional, gRPC API Server timeout in seconds to report metrics.
Raises:
ValueError: The Trial name is not passed to environment variables.
RuntimeError: Unable to push Trial metrics to Katib DB or
@@ -48,9 +48,7 @@ def report_metrics(
namespace = utils.get_current_k8s_namespace()
name = os.getenv("KATIB_TRIAL_NAME")
if name is None:
raise ValueError(
"The Trial name is not passed to environment variables"
)
raise ValueError("The Trial name is not passed to environment variables")
# Get channel for grpc call to db manager
db_manager_address = db_manager_address.split(":")
@@ -61,7 +59,7 @@ def report_metrics(
# Validate metrics value in dict
for value in metrics.values():
utils.validate_metrics_value(value)
# Dial katib db manager to report metrics
with katib_api_pb2.beta_create_DBManager_stub(channel) as client:
try:
@@ -73,11 +71,13 @@ def report_metrics(
metric_logs=[
katib_api_pb2.MetricLog(
time_stamp=timestamp,
metric=katib_api_pb2.Metric(name=name,value=str(value))
metric=katib_api_pb2.Metric(
name=name, value=str(value)
),
)
for name, value in metrics.items()
]
)
),
),
timeout=timeout,
)
sdk/python/v1beta1/kubeflow/katib/api/search.py
+2 -1
View File
@@ -61,5 +61,6 @@ def categorical(list: List):
"""
return models.V1beta1ParameterSpec(
parameter_type="categorical", feasible_space=models.V1beta1FeasibleSpace(list),
parameter_type="categorical",
feasible_space=models.V1beta1FeasibleSpace(list),
)
sdk/python/v1beta1/kubeflow/katib/utils/utils.py
+1
View File
@@ -72,6 +72,7 @@ def print_experiment_status(experiment: models.V1beta1Experiment):
print(f"Current Optimal Trial:\n {experiment.status.current_optimal_trial}")
print(f"Experiment conditions:\n {experiment.status.conditions}")
def validate_metrics_value(value: Any):
"""Validate if the metrics value can be converted to type `float`."""
try: