hanzo-ml/katib
1
0
Fork 0
mirror of https://github.com/hanzo-ml/katib.git synced 2026-09-24 00:31:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enhance pre-commit hooks with flake8 and black (#2407)

Browse Source 
* Add black formater and flake8 linter to pre-commit

Also add's the flake8 config file

Signed-off-by: Ignas Baranauskas <[email protected]>

* Fixes black formating

Signed-off-by: Ignas Baranauskas <[email protected]>

* Fixes flake8 linting errors

Signed-off-by: Ignas Baranauskas <[email protected]>

---------

Signed-off-by: Ignas Baranauskas <[email protected]>
This commit is contained in:
Ignas Baranauskas
2024-08-16 10:13:28 +00:00
committed by GitHub
parent 2e30a54fa5
commit bcb199ddc3
21 changed files with 819 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
+4
View File
@@ -0,0 +1,4 @@
[flake8]
max-line-length = 100
# E203 is ignored to avoid conflicts with Black's formatting, as it's not PEP 8 compliant
extend-ignore = W503, E203
.pre-commit-config.yaml
+10
View File
@@ -11,6 +11,16 @@ repos:
- id: isort - id: isort
name: isort name: isort
entry: isort --profile google entry: isort --profile google
- repo: https://github.com/psf/black
rev: 24.2.0
hooks:
- id: black
files: (sdk|examples)/.*
- repo: https://github.com/pycqa/flake8
rev: 7.1.1
hooks:
- id: flake8
files: (sdk|examples)/.*
exclude: | exclude: |
(?x)^( (?x)^(
.*zz_generated.deepcopy.*| .*zz_generated.deepcopy.*|
examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py
+33 -55
View File
@@ -21,10 +21,12 @@
# This Experiment is similar to this: # This Experiment is similar to this:
# https://github.com/kubeflow/katib/blob/master/examples/v1beta1/kubeflow-training-operator/mpijob-horovod.yaml # https://github.com/kubeflow/katib/blob/master/examples/v1beta1/kubeflow-training-operator/mpijob-horovod.yaml
# Check the training container source code here: https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod. # Check the training container source code here:
# https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod.
# Note: To run this example, your Kubernetes cluster should run MPIJob operator. # Note: To run this example, your Kubernetes cluster should run MPIJob operator.
# Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/ # Follow this guide to install MPIJob on your cluster:
# https://www.kubeflow.org/docs/components/training/mpi/
import kfp import kfp
from kfp import components from kfp import components
@@ -42,13 +44,12 @@ from kubeflow.katib import V1beta1TrialTemplate
@dsl.pipeline( @dsl.pipeline(
name="Launch Katib MPIJob Experiment", name="Launch Katib MPIJob Experiment",
description="An example to launch Katib Experiment with MPIJob" description="An example to launch Katib Experiment with MPIJob",
) )
def horovod_mnist_hpo( def horovod_mnist_hpo(
experiment_name: str = "mpi-horovod-mnist", experiment_name: str = "mpi-horovod-mnist",
experiment_namespace: str = "kubeflow-user-example-com", experiment_namespace: str = "kubeflow-user-example-com",
): ):
# Trial count specification. # Trial count specification.
max_trial_count = 6 max_trial_count = 6
max_failed_trial_count = 3 max_failed_trial_count = 3
@@ -64,12 +65,7 @@ def horovod_mnist_hpo(
# Algorithm specification. # Algorithm specification.
algorithm = V1beta1AlgorithmSpec( algorithm = V1beta1AlgorithmSpec(
algorithm_name="bayesianoptimization", algorithm_name="bayesianoptimization",
algorithm_settings=[ algorithm_settings=[V1beta1AlgorithmSetting(name="random_state", value="10")],
V1beta1AlgorithmSetting(
name="random_state",
value="10"
)
]
) )
# Experiment search space. # Experiment search space.
@@ -78,19 +74,12 @@ def horovod_mnist_hpo(
V1beta1ParameterSpec( V1beta1ParameterSpec(
name="lr", name="lr",
parameter_type="double", parameter_type="double",
feasible_space=V1beta1FeasibleSpace( feasible_space=V1beta1FeasibleSpace(min="0.001", max="0.003"),
min="0.001",
max="0.003"
),
), ),
V1beta1ParameterSpec( V1beta1ParameterSpec(
name="num-steps", name="num-steps",
parameter_type="int", parameter_type="int",
feasible_space=V1beta1FeasibleSpace( feasible_space=V1beta1FeasibleSpace(min="50", max="150", step="10"),
min="50",
max="150",
step="10"
),
), ),
] ]
@@ -106,18 +95,14 @@ def horovod_mnist_hpo(
"replicas": 1, "replicas": 1,
"template": { "template": {
"metadata": { "metadata": {
"annotations": { "annotations": {"sidecar.istio.io/inject": "false"}
"sidecar.istio.io/inject": "false"
}
}, },
"spec": { "spec": {
"containers": [ "containers": [
{ {
"image": "docker.io/kubeflow/mpi-horovod-mnist", "image": "docker.io/kubeflow/mpi-horovod-mnist",
"name": "mpi-launcher", "name": "mpi-launcher",
"command": [ "command": ["mpirun"],
"mpirun"
],
"args": [ "args": [
"-np", "-np",
"2", "2",
@@ -141,26 +126,21 @@ def horovod_mnist_hpo(
"--lr", "--lr",
"${trialParameters.learningRate}", "${trialParameters.learningRate}",
"--num-steps", "--num-steps",
"${trialParameters.numberSteps}" "${trialParameters.numberSteps}",
], ],
"resources": { "resources": {
"limits": { "limits": {"cpu": "500m", "memory": "2Gi"}
"cpu": "500m", },
"memory": "2Gi"
}
}
} }
] ]
} },
} },
}, },
"Worker": { "Worker": {
"replicas": 2, "replicas": 2,
"template": { "template": {
"metadata": { "metadata": {
"annotations": { "annotations": {"sidecar.istio.io/inject": "false"}
"sidecar.istio.io/inject": "false"
}
}, },
"spec": { "spec": {
"containers": [ "containers": [
@@ -168,25 +148,20 @@ def horovod_mnist_hpo(
"image": "docker.io/kubeflow/mpi-horovod-mnist", "image": "docker.io/kubeflow/mpi-horovod-mnist",
"name": "mpi-worker", "name": "mpi-worker",
"resources": { "resources": {
"limits": { "limits": {"cpu": "500m", "memory": "4Gi"}
"cpu": "500m", },
"memory": "4Gi"
}
}
} }
] ]
} },
} },
} },
} },
} },
} }
# Configure parameters for the Trial template. # Configure parameters for the Trial template.
trial_template = V1beta1TrialTemplate( trial_template = V1beta1TrialTemplate(
primary_pod_labels={ primary_pod_labels={"mpi-job-role": "launcher"},
"mpi-job-role": "launcher"
},
primary_container_name="mpi-launcher", primary_container_name="mpi-launcher",
success_condition='status.conditions.#(type=="Succeeded")#|#(status=="True")#', success_condition='status.conditions.#(type=="Succeeded")#|#(status=="True")#',
failure_condition='status.conditions.#(type=="Failed")#|#(status=="True")#', failure_condition='status.conditions.#(type=="Failed")#|#(status=="True")#',
@@ -194,15 +169,15 @@ def horovod_mnist_hpo(
V1beta1TrialParameterSpec( V1beta1TrialParameterSpec(
name="learningRate", name="learningRate",
description="Learning rate for the training model", description="Learning rate for the training model",
reference="lr" reference="lr",
), ),
V1beta1TrialParameterSpec( V1beta1TrialParameterSpec(
name="numberSteps", name="numberSteps",
description="Number of training steps", description="Number of training steps",
reference="num-steps" reference="num-steps",
), ),
], ],
trial_spec=trial_spec trial_spec=trial_spec,
) )
# Create Experiment specification. # Create Experiment specification.
@@ -213,13 +188,15 @@ def horovod_mnist_hpo(
objective=objective, objective=objective,
algorithm=algorithm, algorithm=algorithm,
parameters=parameters, parameters=parameters,
trial_template=trial_template trial_template=trial_template,
) )
# Get the Katib launcher. # Get the Katib launcher.
# Load component from the URL or from the file. # Load component from the URL or from the file.
katib_experiment_launcher_op = components.load_component_from_url( katib_experiment_launcher_op = components.load_component_from_url(
"https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml") "https://raw.githubusercontent.com/kubeflow/pipelines/master/"
"components/kubeflow/katib-launcher/component.yaml"
)
# katib_experiment_launcher_op = components.load_component_from_file( # katib_experiment_launcher_op = components.load_component_from_file(
# "../../../components/kubeflow/katib-launcher/component.yaml" # "../../../components/kubeflow/katib-launcher/component.yaml"
# ) # )
@@ -231,7 +208,8 @@ def horovod_mnist_hpo(
experiment_name=experiment_name, experiment_name=experiment_name,
experiment_namespace=experiment_namespace, experiment_namespace=experiment_namespace,
experiment_spec=ApiClient().sanitize_for_serialization(experiment_spec), experiment_spec=ApiClient().sanitize_for_serialization(experiment_spec),
experiment_timeout_minutes=60) experiment_timeout_minutes=60,
)
# Output container to print the results. # Output container to print the results.
dsl.ContainerOp( dsl.ContainerOp(
examples/v1beta1/trial-images/darts-cnn-cifar10/architect.py
+26 -18
View File
@@ -17,9 +17,8 @@ import copy
import torch import torch
class Architect(): class Architect:
"""" Architect controls architecture of cell by computing gradients of alphas """ " Architect controls architecture of cell by computing gradients of alphas"""
"""
def __init__(self, model, w_momentum, w_weight_decay, device): def __init__(self, model, w_momentum, w_weight_decay, device):
self.model = model self.model = model
@@ -48,25 +47,32 @@ class Architect():
# Compute gradient # Compute gradient
gradients = torch.autograd.grad(loss, self.model.getWeights()) gradients = torch.autograd.grad(loss, self.model.getWeights())
# Do virtual step (Update gradient) # Do virtual step (Update gradient)
# Below operations do not need gradient tracking # Below operations do not need gradient tracking
with torch.no_grad(): with torch.no_grad():
# dict key is not the value, but the pointer. So original network weight have to # dict key is not the value, but the pointer. So original network weight have to
# be iterated also. # be iterated also.
for w, vw, g in zip(self.model.getWeights(), self.v_model.getWeights(), gradients): for w, vw, g in zip(
m = w_optim.state[w].get("momentum_buffer", 0.) * self.w_momentum self.model.getWeights(), self.v_model.getWeights(), gradients
if(self.device == 'cuda'): ):
vw.copy_(w - torch.cuda.FloatTensor(xi) * (m + g + self.w_weight_decay * w)) m = w_optim.state[w].get("momentum_buffer", 0.0) * self.w_momentum
elif(self.device == 'cpu'): if self.device == "cuda":
vw.copy_(w - torch.FloatTensor(xi) * (m + g + self.w_weight_decay * w)) vw.copy_(
w
- torch.cuda.FloatTensor(xi) * (m + g + self.w_weight_decay * w)
)
elif self.device == "cpu":
vw.copy_(
w - torch.FloatTensor(xi) * (m + g + self.w_weight_decay * w)
)
# Sync alphas # Sync alphas
for a, va in zip(self.model.getAlphas(), self.v_model.getAlphas()): for a, va in zip(self.model.getAlphas(), self.v_model.getAlphas()):
va.copy_(a) va.copy_(a)
def unrolled_backward(self, train_x, train_y, valid_x, valid_y, xi, w_optim): def unrolled_backward(self, train_x, train_y, valid_x, valid_y, xi, w_optim):
""" Compute unrolled loss and backward its gradients """Compute unrolled loss and backward its gradients
Args: Args:
xi: learning rate for virtual gradient step (same as model lr) xi: learning rate for virtual gradient step (same as model lr)
w_optim: weights optimizer - for virtual step w_optim: weights optimizer - for virtual step
@@ -77,23 +83,23 @@ class Architect():
# Calculate unrolled loss # Calculate unrolled loss
# Loss for validation with w'. L_valid(w') # Loss for validation with w'. L_valid(w')
loss = self.v_model.loss(valid_x, valid_y) loss = self.v_model.loss(valid_x, valid_y)
# Calculate gradient # Calculate gradient
v_alphas = tuple(self.v_model.getAlphas()) v_alphas = tuple(self.v_model.getAlphas())
v_weights = tuple(self.v_model.getWeights()) v_weights = tuple(self.v_model.getWeights())
v_grads = torch.autograd.grad(loss, v_alphas + v_weights) v_grads = torch.autograd.grad(loss, v_alphas + v_weights)
dalpha = v_grads[:len(v_alphas)] dalpha = v_grads[: len(v_alphas)]
dws = v_grads[len(v_alphas):] dws = v_grads[len(v_alphas) :]
hessian = self.compute_hessian(dws, train_x, train_y) hessian = self.compute_hessian(dws, train_x, train_y)
# Update final gradient = dalpha - xi * hessian # Update final gradient = dalpha - xi * hessian
with torch.no_grad(): with torch.no_grad():
for alpha, da, h in zip(self.model.getAlphas(), dalpha, hessian): for alpha, da, h in zip(self.model.getAlphas(), dalpha, hessian):
if(self.device == 'cuda'): if self.device == "cuda":
alpha.grad = da - torch.cuda.FloatTensor(xi) * h alpha.grad = da - torch.cuda.FloatTensor(xi) * h
elif(self.device == 'cpu'): elif self.device == "cpu":
alpha.grad = da - torch.cpu.FloatTensor(xi) * h alpha.grad = da - torch.cpu.FloatTensor(xi) * h
def compute_hessian(self, dws, train_x, train_y): def compute_hessian(self, dws, train_x, train_y):
@@ -121,7 +127,7 @@ class Architect():
with torch.no_grad(): with torch.no_grad():
for p, dw in zip(self.model.getWeights(), dws): for p, dw in zip(self.model.getWeights(), dws):
# TODO (andreyvelich): Do we need this * 2.0 ? # TODO (andreyvelich): Do we need this * 2.0 ?
p -= 2. * eps * dw p -= 2.0 * eps * dw
loss = self.model.loss(train_x, train_y) loss = self.model.loss(train_x, train_y)
# dalpha { L_train(w-, alpha) } # dalpha { L_train(w-, alpha) }
@@ -132,5 +138,7 @@ class Architect():
for p, dw in zip(self.model.getWeights(), dws): for p, dw in zip(self.model.getWeights(), dws):
p += eps * dw p += eps * dw
hessian = [(p-n) / (2. * eps) for p, n in zip(dalpha_positive, dalpha_negative)] hessian = [
(p - n) / (2.0 * eps) for p, n in zip(dalpha_positive, dalpha_negative)
]
return hessian return hessian
examples/v1beta1/trial-images/darts-cnn-cifar10/model.py
+54 -16
View File
@@ -21,11 +21,20 @@ import torch.nn.functional as F
class Cell(nn.Module): class Cell(nn.Module):
""" Cell for search """Cell for search
Each edge is mixed and continuous relaxed. Each edge is mixed and continuous relaxed.
""" """
def __init__(self, num_nodes, c_prev_prev, c_prev, c_cur, reduction_prev, reduction_cur, search_space): def __init__(
self,
num_nodes,
c_prev_prev,
c_prev,
c_cur,
reduction_prev,
reduction_cur,
search_space,
):
""" """
Args: Args:
num_nodes: Number of intermediate cell nodes num_nodes: Number of intermediate cell nodes
@@ -45,7 +54,9 @@ class Cell(nn.Module):
if reduction_prev: if reduction_prev:
self.preprocess0 = FactorizedReduce(c_prev_prev, c_cur) self.preprocess0 = FactorizedReduce(c_prev_prev, c_cur)
else: else:
self.preprocess0 = StdConv(c_prev_prev, c_cur, kernel_size=1, stride=1, padding=0) self.preprocess0 = StdConv(
c_prev_prev, c_cur, kernel_size=1, stride=1, padding=0
)
self.preprocess1 = StdConv(c_prev, c_cur, kernel_size=1, stride=1, padding=0) self.preprocess1 = StdConv(c_prev, c_cur, kernel_size=1, stride=1, padding=0)
# Generate dag from mixed operations # Generate dag from mixed operations
@@ -54,7 +65,7 @@ class Cell(nn.Module):
for i in range(self.num_nodes): for i in range(self.num_nodes):
self.dag_ops.append(nn.ModuleList()) self.dag_ops.append(nn.ModuleList())
# Include 2 input nodes # Include 2 input nodes
for j in range(2+i): for j in range(2 + i):
# Reduction with stride = 2 must be only for the input node # Reduction with stride = 2 must be only for the input node
stride = 2 if reduction_cur and j < 2 else 1 stride = 2 if reduction_cur and j < 2 else 1
op = MixedOp(c_cur, stride, search_space) op = MixedOp(c_cur, stride, search_space)
@@ -66,7 +77,9 @@ class Cell(nn.Module):
states = [s0, s1] states = [s0, s1]
for edges, w_list in zip(self.dag_ops, w_dag): for edges, w_list in zip(self.dag_ops, w_dag):
state_cur = sum(edges[i](s, w) for i, (s, w) in enumerate((zip(states, w_list)))) state_cur = sum(
edges[i](s, w) for i, (s, w) in enumerate((zip(states, w_list)))
)
states.append(state_cur) states.append(state_cur)
state_out = torch.cat(states[2:], dim=1) state_out = torch.cat(states[2:], dim=1)
@@ -75,8 +88,17 @@ class Cell(nn.Module):
class NetworkCNN(nn.Module): class NetworkCNN(nn.Module):
def __init__(self, init_channels, input_channels, num_classes, def __init__(
num_layers, criterion, search_space, num_nodes, stem_multiplier): self,
init_channels,
input_channels,
num_classes,
num_layers,
criterion,
search_space,
num_nodes,
stem_multiplier,
):
super(NetworkCNN, self).__init__() super(NetworkCNN, self).__init__()
self.init_channels = init_channels self.init_channels = init_channels
@@ -87,11 +109,11 @@ class NetworkCNN(nn.Module):
self.num_nodes = num_nodes self.num_nodes = num_nodes
self.stem_multiplier = stem_multiplier self.stem_multiplier = stem_multiplier
c_cur = self.stem_multiplier*self.init_channels c_cur = self.stem_multiplier * self.init_channels
self.stem = nn.Sequential( self.stem = nn.Sequential(
nn.Conv2d(input_channels, c_cur, 3, padding=1, bias=False), nn.Conv2d(input_channels, c_cur, 3, padding=1, bias=False),
nn.BatchNorm2d(c_cur) nn.BatchNorm2d(c_cur),
) )
# In first Cell stem is used for s0 and s1 # In first Cell stem is used for s0 and s1
@@ -110,14 +132,24 @@ class NetworkCNN(nn.Module):
# For Network with two layers: First layer - Normal, Second - Reduction # For Network with two layers: First layer - Normal, Second - Reduction
# For Other Networks: [1/3, 2/3] Layers - Reduction cell with double channels # For Other Networks: [1/3, 2/3] Layers - Reduction cell with double channels
# Others - Normal cell # Others - Normal cell
if ((self.num_layers == 2 and i == 1) or if (self.num_layers == 2 and i == 1) or (
(self.num_layers > 2 and i in [self.num_layers//3, 2*self.num_layers//3])): self.num_layers > 2
and i in [self.num_layers // 3, 2 * self.num_layers // 3]
):
c_cur *= 2 c_cur *= 2
reduction_cur = True reduction_cur = True
else: else:
reduction_cur = False reduction_cur = False
cell = Cell(self.num_nodes, c_prev_prev, c_prev, c_cur, reduction_prev, reduction_cur, search_space) cell = Cell(
self.num_nodes,
c_prev_prev,
c_prev,
c_cur,
reduction_prev,
reduction_cur,
search_space,
)
reduction_prev = reduction_cur reduction_prev = reduction_cur
self.cells.append(cell) self.cells.append(cell)
@@ -134,9 +166,11 @@ class NetworkCNN(nn.Module):
self.alpha_reduce = nn.ParameterList() self.alpha_reduce = nn.ParameterList()
for i in range(self.num_nodes): for i in range(self.num_nodes):
self.alpha_normal.append(nn.Parameter(1e-3*torch.randn(i+2, num_ops))) self.alpha_normal.append(nn.Parameter(1e-3 * torch.randn(i + 2, num_ops)))
if self.num_layers > 1: if self.num_layers > 1:
self.alpha_reduce.append(nn.Parameter(1e-3*torch.randn(i+2, num_ops))) self.alpha_reduce.append(
nn.Parameter(1e-3 * torch.randn(i + 2, num_ops))
)
# Setup alphas list # Setup alphas list
self.alphas = [] self.alphas = []
@@ -192,5 +226,9 @@ class NetworkCNN(nn.Module):
# concat all intermediate nodes # concat all intermediate nodes
concat = range(2, 2 + self.num_nodes) concat = range(2, 2 + self.num_nodes)
return search_space.genotype(normal=gene_normal, normal_concat=concat, return search_space.genotype(
reduce=gene_reduce, reduce_concat=concat) normal=gene_normal,
normal_concat=concat,
reduce=gene_reduce,
reduce_concat=concat,
)
examples/v1beta1/trial-images/darts-cnn-cifar10/operations.py
+62 -30
View File
@@ -16,18 +16,30 @@ import torch
import torch.nn as nn import torch.nn as nn
OPS = { OPS = {
'none': lambda channels, stride: Zero(stride), "none": lambda channels, stride: Zero(stride),
'avg_pooling_3x3': lambda channels, stride: PoolBN('avg', channels, kernel_size=3, stride=stride, padding=1), "avg_pooling_3x3": lambda channels, stride: PoolBN(
'max_pooling_3x3': lambda channels, stride: PoolBN('max', channels, kernel_size=3, stride=stride, padding=1), "avg", channels, kernel_size=3, stride=stride, padding=1
'skip_connection': lambda channels, stride: Identity() if stride == 1 else FactorizedReduce(channels, channels), ),
'separable_convolution_3x3': lambda channels, stride: SepConv(channels, kernel_size=3, stride=stride, padding=1), "max_pooling_3x3": lambda channels, stride: PoolBN(
'separable_convolution_5x5': lambda channels, stride: SepConv(channels, kernel_size=5, stride=stride, padding=2), "max", channels, kernel_size=3, stride=stride, padding=1
),
"skip_connection": lambda channels, stride: (
Identity() if stride == 1 else FactorizedReduce(channels, channels)
),
"separable_convolution_3x3": lambda channels, stride: SepConv(
channels, kernel_size=3, stride=stride, padding=1
),
"separable_convolution_5x5": lambda channels, stride: SepConv(
channels, kernel_size=5, stride=stride, padding=2
),
# 3x3 -> 5x5 # 3x3 -> 5x5
'dilated_convolution_3x3': lambda channels, stride: DilConv(channels, "dilated_convolution_3x3": lambda channels, stride: DilConv(
kernel_size=3, stride=stride, padding=2, dilation=2), channels, kernel_size=3, stride=stride, padding=2, dilation=2
),
# 5x5 -> 9x9 # 5x5 -> 9x9
'dilated_convolution_5x5': lambda channels, stride: DilConv(channels, "dilated_convolution_5x5": lambda channels, stride: DilConv(
kernel_size=5, stride=stride, padding=4, dilation=2), channels, kernel_size=5, stride=stride, padding=4, dilation=2
),
} }
@@ -42,9 +54,9 @@ class Zero(nn.Module):
def forward(self, x): def forward(self, x):
if self.stride == 1: if self.stride == 1:
return x * 0. return x * 0.0
# Resize by stride # Resize by stride
return x[:, :, ::self.stride, ::self.stride] * 0. return x[:, :, :: self.stride, :: self.stride] * 0.0
class PoolBN(nn.Module): class PoolBN(nn.Module):
@@ -55,15 +67,14 @@ class PoolBN(nn.Module):
def __init__(self, pool_type, channels, kernel_size, stride, padding): def __init__(self, pool_type, channels, kernel_size, stride, padding):
super(PoolBN, self).__init__() super(PoolBN, self).__init__()
if pool_type == "avg": if pool_type == "avg":
self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False) self.pool = nn.AvgPool2d(
kernel_size, stride, padding, count_include_pad=False
)
elif pool_type == "max": elif pool_type == "max":
self.pool = nn.MaxPool2d(kernel_size, stride, padding) self.pool = nn.MaxPool2d(kernel_size, stride, padding)
self.bn = nn.BatchNorm2d(channels, affine=False) self.bn = nn.BatchNorm2d(channels, affine=False)
self.net = nn.Sequential( self.net = nn.Sequential(self.pool, self.bn)
self.pool,
self.bn
)
def forward(self, x): def forward(self, x):
# out = self.pool(x), # out = self.pool(x),
@@ -91,8 +102,12 @@ class FactorizedReduce(nn.Module):
def __init__(self, c_in, c_out): def __init__(self, c_in, c_out):
super(FactorizedReduce, self).__init__() super(FactorizedReduce, self).__init__()
self.relu = nn.ReLU() self.relu = nn.ReLU()
self.conv1 = nn.Conv2d(c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False) self.conv1 = nn.Conv2d(
self.conv2 = nn.Conv2d(c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False) c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False
)
self.conv2 = nn.Conv2d(
c_in, c_out // 2, kernel_size=1, stride=2, padding=0, bias=False
)
self.bn = nn.BatchNorm2d(c_out, affine=False) self.bn = nn.BatchNorm2d(c_out, affine=False)
def forward(self, x): def forward(self, x):
@@ -105,7 +120,7 @@ class FactorizedReduce(nn.Module):
class StdConv(nn.Module): class StdConv(nn.Module):
""" Standard convolition """Standard convolition
ReLU - Conv - BN ReLU - Conv - BN
""" """
@@ -113,8 +128,15 @@ class StdConv(nn.Module):
super(StdConv, self).__init__() super(StdConv, self).__init__()
self.net = nn.Sequential( self.net = nn.Sequential(
nn.ReLU(), nn.ReLU(),
nn.Conv2d(c_in, c_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=False), nn.Conv2d(
nn.BatchNorm2d(c_out, affine=False) c_in,
c_out,
kernel_size=kernel_size,
stride=stride,
padding=padding,
bias=False,
),
nn.BatchNorm2d(c_out, affine=False),
) )
def forward(self, x): def forward(self, x):
@@ -122,7 +144,7 @@ class StdConv(nn.Module):
class DilConv(nn.Module): class DilConv(nn.Module):
""" (Dilated) depthwise separable conv """(Dilated) depthwise separable conv
ReLU - (Dilated) depthwise separable - Pointwise - BN ReLU - (Dilated) depthwise separable - Pointwise - BN
If dilation == 2, 3x3 conv => 5x5 receptive field If dilation == 2, 3x3 conv => 5x5 receptive field
@@ -134,9 +156,20 @@ class DilConv(nn.Module):
self.net = nn.Sequential( self.net = nn.Sequential(
nn.ReLU(), nn.ReLU(),
nn.Conv2d(channels, channels, kernel_size, stride, padding, dilation=dilation, groups=channels, bias=False), nn.Conv2d(
nn.Conv2d(channels, channels, kernel_size=1, stride=1, padding=0, bias=False), channels,
nn.BatchNorm2d(channels, affine=False) channels,
kernel_size,
stride,
padding,
dilation=dilation,
groups=channels,
bias=False,
),
nn.Conv2d(
channels, channels, kernel_size=1, stride=1, padding=0, bias=False
),
nn.BatchNorm2d(channels, affine=False),
) )
def forward(self, x): def forward(self, x):
@@ -144,7 +177,7 @@ class DilConv(nn.Module):
class SepConv(nn.Module): class SepConv(nn.Module):
""" Depthwise separable conv """Depthwise separable conv
DilConv (dilation=1) * 2 DilConv (dilation=1) * 2
""" """
@@ -152,7 +185,7 @@ class SepConv(nn.Module):
super(SepConv, self).__init__() super(SepConv, self).__init__()
self.net = nn.Sequential( self.net = nn.Sequential(
DilConv(channels, kernel_size, stride=stride, padding=padding, dilation=1), DilConv(channels, kernel_size, stride=stride, padding=padding, dilation=1),
DilConv(channels, kernel_size, stride=1, padding=padding, dilation=1) DilConv(channels, kernel_size, stride=1, padding=padding, dilation=1),
) )
def forward(self, x): def forward(self, x):
@@ -160,8 +193,7 @@ class SepConv(nn.Module):
class MixedOp(nn.Module): class MixedOp(nn.Module):
""" Mixed operation """Mixed operation"""
"""
def __init__(self, channels, stride, search_space): def __init__(self, channels, stride, search_space):
super(MixedOp, self).__init__() super(MixedOp, self).__init__()
examples/v1beta1/trial-images/darts-cnn-cifar10/run_trial.py
+118 -39
View File
@@ -27,15 +27,27 @@ import utils
def main(): def main():
parser = argparse.ArgumentParser(description='TrainingContainer') parser = argparse.ArgumentParser(description="TrainingContainer")
parser.add_argument('--algorithm-settings', type=str, default="", help="algorithm settings") parser.add_argument(
parser.add_argument('--search-space', type=str, default="", help="search space for the neural architecture search") "--algorithm-settings", type=str, default="", help="algorithm settings"
parser.add_argument('--num-layers', type=str, default="", help="number of layers of the neural network") )
parser.add_argument(
"--search-space",
type=str,
default="",
help="search space for the neural architecture search",
)
parser.add_argument(
"--num-layers",
type=str,
default="",
help="number of layers of the neural network",
)
args = parser.parse_args() args = parser.parse_args()
# Get Algorithm Settings # Get Algorithm Settings
algorithm_settings = args.algorithm_settings.replace("\'", "\"") algorithm_settings = args.algorithm_settings.replace("'", '"')
algorithm_settings = json.loads(algorithm_settings) algorithm_settings = json.loads(algorithm_settings)
print(">>> Algorithm settings") print(">>> Algorithm settings")
for key, value in algorithm_settings.items(): for key, value in algorithm_settings.items():
@@ -69,7 +81,7 @@ def main():
stem_multiplier = int(algorithm_settings["stem_multiplier"]) stem_multiplier = int(algorithm_settings["stem_multiplier"])
# Get Search Space # Get Search Space
search_space = args.search_space.replace("\'", "\"") search_space = args.search_space.replace("'", '"')
search_space = json.loads(search_space) search_space = json.loads(search_space)
search_space = SearchSpace(search_space) search_space = SearchSpace(search_space)
@@ -103,16 +115,28 @@ def main():
criterion = nn.CrossEntropyLoss().to(device) criterion = nn.CrossEntropyLoss().to(device)
model = NetworkCNN(init_channels, input_channels, num_classes, num_layers, model = NetworkCNN(
criterion, search_space, num_nodes, stem_multiplier) init_channels,
input_channels,
num_classes,
num_layers,
criterion,
search_space,
num_nodes,
stem_multiplier,
)
model = model.to(device) model = model.to(device)
# Weights optimizer # Weights optimizer
w_optim = torch.optim.SGD(model.getWeights(), w_lr, momentum=w_momentum, weight_decay=w_weight_decay) w_optim = torch.optim.SGD(
model.getWeights(), w_lr, momentum=w_momentum, weight_decay=w_weight_decay
)
# Alphas optimizer # Alphas optimizer
alpha_optim = torch.optim.Adam(model.getAlphas(), alpha_lr, betas=(0.5, 0.999), weight_decay=alpha_weight_decay) alpha_optim = torch.optim.Adam(
model.getAlphas(), alpha_lr, betas=(0.5, 0.999), weight_decay=alpha_weight_decay
)
# Split data to train/validation # Split data to train/validation
num_train = len(train_data) num_train = len(train_data)
@@ -122,27 +146,30 @@ def main():
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split]) train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:]) valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(train_data, train_loader = torch.utils.data.DataLoader(
batch_size=batch_size, train_data,
sampler=train_sampler, batch_size=batch_size,
num_workers=num_workers, sampler=train_sampler,
pin_memory=True) num_workers=num_workers,
pin_memory=True,
)
valid_loader = torch.utils.data.DataLoader(train_data, valid_loader = torch.utils.data.DataLoader(
batch_size=batch_size, train_data,
sampler=valid_sampler, batch_size=batch_size,
num_workers=num_workers, sampler=valid_sampler,
pin_memory=True) num_workers=num_workers,
pin_memory=True,
)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, w_optim, num_epochs, eta_min=w_lr_min
num_epochs, )
eta_min=w_lr_min)
architect = Architect(model, w_momentum, w_weight_decay, device) architect = Architect(model, w_momentum, w_weight_decay, device)
# Start training # Start training
best_top1 = 0. best_top1 = 0.0
for epoch in range(num_epochs): for epoch in range(num_epochs):
lr = lr_scheduler.get_last_lr() lr = lr_scheduler.get_last_lr()
@@ -151,14 +178,28 @@ def main():
# Training # Training
print(">>> Training") print(">>> Training")
train(train_loader, valid_loader, model, architect, w_optim, alpha_optim, train(
lr, epoch, num_epochs, device, w_grad_clip, print_step) train_loader,
valid_loader,
model,
architect,
w_optim,
alpha_optim,
lr,
epoch,
num_epochs,
device,
w_grad_clip,
print_step,
)
lr_scheduler.step() lr_scheduler.step()
# Validation # Validation
print("\n>>> Validation") print("\n>>> Validation")
cur_step = (epoch + 1) * len(train_loader) cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_step) top1 = validate(
valid_loader, model, epoch, cur_step, num_epochs, device, print_step
)
# Print genotype # Print genotype
genotype = model.genotype(search_space) genotype = model.genotype(search_space)
@@ -173,18 +214,36 @@ def main():
print("\nBest-Genotype={}".format(str(best_genotype).replace(" ", ""))) print("\nBest-Genotype={}".format(str(best_genotype).replace(" ", "")))
def train(train_loader, valid_loader, model, architect, w_optim, alpha_optim, def train(
lr, epoch, num_epochs, device, w_grad_clip, print_step): train_loader,
valid_loader,
model,
architect,
w_optim,
alpha_optim,
lr,
epoch,
num_epochs,
device,
w_grad_clip,
print_step,
):
top1 = utils.AverageMeter() top1 = utils.AverageMeter()
top5 = utils.AverageMeter() top5 = utils.AverageMeter()
losses = utils.AverageMeter() losses = utils.AverageMeter()
cur_step = epoch * len(train_loader) cur_step = epoch * len(train_loader)
model.train() model.train()
for step, ((train_x, train_y), (valid_x, valid_y)) in enumerate(zip(train_loader, valid_loader)): for step, ((train_x, train_y), (valid_x, valid_y)) in enumerate(
zip(train_loader, valid_loader)
):
train_x, train_y = train_x.to(device, non_blocking=True), train_y.to(device, non_blocking=True) train_x, train_y = train_x.to(device, non_blocking=True), train_y.to(
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(device, non_blocking=True) device, non_blocking=True
)
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(
device, non_blocking=True
)
train_size = train_x.size(0) train_size = train_x.size(0)
@@ -213,12 +272,21 @@ def train(train_loader, valid_loader, model, architect, w_optim, alpha_optim,
print( print(
"Train: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} " "Train: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} "
"Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format( "Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format(
epoch+1, num_epochs, step, len(train_loader)-1, losses=losses, epoch + 1,
top1=top1, top5=top5)) num_epochs,
step,
len(train_loader) - 1,
losses=losses,
top1=top1,
top5=top5,
)
)
cur_step += 1 cur_step += 1
print("Train: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch+1, num_epochs, top1.avg)) print(
"Train: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch + 1, num_epochs, top1.avg)
)
def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_step): def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_step):
@@ -230,7 +298,9 @@ def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_ste
with torch.no_grad(): with torch.no_grad():
for step, (valid_x, valid_y) in enumerate(valid_loader): for step, (valid_x, valid_y) in enumerate(valid_loader):
valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(device, non_blocking=True) valid_x, valid_y = valid_x.to(device, non_blocking=True), valid_y.to(
device, non_blocking=True
)
valid_size = valid_x.size(0) valid_size = valid_x.size(0)
@@ -246,10 +316,19 @@ def validate(valid_loader, model, epoch, cur_step, num_epochs, device, print_ste
print( print(
"Validation: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} " "Validation: [{:2d}/{}] Step {:03d}/{:03d} Loss {losses.avg:.3f} "
"Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format( "Prec@(1,5) ({top1.avg:.1%}, {top5.avg:.1%})".format(
epoch+1, num_epochs, step, len(valid_loader)-1, losses=losses, epoch + 1,
top1=top1, top5=top5)) num_epochs,
step,
len(valid_loader) - 1,
losses=losses,
top1=top1,
top5=top5,
)
)
print("Valid: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch+1, num_epochs, top1.avg)) print(
"Valid: [{:2d}/{}] Final Prec@1 {:.4%}".format(epoch + 1, num_epochs, top1.avg)
)
return top1.avg return top1.avg
examples/v1beta1/trial-images/darts-cnn-cifar10/search_space.py
+5 -3
View File
@@ -17,14 +17,16 @@ from collections import namedtuple
import torch import torch
class SearchSpace(): class SearchSpace:
def __init__(self, search_space): def __init__(self, search_space):
self.primitives = search_space self.primitives = search_space
self.primitives.append("none") self.primitives.append("none")
print(">>> All Primitives") print(">>> All Primitives")
print("{}\n".format(self.primitives)) print("{}\n".format(self.primitives))
self.genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') self.genotype = namedtuple(
"Genotype", "normal normal_concat reduce reduce_concat"
)
def parse(self, alpha, k): def parse(self, alpha, k):
""" """
@@ -46,7 +48,7 @@ class SearchSpace():
""" """
gene = [] gene = []
assert self.primitives[-1] == 'none' # assume last PRIMITIVE is 'none' assert self.primitives[-1] == "none" # assume last PRIMITIVE is 'none'
# 1) Convert the mixed op to discrete edge (single op) by choosing top-1 weight edge # 1) Convert the mixed op to discrete edge (single op) by choosing top-1 weight edge
# 2) Choose top-k edges per node by edge score (top-1 weight in edge) # 2) Choose top-k edges per node by edge score (top-1 weight in edge)
examples/v1beta1/trial-images/darts-cnn-cifar10/utils.py
+10 -14
View File
@@ -16,21 +16,21 @@ import torchvision.datasets as dset
import torchvision.transforms as transforms import torchvision.transforms as transforms
class AverageMeter(): class AverageMeter:
""" Computes and stores the average and current value """ """Computes and stores the average and current value"""
def __init__(self): def __init__(self):
self.reset() self.reset()
def reset(self): def reset(self):
""" Reset all statistics """ """Reset all statistics"""
self.val = 0 self.val = 0
self.avg = 0 self.avg = 0
self.sum = 0 self.sum = 0
self.count = 0 self.count = 0
def update(self, val, n=1): def update(self, val, n=1):
""" Update statistics """ """Update statistics"""
self.val = val self.val = val
self.sum += val * n self.sum += val * n
self.count += n self.count += n
@@ -38,7 +38,7 @@ class AverageMeter():
def accuracy(output, target, topk=(1,)): def accuracy(output, target, topk=(1,)):
""" Computes the precision@k for the specified values of k """ """Computes the precision@k for the specified values of k"""
maxk = max(topk) maxk = max(topk)
batch_size = target.size(0) batch_size = target.size(0)
@@ -67,18 +67,14 @@ def get_dataset():
# Do preprocessing # Do preprocessing
MEAN = [0.49139968, 0.48215827, 0.44653124] MEAN = [0.49139968, 0.48215827, 0.44653124]
STD = [0.24703233, 0.24348505, 0.26158768] STD = [0.24703233, 0.24348505, 0.26158768]
transf = [ transf = [transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip()]
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip()
]
normalize = [ normalize = [transforms.ToTensor(), transforms.Normalize(MEAN, STD)]
transforms.ToTensor(),
transforms.Normalize(MEAN, STD)
]
train_transform = transforms.Compose(transf + normalize) train_transform = transforms.Compose(transf + normalize)
train_data = dataset_cls(root="./data", train=True, download=True, transform=train_transform) train_data = dataset_cls(
root="./data", train=True, download=True, transform=train_transform
)
return input_channels, num_classes, train_data return input_channels, num_classes, train_data
examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py
+16 -28
View File
@@ -14,20 +14,11 @@
import json import json
from keras import backend as K
from keras.layers import Activation
from keras.layers import AveragePooling2D
from keras.layers import BatchNormalization
from keras.layers import concatenate
from keras.layers import Conv2D
from keras.layers import Dense from keras.layers import Dense
from keras.layers import Dropout from keras.layers import Dropout
from keras.layers import GlobalAveragePooling2D from keras.layers import GlobalAveragePooling2D
from keras.layers import Input from keras.layers import Input
from keras.layers import MaxPooling2D
from keras.layers import ZeroPadding2D
from keras.models import Model from keras.models import Model
import numpy as np
from op_library import concat from op_library import concat
from op_library import conv from op_library import conv
from op_library import dw_conv from op_library import dw_conv
@@ -39,16 +30,14 @@ class ModelConstructor(object):
def __init__(self, arc_json, nn_json): def __init__(self, arc_json, nn_json):
self.arch = json.loads(arc_json) self.arch = json.loads(arc_json)
nn_config = json.loads(nn_json) nn_config = json.loads(nn_json)
self.num_layers = nn_config['num_layers'] self.num_layers = nn_config["num_layers"]
self.input_sizes = nn_config['input_sizes'] self.input_sizes = nn_config["input_sizes"]
self.output_size = nn_config['output_sizes'][-1] self.output_size = nn_config["output_sizes"][-1]
self.embedding = nn_config['embedding'] self.embedding = nn_config["embedding"]
def build_model(self): def build_model(self):
# a list of the data all layers # a list of the data all layers
all_layers = [0 for _ in range(self.num_layers + 1)] all_layers = [0 for _ in range(self.num_layers + 1)]
# a list of all the dimensions of all layers
all_dims = [0 for _ in range(self.num_layers + 1)]
# ================= Stacking layers ================= # ================= Stacking layers =================
# Input Layer. Layer 0 # Input Layer. Layer 0
@@ -56,38 +45,37 @@ class ModelConstructor(object):
all_layers[0] = input_layer all_layers[0] = input_layer
# Intermediate Layers. Starting from layer 1. # Intermediate Layers. Starting from layer 1.
for l in range(1, self.num_layers + 1): for l_index in range(1, self.num_layers + 1):
input_layers = list() input_layers = list()
opt = self.arch[l - 1][0] opt = self.arch[l_index - 1][0]
opt_config = self.embedding[str(opt)] opt_config = self.embedding[str(opt)]
skip = self.arch[l - 1][1:l+1] skip = self.arch[l_index - 1][1 : l_index + 1]
# set up the connection to the previous layer first # set up the connection to the previous layer first
input_layers.append(all_layers[l - 1]) input_layers.append(all_layers[l_index - 1])
# then add skip connections # then add skip connections
for i in range(l - 1): for i in range(l_index - 1):
if l > 1 and skip[i] == 1: if l_index > 1 and skip[i] == 1:
input_layers.append(all_layers[i]) input_layers.append(all_layers[i])
layer_input = concat(input_layers) layer_input = concat(input_layers)
if opt_config['opt_type'] == 'convolution': if opt_config["opt_type"] == "convolution":
layer_output = conv(layer_input, opt_config) layer_output = conv(layer_input, opt_config)
if opt_config['opt_type'] == 'separable_convolution': if opt_config["opt_type"] == "separable_convolution":
layer_output = sp_conv(layer_input, opt_config) layer_output = sp_conv(layer_input, opt_config)
if opt_config['opt_type'] == 'depthwise_convolution': if opt_config["opt_type"] == "depthwise_convolution":
layer_output = dw_conv(layer_input, opt_config) layer_output = dw_conv(layer_input, opt_config)
elif opt_config['opt_type'] == 'reduction': elif opt_config["opt_type"] == "reduction":
layer_output = reduction(layer_input, opt_config) layer_output = reduction(layer_input, opt_config)
all_layers[l] = layer_output all_layers[l_index] = layer_output
# Final Layer # Final Layer
# Global Average Pooling, then Fully connected with softmax. # Global Average Pooling, then Fully connected with softmax.
avgpooled = GlobalAveragePooling2D()(all_layers[self.num_layers]) avgpooled = GlobalAveragePooling2D()(all_layers[self.num_layers])
dropped = Dropout(0.4)(avgpooled) dropped = Dropout(0.4)(avgpooled)
logits = Dense(units=self.output_size, logits = Dense(units=self.output_size, activation="softmax")(dropped)
activation='softmax')(dropped)
# Encapsulate the model # Encapsulate the model
self.model = Model(inputs=input_layer, outputs=logits) self.model = Model(inputs=input_layer, outputs=logits)
examples/v1beta1/trial-images/enas-cnn-cifar10/RunTrial.py
+60 -33
View File
@@ -24,22 +24,42 @@ from tensorflow.keras.layers import Rescaling
from tensorflow.keras.utils import to_categorical from tensorflow.keras.utils import to_categorical
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='TrainingContainer') parser = argparse.ArgumentParser(description="TrainingContainer")
parser.add_argument('--architecture', type=str, default="", metavar='N', parser.add_argument(
help='architecture of the neural network') "--architecture",
parser.add_argument('--nn_config', type=str, default="", metavar='N', type=str,
help='configurations and search space embeddings') default="",
parser.add_argument('--num_epochs', type=int, default=10, metavar='N', metavar="N",
help='number of epoches that each child will be trained') help="architecture of the neural network",
parser.add_argument('--num_gpus', type=int, default=1, metavar='N', )
help='number of GPU that used for training') parser.add_argument(
"--nn_config",
type=str,
default="",
metavar="N",
help="configurations and search space embeddings",
)
parser.add_argument(
"--num_epochs",
type=int,
default=10,
metavar="N",
help="number of epoches that each child will be trained",
)
parser.add_argument(
"--num_gpus",
type=int,
default=1,
metavar="N",
help="number of GPU that used for training",
)
args = parser.parse_args() args = parser.parse_args()
arch = args.architecture.replace("\'", "\"") arch = args.architecture.replace("'", '"')
print(">>> arch received by trial") print(">>> arch received by trial")
print(arch) print(arch)
nn_config = args.nn_config.replace("\'", "\"") nn_config = args.nn_config.replace("'", '"')
print(">>> nn_config received by trial") print(">>> nn_config received by trial")
print(nn_config) print(nn_config)
@@ -54,36 +74,40 @@ if __name__ == "__main__":
print("\n>>> Constructing Model...") print("\n>>> Constructing Model...")
constructor = ModelConstructor(arch, nn_config) constructor = ModelConstructor(arch, nn_config)
num_physical_gpus = len(tf.config.experimental.list_physical_devices('GPU')) num_physical_gpus = len(tf.config.experimental.list_physical_devices("GPU"))
if 1 <= num_gpus <= num_physical_gpus: if 1 <= num_gpus <= num_physical_gpus:
devices = ["/gpu:"+str(i) for i in range(num_physical_gpus)] devices = ["/gpu:" + str(i) for i in range(num_physical_gpus)]
else: else:
num_physical_cpu = len(tf.config.experimental.list_physical_devices('CPU')) num_physical_cpu = len(tf.config.experimental.list_physical_devices("CPU"))
devices = ["/cpu:"+str(j) for j in range(num_physical_cpu)] devices = ["/cpu:" + str(j) for j in range(num_physical_cpu)]
strategy = tf.distribute.MirroredStrategy(devices) strategy = tf.distribute.MirroredStrategy(devices)
with strategy.scope(): with strategy.scope():
test_model = constructor.build_model() test_model = constructor.build_model()
test_model.summary() test_model.summary()
test_model.compile(loss=keras.losses.categorical_crossentropy, test_model.compile(
optimizer=keras.optimizers.Adam(learning_rate=1e-3), loss=keras.losses.categorical_crossentropy,
metrics=['accuracy']) optimizer=keras.optimizers.Adam(learning_rate=1e-3),
metrics=["accuracy"],
)
print(">>> Model Constructed Successfully\n") print(">>> Model Constructed Successfully\n")
(x_train, y_train), (x_test, y_test) = cifar10.load_data() (x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32') x_train = x_train.astype("float32")
x_test = x_test.astype('float32') x_test = x_test.astype("float32")
x_train /= 255 x_train /= 255
x_test /= 255 x_test /= 255
y_train = to_categorical(y_train) y_train = to_categorical(y_train)
y_test = to_categorical(y_test) y_test = to_categorical(y_test)
augmentation = tf.keras.Sequential([ augmentation = tf.keras.Sequential(
Rescaling(1./255), [
RandomFlip('horizontal'), Rescaling(1.0 / 255),
RandomTranslation(height_factor=0.1, width_factor=0.1), RandomFlip("horizontal"),
]) RandomTranslation(height_factor=0.1, width_factor=0.1),
]
)
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)) train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
train_dataset = train_dataset.map(lambda x, y: (augmentation(x, training=True), y)) train_dataset = train_dataset.map(lambda x, y: (augmentation(x, training=True), y))
@@ -93,11 +117,14 @@ if __name__ == "__main__":
print(">>> Data Loaded. Training starts.") print(">>> Data Loaded. Training starts.")
for e in range(num_epochs): for e in range(num_epochs):
print("\nTotal Epoch {}/{}".format(e + 1, num_epochs)) print("\nTotal Epoch {}/{}".format(e + 1, num_epochs))
history = test_model.fit(train_dataset, history = test_model.fit(
steps_per_epoch=int(len(x_train) / 128) + 1, train_dataset,
epochs=1, verbose=1, steps_per_epoch=int(len(x_train) / 128) + 1,
validation_data=(x_test, y_test)) epochs=1,
print("Training-Accuracy={}".format(history.history['accuracy'][-1])) verbose=1,
print("Training-Loss={}".format(history.history['loss'][-1])) validation_data=(x_test, y_test),
print("Validation-Accuracy={}".format(history.history['val_accuracy'][-1])) )
print("Validation-Loss={}".format(history.history['val_loss'][-1])) print("Training-Accuracy={}".format(history.history["accuracy"][-1]))
print("Training-Loss={}".format(history.history["loss"][-1]))
print("Validation-Accuracy={}".format(history.history["val_accuracy"][-1]))
print("Validation-Loss={}".format(history.history["val_loss"][-1]))
examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py
+51 -46
View File
@@ -18,10 +18,7 @@ from keras.layers import AveragePooling2D
from keras.layers import BatchNormalization from keras.layers import BatchNormalization
from keras.layers import concatenate from keras.layers import concatenate
from keras.layers import Conv2D from keras.layers import Conv2D
from keras.layers import Dense
from keras.layers import DepthwiseConv2D from keras.layers import DepthwiseConv2D
from keras.layers import GlobalAveragePooling2D
from keras.layers import Input
from keras.layers import MaxPooling2D from keras.layers import MaxPooling2D
from keras.layers import SeparableConv2D from keras.layers import SeparableConv2D
from keras.layers import ZeroPadding2D from keras.layers import ZeroPadding2D
@@ -46,10 +43,13 @@ def concat(inputs):
diff = max_dim - total_dim[i][1] diff = max_dim - total_dim[i][1]
half_diff = int(diff / 2) half_diff = int(diff / 2)
if diff % 2 == 0: if diff % 2 == 0:
padded_input[i] = ZeroPadding2D(padding=(half_diff, half_diff))(inputs[i]) padded_input[i] = ZeroPadding2D(padding=(half_diff, half_diff))(
inputs[i]
)
else: else:
padded_input[i] = ZeroPadding2D(padding=((half_diff, half_diff + 1), padded_input[i] = ZeroPadding2D(
(half_diff, half_diff + 1)))(inputs[i]) padding=((half_diff, half_diff + 1), (half_diff, half_diff + 1))
)(inputs[i])
else: else:
padded_input[i] = inputs[i] padded_input[i] = inputs[i]
@@ -59,21 +59,22 @@ def concat(inputs):
def conv(x, config): def conv(x, config):
parameters = { parameters = {
"num_filter": 64, "num_filter": 64,
"filter_size": 3, "filter_size": 3,
"stride": 1, "stride": 1,
} }
for k in parameters.keys(): for k in parameters.keys():
if k in config: if k in config:
parameters[k] = int(config[k]) parameters[k] = int(config[k])
activated = Activation('relu')(x) activated = Activation("relu")(x)
conved = Conv2D( conved = Conv2D(
filters=parameters['num_filter'], filters=parameters["num_filter"],
kernel_size=parameters['filter_size'], kernel_size=parameters["filter_size"],
strides=parameters['stride'], strides=parameters["stride"],
padding='same')(activated) padding="same",
)(activated)
result = BatchNormalization()(conved) result = BatchNormalization()(conved)
@@ -82,9 +83,9 @@ def conv(x, config):
def sp_conv(x, config): def sp_conv(x, config):
parameters = { parameters = {
"num_filter": 64, "num_filter": 64,
"filter_size": 3, "filter_size": 3,
"stride": 1, "stride": 1,
"depth_multiplier": 1, "depth_multiplier": 1,
} }
@@ -92,36 +93,39 @@ def sp_conv(x, config):
if k in config: if k in config:
parameters[k] = int(config[k]) parameters[k] = int(config[k])
activated = Activation('relu')(x) activated = Activation("relu")(x)
conved = SeparableConv2D( conved = SeparableConv2D(
filters=parameters['num_filter'], filters=parameters["num_filter"],
kernel_size=parameters['filter_size'], kernel_size=parameters["filter_size"],
strides=parameters['stride'], strides=parameters["stride"],
depth_multiplier=parameters['depth_multiplier'], depth_multiplier=parameters["depth_multiplier"],
padding='same')(activated) padding="same",
)(activated)
result = BatchNormalization()(conved) result = BatchNormalization()(conved)
return result return result
def dw_conv(x, config): def dw_conv(x, config):
parameters = { parameters = {
"filter_size": 3, "filter_size": 3,
"stride": 1, "stride": 1,
"depth_multiplier": 1, "depth_multiplier": 1,
} }
for k in parameters.keys(): for k in parameters.keys():
if k in config: if k in config:
parameters[k] = int(config[k]) parameters[k] = int(config[k])
activated = Activation('relu')(x) activated = Activation("relu")(x)
conved = DepthwiseConv2D( conved = DepthwiseConv2D(
kernel_size=parameters['filter_size'], kernel_size=parameters["filter_size"],
strides=parameters['stride'], strides=parameters["stride"],
depth_multiplier=parameters['depth_multiplier'], depth_multiplier=parameters["depth_multiplier"],
padding='same')(activated) padding="same",
)(activated)
result = BatchNormalization()(conved) result = BatchNormalization()(conved)
@@ -134,31 +138,32 @@ def reduction(x, config):
# such situation is very likely to appear though # such situation is very likely to appear though
dim = K.int_shape(x) dim = K.int_shape(x)
if dim[1] == 1 or dim[2] == 1: if dim[1] == 1 or dim[2] == 1:
print("WARNING: One or more dimensions of the input of the reduction layer is 1. It cannot be further reduced. A identity layer will be used instead.") print(
"WARNING: One or more dimensions of the input of the reduction layer is 1. "
"It cannot be further reduced. A identity layer will be used instead."
)
return x return x
parameters = { parameters = {
'reduction_type': "max_pooling", "reduction_type": "max_pooling",
'pool_size': 2, "pool_size": 2,
'stride': None, "stride": None,
} }
if 'reduction_type' in config: if "reduction_type" in config:
parameters['reduction_type'] = config['reduction_type'] parameters["reduction_type"] = config["reduction_type"]
if 'pool_size' in config: if "pool_size" in config:
parameters['pool_size'] = int(config['pool_size']) parameters["pool_size"] = int(config["pool_size"])
if 'stride' in config: if "stride" in config:
parameters['stride'] = int(config['stride']) parameters["stride"] = int(config["stride"])
if parameters['reduction_type'] == 'max_pooling': if parameters["reduction_type"] == "max_pooling":
result = MaxPooling2D( result = MaxPooling2D(
pool_size=parameters['pool_size'], pool_size=parameters["pool_size"], strides=parameters["stride"]
strides=parameters['stride']
)(x) )(x)
elif parameters['reduction_type'] == 'avg_pooling': elif parameters["reduction_type"] == "avg_pooling":
result = AveragePooling2D( result = AveragePooling2D(
pool_size=parameters['pool_size'], pool_size=parameters["pool_size"], strides=parameters["stride"]
strides=parameters['stride']
)(x) )(x)
return result return result
examples/v1beta1/trial-images/pytorch-mnist/mnist.py
+126 -57
View File
@@ -35,7 +35,7 @@ class Net(nn.Module):
super(Net, self).__init__() super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 20, 5, 1) self.conv1 = nn.Conv2d(1, 20, 5, 1)
self.conv2 = nn.Conv2d(20, 50, 5, 1) self.conv2 = nn.Conv2d(20, 50, 5, 1)
self.fc1 = nn.Linear(4*4*50, 500) self.fc1 = nn.Linear(4 * 4 * 50, 500)
self.fc2 = nn.Linear(500, 10) self.fc2 = nn.Linear(500, 10)
def forward(self, x): def forward(self, x):
@@ -43,7 +43,7 @@ class Net(nn.Module):
x = F.max_pool2d(x, 2, 2) x = F.max_pool2d(x, 2, 2)
x = F.relu(self.conv2(x)) x = F.relu(self.conv2(x))
x = F.max_pool2d(x, 2, 2) x = F.max_pool2d(x, 2, 2)
x = x.view(-1, 4*4*50) x = x.view(-1, 4 * 4 * 50)
x = F.relu(self.fc1(x)) x = F.relu(self.fc1(x))
x = self.fc2(x) x = self.fc2(x)
return F.log_softmax(x, dim=1) return F.log_softmax(x, dim=1)
@@ -60,10 +60,14 @@ def train(args, model, device, train_loader, optimizer, epoch):
optimizer.step() optimizer.step()
if batch_idx % args.log_interval == 0: if batch_idx % args.log_interval == 0:
msg = "Train Epoch: {} [{}/{} ({:.0f}%)]\tloss={:.4f}".format( msg = "Train Epoch: {} [{}/{} ({:.0f}%)]\tloss={:.4f}".format(
epoch, batch_idx * len(data), len(train_loader.dataset), epoch,
100. * batch_idx / len(train_loader), loss.item()) batch_idx * len(data),
len(train_loader.dataset),
100.0 * batch_idx / len(train_loader),
loss.item(),
)
logging.info(msg) logging.info(msg)
niter = epoch * len(train_loader) + batch_idx niter = epoch * len(train_loader) + batch_idx # noqa: F841
def test(args, model, device, test_loader, epoch, hpt): def test(args, model, device, test_loader, epoch, hpt):
@@ -74,24 +78,31 @@ def test(args, model, device, test_loader, epoch, hpt):
for data, target in test_loader: for data, target in test_loader:
data, target = data.to(device), target.to(device) data, target = data.to(device), target.to(device)
output = model(data) output = model(data)
test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss test_loss += F.nll_loss(
pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability output, target, reduction="sum"
).item() # sum up batch loss
pred = output.max(1, keepdim=True)[
1
] # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item() correct += pred.eq(target.view_as(pred)).sum().item()
test_loss /= len(test_loader.dataset) test_loss /= len(test_loader.dataset)
test_accuracy = float(correct) / len(test_loader.dataset) test_accuracy = float(correct) / len(test_loader.dataset)
logging.info("{{metricName: accuracy, metricValue: {:.4f}}};{{metricName: loss, metricValue: {:.4f}}}\n".format(
test_accuracy, test_loss)) logging.info(
"{{metricName: accuracy, metricValue: {:.4f}}};"
"{{metricName: loss, metricValue: {:.4f}}}\n".format(test_accuracy, test_loss)
)
if args.logger == "hypertune": if args.logger == "hypertune":
hpt.report_hyperparameter_tuning_metric( hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='loss', hyperparameter_metric_tag="loss", metric_value=test_loss, global_step=epoch
metric_value=test_loss, )
global_step=epoch)
hpt.report_hyperparameter_tuning_metric( hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='accuracy', hyperparameter_metric_tag="accuracy",
metric_value=test_accuracy, metric_value=test_accuracy,
global_step=epoch) global_step=epoch,
)
def should_distribute(): def should_distribute():
@@ -105,33 +116,82 @@ def is_distributed():
def main(): def main():
# Training settings # Training settings
parser = argparse.ArgumentParser(description="PyTorch MNIST Example") parser = argparse.ArgumentParser(description="PyTorch MNIST Example")
parser.add_argument("--batch-size", type=int, default=64, metavar="N", parser.add_argument(
help="input batch size for training (default: 64)") "--batch-size",
parser.add_argument("--test-batch-size", type=int, default=1000, metavar="N", type=int,
help="input batch size for testing (default: 1000)") default=64,
parser.add_argument("--epochs", type=int, default=10, metavar="N", metavar="N",
help="number of epochs to train (default: 10)") help="input batch size for training (default: 64)",
parser.add_argument("--lr", type=float, default=0.01, metavar="LR", )
help="learning rate (default: 0.01)") parser.add_argument(
parser.add_argument("--momentum", type=float, default=0.5, metavar="M", "--test-batch-size",
help="SGD momentum (default: 0.5)") type=int,
parser.add_argument("--no-cuda", action="store_true", default=False, default=1000,
help="disables CUDA training") metavar="N",
parser.add_argument("--seed", type=int, default=1, metavar="S", help="input batch size for testing (default: 1000)",
help="random seed (default: 1)") )
parser.add_argument("--log-interval", type=int, default=10, metavar="N", parser.add_argument(
help="how many batches to wait before logging training status") "--epochs",
parser.add_argument("--log-path", type=str, default="", type=int,
help="Path to save logs. Print to StdOut if log-path is not set") default=10,
parser.add_argument("--save-model", action="store_true", default=False, metavar="N",
help="For Saving the current Model") help="number of epochs to train (default: 10)",
parser.add_argument("--logger", type=str, choices=["standard", "hypertune"], )
help="Logger", default="standard") parser.add_argument(
"--lr",
type=float,
default=0.01,
metavar="LR",
help="learning rate (default: 0.01)",
)
parser.add_argument(
"--momentum",
type=float,
default=0.5,
metavar="M",
help="SGD momentum (default: 0.5)",
)
parser.add_argument(
"--no-cuda", action="store_true", default=False, help="disables CUDA training"
)
parser.add_argument(
"--seed", type=int, default=1, metavar="S", help="random seed (default: 1)"
)
parser.add_argument(
"--log-interval",
type=int,
default=10,
metavar="N",
help="how many batches to wait before logging training status",
)
parser.add_argument(
"--log-path",
type=str,
default="",
help="Path to save logs. Print to StdOut if log-path is not set",
)
parser.add_argument(
"--save-model",
action="store_true",
default=False,
help="For Saving the current Model",
)
parser.add_argument(
"--logger",
type=str,
choices=["standard", "hypertune"],
help="Logger",
default="standard",
)
if dist.is_available(): if dist.is_available():
parser.add_argument("--backend", type=str, help="Distributed backend", parser.add_argument(
choices=[dist.Backend.GLOO, dist.Backend.NCCL, dist.Backend.MPI], "--backend",
default=dist.Backend.GLOO) type=str,
help="Distributed backend",
choices=[dist.Backend.GLOO, dist.Backend.NCCL, dist.Backend.MPI],
default=dist.Backend.GLOO,
)
args = parser.parse_args() args = parser.parse_args()
# Use this format (%Y-%m-%dT%H:%M:%SZ) to record timestamp of the metrics. # Use this format (%Y-%m-%dT%H:%M:%SZ) to record timestamp of the metrics.
@@ -140,16 +200,18 @@ def main():
logging.basicConfig( logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s", format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%dT%H:%M:%SZ", datefmt="%Y-%m-%dT%H:%M:%SZ",
level=logging.DEBUG) level=logging.DEBUG,
)
else: else:
logging.basicConfig( logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s", format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%dT%H:%M:%SZ", datefmt="%Y-%m-%dT%H:%M:%SZ",
level=logging.DEBUG, level=logging.DEBUG,
filename=args.log_path) filename=args.log_path,
)
if args.logger == "hypertune" and args.log_path != "": if args.logger == "hypertune" and args.log_path != "":
os.environ['CLOUD_ML_HP_METRIC_FILE'] = args.log_path os.environ["CLOUD_ML_HP_METRIC_FILE"] = args.log_path
# For JSON logging # For JSON logging
hpt = hypertune.HyperTune() hpt = hypertune.HyperTune()
@@ -169,27 +231,34 @@ def main():
kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {} kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader( train_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST("./data", datasets.FashionMNIST(
train=True, "./data",
download=True, train=True,
transform=transforms.Compose([ download=True,
transforms.ToTensor() transform=transforms.Compose([transforms.ToTensor()]),
])), ),
batch_size=args.batch_size, shuffle=True, **kwargs) batch_size=args.batch_size,
shuffle=True,
**kwargs,
)
test_loader = torch.utils.data.DataLoader( test_loader = torch.utils.data.DataLoader(
datasets.FashionMNIST("./data", datasets.FashionMNIST(
train=False, "./data", train=False, transform=transforms.Compose([transforms.ToTensor()])
transform=transforms.Compose([ ),
transforms.ToTensor() batch_size=args.test_batch_size,
])), shuffle=False,
batch_size=args.test_batch_size, shuffle=False, **kwargs) **kwargs,
)
model = Net().to(device) model = Net().to(device)
if is_distributed(): if is_distributed():
Distributor = nn.parallel.DistributedDataParallel if use_cuda \ Distributor = (
nn.parallel.DistributedDataParallel
if use_cuda
else nn.parallel.DistributedDataParallelCPU else nn.parallel.DistributedDataParallelCPU
)
model = Distributor(model) model = Distributor(model)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
@@ -198,7 +267,7 @@ def main():
train(args, model, device, train_loader, optimizer, epoch) train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader, epoch, hpt) test(args, model, device, test_loader, epoch, hpt)
if (args.save_model): if args.save_model:
torch.save(model.state_dict(), "mnist_cnn.pt") torch.save(model.state_dict(), "mnist_cnn.pt")
examples/v1beta1/trial-images/tf-mnist-with-summaries/mnist.py
+100 -37
View File
@@ -25,9 +25,9 @@ from tensorflow.keras.layers import Flatten
class MyModel(Model): class MyModel(Model):
def __init__(self): def __init__(self):
super(MyModel, self).__init__() super(MyModel, self).__init__()
self.conv1 = Conv2D(32, 3, activation='relu') self.conv1 = Conv2D(32, 3, activation="relu")
self.flatten = Flatten() self.flatten = Flatten()
self.d1 = Dense(128, activation='relu') self.d1 = Dense(128, activation="relu")
self.d2 = Dense(10) self.d2 = Dense(10)
def call(self, x): def call(self, x):
@@ -37,7 +37,17 @@ class MyModel(Model):
return self.d2(x) return self.d2(x)
def train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summary_writer, train_loss, train_accuracy): def train_step(
args,
model,
optimizer,
train_ds,
epoch,
loss_object,
train_summary_writer,
train_loss,
train_accuracy,
):
for step, (images, labels) in enumerate(train_ds): for step, (images, labels) in enumerate(train_ds):
with tf.GradientTape() as tape: with tf.GradientTape() as tape:
# training=True is only needed if there are layers with different # training=True is only needed if there are layers with different
@@ -51,18 +61,25 @@ def train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summa
train_accuracy(labels, predictions) train_accuracy(labels, predictions)
if step % args.log_interval == 0: if step % args.log_interval == 0:
print("Train Epoch: {} [{}/60000 ({:.0f}%)]\tloss={:.4f}, accuracy={:.4f}".format( print(
epoch + 1, step * args.batch_size, 100. * step * args.batch_size / 60000, "Train Epoch: {} [{}/60000 ({:.0f}%)]\tloss={:.4f}, accuracy={:.4f}".format(
train_loss.result(), train_accuracy.result() * 100) epoch + 1,
step * args.batch_size,
100.0 * step * args.batch_size / 60000,
train_loss.result(),
train_accuracy.result() * 100,
)
) )
with train_summary_writer.as_default(): with train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss.result(), step=epoch) tf.summary.scalar("loss", train_loss.result(), step=epoch)
tf.summary.scalar('accuracy', train_accuracy.result(), step=epoch) tf.summary.scalar("accuracy", train_accuracy.result(), step=epoch)
def test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy): def test_step(
for (images, labels) in test_ds: model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy
):
for images, labels in test_ds:
# training=False is only needed if there are layers with different # training=False is only needed if there are layers with different
# behavior during training versus inference (e.g. Dropout). # behavior during training versus inference (e.g. Dropout).
predictions = model(images, training=False) predictions = model(images, training=False)
@@ -72,30 +89,53 @@ def test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss
test_accuracy(labels, predictions) test_accuracy(labels, predictions)
with test_summary_writer.as_default(): with test_summary_writer.as_default():
tf.summary.scalar('loss', test_loss.result(), step=epoch) tf.summary.scalar("loss", test_loss.result(), step=epoch)
tf.summary.scalar('accuracy', test_accuracy.result(), step=epoch) tf.summary.scalar("accuracy", test_accuracy.result(), step=epoch)
print("Test Loss: {:.4f}, Test Accuracy: {:.4f}\n".format( print(
test_loss.result(), test_accuracy.result() * 100) "Test Loss: {:.4f}, Test Accuracy: {:.4f}\n".format(
test_loss.result(), test_accuracy.result() * 100
)
) )
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--batch-size', type=int, default=64,
help='input batch size for training (default: 64)')
parser.add_argument('--learning-rate', type=float, default=0.001,
help='learning rate (default: 0.001)')
parser.add_argument("--epochs", type=int, default=10, metavar="N",
help="number of epochs to train (default: 10)")
parser.add_argument("--log-interval", type=int, default=100, metavar="N",
help="how many batches to wait before logging training status (default: 100)")
parser.add_argument( parser.add_argument(
'--log-path', "--batch-size",
type=int,
default=64,
help="input batch size for training (default: 64)",
)
parser.add_argument(
"--learning-rate",
type=float,
default=0.001,
help="learning rate (default: 0.001)",
)
parser.add_argument(
"--epochs",
type=int,
default=10,
metavar="N",
help="number of epochs to train (default: 10)",
)
parser.add_argument(
"--log-interval",
type=int,
default=100,
metavar="N",
help="how many batches to wait before logging training status (default: 100)",
)
parser.add_argument(
"--log-path",
type=str, type=str,
default=os.path.join(os.getenv('TEST_TMPDIR', '/tmp'), default=os.path.join(
'tensorflow/mnist/logs/mnist_with_summaries'), os.getenv("TEST_TMPDIR", "/tmp"),
help='Summaries log PATH') "tensorflow/mnist/logs/mnist_with_summaries",
),
help="Summaries log PATH",
)
args = parser.parse_args() args = parser.parse_args()
# Setup dataset # Setup dataset
@@ -105,12 +145,18 @@ def main():
# Add a channels dimension # Add a channels dimension
x_train = x_train[..., tf.newaxis].astype("float32") x_train = x_train[..., tf.newaxis].astype("float32")
x_test = x_test[..., tf.newaxis].astype("float32") x_test = x_test[..., tf.newaxis].astype("float32")
train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(args.batch_size) train_ds = (
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(args.batch_size) tf.data.Dataset.from_tensor_slices((x_train, y_train))
.shuffle(10000)
.batch(args.batch_size)
)
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(
args.batch_size
)
# Setup tensorflow summaries # Setup tensorflow summaries
train_log_dir = os.path.join(args.log_path, 'train') train_log_dir = os.path.join(args.log_path, "train")
test_log_dir = os.path.join(args.log_path, 'test') test_log_dir = os.path.join(args.log_path, "test")
train_summary_writer = tf.summary.create_file_writer(train_log_dir) train_summary_writer = tf.summary.create_file_writer(train_log_dir)
test_summary_writer = tf.summary.create_file_writer(test_log_dir) test_summary_writer = tf.summary.create_file_writer(test_log_dir)
@@ -119,20 +165,37 @@ def main():
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate) optimizer = tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
train_loss = tf.keras.metrics.Mean(name='train_loss') train_loss = tf.keras.metrics.Mean(name="train_loss")
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="train_accuracy")
test_loss = tf.keras.metrics.Mean(name='test_loss') test_loss = tf.keras.metrics.Mean(name="test_loss")
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy') test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="test_accuracy")
for epoch in range(args.epochs): for epoch in range(args.epochs):
# Reset the metrics at the start of the next epoch # Reset the metrics at the start of the next epoch
train_summary_writer.flush() train_summary_writer.flush()
test_summary_writer.flush() test_summary_writer.flush()
train_step(args, model, optimizer, train_ds, epoch, loss_object, train_summary_writer, train_step(
train_loss, train_accuracy) args,
test_step(model, test_ds, epoch, loss_object, test_summary_writer, test_loss, test_accuracy) model,
optimizer,
train_ds,
epoch,
loss_object,
train_summary_writer,
train_loss,
train_accuracy,
)
test_step(
model,
test_ds,
epoch,
loss_object,
test_summary_writer,
test_loss,
test_accuracy,
)
if __name__ == "__main__": if __name__ == "__main__":
hack/gen-python-sdk/post_gen.py
+63 -46
View File
@@ -18,14 +18,14 @@ import sys
IGNORE_LINES = [ IGNORE_LINES = [
"from kubeflow.katib.models.v1_unstructured_unstructured import V1UnstructuredUnstructured", "from kubeflow.katib.models.v1_unstructured_unstructured import V1UnstructuredUnstructured",
"from kubeflow.katib.models.v1_time import V1Time" "from kubeflow.katib.models.v1_time import V1Time",
] ]
def _rewrite_helper(input_file, output_file, rewrite_rules): def _rewrite_helper(input_file, output_file, rewrite_rules):
rules = rewrite_rules or [] rules = rewrite_rules or []
lines = [] lines = []
with open(input_file, 'r') as f: with open(input_file, "r") as f:
while True: while True:
line = f.readline() line = f.readline()
if not line: if not line:
@@ -34,11 +34,11 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
for rule in rules: for rule in rules:
line = rule(line) line = rule(line)
# Remove ignored lines. # Remove ignored lines.
if not any(l in line for l in IGNORE_LINES): if not any(li in line for li in IGNORE_LINES):
lines.append(line) lines.append(line)
# Add Katib APIs to the init file. # Add Katib APIs to the init file.
if (output_file == "sdk/python/v1beta1/kubeflow/katib/__init__.py"): if output_file == "sdk/python/v1beta1/kubeflow/katib/__init__.py":
lines.append("# Import Katib API client.\n") lines.append("# Import Katib API client.\n")
lines.append("from kubeflow.katib.api.katib_client import KatibClient\n") lines.append("from kubeflow.katib.api.katib_client import KatibClient\n")
lines.append("# Import Katib report metrics functions") lines.append("# Import Katib report metrics functions")
@@ -46,62 +46,79 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
lines.append("# Import Katib helper functions.\n") lines.append("# Import Katib helper functions.\n")
lines.append("import kubeflow.katib.api.search as search\n") lines.append("import kubeflow.katib.api.search as search\n")
lines.append("# Import Katib helper constants.\n") lines.append("# Import Katib helper constants.\n")
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW\n") lines.append(
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW_GPU\n") "from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW\n"
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_PYTORCH\n") )
lines.append("from kubeflow.katib.constants.constants import BASE_IMAGE_MXNET\n") lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_TENSORFLOW_GPU\n"
)
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_PYTORCH\n"
)
lines.append(
"from kubeflow.katib.constants.constants import BASE_IMAGE_MXNET\n"
)
# Add Kubernetes models to proper deserialization of Katib models. # Add Kubernetes models to proper deserialization of Katib models.
if (output_file == "sdk/python/v1beta1/kubeflow/katib/models/__init__.py"): if output_file == "sdk/python/v1beta1/kubeflow/katib/models/__init__.py":
lines.append("\n") lines.append("\n")
lines.append("# Import Kubernetes models.\n") lines.append("# Import Kubernetes models.\n")
lines.append("from kubernetes.client import *\n") lines.append("from kubernetes.client import *\n")
with open(output_file, 'w') as f: with open(output_file, "w") as f:
f.writelines(lines) f.writelines(lines)
def update_python_sdk(src, dest, versions=('v1beta1')): def update_python_sdk(src, dest, versions=("v1beta1")):
# tiny transformers to refine generated codes # tiny transformers to refine generated codes
rewrite_rules = [ rewrite_rules = [
# Models rules. # Models rules.
lambda l: l.replace('import katib', 'import kubeflow.katib'), lambda line: line.replace("import katib", "import kubeflow.katib"),
lambda l: l.replace('from katib', 'from kubeflow.katib'), lambda line: line.replace("from katib", "from kubeflow.katib"),
# For the api_client.py. # For the api_client.py.
lambda l: l.replace('klass = getattr(katib.models, klass)', 'klass = getattr(kubeflow.katib.models, klass)'), lambda line: line.replace(
"klass = getattr(katib.models, klass)",
"klass = getattr(kubeflow.katib.models, klass)",
),
# Doc rules. # Doc rules.
lambda l: l.replace('[**datetime**](V1Time.md)', '**datetime**'), lambda line: line.replace("[**datetime**](V1Time.md)", "**datetime**"),
lambda l: l.replace('[**object**](V1UnstructuredUnstructured.md)', '**object**'), lambda line: line.replace(
"[**object**](V1UnstructuredUnstructured.md)", "**object**"
lambda l: l.replace('[**V1Container**](V1Container.md)', ),
'[**V1Container**](https://github.com/kubernetes-client/' lambda line: line.replace(
'python/blob/master/kubernetes/docs/V1Container.md)'), "[**V1Container**](V1Container.md)",
"[**V1Container**](https://github.com/kubernetes-client/"
lambda l: l.replace('[**V1ObjectMeta**](V1ObjectMeta.md)', "python/blob/master/kubernetes/docs/V1Container.md)",
'[**V1ObjectMeta**](https://github.com/kubernetes-client/' ),
'python/blob/master/kubernetes/docs/V1ObjectMeta.md)'), lambda line: line.replace(
"[**V1ObjectMeta**](V1ObjectMeta.md)",
lambda l: l.replace('[**V1ListMeta**](V1ListMeta.md)', "[**V1ObjectMeta**](https://github.com/kubernetes-client/"
'[**V1ListMeta**](https://github.com/kubernetes-client/' "python/blob/master/kubernetes/docs/V1ObjectMeta.md)",
'python/blob/master/kubernetes/docs/V1ListMeta.md)'), ),
lambda line: line.replace(
lambda l: l.replace('[**V1HTTPGetAction**](V1HTTPGetAction.md)', "[**V1ListMeta**](V1ListMeta.md)",
'[**V1HTTPGetAction**](https://github.com/kubernetes-client/' "[**V1ListMeta**](https://github.com/kubernetes-client/"
'python/blob/master/kubernetes/docs/V1HTTPGetAction.md)') "python/blob/master/kubernetes/docs/V1ListMeta.md)",
),
lambda line: line.replace(
"[**V1HTTPGetAction**](V1HTTPGetAction.md)",
"[**V1HTTPGetAction**](https://github.com/kubernetes-client/"
"python/blob/master/kubernetes/docs/V1HTTPGetAction.md)",
),
] ]
# TODO (andreyvelich): Currently test can't be generated properly. # TODO (andreyvelich): Currently test can't be generated properly.
src_dirs = [ src_dirs = [
os.path.join(src, 'katib'), os.path.join(src, "katib"),
os.path.join(src, 'katib', 'models'), os.path.join(src, "katib", "models"),
# os.path.join(src, 'test'), # os.path.join(src, 'test'),
os.path.join(src, 'docs') os.path.join(src, "docs"),
] ]
dest_dirs = [ dest_dirs = [
os.path.join(dest, 'kubeflow', 'katib'), os.path.join(dest, "kubeflow", "katib"),
os.path.join(dest, 'kubeflow', 'katib', 'models'), os.path.join(dest, "kubeflow", "katib", "models"),
# os.path.join(dest, 'test'), # os.path.join(dest, 'test'),
os.path.join(dest, 'docs') os.path.join(dest, "docs"),
] ]
for src_dir, dest_dir in zip(src_dirs, dest_dirs): for src_dir, dest_dir in zip(src_dirs, dest_dirs):
@@ -128,13 +145,13 @@ def update_python_sdk(src, dest, versions=('v1beta1')):
update_buffer = [] update_buffer = []
# Get data from generated doc # Get data from generated doc
with open(os.path.join(src, 'README.md'), 'r') as src_f: with open(os.path.join(src, "README.md"), "r") as src_f:
anchor = 0 anchor = 0
for line in src_f.readlines(): for line in src_f.readlines():
if line.startswith('## Documentation For Models'): if line.startswith("## Documentation For Models"):
if anchor == 0: if anchor == 0:
anchor = 1 anchor = 1
elif line.startswith('##') and anchor == 1: elif line.startswith("##") and anchor == 1:
anchor = 2 anchor = 2
if anchor == 0: if anchor == 0:
continue continue
@@ -148,24 +165,24 @@ def update_python_sdk(src, dest, versions=('v1beta1')):
update_buffer = update_buffer[:-1] update_buffer = update_buffer[:-1]
# Update README with new models # Update README with new models
with open(os.path.join(dest, 'README.md'), 'r') as dest_f: with open(os.path.join(dest, "README.md"), "r") as dest_f:
anchor = 0 anchor = 0
for line in dest_f.readlines(): for line in dest_f.readlines():
if line.startswith('## Documentation For Models'): if line.startswith("## Documentation For Models"):
if anchor == 0: if anchor == 0:
buffer.extend(update_buffer) buffer.extend(update_buffer)
anchor = 1 anchor = 1
elif line.startswith('##') and anchor == 1: elif line.startswith("##") and anchor == 1:
anchor = 2 anchor = 2
if anchor == 1: if anchor == 1:
continue continue
buffer.append(line) buffer.append(line)
with open(os.path.join(dest, 'README.md'), 'w') as dest_f: with open(os.path.join(dest, "README.md"), "w") as dest_f:
dest_f.writelines(buffer) dest_f.writelines(buffer)
# Clear working dictionary # Clear working dictionary
shutil.rmtree(src) shutil.rmtree(src)
if __name__ == '__main__': if __name__ == "__main__":
update_python_sdk(src=sys.argv[1], dest=sys.argv[2]) update_python_sdk(src=sys.argv[1], dest=sys.argv[2])
sdk/python/v1beta1/kubeflow/__init__.py
+1 -1
View File
@@ -1 +1 @@
__path__ = __import__('pkgutil').extend_path(__path__, __name__) __path__ = __import__("pkgutil").extend_path(__path__, __name__)
sdk/python/v1beta1/kubeflow/katib/api/katib_client.py
+45 -19
View File
@@ -106,7 +106,7 @@ class KatibClient(object):
namespace = namespace or self.namespace namespace = namespace or self.namespace
experiment_name = None experiment_name = None
if type(experiment) == models.V1beta1Experiment: if type(experiment) is models.V1beta1Experiment:
if experiment.metadata.name is not None: if experiment.metadata.name is not None:
experiment_name = experiment.metadata.name experiment_name = experiment.metadata.name
elif experiment.metadata.generate_name is not None: elif experiment.metadata.generate_name is not None:
@@ -137,7 +137,8 @@ class KatibClient(object):
except Exception as e: except Exception as e:
if hasattr(e, "status") and e.status == 409: if hasattr(e, "status") and e.status == 409:
raise Exception( raise Exception(
f"A Katib Experiment with the name {namespace}/{experiment_name} already exists." f"A Katib Experiment with the name "
f"{namespace}/{experiment_name} already exists."
) )
raise RuntimeError( raise RuntimeError(
f"Failed to create Katib Experiment: {namespace}/{experiment_name}" f"Failed to create Katib Experiment: {namespace}/{experiment_name}"
@@ -152,7 +153,8 @@ class KatibClient(object):
IPython.display.display( IPython.display.display(
IPython.display.HTML( IPython.display.HTML(
"Katib Experiment {} " "Katib Experiment {} "
'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" target="_blank">here</a>'.format( 'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" '
'target="_blank">here</a>'.format(
experiment_name, experiment_name,
namespace, namespace,
experiment_name, experiment_name,
@@ -215,7 +217,8 @@ class KatibClient(object):
https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1EnvFromSource.md) https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1EnvFromSource.md)
algorithm_name: Search algorithm for the HyperParameter tuning. algorithm_name: Search algorithm for the HyperParameter tuning.
algorithm_settings: Settings for the search algorithm given. algorithm_settings: Settings for the search algorithm given.
For available fields, check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail. For available fields, check this doc:
https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail.
objective_metric_name: Objective metric that Katib optimizes. objective_metric_name: Objective metric that Katib optimizes.
additional_metric_names: List of metrics that Katib collects from the additional_metric_names: List of metrics that Katib collects from the
objective function in addition to objective metric. objective function in addition to objective metric.
@@ -223,7 +226,8 @@ class KatibClient(object):
Must be one of `minimize` or `maximize`. Must be one of `minimize` or `maximize`.
objective_goal: Objective goal that Experiment should reach to be Succeeded. objective_goal: Objective goal that Experiment should reach to be Succeeded.
max_trial_count: Maximum number of Trials to run. For the default max_trial_count: Maximum number of Trials to run. For the default
values check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec. values check this doc:
https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec.
parallel_trial_count: Number of Trials that Experiment runs in parallel. parallel_trial_count: Number of Trials that Experiment runs in parallel.
max_failed_trial_count: Maximum number of Trials allowed to fail. max_failed_trial_count: Maximum number of Trials allowed to fail.
resources_per_trial: A parameter that lets you specify how much resources_per_trial: A parameter that lets you specify how much
@@ -249,7 +253,7 @@ class KatibClient(object):
to the base image packages. These packages are installed before to the base image packages. These packages are installed before
executing the objective function. executing the objective function.
pip_index_url: The PyPI url from which to install Python packages. pip_index_url: The PyPI url from which to install Python packages.
metrics_collector_config: Specify the config of metrics collector, metrics_collector_config: Specify the config of metrics collector,
for example, `metrics_collector_config = {"kind": "Push"}`. for example, `metrics_collector_config = {"kind": "Push"}`.
Currently, we only support `StdOut` and `Push` metrics collector. Currently, we only support `StdOut` and `Push` metrics collector.
@@ -330,10 +334,14 @@ class KatibClient(object):
# Otherwise, add value to the function input. # Otherwise, add value to the function input.
input_params[p_name] = p_value input_params[p_name] = p_value
# Wrap objective function to execute it from the file. For example # Wrap objective function to execute it from the file. For example:
# def objective(parameters): # def objective(parameters):
# print(f'Parameters are {parameters}') # print(f'Parameters are {parameters}')
# objective({'lr': '${trialParameters.lr}', 'epochs': '${trialParameters.epochs}', 'is_dist': False}) # objective({
# 'lr': '${trialParameters.lr}',
# 'epochs': '${trialParameters.epochs}',
# 'is_dist': False
# })
objective_code = f"{objective_code}\n{objective.__name__}({input_params})\n" objective_code = f"{objective_code}\n{objective.__name__}({input_params})\n"
# Prepare execute script template. # Prepare execute script template.
@@ -385,7 +393,8 @@ class KatibClient(object):
) )
# Add metrics collector to the Katib Experiment. # Add metrics collector to the Katib Experiment.
# Up to now, We only support parameter `kind`, of which default value is `StdOut`, to specify the kind of metrics collector. # Up to now, we only support parameter `kind`, of which default value
# is `StdOut`, to specify the kind of metrics collector.
experiment.spec.metrics_collector_spec = models.V1beta1MetricsCollectorSpec( experiment.spec.metrics_collector_spec = models.V1beta1MetricsCollectorSpec(
collector=models.V1beta1CollectorSpec(kind=metrics_collector_config["kind"]) collector=models.V1beta1CollectorSpec(kind=metrics_collector_config["kind"])
) )
@@ -764,7 +773,9 @@ class KatibClient(object):
) )
): ):
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n") logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment return experiment
# Raise exception if Experiment is Failed. # Raise exception if Experiment is Failed.
@@ -784,7 +795,9 @@ class KatibClient(object):
) )
): ):
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n") logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment return experiment
# Check if Experiment reaches Running condition. # Check if Experiment reaches Running condition.
@@ -795,7 +808,9 @@ class KatibClient(object):
) )
): ):
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n") logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment return experiment
# Check if Experiment reaches Restarting condition. # Check if Experiment reaches Restarting condition.
@@ -806,7 +821,9 @@ class KatibClient(object):
) )
): ):
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n") logger.debug(
f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
)
return experiment return experiment
# Check if Experiment reaches Succeeded condition. # Check if Experiment reaches Succeeded condition.
@@ -817,18 +834,24 @@ class KatibClient(object):
) )
): ):
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug(f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n")
logger.debug(
f"waiting for experiment: {namespace}/{name} "
f"to reach {expected_condition} condition\n\n\n"
)
return experiment return experiment
# Otherwise, print the current Experiment results and sleep for the pooling interval. # Otherwise, print the current Experiment results and sleep for the pooling interval.
utils.print_experiment_status(experiment) utils.print_experiment_status(experiment)
logger.debug( logger.debug(
f"Waiting for Experiment: {namespace}/{name} to reach {expected_condition} condition\n\n\n" f"waiting for experiment: {namespace}/{name} "
f"to reach {expected_condition} condition\n\n\n"
) )
time.sleep(polling_interval) time.sleep(polling_interval)
raise TimeoutError( raise TimeoutError(
f"Timeout waiting for Experiment: {namespace}/{name} to reach {expected_condition} state" f"Timeout waiting for Experiment: {namespace}/{name} "
f"to reach {expected_condition} state"
) )
def edit_experiment_budget( def edit_experiment_budget(
@@ -844,7 +867,8 @@ class KatibClient(object):
budget to resume Succeeded Experiments with `LongRunning` and `FromVolume` budget to resume Succeeded Experiments with `LongRunning` and `FromVolume`
resume policies. resume policies.
Learn about resuming Experiments here: https://www.kubeflow.org/docs/components/katib/resume-experiment/ Learn about resuming Experiments here:
https://www.kubeflow.org/docs/components/katib/resume-experiment/
Args: Args:
name: Name for the Experiment. name: Name for the Experiment.
@@ -1257,10 +1281,12 @@ class KatibClient(object):
use the default Katib DB Manager address: `katib-db-manager.kubeflow:6789`. use the default Katib DB Manager address: `katib-db-manager.kubeflow:6789`.
If you run this API outside the cluster, you have to port-forward the If you run this API outside the cluster, you have to port-forward the
Katib DB Manager before getting the Trial metrics: `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`. Katib DB Manager before getting the Trial metrics:
`kubectl port-forward svc/katib-db-manager -n kubeflow 6789`.
In that case, you can use this Katib DB Manager address: `localhost:6789`. In that case, you can use this Katib DB Manager address: `localhost:6789`.
You can use `curl` to verify that Katib DB Manager is reachable: `curl <db-manager-address>`. You can use `curl` to verify that Katib DB Manager is reachable:
`curl <db-manager-address>`.
Args: Args:
name: Name for the Trial. name: Name for the Trial.
sdk/python/v1beta1/kubeflow/katib/api/katib_client_test.py
+25 -41
View File
@@ -38,16 +38,12 @@ def create_namespaced_custom_object_response(*args, **kwargs):
def generate_trial_template() -> V1beta1TrialTemplate: def generate_trial_template() -> V1beta1TrialTemplate:
trial_spec={ trial_spec = {
"apiVersion": "batch/v1", "apiVersion": "batch/v1",
"kind": "Job", "kind": "Job",
"spec": { "spec": {
"template": { "template": {
"metadata": { "metadata": {"annotations": {"sidecar.istio.io/inject": "false"}},
"annotations": {
"sidecar.istio.io/inject": "false"
}
},
"spec": { "spec": {
"containers": [ "containers": [
{ {
@@ -60,13 +56,13 @@ def generate_trial_template() -> V1beta1TrialTemplate:
"--batch-size=64", "--batch-size=64",
"--lr=${trialParameters.learningRate}", "--lr=${trialParameters.learningRate}",
"--momentum=${trialParameters.momentum}", "--momentum=${trialParameters.momentum}",
] ],
} }
], ],
"restartPolicy": "Never" "restartPolicy": "Never",
} },
} }
} },
} }
return V1beta1TrialTemplate( return V1beta1TrialTemplate(
@@ -75,15 +71,15 @@ def generate_trial_template() -> V1beta1TrialTemplate:
V1beta1TrialParameterSpec( V1beta1TrialParameterSpec(
name="learningRate", name="learningRate",
description="Learning rate for the training model", description="Learning rate for the training model",
reference="lr" reference="lr",
), ),
V1beta1TrialParameterSpec( V1beta1TrialParameterSpec(
name="momentum", name="momentum",
description="Momentum for the training model", description="Momentum for the training model",
reference="momentum" reference="momentum",
), ),
], ],
trial_spec=trial_spec trial_spec=trial_spec,
) )
@@ -106,60 +102,49 @@ def generate_experiment(
objective=objective_spec, objective=objective_spec,
parameters=parameters, parameters=parameters,
trial_template=trial_template, trial_template=trial_template,
) ),
) )
def create_experiment( def create_experiment(
name: Optional[str] = None, name: Optional[str] = None, generate_name: Optional[str] = None
generate_name: Optional[str] = None
) -> V1beta1Experiment: ) -> V1beta1Experiment:
experiment_namespace = "test" experiment_namespace = "test"
if name is not None: if name is not None:
metadata = V1ObjectMeta(name=name, namespace=experiment_namespace) metadata = V1ObjectMeta(name=name, namespace=experiment_namespace)
elif generate_name is not None: elif generate_name is not None:
metadata = V1ObjectMeta(generate_name=generate_name, namespace=experiment_namespace) metadata = V1ObjectMeta(
generate_name=generate_name, namespace=experiment_namespace
)
else: else:
metadata = V1ObjectMeta(namespace=experiment_namespace) metadata = V1ObjectMeta(namespace=experiment_namespace)
algorithm_spec=V1beta1AlgorithmSpec( algorithm_spec = V1beta1AlgorithmSpec(algorithm_name="random")
algorithm_name="random"
)
objective_spec=V1beta1ObjectiveSpec( objective_spec = V1beta1ObjectiveSpec(
type="minimize", type="minimize",
goal= 0.001, goal=0.001,
objective_metric_name="loss", objective_metric_name="loss",
) )
parameters=[ parameters = [
V1beta1ParameterSpec( V1beta1ParameterSpec(
name="lr", name="lr",
parameter_type="double", parameter_type="double",
feasible_space=V1beta1FeasibleSpace( feasible_space=V1beta1FeasibleSpace(min="0.01", max="0.06"),
min="0.01",
max="0.06"
),
), ),
V1beta1ParameterSpec( V1beta1ParameterSpec(
name="momentum", name="momentum",
parameter_type="double", parameter_type="double",
feasible_space=V1beta1FeasibleSpace( feasible_space=V1beta1FeasibleSpace(min="0.5", max="0.9"),
min="0.5",
max="0.9"
),
), ),
] ]
trial_template = generate_trial_template() trial_template = generate_trial_template()
experiment = generate_experiment( experiment = generate_experiment(
metadata, metadata, algorithm_spec, objective_spec, parameters, trial_template
algorithm_spec,
objective_spec,
parameters,
trial_template
) )
return experiment return experiment
@@ -246,15 +231,14 @@ def katib_client():
side_effect=create_namespaced_custom_object_response side_effect=create_namespaced_custom_object_response
) )
), ),
), patch( ), patch("kubernetes.config.load_kube_config", return_value=Mock()):
"kubernetes.config.load_kube_config",
return_value=Mock()
):
client = KatibClient() client = KatibClient()
yield client yield client
@pytest.mark.parametrize("test_name,kwargs,expected_output", test_create_experiment_data) @pytest.mark.parametrize(
"test_name,kwargs,expected_output", test_create_experiment_data
)
def test_create_experiment(katib_client, test_name, kwargs, expected_output): def test_create_experiment(katib_client, test_name, kwargs, expected_output):
""" """
test create_experiment function of katib client test create_experiment function of katib client
sdk/python/v1beta1/kubeflow/katib/api/report_metrics.py
+7 -7
View File
@@ -37,7 +37,7 @@ def report_metrics(
For examle, `metrics = {"loss": 0.01, "accuracy": 0.99}`. For examle, `metrics = {"loss": 0.01, "accuracy": 0.99}`.
db-manager-address: Address for the Katib DB Manager in this format: `ip-address:port`. db-manager-address: Address for the Katib DB Manager in this format: `ip-address:port`.
timeout: Optional, gRPC API Server timeout in seconds to report metrics. timeout: Optional, gRPC API Server timeout in seconds to report metrics.
Raises: Raises:
ValueError: The Trial name is not passed to environment variables. ValueError: The Trial name is not passed to environment variables.
RuntimeError: Unable to push Trial metrics to Katib DB or RuntimeError: Unable to push Trial metrics to Katib DB or
@@ -48,9 +48,7 @@ def report_metrics(
namespace = utils.get_current_k8s_namespace() namespace = utils.get_current_k8s_namespace()
name = os.getenv("KATIB_TRIAL_NAME") name = os.getenv("KATIB_TRIAL_NAME")
if name is None: if name is None:
raise ValueError( raise ValueError("The Trial name is not passed to environment variables")
"The Trial name is not passed to environment variables"
)
# Get channel for grpc call to db manager # Get channel for grpc call to db manager
db_manager_address = db_manager_address.split(":") db_manager_address = db_manager_address.split(":")
@@ -61,7 +59,7 @@ def report_metrics(
# Validate metrics value in dict # Validate metrics value in dict
for value in metrics.values(): for value in metrics.values():
utils.validate_metrics_value(value) utils.validate_metrics_value(value)
# Dial katib db manager to report metrics # Dial katib db manager to report metrics
with katib_api_pb2.beta_create_DBManager_stub(channel) as client: with katib_api_pb2.beta_create_DBManager_stub(channel) as client:
try: try:
@@ -73,11 +71,13 @@ def report_metrics(
metric_logs=[ metric_logs=[
katib_api_pb2.MetricLog( katib_api_pb2.MetricLog(
time_stamp=timestamp, time_stamp=timestamp,
metric=katib_api_pb2.Metric(name=name,value=str(value)) metric=katib_api_pb2.Metric(
name=name, value=str(value)
),
) )
for name, value in metrics.items() for name, value in metrics.items()
] ]
) ),
), ),
timeout=timeout, timeout=timeout,
) )
sdk/python/v1beta1/kubeflow/katib/api/search.py
+2 -1
View File
@@ -61,5 +61,6 @@ def categorical(list: List):
""" """
return models.V1beta1ParameterSpec( return models.V1beta1ParameterSpec(
parameter_type="categorical", feasible_space=models.V1beta1FeasibleSpace(list), parameter_type="categorical",
feasible_space=models.V1beta1FeasibleSpace(list),
) )
sdk/python/v1beta1/kubeflow/katib/utils/utils.py
+1
View File
@@ -72,6 +72,7 @@ def print_experiment_status(experiment: models.V1beta1Experiment):
print(f"Current Optimal Trial:\n {experiment.status.current_optimal_trial}") print(f"Current Optimal Trial:\n {experiment.status.current_optimal_trial}")
print(f"Experiment conditions:\n {experiment.status.conditions}") print(f"Experiment conditions:\n {experiment.status.conditions}")
def validate_metrics_value(value: Any): def validate_metrics_value(value: Any):
"""Validate if the metrics value can be converted to type `float`.""" """Validate if the metrics value can be converted to type `float`."""
try: try: