hanzoai/bootnode
2
0
Fork 0
mirror of https://github.com/hanzoai/bootnode.git synced 2026-09-30 20:46:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

add deployment support

Browse Source
This commit is contained in:
David Tai
2019-04-18 13:46:18 -05:00
parent ff65edd8f1
commit 181b0da2ea
8 changed files with 220 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,3 +1,4 @@
config/*/cluster.yaml
*.pyc
__pycache__
kubeconfig
Makefile
+16 -13
View File
@@ -3,12 +3,13 @@ geth := /bin/geth --datadir=/data
net ?= testnet
cfg := config/$(net)
region := us-central1
zone := us-central1-a
zone ?= us-central1-a
number ?= 657758440341
node ?= geth
project_id := hanzo-ai
project_id ?= hanzo-ai
pod ?= $(node)-$(net)-$(number)
export KUBECONFIG=config/$(node)-$(net)/cluster.yaml
# export KUBECONFIG=config/$(node)-$(net)/cluster.yaml
all: deploy
@@ -17,6 +18,12 @@ gcloud-auth:
gcloud auth application-default login
gcloud config set project $(project_id)
# Get credentials for kubectl for current cluster
get-credentials:
gcloud container clusters get-credentials $(node)-$(net) --zone $(zone) --project $(project_id)
@echo "KUBECONFIG=$(KUBECONFIG)" > kubeconfig
echo "`source kubeconfig` to update KUBECONFIG environemental variable"
#build image
build:
gcloud builds submit \
@@ -61,22 +68,22 @@ status:
kubectl get pods -o yaml
logs:
kubectl logs $(node)-$(net)-$(number) -f
kubectl logs $(pod) -f
ssh:
kubectl exec -it $(node)-$(net)-$(number) -- /bin/bash
kubectl exec -it $(pod) -- /bin/bash
attach:
kubectl exec -it $(node)-$(net)-$(number) -- $(geth) attach
kubectl exec -it $(pod) -- $(geth) attach
nodeinfo:
kubectl exec -it $(node)-$(net)-$(number) -- $(geth) attach --exec 'admin.nodeInfo'
kubectl exec -it $(pod) -- $(geth) attach --exec 'admin.nodeInfo'
syncstatus:
kubectl exec -it $(node)-$(net)-$(number) -- $(geth) attach --exec 'eth.syncing'
kubectl exec -it $(pod) -- $(geth) attach --exec 'eth.syncing'
blocknumber:
kubectl exec -it $(node)-$(net)-$(number) -- $(geth) attach --exec 'eth.blockNumber'
kubectl exec -it $(pod) -- $(geth) attach --exec 'eth.blockNumber'
# Add secret for hanzo-ai gcr.io. This enables our cluster to pull images from
# our shared image repo. This should only be run once after cluster creation.
@@ -84,10 +91,6 @@ add-gcr-key:
kubectl create -f config/secret.yaml
kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "gcr-secret"}]}'
# Get credentials for kubectl for current cluster
get-credentials:
gcloud container clusters get-credentials $(node)-$(net) --zone $(zone) --project $(project_id)
# Default to whatever zone you launched your cluster in to reduce key strokes
region-zone-defaults:
gcloud compute project-info add-metadata --metadata=google-compute-default-region=$(region)
bootnode/bootnode.py
+59 -25
View File
@@ -79,30 +79,6 @@ class Bootnode(object):
if blockchain.is_blockchain(chain):
return blockchain
def create_pod(self, name=None):
"""
Create a new pod for a specific chain on a specific network of that
chain. Ex. geth-mainnet
"""
if not name:
name = '{0}-{1}-{2}'.format(self.chain.get_name(), self.network, secrets.randbelow(1000000000000))
print('Creating pod {0}'.format(name))
config = self.chain(name, self.network, self.cluster)
disk_name = config.spec.volumes[0].gcePersistentDisk.pdName
snap = self.gcloud.get_last_snapshot(self.network)
if snap:
snap.create_disk(disk_name)
else:
self.gcloud.create_disk(disk_name)
# pool = self.kube.get_pool(network)
# if not pool:
# self.kube.create_pool(network)
print(self.kube.create_pod(config))
def create_load_balancer(self, name=None):
"""
Create a new load balancer for a given set of pods.
@@ -137,6 +113,31 @@ class Bootnode(object):
self.kube.create_service(service)
self.kube.create_ingress(ingress)
def create_pod(self, name=None):
"""
Create a new pod for a specific chain on a specific network of that
chain. Ex. geth-mainnet
"""
if not name:
name = '{0}-{1}-{2}'.format(self.chain.get_name(), self.network, secrets.randbelow(1000000000000))
print('Creating pod {0}'.format(name))
config = self.chain(name, self.network, self.cluster).spec.template
disk_name = config.spec.volumes[0].gcePersistentDisk.pdName
snap = self.gcloud.get_last_snapshot(self.network)
if snap:
snap.create_disk(disk_name)
else:
self.gcloud.create_disk(disk_name)
# pool = self.kube.get_pool(network)
# if not pool:
# self.kube.create_pool(network)
print(self.kube.create_pod(config))
def delete_pod(self, name):
self.kube.delete_pod(name)
@@ -152,6 +153,39 @@ class Bootnode(object):
def get_synced_pod(self):
table(self.kube.get_synced_pod(network=self.network), 'name', 'phase', 'block_number', 'ip')
def create_deployment(self, name=None):
"""
Create a new deployment for a specific chain on a specific network of that
chain. Ex. geth-mainnet
"""
if not name:
name = '{0}-{1}-{2}'.format(self.chain.get_name(), self.network, secrets.randbelow(1000000000000))
print('Creating deployment {0}'.format(name))
config = self.chain(name, self.network, self.cluster)
disk_name = config.spec.template.spec.volumes[0].gcePersistentDisk.pdName
snap = self.gcloud.get_last_snapshot(self.network)
if snap:
snap.create_disk(disk_name)
else:
self.gcloud.create_disk(disk_name)
# pool = self.kube.get_pool(network)
# if not pool:
# self.kube.create_pool(network)
print(self.kube.create_deployment(config))
def delete_deployment(self, name):
self.kube.delete_deployment(name)
def list_deployments(self, network=None):
table(self.kube.list_deployments(network=network), 'name', 'phase', 'block_number', 'ip')
def get_deployment(self, name):
table(self.kube.get_deployment(name), 'name', 'phase', 'ip')
def get_block_number(self, name):
pod = self.kube.get_pod(name)
print(pod.block_number())
@@ -163,7 +197,7 @@ class Bootnode(object):
chain. Ex. geth-mainnet
"""
print(self.gcloud.create_cluster(self.cluster))
print(self.gcloud.create_cluster(self.chain.get_name(), self.network))
def delete_cluster(self):
"""
bootnode/gcloud.py
+1 -1
View File
@@ -297,7 +297,7 @@ class Gcloud(object):
zone = self.zone
name = "{}-{}".format(chain, network)
version = "1.11.3-gke.18"
version = "1.12.6-gke.10"
body = {
"cluster": {
bootnode/kubernetes.py
+17 -1
View File
@@ -2,7 +2,7 @@ from kubernetes import client, config
from kubernetes.stream import stream
POD_NAMESPACE = 'default'
DEPLOYMENT_NAMESPACE=POD_NAMESPACE
class Node(object):
def __init__(self, node, api=None):
@@ -76,6 +76,7 @@ class Kubernetes(object):
config.load_kube_config(config_path)
self.api = client.CoreV1Api()
self.extApi = client.ExtensionsV1beta1Api()
def exec(self, pod_name, command, namespace=POD_NAMESPACE, stdin=False,
stderr=True, stdout=True, tty=False):
@@ -98,6 +99,21 @@ class Kubernetes(object):
return [p for p in pods if p.network == network]
def create_deployment(self, config):
return self.extApi.create_namespaced_deployment(DEPLOYMENT_NAMESPACE, body=config)
def delete_deployment(self, name):
return self.extApi.delete_namespaced_deployment(name, DEPLOYMENT_NAMESPACE, body=client.V1DeleteOptions())
def list_deployments(self, network=None):
deployments = [deployment(p, self) for p in
self.extApi.list_namespaced_deployment(DEPLOYMENT_NAMESPACE).items]
if not network:
return deployments
return [p for p in deployments if p.network == network]
def get_pod(self, name):
for pod in self.list_pods():
if pod.name == name:
bootnode/template.py
+106 -19
View File
@@ -7,17 +7,27 @@ class Dict(dict):
self.__dict__ = self
class Selector(Dict):
def __init__(self, matchLabels={}, match_expressions=[]):
Dict.__init__(self, matchLabels=matchLabels,
match_expressions=match_expressions)
self.matchLabels = matchLabels
self.match_expressions=[]
class Metadata(Dict):
def __init__(self, name='', cluster='', blockchain='', network='',
annotations=None):
Dict.__init__(self, name=name, cluster=cluster, blockchain=blockchain, network=network)
labels=None, annotations=None):
Dict.__init__(self, name=name, cluster=cluster, blockchain=blockchain,
network=network, labels=labels)
self.name = name
self.cluster = cluster
self.blockchain = blockchain
self.network = network
self.annotations = annotations
self.labels = labels
class Spec(Dict):
class PodSpec(Dict):
def __init__(self, containers=(), volumes=(), ports=(), selector=None,
backend=None):
Dict.__init__(self, selector=selector, containers=containers,
@@ -27,7 +37,6 @@ class Spec(Dict):
self.volumes = volumes
self.backend = backend
class Resources(Dict):
def __init__(self, requests=None, limits=None):
Dict.__init__(self, requests=requests, limits=limits)
@@ -99,9 +108,9 @@ class Port(Dict):
self.targetPort = targetPort
class Config(Dict):
class PodTemplateSpec(Dict):
def __init__(self, apiVersion='v1', kind=None, metadata=Metadata(),
spec=Spec()):
spec=PodSpec()):
Dict.__init__(self, apiVersion=apiVersion, kind=kind,
metadata=metadata, spec=spec)
self.apiVersion = apiVersion
@@ -109,29 +118,70 @@ class Config(Dict):
self.metadata = metadata
self.spec = spec
class Ingress(Config):
class Ingress(PodTemplateSpec):
def __init__(self, metadata=None, spec=None):
super(Ingress, self).__init__(apiVersion='extensions/v1beta1',
super(Ingress, self).__init__(apiVersion='extensions/extensions/v1beta1beta1',
kind='Ingress', metadata=metadata, spec=spec)
class Service(Config):
class Service(PodTemplateSpec):
def __init__(self, metadata=None, spec=None):
super(Service, self).__init__(apiVersion='v1', kind='Service',
super(Service, self).__init__(apiVersion='extensions/v1beta1', kind='Service',
metadata=metadata, spec=spec)
class Pod(Config):
class Pod(PodTemplateSpec):
def __init__(self, metadata=None, spec=None):
super(Pod, self).__init__(apiVersion='v1', kind='Pod',
metadata=metadata, spec=spec)
class DeploymentStrategy(Dict):
def __init__(self, type='RollingUpdate'):
Dict.__init__(self, type=type)
class Blockchain(Pod):
self.type = type
class DeploymentSpec(Dict):
def __init__(self,
min_ready_seconds=0,
paused=False,
progress_deadline_seconds=600,
replicas=1,
revision_history_limit=10,
selector={},
strategy=DeploymentStrategy(),
template=PodTemplateSpec()):
Dict.__init__(self,
min_ready_seconds=min_ready_seconds,
paused=paused,
progress_deadline_seconds=progress_deadline_seconds,
replicas=replicas,
revision_history_limit=revision_history_limit,
selector=selector,
strategy=strategy,
template=template)
self.min_ready_seconds = min_ready_seconds
self.paused = paused
self.progress_deadline_seconds = progress_deadline_seconds
self.replicas = replicas
self.revision_history_limit = revision_history_limit
self.selector = selector
self.strategy = strategy
self.template = template
class Deployment(PodTemplateSpec):
def __init__(self, metadata=None, spec=None):
super(Deployment, self).__init__(apiVersion='extensions/v1beta1', kind='Deployment',
metadata=metadata, spec=spec)
class Blockchain(Deployment):
def __init__(self, name, cluster, blockchain, network, image, command, args, path,
requests=None, limits=None):
"""
Takes blockchain parameters and generates a DeploymentSpec json
"""
self.name = name
self.cluster = cluster
self.blockchain = blockchain
@@ -141,15 +191,41 @@ class Blockchain(Pod):
self.args = args
self.path = path
self.metadata = Metadata(name=name, cluster=cluster, blockchain=blockchain, network=network)
self.deploymentMetadata = Metadata(
name=name,
cluster=cluster,
blockchain=blockchain,
network=network)
self.podMetadata = Metadata(
name=name,
cluster=cluster,
blockchain=blockchain,
network=network,
labels={
'app': name
})
client = os.path.basename(command)
self.spec = Spec(
self.podSpec = PodSpec(
selector={},
containers=[],
volumes=[],
)
self.deploymentSpec = DeploymentSpec(
selector=Selector(
matchLabels={
'app': name
}
),
template=PodTemplateSpec(
metadata=self.podMetadata,
spec=self.podSpec,
)
)
container = Container(
name=name,
image=image,
@@ -164,7 +240,7 @@ class Blockchain(Pod):
if limits:
container.resources.limits = limits
self.spec.containers.append(container)
self.podSpec.containers.append(container)
volume = Volume(
name=name + '-pv',
@@ -174,9 +250,10 @@ class Blockchain(Pod):
)
)
self.spec.volumes.append(volume)
self.podSpec.volumes.append(volume)
super(Blockchain, self).__init__(self.metadata, self.spec)
# Create a PodSpec
super(Blockchain, self).__init__(metadata=self.deploymentMetadata, spec=self.deploymentSpec)
@classmethod
def to_network_id(cls, network):
@@ -206,6 +283,10 @@ class Ethereum(Blockchain):
size=None, rpc=True, ws=True, rpcport=8545, wsport=8546,
rpccorsdomain='*', wsorigins='*', requests=None, limits=None):
"""
Takes Ethereum parameters and generates a DeploymentSpec json
"""
# Normalize networks
network, network_id = Ethereum.normalize_network(network)
@@ -273,6 +354,7 @@ class Ethereum(Blockchain):
'--maxpeers=100',
])
# Generate a PodSpec
super(Ethereum, self).__init__(name, cluster, 'ethereum', network, image,
command, args, path, requests, limits)
@@ -321,6 +403,11 @@ class Ethereum(Blockchain):
class Bitcoin(Blockchain):
def __init__(self, name, network, image, command, args, path,
resources=None, limits=None):
"""
Takes Bitcoin parameters and generates a DeploymentSpec json
"""
#Generate PodSpec
super(Bitcoin, self).__init__(name, 'bitcoin', network, image, command,
args, path, resources, limits)
config/cluster.yaml
+2 -2
View File
@@ -17,10 +17,10 @@ users:
user:
auth-provider:
config:
access-token: ya29.GlxmBuWg5wZdtpwLCJwUpzyIHiUJ-VBgTaHV_x7tIdwAxAz7Aug8lWuwGTH3w0eo0MM8DjTMC6ACIsyo6jWahz6V5Y9y04g17QmSTpssEG8oeb_aJ1QKKiNplkB_xw
access-token: ya29.GlzvBgXYo06yvavGrjZyT92AR4rDO6GTF-Y33PDNrxVn2Xln4nL9l9sCdZP4Ua8xNGax44da-jO8LZl3AODMA62Sab1XMqg_oFEaQFiHOKfF4fKljfMAS6A2b_hTrA
cmd-args: config config-helper --format=json
cmd-path: /usr/local/Caskroom/google-cloud-sdk/latest/google-cloud-sdk/bin/gcloud
expiry: 2018-12-02T21:31:18Z
expiry: 2019-04-18T17:42:21Z
expiry-key: '{.credential.token_expiry}'
token-key: '{.credential.access_token}'
name: gcp
manage.py
+18
View File
@@ -93,6 +93,24 @@ def parser():
get_block_number.add_argument('name', help='Name of pod')
get_block_number.set_defaults(command='get_block_number')
# Deployments
create_deployment = subparsers.add_parser('create-deployment', help='Create deployment')
create_deployment.set_defaults(command='create_deployment')
delete_deployment = subparsers.add_parser('delete-deployment', help='Delete deployment')
delete_deployment.add_argument('name', help='Name of deployment')
delete_deployment.set_defaults(command='delete_deployment')
list_deployments = subparsers.add_parser('deployments', help='List deployments')
list_deployments.set_defaults(command='list_deployments')
get_deployment = subparsers.add_parser('deployment', help='Get deployment')
get_deployment.add_argument('name', help='Name of deployment')
get_deployment.set_defaults(command='get_deployment')
get_last_deployment = subparsers.add_parser('last-deployment', help='Last deployment')
get_last_deployment.set_defaults(command='get_last_deployment')
# Clusters
create_cluster = subparsers.add_parser('create-cluster', help='Create a new cluster')
create_cluster.set_defaults(command='create_cluster')