hanzoai/bootnode
2
0
Fork 0
mirror of https://github.com/hanzoai/bootnode.git synced 2026-10-25 02:31:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

many updates later on friday...

Browse Source
This commit is contained in:
David Tai
2019-05-10 20:58:27 -07:00
parent 34a87918c1
commit 9997ff0a33
10 changed files with 233 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+6 -5
View File
@@ -6,6 +6,7 @@ region := us-central1
zone ?= us-central1-a
number ?= 657758440341
node ?= geth
name ?= node
project_id ?= hanzo-ai
pod ?= $(node)-$(net)-$(number)
@@ -44,9 +45,9 @@ build:
gcloud builds submit \
--machine-type n1-highcpu-8 \
--timeout 30m \
-t gcr.io/$(project_id)/$(node):$(sha1) \
-t gcr.io/$(project_id)/$(node):latest \
$(node)
-t gcr.io/$(project_id)/$(name):$(sha1) \
-t gcr.io/$(project_id)/$(name):latest \
$(name)
# deploy: build
# kubectl set image deployment/$(node)-$(net) $(node)-$(net)=gcr.io/$(project_id)/$(node):$(sha1)
@@ -112,7 +113,7 @@ deps:
# Run Server
serve: kubeconfig
python api.py
python app.py
serve-prod: kubeconfig
python app.py
python main.py
api.py
-165
View File
@@ -1,165 +0,0 @@
from functools import wraps
from flask import Flask, request
from flask_restful import Resource, Api
from flask_cors import CORS
from bootnode import Bootnode
app = Flask(__name__)
CORS(app)
api = Api(app)
SUPPORTED_ZONES = ['us-central1-a', 'europe-west6-a', 'asia-east2-a']
def convert_to_nodes(deployments, services, pods, zone):
nodes_by_id = {}
nodes = []
for d in deployments:
nodes_by_id[d['name']] = len(nodes)
nodes.append({
'blockchain': d['blockchain'],
'network': d['network'],
'id': d['name'],
'instances': [],
'zone': zone,
})
for s in services:
i = nodes_by_id.get(s['name'])
if i is not None:
nodes[i]['ip'] = s['ip']
nodes[i]['ports'] = s['ports']
else:
print('{0} does not exist'.format(s['name']))
for p in pods:
i = nodes_by_id.get('{0}-{1}-{2}'.format(p['blockchain'], p['network'], p['number']))
if i is not None:
nodes[i]['instances'].append({
'name': p['name'],
'status': p['status'],
})
else:
print('{0} does not exist'.format(p['name']))
return nodes
def auth_required(fn):
@wraps(fn)
def wrapped_fn(*args, **kwargs):
print(request.headers.get('Authorization'))
if request.headers.get('Authorization') != 'Bearer fLcLu7OLD81aR9jf':
return {
'status': 'failed',
'error': 'authorization required',
}
return fn(*args, **kwargs)
return wrapped_fn
class Nodes(Resource):
@auth_required
def get(self):
try:
zones = SUPPORTED_ZONES
try:
json = request.get_json()
if json['zone'] is None:
zones = [json['zone']]
except Exception as e:
print('warning: no zone provided, getting all zones: ' + str(e))
nodes = []
for zone in zones:
print('getting nodes in zone: ' + zone)
bootnode = Bootnode('casper', 'testnet', zone)
deployments = [d.to_dict() for d in bootnode.list_deployments()]
services = [s.to_dict() for s in bootnode.list_services()]
pods = [p.to_dict() for p in bootnode.list_pods()]
nodes += convert_to_nodes(deployments, services, pods, zone)
return nodes
except Exception as e:
return {
'status': 'failed',
'error': 'could not get nodes: ' + str(e),
}
@auth_required
def put(self):
try:
json = request.get_json()
print('launching ' + json['number'] + ' nodes in ' + json['zone'])
if json['zone'] not in SUPPORTED_ZONES:
return {
'status': 'failed',
'error': json['zone'] + ' is not a valid zone',
}
bootnode = Bootnode('casper', 'testnet', json['zone'])
number = 1
if json['number'] is not None:
number = int(json['number'])
nodes = []
for i in range(number):
ret = bootnode.create_deployment()
nodes.append(Node().get(ret['deployment'].metadata.name, zone=json['zone']))
return nodes
except Exception as e:
return {
'status': 'failed',
'error': 'could not create a node: ' + str(e)
}
class Node(Resource):
@auth_required
def get(self, node_id, zone='us-central1-a'):
try:
bootnode = Bootnode('casper', 'testnet', zone)
deployment = bootnode.get_deployment(node_id)
service = bootnode.get_service(node_id)
pods = [p.to_dict() for p in bootnode.list_pods(label_selector='app=' + node_id)]
return convert_to_nodes([deployment.to_dict()],
[service.to_dict()], pods, zone)
except Exception as e:
return {
'status': 'failed',
'error': 'node not found: ' + str(e)
}
@auth_required
def delete(self, node_id, zone='us-central1-a'):
try:
json = request.get_json()
print('deleting ' + node_id + ' from zone ' + json['zone'])
zone = json['zone']
bootnode = Bootnode('casper', 'testnet', zone)
bootnode.delete_deployment(node_id)
return {
'status': 'ok',
}
except Exception as e:
return {
'status': 'failed',
'error': 'could not delete: ' + str(e)
}
api.add_resource(Nodes, '/nodes')
api.add_resource(Node, '/nodes/<node_id>')
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port='4000')
app.py
+184 -3
View File
@@ -1,4 +1,185 @@
from waitress import serve
from api import app
from threading import Thread
from functools import wraps
from quart import jsonify, Quart, request
from quart_cors import cors
from bootnode import Bootnode
from util import asynctools, convert
from pymongo import MongoClient
import datetime
import asyncio
serve(app, host='0.0.0.0', port=9090)
app = Quart(__name__)
cors(app)
SUPPORTED_ZONES = ['us-central1-a', 'europe-west6-a', 'asia-east2-a']
# connect to mongo and set up database vars
mongo_client = MongoClient()
bootnode_db = mongo_client.bootnode
nodes_collection = bootnode_db.nodes
updates_collection = bootnode_db.updates
# set up system update loop
def update_nodes_lambda():
print('updating nodes')
try:
date = datetime.datetime.utcnow()
zones = SUPPORTED_ZONES
for zone in zones:
print('getting nodes in zone: ' + zone)
bootnode = Bootnode('casper', 'testnet', zone)
deployments = [d.to_dict() for d in bootnode.list_deployments()]
services = [s.to_dict() for s in bootnode.list_services()]
pods = [p.to_dict() for p in bootnode.list_pods()]
nodes = convert.to_nodes(deployments, services, pods, zone)
for node in nodes:
node['lateUpdated'] = date
nodes_collection.insert_one(node)
except Exception as e:
print('update nodes loop' + str(e))
finally:
updates_collection.update_one(
{
'name': 'nodes',
},
{
'$set': {
'date': date
},
},
True
)
# function to spin off thread
async def update_nodes_loop():
while True:
update_nodes_lambda()
await asyncio.sleep(1)
def update_nodes_daemon():
loop = asyncio.new_event_loop()
task = loop.create_task(update_nodes_loop())
loop.run_until_complete(task)
Thread(target=update_nodes_daemon, daemon=True).start()
def auth_required(fn):
@wraps(fn)
async def wrapped_fn(*args, **kwargs):
print(request.headers.get('Authorization'))
if request.headers.get('Authorization') != 'Bearer fLcLu7OLD81aR9jf':
return jsonify({
'status': 'failed',
'error': 'authorization required',
})
return await fn(*args, **kwargs)
return wrapped_fn
@app.route('/nodes', methods=['GET'])
@auth_required
async def get_nodes():
try:
update = updates_collection.find_one({ 'name': 'nodes' })
print(update)
print('getting node data as of ' + str(update['date']))
nodes = nodes_collection.find({'lateUpdated': update['date']})
ns = []
for node in nodes:
node.pop('_id')
ns.append(node)
return jsonify(ns)
except Exception as e:
return jsonify({
'status': 'failed',
'error': 'could not get nodes: ' + str(e),
})
@app.route('/nodes', methods=['PUT'])
@auth_required
async def put_node():
try:
json = await request.get_json()
print('launching ' + json['number'] + ' nodes in ' + json['zone'])
if json['zone'] not in SUPPORTED_ZONES:
return jsonify({
'status': 'failed',
'error': json['zone'] + ' is not a valid zone',
})
bootnode = Bootnode('casper', 'testnet', json['zone'])
number = 1
if json['number'] is not None:
number = int(json['number'])
def create_deployment():
try:
bootnode.create_deployment()
except Exception as e:
print('could not create deployment: ' + str(e))
nodes = []
for i in range(number):
asyncio.get_running_loop().run_in_executor(None,
create_deployment())
return jsonify({
'status': 'success',
'nodes-starting': number,
})
except Exception as e:
return jsonify({
'status': 'failed',
'error': 'could not create a node: ' + str(e)
})
@app.route('/nodes/<node_id>', methods=['GET'])
@auth_required
async def get_node(node_id, zone='us-central1-a'):
try:
bootnode = Bootnode('casper', 'testnet', zone)
deployment = bootnode.get_deployment(node_id)
service = bootnode.get_service(node_id)
pods = [p.to_dict() for p in bootnode.list_pods(label_selector='app=' + node_id)]
return jsonify(convert.to_nodes([deployment.to_dict()],
[service.to_dict()], pods, zone))
except Exception as e:
return jsonify({
'status': 'failed',
'error': 'node not found: ' + str(e)
})
@app.route('/nodes/<node_id>', methods=['DELETE'])
@auth_required
async def delete_node(node_id, zone='us-central1-a'):
try:
json = await request.get_json()
print('deleting ' + node_id + ' from zone ' + json['zone'])
zone = json['zone']
bootnode = Bootnode('casper', 'testnet', zone)
bootnode.delete_deployment(node_id)
return jsonify({
'status': 'ok',
})
except Exception as e:
return jsonify({
'status': 'failed',
'error': 'could not delete: ' + str(e)
})
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port='4000')
bootnode/bootnode.py
+1 -1
View File
@@ -202,7 +202,7 @@ class Bootnode(object):
print('warning: could not delete service ' + 'service-' + name + ': ' +
str(e))
try:
self.kube.delete_volume(name+'-pd')
self.kube.delete_volume_claim(name+'-pd')
except Exception as e:
print('warning: could not delete volume claim ' + name + '-pd : ' +
str(e))
bootnode/template.py
+33 -15
View File
@@ -302,7 +302,7 @@ class Blockchain(Deployment):
client = os.path.basename(command)
self.podSpec = PodSpec(
podSpec = self.podSpec = PodSpec(
selector={},
containers=[],
volumes=[],
@@ -362,6 +362,7 @@ class Blockchain(Deployment):
self.cluster = cluster
self.blockchain = blockchain
self.network = network
self.podSpec = podSpec
def get_service(self, ports):
return Service(
@@ -608,32 +609,42 @@ class Casper(Blockchain):
# ]
if bootstrapaddress is None:
args.extend(['-s'])
args.extend(['--casper-standalone'])
else:
args.extend([
'--server-bootstrap {0}'.format(bootstrapaddress),
])
if size is None and requests is None:
if network is 'mainnet':
size = 'medium'
elif network is 'testnet':
size = 'small'
else:
size = 'small'
if size is None and requests is None:
if network is 'mainnet':
size = 'medium'
elif network is 'testnet':
size = 'small'
else:
size = 'small'
if size == 'small':
requests = Requests(cpu='1', memory='1Gi')
limits = Limits(cpu='1', memory='1Gi')
if size == 'small':
requests = Requests(cpu='1', memory='1Gi')
limits = Limits(cpu='1', memory='1Gi')
elif size == 'medium':
requests = Requests(cpu='2', memory='2Gi')
limits = Limits(cpu='2', memory='2Gi')
elif size == 'medium':
requests = Requests(cpu='2', memory='2Gi')
limits = Limits(cpu='2', memory='2Gi')
# Generate a PodSpec
super(Casper, self).__init__(name, cluster, 'casper', network, image,
command, args, path, requests, limits)
envoyContainer = Container(
name=name+'-envoy',
image='gcr.io/hanzo-ai/casper-envoy:latest',
command=['envoy'],
args=['--config-path envoyConfig.yaml'],
resources=Resources(limits=Limits(cpu='0.1', memory='0.1Gi')),
)
self.podSpec.containers.append(envoyContainer)
def get_service(self):
ports = []
ports.append(
@@ -657,6 +668,13 @@ class Casper(Blockchain):
port=40404,
targetPort=40404
))
ports.append(
ServicePort(
name='rpc',
protocol='TCP',
port=9001,
targetPort=9001
))
service = super(Casper, self).get_service(ports)
casper/Dockerfile
+1 -46
View File
@@ -1,47 +1,2 @@
# FROM ubuntu:18.04
# RUN apt-get update
# RUN apt-get install curl gnupg openjdk-11-jre-headless -y
# RUN curl -sO http://repo.casperlabs.io/casperlabs/repo/master/casperlabs-node_0.2_all.deb
# RUN curl -sO http://repo.casperlabs.io/casperlabs/repo/master/casperlabs-engine-grpc-server_0.2_amd64.deb
# RUN curl -sO http://repo.casperlabs.io/casperlabs/repo/master/casperlabs-client_0.2_all.deb
# RUN dpkg -i casperlabs-node_0.2_all.deb
# RUN dpkg -i casperlabs-engine-grpc-server_0.2_amd64.deb
# RUN dpkg -i casperlabs-client_0.2_all.deb
# FROM ubuntu:18.04
# RUN apt-get update
# RUN apt-get install curl gnupg openjdk-11-jre-headless -y
# RUN echo "deb https://dl.bintray.com/casperlabs/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
# RUN curl -o casperlabs-public.key.asc https://bintray.com/user/downloadSubjectPublicKey?username=casperlabs
# RUN apt-key add casperlabs-public.key.asc
# RUN apt-get update
# RUN apt install casperlabs -y
FROM openjdk:11-slim
RUN apt-get update
RUN apt-get install psmisc unzip git gnupg apt-transport-https ca-certificates openssl dirmngr -y
RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
RUN apt-get update
RUN apt-get install sbt build-essential cmake curl -y
RUN curl https://sh.rustup.rs -sSf > $HOME/rustup.sh
RUN chmod 775 $HOME/rustup.sh
RUN $HOME/rustup.sh -y
RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.25.0/install.sh | bash
RUN bash -c "source $HOME/.cargo/env; rustup update"
RUN bash -c "source $HOME/.cargo/env; rustup toolchain install nightly"
RUN bash -c "source $HOME/.cargo/env; rustup target add wasm32-unknown-unknown --toolchain nightly"
ENV PROTOC_VERSION=3.6.1
ENV PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip
RUN curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP
RUN unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
RUN rm -f $PROTOC_ZIP
RUN git clone https://github.com/CasperLabs/CasperLabs.git
FROM gcr.io/hanzo-ai/casper-env:latest
COPY scripts/ scripts
RUN ./scripts/build.sh
RUN export
casper/scripts/build.sh
-9
View File
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
mkdir /.casperlabs
cd CasperLabs
sbt -mem 5000 client/universal:stage
sbt -mem 5000 node/universal:stage
cd execution-engine
bash -c "source $HOME/.cargo/env; cargo build"
casper/scripts/start.sh
+2
View File
@@ -18,4 +18,6 @@ KEY_FILE=$(find /.casperlabs -type f -name *.sk | sort | tail -n 1 | head -n 1)
CL_VALIDATOR_PUBLIC_KEY=$(echo "$KEY_FILE" | awk -F '[/.]' '{print $(NF-1)}')
CL_VALIDATOR_PRIVATE_KEY=$(cat "$KEY_FILE")
echo "./node/target/universal/stage/bin/casperlabs-node run --server-data-dir /.casperlabs --casper-validator-public-key $CL_VALIDATOR_PUBLIC_KEY --casper-validator-private-key $CL_VALIDATOR_PRIVATE_KEY --tls-certificate /.casperlabs/node.certificate.pem --tls-key /.casperlabs/node.key.pem --server-host 0.0.0.0 $@"
./node/target/universal/stage/bin/casperlabs-node run --server-data-dir /.casperlabs --casper-validator-public-key $CL_VALIDATOR_PUBLIC_KEY --casper-validator-private-key $CL_VALIDATOR_PRIVATE_KEY --tls-certificate /.casperlabs/node.certificate.pem --tls-key /.casperlabs/node.key.pem --server-host 0.0.0.0 $@
config/cluster.yaml
+2 -2
View File
@@ -25,10 +25,10 @@ users:
user:
auth-provider:
config:
access-token: ya29.Glz9BrkNT5osnnc-_bEoms2dJKEAiVqvEic-QYvSkTy55Dsk9Ag-TpuGLHYdN6-9qZJV7izEF5NYnRiKGa4HYEeqZne8ZJqHL_IoZNQrwBHtF0XMYzcOAo84fELuGg
access-token: ya29.GlwEBwbPyZrCEH40LdxS3FTzgZYigW6BAhYCHymurCcIyxEg80BBmGIVIwNGs_cJuRc9HatxmDEjHrhcaIjHVSz4CoAiGm-S-Ig1z-nLR_jPY_bFiobGT0289aoe-g
cmd-args: config config-helper --format=json
cmd-path: /usr/local/Caskroom/google-cloud-sdk/latest/google-cloud-sdk/bin/gcloud
expiry: 2019-05-01T22:38:43Z
expiry: 2019-05-09T20:37:06Z
expiry-key: '{.credential.token_expiry}'
token-key: '{.credential.access_token}'
name: gcp
requirements.txt
+4 -3
View File
@@ -5,7 +5,8 @@ tabulate
protobuf3-to-dict
google-cloud-container
oauth2client
flask
flask_restful
flask_cors
quart
quart-cors
asyncio
waitress
pymongo